diff options
| author | Jonas Knobloch <jonas.knobloch@t-online.de> | 2026-04-30 13:20:32 +0200 |
|---|---|---|
| committer | Jonas Knobloch <jonas.knobloch@t-online.de> | 2026-04-30 13:20:32 +0200 |
| commit | e2aa50000f5ac1a0672c25a1e116e81ff54009c8 (patch) | |
| tree | 61744560452a0c6f69b1b14f8fd8099b9a48004b /llmc/tokenize.go | |
| parent | 806dcb7f2c01514275dab76116d494345ffb0fd0 (diff) | |
Manually rename first shard
Diffstat (limited to 'llmc/tokenize.go')
| -rw-r--r-- | llmc/tokenize.go | 8 |
1 files changed, 1 insertions, 7 deletions
diff --git a/llmc/tokenize.go b/llmc/tokenize.go index e90de58..1685168 100644 --- a/llmc/tokenize.go +++ b/llmc/tokenize.go @@ -44,13 +44,7 @@ func WriteShards(path, name string, size int, docs <-chan []uint32) (int, error) n := 0 flush := func() error { - split := "train" - - if n == 0 { - split = "val" - } - - shardName := filepath.Join(path, fmt.Sprintf("%s_%s_%06d.bin", name, split, n)) + shardName := filepath.Join(path, fmt.Sprintf("%s_%06d.bin", name, n)) d := DataFile[uint32]{ Model: GPT2, |