From e2aa50000f5ac1a0672c25a1e116e81ff54009c8 Mon Sep 17 00:00:00 2001 From: Jonas Knobloch Date: Thu, 30 Apr 2026 13:20:32 +0200 Subject: Manually rename first shard --- llmc/cmd/data/fineweb.go | 12 +++++++++++- llmc/tokenize.go | 8 +------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/llmc/cmd/data/fineweb.go b/llmc/cmd/data/fineweb.go index 599089d..ae96794 100644 --- a/llmc/cmd/data/fineweb.go +++ b/llmc/cmd/data/fineweb.go @@ -1,7 +1,10 @@ package main import ( + "fmt" "log" + "os" + "path" "go.jknobloc.com/x/dataset" "go.jknobloc.com/x/llm" @@ -37,11 +40,18 @@ func fineWeb() { docs := llmc.TokenizeAll(reader, tokenizer, 50256) - if _, err := llmc.WriteShards(shelf.Abs("llmc/edu_fineweb100B"), "edu_fineweb", 100_000_000, docs); err != nil { + if _, err := llmc.WriteShards(shelf.Abs("llmc/edu_fineweb100B"), "edu_fineweb_train", 100_000_000, docs); err != nil { log.Fatal(err) } if err := reader.Err(); err != nil { log.Fatal(err) } + + oldPath := path.Join(shelf.Abs("llmc/edu_fineweb100B"), fmt.Sprintf("edu_fineweb_train_%06d.bin", 0)) + newPath := path.Join(shelf.Abs("llmc/edu_fineweb100B"), fmt.Sprintf("edu_fineweb_val_%06d.bin", 0)) + + if err := os.Rename(oldPath, newPath); err != nil { + log.Fatal(err) + } } diff --git a/llmc/tokenize.go b/llmc/tokenize.go index e90de58..1685168 100644 --- a/llmc/tokenize.go +++ b/llmc/tokenize.go @@ -44,13 +44,7 @@ func WriteShards(path, name string, size int, docs <-chan []uint32) (int, error) n := 0 flush := func() error { - split := "train" - - if n == 0 { - split = "val" - } - - shardName := filepath.Join(path, fmt.Sprintf("%s_%s_%06d.bin", name, split, n)) + shardName := filepath.Join(path, fmt.Sprintf("%s_%06d.bin", name, n)) d := DataFile[uint32]{ Model: GPT2, -- cgit v1.2.3