Manually rename first shard

2 files changed, 12 insertions, 8 deletions

diff --git a/llmc/cmd/data/fineweb.go b/llmc/cmd/data/fineweb.go
index 599089d..ae96794 100644
--- a/llmc/cmd/data/fineweb.go
+++ b/llmc/cmd/data/fineweb.go
@@ -1,7 +1,10 @@
 package main
 
 import (
+	"fmt"
 	"log"
+	"os"
+	"path"
 
 	"go.jknobloc.com/x/dataset"
 	"go.jknobloc.com/x/llm"
@@ -37,11 +40,18 @@ func fineWeb() {
 
 	docs := llmc.TokenizeAll(reader, tokenizer, 50256)
 
-	if _, err := llmc.WriteShards(shelf.Abs("llmc/edu_fineweb100B"), "edu_fineweb", 100_000_000, docs); err != nil {
+	if _, err := llmc.WriteShards(shelf.Abs("llmc/edu_fineweb100B"), "edu_fineweb_train", 100_000_000, docs); err != nil {
 		log.Fatal(err)
 	}
 
 	if err := reader.Err(); err != nil {
 		log.Fatal(err)
 	}
+
+	oldPath := path.Join(shelf.Abs("llmc/edu_fineweb100B"), fmt.Sprintf("edu_fineweb_train_%06d.bin", 0))
+	newPath := path.Join(shelf.Abs("llmc/edu_fineweb100B"), fmt.Sprintf("edu_fineweb_val_%06d.bin", 0))
+
+	if err := os.Rename(oldPath, newPath); err != nil {
+		log.Fatal(err)
+	}
 }
diff --git a/llmc/tokenize.go b/llmc/tokenize.go
index e90de58..1685168 100644
--- a/llmc/tokenize.go
+++ b/llmc/tokenize.go
@@ -44,13 +44,7 @@ func WriteShards(path, name string, size int, docs <-chan []uint32) (int, error)
 	n := 0
 
 	flush := func() error {
-		split := "train"
-
-		if n == 0 {
-			split = "val"
-		}
-
-		shardName := filepath.Join(path, fmt.Sprintf("%s_%s_%06d.bin", name, split, n))
+		shardName := filepath.Join(path, fmt.Sprintf("%s_%06d.bin", name, n))
 
 		d := DataFile[uint32]{
 			Model:  GPT2,