From e2aa50000f5ac1a0672c25a1e116e81ff54009c8 Mon Sep 17 00:00:00 2001
From: Jonas Knobloch <jonas.knobloch@t-online.de>
Date: Thu, 30 Apr 2026 13:20:32 +0200
Subject: Manually rename first shard

---
 llmc/tokenize.go | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

(limited to 'llmc/tokenize.go')

diff --git a/llmc/tokenize.go b/llmc/tokenize.go
index e90de58..1685168 100644
--- a/llmc/tokenize.go
+++ b/llmc/tokenize.go
@@ -44,13 +44,7 @@ func WriteShards(path, name string, size int, docs <-chan []uint32) (int, error)
 	n := 0
 
 	flush := func() error {
-		split := "train"
-
-		if n == 0 {
-			split = "val"
-		}
-
-		shardName := filepath.Join(path, fmt.Sprintf("%s_%s_%06d.bin", name, split, n))
+		shardName := filepath.Join(path, fmt.Sprintf("%s_%06d.bin", name, n))
 
 		d := DataFile[uint32]{
 			Model:  GPT2,
-- 
cgit v1.3.1