From 806dcb7f2c01514275dab76116d494345ffb0fd0 Mon Sep 17 00:00:00 2001 From: Jonas Knobloch Date: Thu, 30 Apr 2026 13:10:03 +0200 Subject: Return number of written shards --- llmc/cmd/data/fineweb.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'llmc/cmd/data/fineweb.go') diff --git a/llmc/cmd/data/fineweb.go b/llmc/cmd/data/fineweb.go index a1bf7df..599089d 100644 --- a/llmc/cmd/data/fineweb.go +++ b/llmc/cmd/data/fineweb.go @@ -37,7 +37,7 @@ func fineWeb() { docs := llmc.TokenizeAll(reader, tokenizer, 50256) - if err := llmc.WriteShards(shelf.Abs("llmc/edu_fineweb100B"), "edu_fineweb", 100_000_000, docs); err != nil { + if _, err := llmc.WriteShards(shelf.Abs("llmc/edu_fineweb100B"), "edu_fineweb", 100_000_000, docs); err != nil { log.Fatal(err) } -- cgit v1.3.1