diff options
| author | Jonas Knobloch <jonas.knobloch@t-online.de> | 2026-04-30 13:10:03 +0200 |
|---|---|---|
| committer | Jonas Knobloch <jonas.knobloch@t-online.de> | 2026-04-30 13:19:08 +0200 |
| commit | 806dcb7f2c01514275dab76116d494345ffb0fd0 (patch) | |
| tree | d436e8ca6fd71942d72d37d403f78b7abdf52a1b /llmc/cmd/data | |
| parent | 4740e5aa19012e5f8e67ac8cb857d9741208c364 (diff) | |
Return number of written shards
Diffstat (limited to 'llmc/cmd/data')
| -rw-r--r-- | llmc/cmd/data/fineweb.go | 2 |
1 files changed, 1 insertions, 1 deletions
diff --git a/llmc/cmd/data/fineweb.go b/llmc/cmd/data/fineweb.go index a1bf7df..599089d 100644 --- a/llmc/cmd/data/fineweb.go +++ b/llmc/cmd/data/fineweb.go @@ -37,7 +37,7 @@ func fineWeb() { docs := llmc.TokenizeAll(reader, tokenizer, 50256) - if err := llmc.WriteShards(shelf.Abs("llmc/edu_fineweb100B"), "edu_fineweb", 100_000_000, docs); err != nil { + if _, err := llmc.WriteShards(shelf.Abs("llmc/edu_fineweb100B"), "edu_fineweb", 100_000_000, docs); err != nil { log.Fatal(err) } |