diff options
| author | Jonas Knobloch <jonas.knobloch@t-online.de> | 2026-04-24 11:49:26 +0200 |
|---|---|---|
| committer | Jonas Knobloch <jonas.knobloch@t-online.de> | 2026-04-24 11:50:02 +0200 |
| commit | 12f15fdd8c048e5958e294f12a9f163f6f950b3e (patch) | |
| tree | 16bb23c134f1fe476d977dd846e6dda6c7667c48 /llmc/cmd/data/fineweb.go | |
| parent | 408536516bdfedc93d93226dc08280ed224361fc (diff) | |
Recover tokenization based on config
Diffstat (limited to 'llmc/cmd/data/fineweb.go')
| -rw-r--r-- | llmc/cmd/data/fineweb.go | 5 |
1 files changed, 4 insertions, 1 deletions
diff --git a/llmc/cmd/data/fineweb.go b/llmc/cmd/data/fineweb.go index 3d3d51b..a1bf7df 100644 --- a/llmc/cmd/data/fineweb.go +++ b/llmc/cmd/data/fineweb.go @@ -26,7 +26,10 @@ func fineWeb() { var tokenizer llm.Tokenizer - if t, err := bpe.NewTokenizerFromFiles(shelf.Abs("models/gpt2/vocab.json"), shelf.Abs("models/gpt2/merges.txt")); err != nil { + v := shelf.Abs("models/gpt2/vocab.json") + m := shelf.Abs("models/gpt2/merges.txt") + + if t, err := bpe.NewTokenizerFromFiles(v, m, bpe.DefaultConfig()); err != nil { log.Fatal(err) } else { tokenizer = t |