From 12f15fdd8c048e5958e294f12a9f163f6f950b3e Mon Sep 17 00:00:00 2001 From: Jonas Knobloch Date: Fri, 24 Apr 2026 11:49:26 +0200 Subject: Recover tokenization based on config --- llmc/cmd/data/fineweb.go | 5 ++++- llmc/cmd/data/tinyshakespeare.go | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) (limited to 'llmc/cmd/data') diff --git a/llmc/cmd/data/fineweb.go b/llmc/cmd/data/fineweb.go index 3d3d51b..a1bf7df 100644 --- a/llmc/cmd/data/fineweb.go +++ b/llmc/cmd/data/fineweb.go @@ -26,7 +26,10 @@ func fineWeb() { var tokenizer llm.Tokenizer - if t, err := bpe.NewTokenizerFromFiles(shelf.Abs("models/gpt2/vocab.json"), shelf.Abs("models/gpt2/merges.txt")); err != nil { + v := shelf.Abs("models/gpt2/vocab.json") + m := shelf.Abs("models/gpt2/merges.txt") + + if t, err := bpe.NewTokenizerFromFiles(v, m, bpe.DefaultConfig()); err != nil { log.Fatal(err) } else { tokenizer = t diff --git a/llmc/cmd/data/tinyshakespeare.go b/llmc/cmd/data/tinyshakespeare.go index 70d4fd4..d71151b 100644 --- a/llmc/cmd/data/tinyshakespeare.go +++ b/llmc/cmd/data/tinyshakespeare.go @@ -23,7 +23,10 @@ func tinyShakespeare() { var tokenizer *bpe.Tokenizer - if t, err := bpe.NewTokenizerFromFiles(shelf.Abs("models/gpt2/vocab.json"), shelf.Abs("models/gpt2/merges.txt")); err != nil { + v := shelf.Abs("models/gpt2/vocab.json") + m := shelf.Abs("models/gpt2/merges.txt") + + if t, err := bpe.NewTokenizerFromFiles(v, m, bpe.DefaultConfig()); err != nil { log.Fatal(err) } else { tokenizer = t -- cgit v1.3.1