From 12f15fdd8c048e5958e294f12a9f163f6f950b3e Mon Sep 17 00:00:00 2001 From: Jonas Knobloch Date: Fri, 24 Apr 2026 11:49:26 +0200 Subject: Recover tokenization based on config --- tokenizer/bpe/cmd/tokenize/main.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'tokenizer/bpe/cmd/tokenize') diff --git a/tokenizer/bpe/cmd/tokenize/main.go b/tokenizer/bpe/cmd/tokenize/main.go index 13926b7..e6cab4b 100644 --- a/tokenizer/bpe/cmd/tokenize/main.go +++ b/tokenizer/bpe/cmd/tokenize/main.go @@ -100,7 +100,10 @@ func data() *dataset.ParquetReader { func tokenizer() llm.Tokenizer { var tok *bpe.Tokenizer - if t, err := bpe.NewTokenizerFromFiles(shelf.Abs("models/gpt2/vocab.json"), shelf.Abs("models/gpt2/merges.txt")); err != nil { + v := shelf.Abs("models/gpt2/vocab.json") + m := shelf.Abs("models/gpt2/merges.txt") + + if t, err := bpe.NewTokenizerFromFiles(v, m, bpe.DefaultConfig()); err != nil { log.Fatal(err) } else { tok = t -- cgit v1.3.1