From 12f15fdd8c048e5958e294f12a9f163f6f950b3e Mon Sep 17 00:00:00 2001 From: Jonas Knobloch Date: Fri, 24 Apr 2026 11:49:26 +0200 Subject: Recover tokenization based on config --- llmc/cmd/debug/debug.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'llmc/cmd/debug/debug.go') diff --git a/llmc/cmd/debug/debug.go b/llmc/cmd/debug/debug.go index d1a621a..f076949 100644 --- a/llmc/cmd/debug/debug.go +++ b/llmc/cmd/debug/debug.go @@ -16,7 +16,10 @@ func main() { _ = must(llmc.Deserialize(shelf.Abs("llmc/tinyshakespeare/tiny_shakespeare_train.bin"), &data)) _ = must(llmc.Deserialize(shelf.Abs("test/llmc/tinyshakespeare/tiny_shakespeare_train.bin"), &gold)) - t := must(bpe.NewTokenizerFromFiles(shelf.Abs("models/gpt2/vocab.json"), shelf.Abs("models/gpt2/merges.txt"))) + v := shelf.Abs("models/gpt2/vocab.json") + m := shelf.Abs("models/gpt2/merges.txt") + + t := must(bpe.NewTokenizerFromFiles(v, m, bpe.DefaultConfig())) itoa := bpe.Itoa(t) -- cgit v1.3.1