From f796560cb4de0b6de118c00be54c58206bad5b60 Mon Sep 17 00:00:00 2001 From: Jonas Knobloch Date: Tue, 24 Mar 2026 15:37:50 +0100 Subject: Expose finite state automaton --- tokenizer/bpe/utility.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tokenizer/bpe/utility.go') diff --git a/tokenizer/bpe/utility.go b/tokenizer/bpe/utility.go index 8ad0ca2..f47c073 100644 --- a/tokenizer/bpe/utility.go +++ b/tokenizer/bpe/utility.go @@ -11,7 +11,7 @@ func NewTokenizerFromFiles(vocab, merges string) (*Tokenizer, error) { tokenizer := mbpe.NewTokenizer(model) - pre := mbpe.NewByteLevel(false) + pre := NewByteLevel(false) tokenizer.SetPreTokenizer(pre) tokenizer.SetDecoder(pre) -- cgit v1.3.1