diff options
| author | Jonas Knobloch <jonas.knobloch@t-online.de> | 2026-03-24 15:37:50 +0100 |
|---|---|---|
| committer | Jonas Knobloch <jonas.knobloch@t-online.de> | 2026-03-24 15:42:10 +0100 |
| commit | f796560cb4de0b6de118c00be54c58206bad5b60 (patch) | |
| tree | da12930d66ec6511abb723b3ef811a310c07bdb0 /tokenizer/bpe/utility.go | |
| parent | a63589d6584548b286825f4fb23202c35382032c (diff) | |
Expose finite state automaton
Diffstat (limited to 'tokenizer/bpe/utility.go')
| -rw-r--r-- | tokenizer/bpe/utility.go | 2 |
1 files changed, 1 insertions, 1 deletions
diff --git a/tokenizer/bpe/utility.go b/tokenizer/bpe/utility.go index 8ad0ca2..f47c073 100644 --- a/tokenizer/bpe/utility.go +++ b/tokenizer/bpe/utility.go @@ -11,7 +11,7 @@ func NewTokenizerFromFiles(vocab, merges string) (*Tokenizer, error) { tokenizer := mbpe.NewTokenizer(model) - pre := mbpe.NewByteLevel(false) + pre := NewByteLevel(false) tokenizer.SetPreTokenizer(pre) tokenizer.SetDecoder(pre) |