diff options
Diffstat (limited to 'tokenizer/bpe/utility.go')
| -rw-r--r-- | tokenizer/bpe/utility.go | 2 |
1 files changed, 1 insertions, 1 deletions
diff --git a/tokenizer/bpe/utility.go b/tokenizer/bpe/utility.go index 8ad0ca2..f47c073 100644 --- a/tokenizer/bpe/utility.go +++ b/tokenizer/bpe/utility.go @@ -11,7 +11,7 @@ func NewTokenizerFromFiles(vocab, merges string) (*Tokenizer, error) { tokenizer := mbpe.NewTokenizer(model) - pre := mbpe.NewByteLevel(false) + pre := NewByteLevel(false) tokenizer.SetPreTokenizer(pre) tokenizer.SetDecoder(pre) |