blob: f47c073646af64568d9d7bec71acc6abe3ccd7f7 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
|
package bpe
import "github.com/jonasknobloch/mbpe"
func NewTokenizerFromFiles(vocab, merges string) (*Tokenizer, error) {
model := mbpe.NewMBPE()
if err := model.Load(vocab, merges); err != nil {
return nil, err
}
tokenizer := mbpe.NewTokenizer(model)
pre := NewByteLevel(false)
tokenizer.SetPreTokenizer(pre)
tokenizer.SetDecoder(pre)
return NewTokenizer(tokenizer), nil
}
|
