blob: fbe99fcd182ac31c754196ecf6277449faec9709 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
|
package bpe
import "github.com/jonasknobloch/mbpe"
func NewTokenizerFromFiles(vocab, merges string) (*Tokenizer, error) {
model := mbpe.NewMBPE()
if err := model.Load(vocab, merges); err != nil {
return nil, err
}
tokenizer := mbpe.NewTokenizer(model)
pre := NewByteLevel(false)
tokenizer.SetPreTokenizer(pre)
tokenizer.SetDecoder(pre)
return NewTokenizer(tokenizer), nil
}
func Vocab(t *Tokenizer) []string {
m, ok := t.mbpe.Model().(*mbpe.MBPE)
if !ok {
panic("unimplemented")
}
return m.Vocab()
}
func Merges(t *Tokenizer) [][2]string {
m, ok := t.mbpe.Model().(*mbpe.MBPE)
if !ok {
panic("unimplemented")
}
return m.Merges()
}
|
