tokenizer/bpe/utility.go


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20

package bpe

import "github.com/jonasknobloch/mbpe"

func NewTokenizerFromFiles(vocab, merges string) (*Tokenizer, error) {
	model := mbpe.NewMBPE()

	if err := model.Load(vocab, merges); err != nil {
		return nil, err
	}

	tokenizer := mbpe.NewTokenizer(model)

	pre := mbpe.NewByteLevel(false)

	tokenizer.SetPreTokenizer(pre)
	tokenizer.SetDecoder(pre)

	return NewTokenizer(tokenizer), nil
}