blob: 1b65de2aa7318c1d8c0c5e20c21c318bf99c0895 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
|
package bpe
import "github.com/jonasknobloch/mbpe"
type Tokenizer struct {
mbpe *mbpe.Tokenizer
}
func NewTokenizer(mbpe *mbpe.Tokenizer) *Tokenizer {
return &Tokenizer{
mbpe: mbpe,
}
}
func (t *Tokenizer) Encode(s string) []int {
return t.mbpe.Tokenize(s)
}
func (t *Tokenizer) Decode(ids []int) string {
panic("unimplemented") // TODO implement
}
func (t *Tokenizer) Tokenize(s string) []int {
return t.Encode(s)
}
|
