blob: 39dd7f4e0640c66c3adc9c861a84692482f8ad24 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
|
package bpe
import "github.com/jonasknobloch/mbpe"
type Tokenizer struct {
mbpe *mbpe.Tokenizer
}
func NewTokenizer(mbpe *mbpe.Tokenizer) *Tokenizer {
return &Tokenizer{
mbpe: mbpe,
}
}
func (t *Tokenizer) Encode(s string) []int {
return t.mbpe.Tokenize(s)
}
func (t *Tokenizer) Decode(ids []int) string {
panic("unimplemented") // TODO implement
}
func (t *Tokenizer) Tokenize(s string) (ids []int) {
defer func() {
if r := recover(); r != nil {
// defer UnknownRunes(t, s)
ids = []int{}
}
}()
return t.Encode(s)
}
|
