diff options
| -rw-r--r-- | tokenizer/bpe/tokenizer.go | 10 |
1 files changed, 9 insertions, 1 deletions
diff --git a/tokenizer/bpe/tokenizer.go b/tokenizer/bpe/tokenizer.go index 39dd7f4..9bc351c 100644 --- a/tokenizer/bpe/tokenizer.go +++ b/tokenizer/bpe/tokenizer.go @@ -17,7 +17,15 @@ func (t *Tokenizer) Encode(s string) []int { } func (t *Tokenizer) Decode(ids []int) string { - panic("unimplemented") // TODO implement + d := t.mbpe.Decoder() + + m, ok := t.mbpe.Model().(*mbpe.MBPE) + + if !ok { + panic("unsupported model") + } + + return d.Decode(m.ToString(ids)) } func (t *Tokenizer) Tokenize(s string) (ids []int) { |