diff options
| author | Jonas Knobloch <jonas.knobloch@t-online.de> | 2026-04-21 19:49:54 +0200 |
|---|---|---|
| committer | Jonas Knobloch <jonas.knobloch@t-online.de> | 2026-04-22 12:54:45 +0200 |
| commit | d72bda93664657c341f605a9b238dcb88d63dcf8 (patch) | |
| tree | 2968600db7cd40fcc30699c34c63b149d6fcfcd8 | |
| parent | 2f8e06698271c077fdd9a88dfe3ad5db90f55f83 (diff) | |
Implement decoding
| -rw-r--r-- | tokenizer/bpe/tokenizer.go | 10 |
1 files changed, 9 insertions, 1 deletions
diff --git a/tokenizer/bpe/tokenizer.go b/tokenizer/bpe/tokenizer.go index 39dd7f4..9bc351c 100644 --- a/tokenizer/bpe/tokenizer.go +++ b/tokenizer/bpe/tokenizer.go @@ -17,7 +17,15 @@ func (t *Tokenizer) Encode(s string) []int { } func (t *Tokenizer) Decode(ids []int) string { - panic("unimplemented") // TODO implement + d := t.mbpe.Decoder() + + m, ok := t.mbpe.Model().(*mbpe.MBPE) + + if !ok { + panic("unsupported model") + } + + return d.Decode(m.ToString(ids)) } func (t *Tokenizer) Tokenize(s string) (ids []int) { |