diff options
| author | Jonas Knobloch <jonas.knobloch@t-online.de> | 2026-03-31 16:21:28 +0200 |
|---|---|---|
| committer | Jonas Knobloch <jonas.knobloch@t-online.de> | 2026-03-31 17:29:27 +0200 |
| commit | ce9c65cb1e13f7ae6eed4b419232a4da9dd63fdb (patch) | |
| tree | 15aad75e696c50e72d0a0bb6fe9daaf6f87ee1ab /tokenizer/bpe/tokenizer.go | |
| parent | a52a3cd0e6d6d999e93877150adf4c4ec19deff0 (diff) | |
Handle unknown runes
Diffstat (limited to 'tokenizer/bpe/tokenizer.go')
| -rw-r--r-- | tokenizer/bpe/tokenizer.go | 10 |
1 files changed, 9 insertions, 1 deletions
diff --git a/tokenizer/bpe/tokenizer.go b/tokenizer/bpe/tokenizer.go index 1b65de2..39dd7f4 100644 --- a/tokenizer/bpe/tokenizer.go +++ b/tokenizer/bpe/tokenizer.go @@ -20,6 +20,14 @@ func (t *Tokenizer) Decode(ids []int) string { panic("unimplemented") // TODO implement } -func (t *Tokenizer) Tokenize(s string) []int { +func (t *Tokenizer) Tokenize(s string) (ids []int) { + defer func() { + if r := recover(); r != nil { + // defer UnknownRunes(t, s) + + ids = []int{} + } + }() + return t.Encode(s) } |