1 files changed, 9 insertions, 1 deletions
diff --git a/tokenizer/bpe/tokenizer.go b/tokenizer/bpe/tokenizer.go
index 1b65de2..39dd7f4 100644
--- a/tokenizer/bpe/tokenizer.go
+++ b/tokenizer/bpe/tokenizer.go
@@ -20,6 +20,14 @@ func (t *Tokenizer) Decode(ids []int) string {
 	panic("unimplemented") // TODO implement
 }
 
-func (t *Tokenizer) Tokenize(s string) []int {
+func (t *Tokenizer) Tokenize(s string) (ids []int) {
+	defer func() {
+		if r := recover(); r != nil {
+			// defer UnknownRunes(t, s)
+
+			ids = []int{}
+		}
+	}()
+
 	return t.Encode(s)
 }