From ce9c65cb1e13f7ae6eed4b419232a4da9dd63fdb Mon Sep 17 00:00:00 2001
From: Jonas Knobloch <jonas.knobloch@t-online.de>
Date: Tue, 31 Mar 2026 16:21:28 +0200
Subject: Handle unknown runes

---
 tokenizer/bpe/tokenizer.go | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

(limited to 'tokenizer/bpe/tokenizer.go')

diff --git a/tokenizer/bpe/tokenizer.go b/tokenizer/bpe/tokenizer.go
index 1b65de2..39dd7f4 100644
--- a/tokenizer/bpe/tokenizer.go
+++ b/tokenizer/bpe/tokenizer.go
@@ -20,6 +20,14 @@ func (t *Tokenizer) Decode(ids []int) string {
 	panic("unimplemented") // TODO implement
 }
 
-func (t *Tokenizer) Tokenize(s string) []int {
+func (t *Tokenizer) Tokenize(s string) (ids []int) {
+	defer func() {
+		if r := recover(); r != nil {
+			// defer UnknownRunes(t, s)
+
+			ids = []int{}
+		}
+	}()
+
 	return t.Encode(s)
 }
-- 
cgit v1.3.1