From 12f15fdd8c048e5958e294f12a9f163f6f950b3e Mon Sep 17 00:00:00 2001
From: Jonas Knobloch <jonas.knobloch@t-online.de>
Date: Fri, 24 Apr 2026 11:49:26 +0200
Subject: Recover tokenization based on config

---
 tokenizer/bpe/tokenizer.go | 24 ++++++++++++++----------
 1 file changed, 14 insertions(+), 10 deletions(-)

(limited to 'tokenizer/bpe/tokenizer.go')

diff --git a/tokenizer/bpe/tokenizer.go b/tokenizer/bpe/tokenizer.go
index 9bc351c..7e2459a 100644
--- a/tokenizer/bpe/tokenizer.go
+++ b/tokenizer/bpe/tokenizer.go
@@ -3,12 +3,14 @@ package bpe
 import "github.com/jonasknobloch/mbpe"
 
 type Tokenizer struct {
-	mbpe *mbpe.Tokenizer
+	mbpe   *mbpe.Tokenizer
+	config Config
 }
 
-func NewTokenizer(mbpe *mbpe.Tokenizer) *Tokenizer {
+func NewTokenizer(mbpe *mbpe.Tokenizer, cfg Config) *Tokenizer {
 	return &Tokenizer{
-		mbpe: mbpe,
+		mbpe:   mbpe,
+		config: cfg,
 	}
 }
 
@@ -29,13 +31,15 @@ func (t *Tokenizer) Decode(ids []int) string {
 }
 
 func (t *Tokenizer) Tokenize(s string) (ids []int) {
-	defer func() {
-		if r := recover(); r != nil {
-			// defer UnknownRunes(t, s)
-
-			ids = []int{}
-		}
-	}()
+	if t.config.Recover {
+		defer func() {
+			if r := recover(); r != nil {
+				// defer UnknownRunes(t, s)
+
+				ids = []int{}
+			}
+		}()
+	}
 
 	return t.Encode(s)
 }
-- 
cgit v1.3.1