From 12f15fdd8c048e5958e294f12a9f163f6f950b3e Mon Sep 17 00:00:00 2001
From: Jonas Knobloch <jonas.knobloch@t-online.de>
Date: Fri, 24 Apr 2026 11:49:26 +0200
Subject: Recover tokenization based on config

---
 tokenizer/bpe/config.go | 11 +++++++++++
 1 file changed, 11 insertions(+)
 create mode 100644 tokenizer/bpe/config.go

(limited to 'tokenizer/bpe/config.go')

diff --git a/tokenizer/bpe/config.go b/tokenizer/bpe/config.go
new file mode 100644
index 0000000..aef62c9
--- /dev/null
+++ b/tokenizer/bpe/config.go
@@ -0,0 +1,11 @@
+package bpe
+
+type Config struct {
+	Recover bool
+}
+
+func DefaultConfig() Config {
+	return Config{
+		Recover: false,
+	}
+}
-- 
cgit v1.3.1