From 330c6387962b197c5ca7a8051d4864feaecc0f25 Mon Sep 17 00:00:00 2001
From: Jonas Knobloch <jonas.knobloch@t-online.de>
Date: Fri, 11 Sep 2026 18:38:02 +0200
Subject: WIP

---
 tokenizer/bpe/special.go | 18 ++++++++++++++++++
 tokenizer/bpe/utility.go | 10 ++++++++++
 2 files changed, 28 insertions(+)
 create mode 100644 tokenizer/bpe/special.go

diff --git a/tokenizer/bpe/special.go b/tokenizer/bpe/special.go
new file mode 100644
index 0000000..405199f
--- /dev/null
+++ b/tokenizer/bpe/special.go
@@ -0,0 +1,18 @@
+package bpe
+
+// Add helper to ensure special token
+// wip-special-tokens
+
+func EnsureSpecialToken(t *Tokenizer, token string) int {
+	vocab := Vocab(t)
+
+	for i, v := range vocab {
+		if v == token {
+			return i
+		}
+	}
+
+	AddToken(t, token)
+
+	return len(vocab) - 1
+}
diff --git a/tokenizer/bpe/utility.go b/tokenizer/bpe/utility.go
index 79e6faf..cb79d76 100644
--- a/tokenizer/bpe/utility.go
+++ b/tokenizer/bpe/utility.go
@@ -42,3 +42,13 @@ func Merges(t *Tokenizer) [][2]string {
 
 	return m.Merges()
 }
+
+func AddToken(t *Tokenizer, token string) {
+	m, ok := t.mbpe.Model().(*mbpe.MBPE)
+
+	if !ok {
+		panic("unimplemented")
+	}
+
+	m.AddToken(token)
+}
-- 
cgit v1.2.3