WIP

1 files changed, 18 insertions, 0 deletions

diff --git a/tokenizer/bpe/special.go b/tokenizer/bpe/special.go
new file mode 100644
index 0000000..405199f
--- /dev/null
+++ b/tokenizer/bpe/special.go
@@ -0,0 +1,18 @@
+package bpe
+
+// Add helper to ensure special token
+// wip-special-tokens
+
+func EnsureSpecialToken(t *Tokenizer, token string) int {
+	vocab := Vocab(t)
+
+	for i, v := range vocab {
+		if v == token {
+			return i
+		}
+	}
+
+	AddToken(t, token)
+
+	return len(vocab) - 1
+}