1 files changed, 10 insertions, 0 deletions
diff --git a/tokenizer/bpe/utility.go b/tokenizer/bpe/utility.go
index 79e6faf..cb79d76 100644
--- a/tokenizer/bpe/utility.go
+++ b/tokenizer/bpe/utility.go
@@ -42,3 +42,13 @@ func Merges(t *Tokenizer) [][2]string {
 
 	return m.Merges()
 }
+
+func AddToken(t *Tokenizer, token string) {
+	m, ok := t.mbpe.Model().(*mbpe.MBPE)
+
+	if !ok {
+		panic("unimplemented")
+	}
+
+	m.AddToken(token)
+}