From 3c5f4c4279e6c9d6479e40eff4fbe1add59c1b3a Mon Sep 17 00:00:00 2001
From: Jonas Knobloch <jonas.knobloch@t-online.de>
Date: Wed, 9 Sep 2026 02:41:24 +0200
Subject: WIP

---
 tokenizer/bpe/utility.go | 4 ++++
 1 file changed, 4 insertions(+)

(limited to 'tokenizer')

diff --git a/tokenizer/bpe/utility.go b/tokenizer/bpe/utility.go
index f8fcdbc..79e6faf 100644
--- a/tokenizer/bpe/utility.go
+++ b/tokenizer/bpe/utility.go
@@ -19,6 +19,10 @@ func NewTokenizerFromFiles(vocab, merges string, cfg Config) (*Tokenizer, error)
 	return NewTokenizer(tokenizer, cfg), nil
 }
 
+func MBPE(t *Tokenizer) *mbpe.Tokenizer {
+	return t.mbpe
+}
+
 func Vocab(t *Tokenizer) []string {
 	m, ok := t.mbpe.Model().(*mbpe.MBPE)
 
-- 
cgit v1.3.1