From 6e6a94b12d6d43c1ce4b8e587742627af6d86a43 Mon Sep 17 00:00:00 2001
From: Jonas Knobloch <jonas.knobloch@t-online.de>
Date: Fri, 11 Sep 2026 18:36:45 +0200
Subject: WIP

---
 tokenizer/bpe/utility.go | 4 ++++
 1 file changed, 4 insertions(+)

(limited to 'tokenizer/bpe')

diff --git a/tokenizer/bpe/utility.go b/tokenizer/bpe/utility.go
index f8fcdbc..79e6faf 100644
--- a/tokenizer/bpe/utility.go
+++ b/tokenizer/bpe/utility.go
@@ -19,6 +19,10 @@ func NewTokenizerFromFiles(vocab, merges string, cfg Config) (*Tokenizer, error)
 	return NewTokenizer(tokenizer, cfg), nil
 }
 
+func MBPE(t *Tokenizer) *mbpe.Tokenizer {
+	return t.mbpe
+}
+
 func Vocab(t *Tokenizer) []string {
 	m, ok := t.mbpe.Model().(*mbpe.MBPE)
 
-- 
cgit v1.3.1