From ac670c698540488c818a17c3749abf3ed4cb764a Mon Sep 17 00:00:00 2001
From: Jonas Knobloch <jonas.knobloch@t-online.de>
Date: Tue, 24 Mar 2026 13:02:49 +0100
Subject: Add tokenizer module

* Wrap MBPE tokenizer
---
 tokenizer/bpe/utility.go | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)
 create mode 100644 tokenizer/bpe/utility.go

(limited to 'tokenizer/bpe/utility.go')

diff --git a/tokenizer/bpe/utility.go b/tokenizer/bpe/utility.go
new file mode 100644
index 0000000..8ad0ca2
--- /dev/null
+++ b/tokenizer/bpe/utility.go
@@ -0,0 +1,20 @@
+package bpe
+
+import "github.com/jonasknobloch/mbpe"
+
+func NewTokenizerFromFiles(vocab, merges string) (*Tokenizer, error) {
+	model := mbpe.NewMBPE()
+
+	if err := model.Load(vocab, merges); err != nil {
+		return nil, err
+	}
+
+	tokenizer := mbpe.NewTokenizer(model)
+
+	pre := mbpe.NewByteLevel(false)
+
+	tokenizer.SetPreTokenizer(pre)
+	tokenizer.SetDecoder(pre)
+
+	return NewTokenizer(tokenizer), nil
+}
-- 
cgit v1.3.1