From ac670c698540488c818a17c3749abf3ed4cb764a Mon Sep 17 00:00:00 2001
From: Jonas Knobloch <jonas.knobloch@t-online.de>
Date: Tue, 24 Mar 2026 13:02:49 +0100
Subject: Add tokenizer module

* Wrap MBPE tokenizer
---
 tokenizer/go.mod | 3 +++
 1 file changed, 3 insertions(+)
 create mode 100644 tokenizer/go.mod

(limited to 'tokenizer/go.mod')

diff --git a/tokenizer/go.mod b/tokenizer/go.mod
new file mode 100644
index 0000000..e66e1f8
--- /dev/null
+++ b/tokenizer/go.mod
@@ -0,0 +1,3 @@
+module go.jknobloc.com/x/tokenizer
+
+go 1.25
-- 
cgit v1.3.1