From ac670c698540488c818a17c3749abf3ed4cb764a Mon Sep 17 00:00:00 2001
From: Jonas Knobloch <jonas.knobloch@t-online.de>
Date: Tue, 24 Mar 2026 13:02:49 +0100
Subject: Add tokenizer module

* Wrap MBPE tokenizer
---
 go.work | 1 +
 1 file changed, 1 insertion(+)

(limited to 'go.work')

diff --git a/go.work b/go.work
index c6ed48e..61c84dd 100644
--- a/go.work
+++ b/go.work
@@ -6,5 +6,6 @@ use (
 	./llm
 	./mbpe
 	./tensor
+	./tokenizer
 	./tui
 )
-- 
cgit v1.3.1