From 7073b124c5eb31169442ef18d1627b7ed7401280 Mon Sep 17 00:00:00 2001
From: Jonas Knobloch <jonas.knobloch@t-online.de>
Date: Thu, 13 Nov 2025 23:04:57 +0100
Subject: Scaffold byte-pair correction

---
 llm/tokenizer.go | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 llm/tokenizer.go

(limited to 'llm/tokenizer.go')

diff --git a/llm/tokenizer.go b/llm/tokenizer.go
new file mode 100644
index 0000000..c6ea3c1
--- /dev/null
+++ b/llm/tokenizer.go
@@ -0,0 +1,5 @@
+package llm
+
+type Tokenizer interface {
+	Tokenize(s string) []int
+}
-- 
cgit v1.3.1