1 files changed, 5 insertions, 0 deletions
diff --git a/llm/tokenizer.go b/llm/tokenizer.go
new file mode 100644
index 0000000..c6ea3c1
--- /dev/null
+++ b/llm/tokenizer.go
@@ -0,0 +1,5 @@
+package llm
+
+type Tokenizer interface {
+	Tokenize(s string) []int
+}