From 123a37a04b0076c569cbed7c2669ab1087e2cba7 Mon Sep 17 00:00:00 2001
From: Jonas Knobloch <jonas.knobloch@t-online.de>
Date: Wed, 9 Sep 2026 02:43:55 +0200
Subject: WIP

---
 tokenizer/byte/tokenizer_test.go | 49 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 49 insertions(+)
 create mode 100644 tokenizer/byte/tokenizer_test.go

(limited to 'tokenizer/byte/tokenizer_test.go')

diff --git a/tokenizer/byte/tokenizer_test.go b/tokenizer/byte/tokenizer_test.go
new file mode 100644
index 0000000..d2c8c6c
--- /dev/null
+++ b/tokenizer/byte/tokenizer_test.go
@@ -0,0 +1,49 @@
+package byte
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	"go.jknobloc.com/x/shelf"
+)
+
+func TestNewTokenizer_Encode(t *testing.T) {
+	var file *os.File
+
+	if f, err := os.Open(shelf.Abs("tokenizers/minipile/tokenizer_gpt2_256_m000_minipile/vocab.json")); err != nil {
+		t.Fatal(err)
+	} else {
+		file = f
+	}
+
+	defer file.Close()
+
+	tok, err := NewTokenizer(file)
+
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	fmt.Println(tok.Encode(" \nabc"))
+}
+
+func TestNewTokenizer_Decode(t *testing.T) {
+	var file *os.File
+
+	if f, err := os.Open(shelf.Abs("tokenizers/minipile/tokenizer_gpt2_256_m000_minipile/vocab.json")); err != nil {
+		t.Fatal(err)
+	} else {
+		file = f
+	}
+
+	defer file.Close()
+
+	tok, err := NewTokenizer(file)
+
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	fmt.Println(tok.Decode([]uint8{220, 198, 64, 65, 66}))
+}
-- 
cgit v1.3.1