Merge remote-tracking branch 'origin/wip-entropy' into wip-frequency

1 files changed, 49 insertions, 0 deletions

diff --git a/tokenizer/byte/tokenizer_test.go b/tokenizer/byte/tokenizer_test.go
new file mode 100644
index 0000000..d2c8c6c
--- /dev/null
+++ b/tokenizer/byte/tokenizer_test.go
@@ -0,0 +1,49 @@
+package byte
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	"go.jknobloc.com/x/shelf"
+)
+
+func TestNewTokenizer_Encode(t *testing.T) {
+	var file *os.File
+
+	if f, err := os.Open(shelf.Abs("tokenizers/minipile/tokenizer_gpt2_256_m000_minipile/vocab.json")); err != nil {
+		t.Fatal(err)
+	} else {
+		file = f
+	}
+
+	defer file.Close()
+
+	tok, err := NewTokenizer(file)
+
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	fmt.Println(tok.Encode(" \nabc"))
+}
+
+func TestNewTokenizer_Decode(t *testing.T) {
+	var file *os.File
+
+	if f, err := os.Open(shelf.Abs("tokenizers/minipile/tokenizer_gpt2_256_m000_minipile/vocab.json")); err != nil {
+		t.Fatal(err)
+	} else {
+		file = f
+	}
+
+	defer file.Close()
+
+	tok, err := NewTokenizer(file)
+
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	fmt.Println(tok.Decode([]uint8{220, 198, 64, 65, 66}))
+}