package byte

import (
	"fmt"
	"os"
	"testing"

	"go.jknobloc.com/x/shelf"
)

func TestNewTokenizer_Encode(t *testing.T) {
	var file *os.File

	if f, err := os.Open(shelf.Abs("tokenizers/minipile/tokenizer_gpt2_256_m000_minipile/vocab.json")); err != nil {
		t.Fatal(err)
	} else {
		file = f
	}

	defer file.Close()

	tok, err := NewTokenizer(file)

	if err != nil {
		t.Fatal(err)
	}

	fmt.Println(tok.Encode(" \nabc"))
}

func TestNewTokenizer_Decode(t *testing.T) {
	var file *os.File

	if f, err := os.Open(shelf.Abs("tokenizers/minipile/tokenizer_gpt2_256_m000_minipile/vocab.json")); err != nil {
		t.Fatal(err)
	} else {
		file = f
	}

	defer file.Close()

	tok, err := NewTokenizer(file)

	if err != nil {
		t.Fatal(err)
	}

	fmt.Println(tok.Decode([]uint8{220, 198, 64, 65, 66}))
}