package byte

import (
	"encoding/json"
	"io"

	"github.com/jonasknobloch/mbpe"
)

// type parameter for ID type in bpe tokenzier ?!

// Name? Byte is bad; it is a bijection but so are most if not all tokenizers; its the Alphabet?
// It would be compativle with GenericTokenizer[uint8] without pretokenization

// Tokenizer is a byte level tokenzier covering the all 2^8 bytes; Essentially each individual byte is mapped to a token ID.
// Serialization follows the standard vocab.json (with HF byte replacements) while omitting merges.txt
// Pre-tokenization is not necessary; however the tokenized strings can be much larger -> check allocations
type Tokenizer struct {
	atoi map[byte]uint8
	itoa map[uint8]byte
}

func (t *Tokenizer) Encode(s string) []byte {
	b := make([]byte, len(s))

	for i := range len(s) {
		v, ok := t.atoi[s[i]]

		if !ok {
			panic("unknown byte")
		}

		b[i] = v
	}

	return b
}

func (t *Tokenizer) Decode(ids []uint8) string {
	b := make([]byte, len(ids))

	for i, id := range ids {
		v, ok := t.itoa[id]

		if !ok {
			panic("unknown token ID")
		}

		b[i] = v
	}

	return string(b)
}

func (t *Tokenizer) Tokenize(s string) []int {
	ids := t.Encode(s)

	r := make([]int, len(ids))

	for i, id := range ids {
		r[i] = int(id)
	}

	return r
}

func NewTokenizer(vocab io.Reader) (*Tokenizer, error) {
	v := make(map[string]uint8)

	decoder := json.NewDecoder(vocab)

	if err := decoder.Decode(&v); err != nil {
		return nil, err
	}

	if len(v) != 256 {
		panic("vocabulary size != 256")
	}

	atoi := make(map[byte]uint8, len(v))
	itoa := make(map[uint8]byte, len(v))

	for char, id := range v {
		b := mbpe.CharBytes[char]

		atoi[b] = id
		itoa[id] = b
	}

	return &Tokenizer{
		atoi: atoi,
		itoa: itoa,
	}, nil
}