Add tokenizer module

* Wrap MBPE tokenizer

1 files changed, 25 insertions, 0 deletions

diff --git a/tokenizer/bpe/tokenizer.go b/tokenizer/bpe/tokenizer.go
new file mode 100644
index 0000000..1b65de2
--- /dev/null
+++ b/tokenizer/bpe/tokenizer.go
@@ -0,0 +1,25 @@
+package bpe
+
+import "github.com/jonasknobloch/mbpe"
+
+type Tokenizer struct {
+	mbpe *mbpe.Tokenizer
+}
+
+func NewTokenizer(mbpe *mbpe.Tokenizer) *Tokenizer {
+	return &Tokenizer{
+		mbpe: mbpe,
+	}
+}
+
+func (t *Tokenizer) Encode(s string) []int {
+	return t.mbpe.Tokenize(s)
+}
+
+func (t *Tokenizer) Decode(ids []int) string {
+	panic("unimplemented") // TODO implement
+}
+
+func (t *Tokenizer) Tokenize(s string) []int {
+	return t.Encode(s)
+}