Add tokenizer module

* Wrap MBPE tokenizer

1 files changed, 20 insertions, 0 deletions

diff --git a/tokenizer/bpe/utility.go b/tokenizer/bpe/utility.go
new file mode 100644
index 0000000..8ad0ca2
--- /dev/null
+++ b/tokenizer/bpe/utility.go
@@ -0,0 +1,20 @@
+package bpe
+
+import "github.com/jonasknobloch/mbpe"
+
+func NewTokenizerFromFiles(vocab, merges string) (*Tokenizer, error) {
+	model := mbpe.NewMBPE()
+
+	if err := model.Load(vocab, merges); err != nil {
+		return nil, err
+	}
+
+	tokenizer := mbpe.NewTokenizer(model)
+
+	pre := mbpe.NewByteLevel(false)
+
+	tokenizer.SetPreTokenizer(pre)
+	tokenizer.SetDecoder(pre)
+
+	return NewTokenizer(tokenizer), nil
+}