From ac670c698540488c818a17c3749abf3ed4cb764a Mon Sep 17 00:00:00 2001 From: Jonas Knobloch Date: Tue, 24 Mar 2026 13:02:49 +0100 Subject: Add tokenizer module * Wrap MBPE tokenizer --- tokenizer/bpe/tokenizer.go | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 tokenizer/bpe/tokenizer.go (limited to 'tokenizer/bpe/tokenizer.go') diff --git a/tokenizer/bpe/tokenizer.go b/tokenizer/bpe/tokenizer.go new file mode 100644 index 0000000..1b65de2 --- /dev/null +++ b/tokenizer/bpe/tokenizer.go @@ -0,0 +1,25 @@ +package bpe + +import "github.com/jonasknobloch/mbpe" + +type Tokenizer struct { + mbpe *mbpe.Tokenizer +} + +func NewTokenizer(mbpe *mbpe.Tokenizer) *Tokenizer { + return &Tokenizer{ + mbpe: mbpe, + } +} + +func (t *Tokenizer) Encode(s string) []int { + return t.mbpe.Tokenize(s) +} + +func (t *Tokenizer) Decode(ids []int) string { + panic("unimplemented") // TODO implement +} + +func (t *Tokenizer) Tokenize(s string) []int { + return t.Encode(s) +} -- cgit v1.3.1