From 0c97f22069d77cf3cac92d68173e38f83b258a9a Mon Sep 17 00:00:00 2001 From: Jonas Knobloch Date: Sat, 11 Apr 2026 01:09:16 +0200 Subject: Add lesci module --- tokenizer/bpe/utility.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) (limited to 'tokenizer/bpe/utility.go') diff --git a/tokenizer/bpe/utility.go b/tokenizer/bpe/utility.go index f47c073..fbe99fc 100644 --- a/tokenizer/bpe/utility.go +++ b/tokenizer/bpe/utility.go @@ -18,3 +18,23 @@ func NewTokenizerFromFiles(vocab, merges string) (*Tokenizer, error) { return NewTokenizer(tokenizer), nil } + +func Vocab(t *Tokenizer) []string { + m, ok := t.mbpe.Model().(*mbpe.MBPE) + + if !ok { + panic("unimplemented") + } + + return m.Vocab() +} + +func Merges(t *Tokenizer) [][2]string { + m, ok := t.mbpe.Model().(*mbpe.MBPE) + + if !ok { + panic("unimplemented") + } + + return m.Merges() +} -- cgit v1.3.1