diff options
| author | Jonas Knobloch <jonas.knobloch@t-online.de> | 2026-04-11 01:09:16 +0200 |
|---|---|---|
| committer | Jonas Knobloch <jonas.knobloch@t-online.de> | 2026-04-14 19:38:34 +0200 |
| commit | 0c97f22069d77cf3cac92d68173e38f83b258a9a (patch) | |
| tree | 127bf00ad1f69ce1368089247471fd52b8f9d305 /tokenizer/bpe/utility.go | |
| parent | aa1fdb5f3b5d6c54c3bc44269482c31f4dfafe67 (diff) | |
Add lesci module
Diffstat (limited to 'tokenizer/bpe/utility.go')
| -rw-r--r-- | tokenizer/bpe/utility.go | 20 |
1 files changed, 20 insertions, 0 deletions
diff --git a/tokenizer/bpe/utility.go b/tokenizer/bpe/utility.go index f47c073..fbe99fc 100644 --- a/tokenizer/bpe/utility.go +++ b/tokenizer/bpe/utility.go @@ -18,3 +18,23 @@ func NewTokenizerFromFiles(vocab, merges string) (*Tokenizer, error) { return NewTokenizer(tokenizer), nil } + +func Vocab(t *Tokenizer) []string { + m, ok := t.mbpe.Model().(*mbpe.MBPE) + + if !ok { + panic("unimplemented") + } + + return m.Vocab() +} + +func Merges(t *Tokenizer) [][2]string { + m, ok := t.mbpe.Model().(*mbpe.MBPE) + + if !ok { + panic("unimplemented") + } + + return m.Merges() +} |