diff options
| author | Jonas Knobloch <jonas.knobloch@t-online.de> | 2026-09-09 02:41:24 +0200 |
|---|---|---|
| committer | Jonas Knobloch <jonas.knobloch@t-online.de> | 2026-09-09 02:41:24 +0200 |
| commit | 3c5f4c4279e6c9d6479e40eff4fbe1add59c1b3a (patch) | |
| tree | c12ce174c117b0205bc3426b6cc2501f7e709f92 /tokenizer/bpe | |
| parent | 8fd3f8aeec85f47d43ef36a81456d48640ca33f9 (diff) | |
WIP
Diffstat (limited to 'tokenizer/bpe')
| -rw-r--r-- | tokenizer/bpe/utility.go | 4 |
1 files changed, 4 insertions, 0 deletions
diff --git a/tokenizer/bpe/utility.go b/tokenizer/bpe/utility.go index f8fcdbc..79e6faf 100644 --- a/tokenizer/bpe/utility.go +++ b/tokenizer/bpe/utility.go @@ -19,6 +19,10 @@ func NewTokenizerFromFiles(vocab, merges string, cfg Config) (*Tokenizer, error) return NewTokenizer(tokenizer, cfg), nil } +func MBPE(t *Tokenizer) *mbpe.Tokenizer { + return t.mbpe +} + func Vocab(t *Tokenizer) []string { m, ok := t.mbpe.Model().(*mbpe.MBPE) |