diff options
| author | Jonas Knobloch <jonas.knobloch@t-online.de> | 2026-04-24 11:49:26 +0200 |
|---|---|---|
| committer | Jonas Knobloch <jonas.knobloch@t-online.de> | 2026-04-24 11:50:02 +0200 |
| commit | 12f15fdd8c048e5958e294f12a9f163f6f950b3e (patch) | |
| tree | 16bb23c134f1fe476d977dd846e6dda6c7667c48 /tokenizer/bpe/utility.go | |
| parent | 408536516bdfedc93d93226dc08280ed224361fc (diff) | |
Recover tokenization based on config
Diffstat (limited to 'tokenizer/bpe/utility.go')
| -rw-r--r-- | tokenizer/bpe/utility.go | 4 |
1 files changed, 2 insertions, 2 deletions
diff --git a/tokenizer/bpe/utility.go b/tokenizer/bpe/utility.go index fbe99fc..f8fcdbc 100644 --- a/tokenizer/bpe/utility.go +++ b/tokenizer/bpe/utility.go @@ -2,7 +2,7 @@ package bpe import "github.com/jonasknobloch/mbpe" -func NewTokenizerFromFiles(vocab, merges string) (*Tokenizer, error) { +func NewTokenizerFromFiles(vocab, merges string, cfg Config) (*Tokenizer, error) { model := mbpe.NewMBPE() if err := model.Load(vocab, merges); err != nil { @@ -16,7 +16,7 @@ func NewTokenizerFromFiles(vocab, merges string) (*Tokenizer, error) { tokenizer.SetPreTokenizer(pre) tokenizer.SetDecoder(pre) - return NewTokenizer(tokenizer), nil + return NewTokenizer(tokenizer, cfg), nil } func Vocab(t *Tokenizer) []string { |