From 12f15fdd8c048e5958e294f12a9f163f6f950b3e Mon Sep 17 00:00:00 2001 From: Jonas Knobloch Date: Fri, 24 Apr 2026 11:49:26 +0200 Subject: Recover tokenization based on config --- tokenizer/bpe/utility.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'tokenizer/bpe/utility.go') diff --git a/tokenizer/bpe/utility.go b/tokenizer/bpe/utility.go index fbe99fc..f8fcdbc 100644 --- a/tokenizer/bpe/utility.go +++ b/tokenizer/bpe/utility.go @@ -2,7 +2,7 @@ package bpe import "github.com/jonasknobloch/mbpe" -func NewTokenizerFromFiles(vocab, merges string) (*Tokenizer, error) { +func NewTokenizerFromFiles(vocab, merges string, cfg Config) (*Tokenizer, error) { model := mbpe.NewMBPE() if err := model.Load(vocab, merges); err != nil { @@ -16,7 +16,7 @@ func NewTokenizerFromFiles(vocab, merges string) (*Tokenizer, error) { tokenizer.SetPreTokenizer(pre) tokenizer.SetDecoder(pre) - return NewTokenizer(tokenizer), nil + return NewTokenizer(tokenizer, cfg), nil } func Vocab(t *Tokenizer) []string { -- cgit v1.3.1