diff options
Diffstat (limited to 'tokenizer/bpe/cmd')
| -rw-r--r-- | tokenizer/bpe/cmd/tokenize/main.go | 2 |
1 files changed, 2 insertions, 0 deletions
diff --git a/tokenizer/bpe/cmd/tokenize/main.go b/tokenizer/bpe/cmd/tokenize/main.go index 1b0b868..1dbfaae 100644 --- a/tokenizer/bpe/cmd/tokenize/main.go +++ b/tokenizer/bpe/cmd/tokenize/main.go @@ -109,5 +109,7 @@ func tokenizer() llm.Tokenizer { tok = t } + _ = bpe.ByteCoverage(tok) + return tok } |