diff options
Diffstat (limited to 'llm/cmd/eval/main.go')
| -rw-r--r-- | llm/cmd/eval/main.go | 5 |
1 files changed, 4 insertions, 1 deletions
diff --git a/llm/cmd/eval/main.go b/llm/cmd/eval/main.go index d31bc32..bd734b3 100644 --- a/llm/cmd/eval/main.go +++ b/llm/cmd/eval/main.go @@ -52,7 +52,10 @@ func model() *gpt2.Model { func tokenizer() *bpe.Tokenizer { var tok *bpe.Tokenizer - if t, err := bpe.NewTokenizerFromFiles(shelf.Abs("models/gpt2/vocab.json"), shelf.Abs("models/gpt2/merges.txt")); err != nil { + v := shelf.Abs("models/gpt2/vocab.json") + m := shelf.Abs("models/gpt2/merges.txt") + + if t, err := bpe.NewTokenizerFromFiles(v, m, bpe.DefaultConfig()); err != nil { log.Fatal(err) } else { tok = t |