diff options
Diffstat (limited to 'llm/cmd/eval/main.go')
| -rw-r--r-- | llm/cmd/eval/main.go | 7 |
1 files changed, 4 insertions, 3 deletions
diff --git a/llm/cmd/eval/main.go b/llm/cmd/eval/main.go index 8b61927..4d27ada 100644 --- a/llm/cmd/eval/main.go +++ b/llm/cmd/eval/main.go @@ -5,6 +5,7 @@ import ( "go.jknobloc.com/x/dataset" "go.jknobloc.com/x/gpt2" + "go.jknobloc.com/x/shelf" "go.jknobloc.com/x/tokenizer/bpe" ) @@ -24,7 +25,7 @@ func main() { func data() *dataset.ParquetReader { var miniPile *dataset.ParquetReader - if r, err := dataset.NewParquetReader("dataset/cmd/dataset/tmp/minipile/validation"); err != nil { + if r, err := dataset.NewParquetReader(shelf.Abs("data/minipile/validation")); err != nil { log.Fatal(err) } else { miniPile = r @@ -40,7 +41,7 @@ func model() *gpt2.Model { WithLogProbs: true, } - m := gpt2.NewModel("gpt2/models/base/model_eval.onnx", gpt2.DefaultConfig(), opts) + m := gpt2.NewModel(shelf.Abs("models/gpt2/model_eval.onnx"), gpt2.DefaultConfig(), opts) if err := m.Init(); err != nil { log.Fatal(err) @@ -52,7 +53,7 @@ func model() *gpt2.Model { func tokenizer() *bpe.Tokenizer { var tok *bpe.Tokenizer - if t, err := bpe.NewTokenizerFromFiles("gpt2/models/base/vocab.json", "gpt2/models/base/merges.txt"); err != nil { + if t, err := bpe.NewTokenizerFromFiles(shelf.Abs("models/gpt2/vocab.json"), shelf.Abs("models/gpt2/merges.txt")); err != nil { log.Fatal(err) } else { tok = t |