diff options
Diffstat (limited to 'research/lesci/cmd')
| -rw-r--r-- | research/lesci/cmd/lesci/main.go | 9 |
1 files changed, 5 insertions, 4 deletions
diff --git a/research/lesci/cmd/lesci/main.go b/research/lesci/cmd/lesci/main.go index ab0a967..c52013a 100644 --- a/research/lesci/cmd/lesci/main.go +++ b/research/lesci/cmd/lesci/main.go @@ -8,6 +8,7 @@ import ( "go.jknobloc.com/x/dataset" "go.jknobloc.com/x/gpt2" "go.jknobloc.com/x/research/lesci" + "go.jknobloc.com/x/shelf" "go.jknobloc.com/x/tokenizer/bpe" ) @@ -40,15 +41,15 @@ func main() { } func setup(control, treatment int) (*lesci.Experiment, *gpt2.Model) { - a := fmt.Sprintf("gpt2/models/onnx_eval/gpt2_%d_m000_babylm_v2", control) - b := fmt.Sprintf("gpt2/models/onnx_eval/gpt2_%d_m000_babylm_v2", 100512) + a := fmt.Sprintf(shelf.Abs("models/mbpe/gpt2_%d_m000_babylm_v2"), control) + b := fmt.Sprintf(shelf.Abs("models/mbpe/gpt2_%d_m000_babylm_v2"), 100512) m := must(model(path.Join(a, "model_eval.onnx"), control)) t := must(bpe.NewTokenizerFromFiles(path.Join(a, "vocab.json"), path.Join(a, "merges.txt"))) c := must(bpe.NewTokenizerFromFiles(path.Join(b, "vocab.json"), path.Join(b, "merges.txt"))) - d := must(dataset.NewFileReader("dataset/cmd/dataset/tmp/babylm/train_100M", "*.train")) + d := must(dataset.NewFileReader(shelf.Abs("data/babylm/train_100M"), "*.train")) - o := fmt.Sprintf("out/lesci/m000/babylm_%d_%d", control, treatment) + o := fmt.Sprintf(shelf.Abs("results/lesci/m000/babylm_%d_%d"), control, treatment) return must(lesci.NewExperiment(m, t, c, d, o, control, 5000)), m } |