Access artifacts via shelf

1 files changed, 5 insertions, 4 deletions

diff --git a/llmc/cmd/data/tinyshakespeare.go b/llmc/cmd/data/tinyshakespeare.go
index b2d062c..70d4fd4 100644
--- a/llmc/cmd/data/tinyshakespeare.go
+++ b/llmc/cmd/data/tinyshakespeare.go
@@ -6,13 +6,14 @@ import (
 
 	"go.jknobloc.com/x/dataset"
 	"go.jknobloc.com/x/llmc"
+	"go.jknobloc.com/x/shelf"
 	"go.jknobloc.com/x/tokenizer/bpe"
 )
 
 func tinyShakespeare() {
 	var reader *dataset.FileReader
 
-	if r, err := dataset.NewFileReader("artifacts/data/raw/tinyshakespeare/input.txt", "*.txt"); err != nil {
+	if r, err := dataset.NewFileReader(shelf.Abs("data/tinyshakespeare/input.txt"), "*.txt"); err != nil {
 		log.Fatal(err)
 	} else {
 		reader = r
@@ -22,7 +23,7 @@ func tinyShakespeare() {
 
 	var tokenizer *bpe.Tokenizer
 
-	if t, err := bpe.NewTokenizerFromFiles("gpt2/models/base/vocab.json", "gpt2/models/base/merges.txt"); err != nil {
+	if t, err := bpe.NewTokenizerFromFiles(shelf.Abs("models/gpt2/vocab.json"), shelf.Abs("models/gpt2/merges.txt")); err != nil {
 		log.Fatal(err)
 	} else {
 		tokenizer = t
@@ -54,11 +55,11 @@ func tinyShakespeare() {
 		Tokens: result[32768:],
 	}
 
-	if _, err := llmc.Serialize(&val, "artifacts/data/llmc/tinyshakespeare/tiny_shakespeare_val.bin"); err != nil {
+	if _, err := llmc.Serialize(&val, shelf.Abs("llmc/tinyshakespeare/tiny_shakespeare_val.bin")); err != nil {
 		log.Fatal(err)
 	}
 
-	if _, err := llmc.Serialize(&train, "artifacts/data/llmc/tinyshakespeare/tiny_shakespeare_train.bin"); err != nil {
+	if _, err := llmc.Serialize(&train, shelf.Abs("llmc/tinyshakespeare/tiny_shakespeare_train.bin")); err != nil {
 		log.Fatal(err)
 	}