2 files changed, 9 insertions, 7 deletions

diff --git a/llmc/cmd/data/fineweb.go b/llmc/cmd/data/fineweb.go
index cabfa74..3d3d51b 100644
--- a/llmc/cmd/data/fineweb.go
+++ b/llmc/cmd/data/fineweb.go
@@ -6,6 +6,7 @@ import (
 	"go.jknobloc.com/x/dataset"
 	"go.jknobloc.com/x/llm"
 	"go.jknobloc.com/x/llmc"
+	"go.jknobloc.com/x/shelf"
 	"go.jknobloc.com/x/tokenizer/bpe"
 )
 
@@ -17,7 +18,7 @@ import (
 func fineWeb() {
 	var reader dataset.Reader
 
-	if r, err := dataset.NewParquetReader("artifacts/data/raw/fineweb-edu/sample-100BT/train"); err != nil {
+	if r, err := dataset.NewParquetReader(shelf.Abs("data/fineweb-edu/sample-100BT/train")); err != nil {
 		log.Fatal(err)
 	} else {
 		reader = r
@@ -25,7 +26,7 @@ func fineWeb() {
 
 	var tokenizer llm.Tokenizer
 
-	if t, err := bpe.NewTokenizerFromFiles("gpt2/models/base/vocab.json", "gpt2/models/base/merges.txt"); err != nil {
+	if t, err := bpe.NewTokenizerFromFiles(shelf.Abs("models/gpt2/vocab.json"), shelf.Abs("models/gpt2/merges.txt")); err != nil {
 		log.Fatal(err)
 	} else {
 		tokenizer = t
@@ -33,7 +34,7 @@ func fineWeb() {
 
 	docs := llmc.TokenizeAll(reader, tokenizer, 50256)
 
-	if err := llmc.WriteShards("artifacts/data/llmc/edu_fineweb100B", "edu_fineweb", 100_000_000, docs); err != nil {
+	if err := llmc.WriteShards(shelf.Abs("llmc/edu_fineweb100B"), "edu_fineweb", 100_000_000, docs); err != nil {
 		log.Fatal(err)
 	}
 
diff --git a/llmc/cmd/data/tinyshakespeare.go b/llmc/cmd/data/tinyshakespeare.go
index b2d062c..70d4fd4 100644
--- a/llmc/cmd/data/tinyshakespeare.go
+++ b/llmc/cmd/data/tinyshakespeare.go
@@ -6,13 +6,14 @@ import (
 
 	"go.jknobloc.com/x/dataset"
 	"go.jknobloc.com/x/llmc"
+	"go.jknobloc.com/x/shelf"
 	"go.jknobloc.com/x/tokenizer/bpe"
 )
 
 func tinyShakespeare() {
 	var reader *dataset.FileReader
 
-	if r, err := dataset.NewFileReader("artifacts/data/raw/tinyshakespeare/input.txt", "*.txt"); err != nil {
+	if r, err := dataset.NewFileReader(shelf.Abs("data/tinyshakespeare/input.txt"), "*.txt"); err != nil {
 		log.Fatal(err)
 	} else {
 		reader = r
@@ -22,7 +23,7 @@ func tinyShakespeare() {
 
 	var tokenizer *bpe.Tokenizer
 
-	if t, err := bpe.NewTokenizerFromFiles("gpt2/models/base/vocab.json", "gpt2/models/base/merges.txt"); err != nil {
+	if t, err := bpe.NewTokenizerFromFiles(shelf.Abs("models/gpt2/vocab.json"), shelf.Abs("models/gpt2/merges.txt")); err != nil {
 		log.Fatal(err)
 	} else {
 		tokenizer = t
@@ -54,11 +55,11 @@ func tinyShakespeare() {
 		Tokens: result[32768:],
 	}
 
-	if _, err := llmc.Serialize(&val, "artifacts/data/llmc/tinyshakespeare/tiny_shakespeare_val.bin"); err != nil {
+	if _, err := llmc.Serialize(&val, shelf.Abs("llmc/tinyshakespeare/tiny_shakespeare_val.bin")); err != nil {
 		log.Fatal(err)
 	}
 
-	if _, err := llmc.Serialize(&train, "artifacts/data/llmc/tinyshakespeare/tiny_shakespeare_train.bin"); err != nil {
+	if _, err := llmc.Serialize(&train, shelf.Abs("llmc/tinyshakespeare/tiny_shakespeare_train.bin")); err != nil {
 		log.Fatal(err)
 	}