Access artifacts via shelf

4 files changed, 17 insertions, 13 deletions

diff --git a/llmc/cmd/data/fineweb.go b/llmc/cmd/data/fineweb.go
index cabfa74..3d3d51b 100644
--- a/llmc/cmd/data/fineweb.go
+++ b/llmc/cmd/data/fineweb.go
@@ -6,6 +6,7 @@ import (
 	"go.jknobloc.com/x/dataset"
 	"go.jknobloc.com/x/llm"
 	"go.jknobloc.com/x/llmc"
+	"go.jknobloc.com/x/shelf"
 	"go.jknobloc.com/x/tokenizer/bpe"
 )
 
@@ -17,7 +18,7 @@ import (
 func fineWeb() {
 	var reader dataset.Reader
 
-	if r, err := dataset.NewParquetReader("artifacts/data/raw/fineweb-edu/sample-100BT/train"); err != nil {
+	if r, err := dataset.NewParquetReader(shelf.Abs("data/fineweb-edu/sample-100BT/train")); err != nil {
 		log.Fatal(err)
 	} else {
 		reader = r
@@ -25,7 +26,7 @@ func fineWeb() {
 
 	var tokenizer llm.Tokenizer
 
-	if t, err := bpe.NewTokenizerFromFiles("gpt2/models/base/vocab.json", "gpt2/models/base/merges.txt"); err != nil {
+	if t, err := bpe.NewTokenizerFromFiles(shelf.Abs("models/gpt2/vocab.json"), shelf.Abs("models/gpt2/merges.txt")); err != nil {
 		log.Fatal(err)
 	} else {
 		tokenizer = t
@@ -33,7 +34,7 @@ func fineWeb() {
 
 	docs := llmc.TokenizeAll(reader, tokenizer, 50256)
 
-	if err := llmc.WriteShards("artifacts/data/llmc/edu_fineweb100B", "edu_fineweb", 100_000_000, docs); err != nil {
+	if err := llmc.WriteShards(shelf.Abs("llmc/edu_fineweb100B"), "edu_fineweb", 100_000_000, docs); err != nil {
 		log.Fatal(err)
 	}
 
diff --git a/llmc/cmd/data/tinyshakespeare.go b/llmc/cmd/data/tinyshakespeare.go
index b2d062c..70d4fd4 100644
--- a/llmc/cmd/data/tinyshakespeare.go
+++ b/llmc/cmd/data/tinyshakespeare.go
@@ -6,13 +6,14 @@ import (
 
 	"go.jknobloc.com/x/dataset"
 	"go.jknobloc.com/x/llmc"
+	"go.jknobloc.com/x/shelf"
 	"go.jknobloc.com/x/tokenizer/bpe"
 )
 
 func tinyShakespeare() {
 	var reader *dataset.FileReader
 
-	if r, err := dataset.NewFileReader("artifacts/data/raw/tinyshakespeare/input.txt", "*.txt"); err != nil {
+	if r, err := dataset.NewFileReader(shelf.Abs("data/tinyshakespeare/input.txt"), "*.txt"); err != nil {
 		log.Fatal(err)
 	} else {
 		reader = r
@@ -22,7 +23,7 @@ func tinyShakespeare() {
 
 	var tokenizer *bpe.Tokenizer
 
-	if t, err := bpe.NewTokenizerFromFiles("gpt2/models/base/vocab.json", "gpt2/models/base/merges.txt"); err != nil {
+	if t, err := bpe.NewTokenizerFromFiles(shelf.Abs("models/gpt2/vocab.json"), shelf.Abs("models/gpt2/merges.txt")); err != nil {
 		log.Fatal(err)
 	} else {
 		tokenizer = t
@@ -54,11 +55,11 @@ func tinyShakespeare() {
 		Tokens: result[32768:],
 	}
 
-	if _, err := llmc.Serialize(&val, "artifacts/data/llmc/tinyshakespeare/tiny_shakespeare_val.bin"); err != nil {
+	if _, err := llmc.Serialize(&val, shelf.Abs("llmc/tinyshakespeare/tiny_shakespeare_val.bin")); err != nil {
 		log.Fatal(err)
 	}
 
-	if _, err := llmc.Serialize(&train, "artifacts/data/llmc/tinyshakespeare/tiny_shakespeare_train.bin"); err != nil {
+	if _, err := llmc.Serialize(&train, shelf.Abs("llmc/tinyshakespeare/tiny_shakespeare_train.bin")); err != nil {
 		log.Fatal(err)
 	}
 
diff --git a/llmc/cmd/debug/debug.go b/llmc/cmd/debug/debug.go
index 289e55f..d1a621a 100644
--- a/llmc/cmd/debug/debug.go
+++ b/llmc/cmd/debug/debug.go
@@ -5,6 +5,7 @@ import (
 	"log"
 
 	"go.jknobloc.com/x/llmc"
+	"go.jknobloc.com/x/shelf"
 	"go.jknobloc.com/x/tokenizer/bpe"
 )
 
@@ -12,10 +13,10 @@ func main() {
 	var data llmc.DataFile[uint16]
 	var gold llmc.DataFile[uint16]
 
-	_ = must(llmc.Deserialize("artifacts/data/llmc/tinyshakespeare/tiny_shakespeare_train.bin", &data))
-	_ = must(llmc.Deserialize("artifacts/test/llmc/tinyshakespeare/tiny_shakespeare_train.bin", &gold))
+	_ = must(llmc.Deserialize(shelf.Abs("llmc/tinyshakespeare/tiny_shakespeare_train.bin"), &data))
+	_ = must(llmc.Deserialize(shelf.Abs("test/llmc/tinyshakespeare/tiny_shakespeare_train.bin"), &gold))
 
-	t := must(bpe.NewTokenizerFromFiles("gpt2/models/base/vocab.json", "gpt2/models/base/merges.txt"))
+	t := must(bpe.NewTokenizerFromFiles(shelf.Abs("models/gpt2/vocab.json"), shelf.Abs("models/gpt2/merges.txt")))
 
 	itoa := bpe.Itoa(t)
 
diff --git a/llmc/cmd/peek/main.go b/llmc/cmd/peek/main.go
index caacb31..0a5eeab 100644
--- a/llmc/cmd/peek/main.go
+++ b/llmc/cmd/peek/main.go
@@ -7,6 +7,7 @@ import (
 	"golang.org/x/exp/constraints"
 
 	"go.jknobloc.com/x/llmc"
+	"go.jknobloc.com/x/shelf"
 	"go.jknobloc.com/x/tokenizer/bpe"
 )
 
@@ -14,10 +15,10 @@ func main() {
 	var a llmc.DataFile[uint16]
 	var b llmc.DataFile[uint16]
 
-	_ = must(llmc.Deserialize("artifacts/data/llmc/edu_fineweb100B/edu_fineweb_val_000000.bin", &a))
-	_ = must(llmc.Deserialize("artifacts/test/llmc/edu_fineweb100B/edu_fineweb_val_000000.bin", &b))
+	_ = must(llmc.Deserialize(shelf.Abs("llmc/edu_fineweb100B/edu_fineweb_val_000000.bin"), &a))
+	_ = must(llmc.Deserialize(shelf.Abs("test/llmc/edu_fineweb100B/edu_fineweb_val_000000.bin"), &b))
 
-	t := must(bpe.NewTokenizerFromFiles("gpt2/models/base/vocab.json", "gpt2/models/base/merges.txt"))
+	t := must(bpe.NewTokenizerFromFiles(shelf.Abs("models/gpt2/vocab.json"), shelf.Abs("models/gpt2/merges.txt")))
 
 	s := decode(&a, 1024, t)
 	k := decode(&b, 1024, t)