From 1e02a5ec7f7794bb542e99ceb804ed88048b6b54 Mon Sep 17 00:00:00 2001 From: Jonas Knobloch Date: Fri, 24 Apr 2026 11:07:01 +0200 Subject: Access artifacts via shelf --- llmc/cmd/peek/main.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'llmc/cmd/peek') diff --git a/llmc/cmd/peek/main.go b/llmc/cmd/peek/main.go index caacb31..0a5eeab 100644 --- a/llmc/cmd/peek/main.go +++ b/llmc/cmd/peek/main.go @@ -7,6 +7,7 @@ import ( "golang.org/x/exp/constraints" "go.jknobloc.com/x/llmc" + "go.jknobloc.com/x/shelf" "go.jknobloc.com/x/tokenizer/bpe" ) @@ -14,10 +15,10 @@ func main() { var a llmc.DataFile[uint16] var b llmc.DataFile[uint16] - _ = must(llmc.Deserialize("artifacts/data/llmc/edu_fineweb100B/edu_fineweb_val_000000.bin", &a)) - _ = must(llmc.Deserialize("artifacts/test/llmc/edu_fineweb100B/edu_fineweb_val_000000.bin", &b)) + _ = must(llmc.Deserialize(shelf.Abs("llmc/edu_fineweb100B/edu_fineweb_val_000000.bin"), &a)) + _ = must(llmc.Deserialize(shelf.Abs("test/llmc/edu_fineweb100B/edu_fineweb_val_000000.bin"), &b)) - t := must(bpe.NewTokenizerFromFiles("gpt2/models/base/vocab.json", "gpt2/models/base/merges.txt")) + t := must(bpe.NewTokenizerFromFiles(shelf.Abs("models/gpt2/vocab.json"), shelf.Abs("models/gpt2/merges.txt"))) s := decode(&a, 1024, t) k := decode(&b, 1024, t) -- cgit v1.3.1