diff options
| author | Jonas Knobloch <jonas.knobloch@t-online.de> | 2026-04-24 11:07:01 +0200 |
|---|---|---|
| committer | Jonas Knobloch <jonas.knobloch@t-online.de> | 2026-04-24 11:07:01 +0200 |
| commit | 1e02a5ec7f7794bb542e99ceb804ed88048b6b54 (patch) | |
| tree | 5db849f4eaa326f9b6511c315180e540f00c393e /llmc/cmd/debug/debug.go | |
| parent | 875ac19922932b12f6357d64a460c06d031000e7 (diff) | |
Access artifacts via shelf
Diffstat (limited to 'llmc/cmd/debug/debug.go')
| -rw-r--r-- | llmc/cmd/debug/debug.go | 7 |
1 files changed, 4 insertions, 3 deletions
diff --git a/llmc/cmd/debug/debug.go b/llmc/cmd/debug/debug.go index 289e55f..d1a621a 100644 --- a/llmc/cmd/debug/debug.go +++ b/llmc/cmd/debug/debug.go @@ -5,6 +5,7 @@ import ( "log" "go.jknobloc.com/x/llmc" + "go.jknobloc.com/x/shelf" "go.jknobloc.com/x/tokenizer/bpe" ) @@ -12,10 +13,10 @@ func main() { var data llmc.DataFile[uint16] var gold llmc.DataFile[uint16] - _ = must(llmc.Deserialize("artifacts/data/llmc/tinyshakespeare/tiny_shakespeare_train.bin", &data)) - _ = must(llmc.Deserialize("artifacts/test/llmc/tinyshakespeare/tiny_shakespeare_train.bin", &gold)) + _ = must(llmc.Deserialize(shelf.Abs("llmc/tinyshakespeare/tiny_shakespeare_train.bin"), &data)) + _ = must(llmc.Deserialize(shelf.Abs("test/llmc/tinyshakespeare/tiny_shakespeare_train.bin"), &gold)) - t := must(bpe.NewTokenizerFromFiles("gpt2/models/base/vocab.json", "gpt2/models/base/merges.txt")) + t := must(bpe.NewTokenizerFromFiles(shelf.Abs("models/gpt2/vocab.json"), shelf.Abs("models/gpt2/merges.txt"))) itoa := bpe.Itoa(t) |