diff options
Diffstat (limited to 'llmc/cmd/data/tinyshakespeare.go')
| -rw-r--r-- | llmc/cmd/data/tinyshakespeare.go | 9 |
1 files changed, 5 insertions, 4 deletions
diff --git a/llmc/cmd/data/tinyshakespeare.go b/llmc/cmd/data/tinyshakespeare.go index b2d062c..70d4fd4 100644 --- a/llmc/cmd/data/tinyshakespeare.go +++ b/llmc/cmd/data/tinyshakespeare.go @@ -6,13 +6,14 @@ import ( "go.jknobloc.com/x/dataset" "go.jknobloc.com/x/llmc" + "go.jknobloc.com/x/shelf" "go.jknobloc.com/x/tokenizer/bpe" ) func tinyShakespeare() { var reader *dataset.FileReader - if r, err := dataset.NewFileReader("artifacts/data/raw/tinyshakespeare/input.txt", "*.txt"); err != nil { + if r, err := dataset.NewFileReader(shelf.Abs("data/tinyshakespeare/input.txt"), "*.txt"); err != nil { log.Fatal(err) } else { reader = r @@ -22,7 +23,7 @@ func tinyShakespeare() { var tokenizer *bpe.Tokenizer - if t, err := bpe.NewTokenizerFromFiles("gpt2/models/base/vocab.json", "gpt2/models/base/merges.txt"); err != nil { + if t, err := bpe.NewTokenizerFromFiles(shelf.Abs("models/gpt2/vocab.json"), shelf.Abs("models/gpt2/merges.txt")); err != nil { log.Fatal(err) } else { tokenizer = t @@ -54,11 +55,11 @@ func tinyShakespeare() { Tokens: result[32768:], } - if _, err := llmc.Serialize(&val, "artifacts/data/llmc/tinyshakespeare/tiny_shakespeare_val.bin"); err != nil { + if _, err := llmc.Serialize(&val, shelf.Abs("llmc/tinyshakespeare/tiny_shakespeare_val.bin")); err != nil { log.Fatal(err) } - if _, err := llmc.Serialize(&train, "artifacts/data/llmc/tinyshakespeare/tiny_shakespeare_train.bin"); err != nil { + if _, err := llmc.Serialize(&train, shelf.Abs("llmc/tinyshakespeare/tiny_shakespeare_train.bin")); err != nil { log.Fatal(err) } |