From 236d184ad8a2af0e6416b381ea55668c89d40df2 Mon Sep 17 00:00:00 2001 From: Jonas Knobloch Date: Mon, 16 Mar 2026 17:15:24 +0100 Subject: Rename dataset reader --- llm/cmd/ppl/ppl.go | 6 +++--- llm/perplexity.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) (limited to 'llm') diff --git a/llm/cmd/ppl/ppl.go b/llm/cmd/ppl/ppl.go index 584cd16..6912e48 100644 --- a/llm/cmd/ppl/ppl.go +++ b/llm/cmd/ppl/ppl.go @@ -30,10 +30,10 @@ func main() { fmt.Println(ppl) } -func data() *dataset.Reader { - var miniPile *dataset.Reader +func data() *dataset.ParquetReader { + var miniPile *dataset.ParquetReader - if r, err := dataset.NewReader("dataset/cmd/dataset/tmp/minipile/validation"); err != nil { + if r, err := dataset.NewParquetReader("dataset/cmd/dataset/tmp/minipile/validation"); err != nil { log.Fatal(err) } else { miniPile = r diff --git a/llm/perplexity.go b/llm/perplexity.go index 9596e8d..2240f2d 100644 --- a/llm/perplexity.go +++ b/llm/perplexity.go @@ -14,7 +14,7 @@ import ( "go.jknobloc.com/x/dataset" ) -func (e *Evaluator) Perplexity(data *dataset.Reader, window, stride int) (float64, error) { +func (e *Evaluator) Perplexity(data *dataset.ParquetReader, window, stride int) (float64, error) { docs := make([]string, 0) n := 0 -- cgit v1.3.1