summaryrefslogtreecommitdiff
path: root/llm
diff options
context:
space:
mode:
authorJonas Knobloch <jonas.knobloch@t-online.de>2026-04-03 16:02:05 +0200
committerJonas Knobloch <jonas.knobloch@t-online.de>2026-04-03 18:45:57 +0200
commitc1644da32ecb9856f15e2c919a84d2666d5ef29c (patch)
tree0c19b78e35c05d7814f855128642a384bd8660cd /llm
parenta61bc9ff890ea50d41a406cd1ffe554341551d2b (diff)
Scope text column to parquet reader
Diffstat (limited to 'llm')
-rw-r--r--llm/cmd/eval/ppl.go2
-rw-r--r--llm/run.go2
2 files changed, 2 insertions, 2 deletions
diff --git a/llm/cmd/eval/ppl.go b/llm/cmd/eval/ppl.go
index 60dd30a..679de90 100644
--- a/llm/cmd/eval/ppl.go
+++ b/llm/cmd/eval/ppl.go
@@ -53,7 +53,7 @@ func joined() dataset.Reader {
docs := make([]string, 0)
- for d := range miniPile.Texts("text") {
+ for d := range miniPile.Texts() {
docs = append(docs, d)
}
diff --git a/llm/run.go b/llm/run.go
index 1843c94..483cd5d 100644
--- a/llm/run.go
+++ b/llm/run.go
@@ -63,7 +63,7 @@ func (e *Evaluator[R]) Run(title string, data dataset.Reader, window, stride int
b := newBatch(e.batchSize)
- for d := range data.Texts("text") {
+ for d := range data.Texts() {
for w, s := range tb.Push(n, d) {
if s == 0 {
s = 1 // first token as context