From fa4e25b4992c25cdec41bdccb8af909b93fc9c39 Mon Sep 17 00:00:00 2001
From: Jonas Knobloch <jonas.knobloch@t-online.de>
Date: Tue, 31 Mar 2026 17:05:43 +0200
Subject: Count skipped texts

---
 llm/run.go | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/llm/run.go b/llm/run.go
index df48ff6..227a365 100644
--- a/llm/run.go
+++ b/llm/run.go
@@ -52,13 +52,25 @@ func (e *Evaluator[R]) Run(title string, data dataset.Reader, window, stride int
 		return int(e.completed.Load())
 	})
 
-	defer pb.Close()
+	s := 0 // skipped
+	n := 0 // total
 
-	n := 0
+	defer func() {
+		fmt.Printf("skipped %d of %d texts\n", s, n)
+	}()
+
+	defer pb.Close()
 
 	for d := range data.Texts("text") {
 		tokens := toInt64(e.tokenizer.Tokenize(d))
 
+		if len(tokens) == 0 {
+			n++
+			s++
+
+			continue
+		}
+
 		e.schedule(n, tokens, window, stride, e.batchSize)
 
 		pb.SetTotal(pb.Total() + e.estimateJobs(tokens, window, stride))
-- 
cgit v1.2.3