From 48cac2575c9ba5377ddeba2f17ddb68277e410d9 Mon Sep 17 00:00:00 2001
From: Jonas Knobloch <jonas.knobloch@t-online.de>
Date: Sat, 27 Jun 2026 20:45:08 +0200
Subject: Clamp perplexity evaluation

---
 llm/cmd/eval/main.go | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/llm/cmd/eval/main.go b/llm/cmd/eval/main.go
index 3a710ac..5e3390f 100644
--- a/llm/cmd/eval/main.go
+++ b/llm/cmd/eval/main.go
@@ -16,12 +16,15 @@ func main() {
 
 	perplexity()
 
+	// pad right: 11.930400581007826 / 22544 tokens
+	// unpadded: 11.246722646921588 / 19451 tokens
+
 	if err := gpt2.DestroyEnvironment(); err != nil {
 		log.Fatal(err)
 	}
 }
 
-func data() *dataset.ParquetReader {
+func data() dataset.Reader {
 	var miniPile *dataset.ParquetReader
 
 	if r, err := dataset.NewParquetReader(shelf.Abs("data/minipile/validation")); err != nil {
@@ -30,7 +33,7 @@ func data() *dataset.ParquetReader {
 		miniPile = r
 	}
 
-	return miniPile
+	return dataset.NewClampedReader(miniPile, 10)
 }
 
 func model() *gpt2.Model {
-- 
cgit v1.2.3