From 57fb765bdc5819f8a367ad75c731a50da68b8d85 Mon Sep 17 00:00:00 2001
From: Jonas Knobloch <jonas.knobloch@t-online.de>
Date: Tue, 17 Mar 2026 20:16:39 +0100
Subject: Update perplexity evaluator

* Introduce results channel
* Introduce jobs channel
* Generalize evaluation via callback
---
 llm/cmd/ppl/ppl.go | 70 ------------------------------------------------------
 1 file changed, 70 deletions(-)
 delete mode 100644 llm/cmd/ppl/ppl.go

(limited to 'llm/cmd/ppl/ppl.go')

diff --git a/llm/cmd/ppl/ppl.go b/llm/cmd/ppl/ppl.go
deleted file mode 100644
index 6912e48..0000000
--- a/llm/cmd/ppl/ppl.go
+++ /dev/null
@@ -1,70 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"log"
-
-	"github.com/jonasknobloch/mbpe"
-
-	"go.jknobloc.com/x/dataset"
-	"go.jknobloc.com/x/gpt2"
-	"go.jknobloc.com/x/llm"
-)
-
-func main() {
-	d := data()
-	m := model()
-	t := tokenizer()
-
-	e := llm.NewEvaluator()
-
-	e.SetTokenizer(t)
-	e.AddModel(m)
-
-	ppl, err := e.Perplexity(d, 1024, 512)
-
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	fmt.Println(ppl)
-}
-
-func data() *dataset.ParquetReader {
-	var miniPile *dataset.ParquetReader
-
-	if r, err := dataset.NewParquetReader("dataset/cmd/dataset/tmp/minipile/validation"); err != nil {
-		log.Fatal(err)
-	} else {
-		miniPile = r
-	}
-
-	return miniPile
-}
-
-func model() *gpt2.Model {
-	m := gpt2.NewModel("gpt2/models/base/model.onnx", "0")
-
-	if err := m.Init(); err != nil {
-		log.Fatal(err)
-	}
-
-	return m
-}
-
-func tokenizer() *mbpe.Tokenizer {
-	m := mbpe.NewMBPE()
-
-	if err := m.Load("gpt2/models/base/vocab.json", "gpt2/models/base/merges.txt"); err != nil {
-		log.Fatal(err)
-	}
-
-	t := mbpe.NewTokenizer(m)
-
-	byteLevel := mbpe.NewByteLevel(false)
-
-	t.SetPreTokenizer(byteLevel)
-	t.SetDecoder(byteLevel)
-
-	return t
-}
-- 
cgit v1.3.1