From bc88dbac521b0de3db441c37f8e946cc2b0d3669 Mon Sep 17 00:00:00 2001
From: Jonas Knobloch <jonas.knobloch@t-online.de>
Date: Mon, 9 Mar 2026 18:59:37 +0100
Subject: Add ppl command

---
 llm/cmd/ppl/ppl.go | 69 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 69 insertions(+)
 create mode 100644 llm/cmd/ppl/ppl.go

(limited to 'llm/cmd')

diff --git a/llm/cmd/ppl/ppl.go b/llm/cmd/ppl/ppl.go
new file mode 100644
index 0000000..622644a
--- /dev/null
+++ b/llm/cmd/ppl/ppl.go
@@ -0,0 +1,69 @@
+package main
+
+import (
+	"fmt"
+	"log"
+
+	"github.com/jonasknobloch/mbpe"
+	"github.com/jonasknobloch/x/dataset"
+	"github.com/jonasknobloch/x/gpt2"
+	"github.com/jonasknobloch/x/llm"
+)
+
+func main() {
+	d := data()
+	m := model()
+	t := tokenizer()
+
+	e := llm.NewEvaluator()
+
+	e.SetTokenizer(t)
+	e.AddModel(m)
+
+	ppl, err := e.Perplexity(d)
+
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	fmt.Println(ppl)
+}
+
+func data() *dataset.Reader {
+	var miniPile *dataset.Reader
+
+	if r, err := dataset.NewReader("dataset/cmd/dataset/tmp/minipile/validation"); err != nil {
+		log.Fatal(err)
+	} else {
+		miniPile = r
+	}
+
+	return miniPile
+}
+
+func model() *gpt2.Model {
+	m := gpt2.NewModel("gpt2/models/base/model.onnx", "0")
+
+	if err := m.Init(); err != nil {
+		log.Fatal(err)
+	}
+
+	return m
+}
+
+func tokenizer() *mbpe.Tokenizer {
+	m := mbpe.NewMBPE()
+
+	if err := m.Load("gpt2/models/base/vocab.json", "gpt2/models/base/merges.txt"); err != nil {
+		log.Fatal(err)
+	}
+
+	t := mbpe.NewTokenizer(m)
+
+	byteLevel := mbpe.NewByteLevel(false)
+
+	t.SetPreTokenizer(byteLevel)
+	t.SetDecoder(byteLevel)
+
+	return t
+}
-- 
cgit v1.3.1