From 321802e7b1f77f684a720d1052befd6d36280875 Mon Sep 17 00:00:00 2001
From: Jonas Knobloch <jonas.knobloch@t-online.de>
Date: Tue, 18 Nov 2025 02:04:41 +0100
Subject: Add perplexity evaluation

---
 bpc/bpc.go          | 22 ++++++++--------------
 bpc/cmd/bpc/main.go |  3 ++-
 2 files changed, 10 insertions(+), 15 deletions(-)

(limited to 'bpc')

diff --git a/bpc/bpc.go b/bpc/bpc.go
index 67804d3..fd4ab6a 100644
--- a/bpc/bpc.go
+++ b/bpc/bpc.go
@@ -3,26 +3,20 @@ package bpc
 import (
 	"fmt"
 	"llm"
+	"log"
 )
 
 func Run(model llm.Causal, tokenizer llm.Tokenizer) {
-	t := tokenizer.Tokenize("The quick brown")
+	e := llm.NewEvaluator()
 
-	logits := make([][]float32, 0)
+	e.AddModel(model)
+	e.SetTokenizer(tokenizer)
 
-	if _, err := model.Generate(toInt64(t), 0, &logits); err != nil {
-		// TODO handle
-	}
-
-	fmt.Println(logits)
-}
-
-func toInt64(s []int) []int64 {
-	r := make([]int64, len(s))
+	ppl, err := e.Perplexity("data/shakespeare.txt")
 
-	for i, v := range s {
-		r[i] = int64(v)
+	if err != nil {
+		log.Fatal(err)
 	}
 
-	return r
+	fmt.Printf("\nPerplexity: %.2f\n", ppl)
 }
diff --git a/bpc/cmd/bpc/main.go b/bpc/cmd/bpc/main.go
index b6ebef1..8f7fc1e 100644
--- a/bpc/cmd/bpc/main.go
+++ b/bpc/cmd/bpc/main.go
@@ -5,6 +5,7 @@ import (
 	"gpt2"
 	"log"
 	mbpe "mbpe-dyn"
+	"os"
 )
 
 func main() {
@@ -24,7 +25,7 @@ func main() {
 }
 
 func model() *gpt2.Model {
-	return gpt2.NewModel("../gpt2/models/base/model.onnx")
+	return gpt2.NewModel("../gpt2/models/base/model.onnx", os.Getenv("BPC_CUDA_DEVICE_ID"))
 }
 
 func tokenizer() *mbpe.Tokenizer {
-- 
cgit v1.3.1