Add perplexity evaluation

1 files changed, 8 insertions, 14 deletions

diff --git a/bpc/bpc.go b/bpc/bpc.go
index 67804d3..fd4ab6a 100644
--- a/bpc/bpc.go
+++ b/bpc/bpc.go
@@ -3,26 +3,20 @@ package bpc
 import (
 	"fmt"
 	"llm"
+	"log"
 )
 
 func Run(model llm.Causal, tokenizer llm.Tokenizer) {
-	t := tokenizer.Tokenize("The quick brown")
+	e := llm.NewEvaluator()
 
-	logits := make([][]float32, 0)
+	e.AddModel(model)
+	e.SetTokenizer(tokenizer)
 
-	if _, err := model.Generate(toInt64(t), 0, &logits); err != nil {
-		// TODO handle
-	}
-
-	fmt.Println(logits)
-}
-
-func toInt64(s []int) []int64 {
-	r := make([]int64, len(s))
+	ppl, err := e.Perplexity("data/shakespeare.txt")
 
-	for i, v := range s {
-		r[i] = int64(v)
+	if err != nil {
+		log.Fatal(err)
 	}
 
-	return r
+	fmt.Printf("\nPerplexity: %.2f\n", ppl)
 }