2 files changed, 81 insertions, 18 deletions
diff --git a/llm/cmd/ppl/ppl.go b/llm/cmd/eval/main.go
index 6912e48..05b3f96 100644
--- a/llm/cmd/ppl/ppl.go
+++ b/llm/cmd/eval/main.go
@@ -1,33 +1,16 @@
 package main
 
 import (
-	"fmt"
 	"log"
 
 	"github.com/jonasknobloch/mbpe"
 
 	"go.jknobloc.com/x/dataset"
 	"go.jknobloc.com/x/gpt2"
-	"go.jknobloc.com/x/llm"
 )
 
 func main() {
-	d := data()
-	m := model()
-	t := tokenizer()
-
-	e := llm.NewEvaluator()
-
-	e.SetTokenizer(t)
-	e.AddModel(m)
-
-	ppl, err := e.Perplexity(d, 1024, 512)
-
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	fmt.Println(ppl)
+	perplexity()
 }
 
 func data() *dataset.ParquetReader {
diff --git a/llm/cmd/eval/ppl.go b/llm/cmd/eval/ppl.go
new file mode 100644
index 0000000..fdbcff3
--- /dev/null
+++ b/llm/cmd/eval/ppl.go
@@ -0,0 +1,80 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"math"
+	"strings"
+	"sync"
+
+	"go.jknobloc.com/x/dataset"
+	"go.jknobloc.com/x/llm"
+)
+
+type pplResult struct {
+	v float64
+	n int
+}
+
+func perplexity() {
+	d := data()
+	m := model()
+	t := tokenizer()
+
+	e := llm.NewEvaluator[pplResult]()
+
+	e.SetTokenizer(t)
+	e.AddModel(m)
+
+	e.SetCallback(func(job llm.Job, logits [][]float32, tokens []int) pplResult {
+		p, n := llm.NegLogLikelihood(logits, tokens)
+
+		return pplResult{
+			v: p,
+			n: n,
+		}
+	})
+
+	results := e.Results()
+
+	var wg sync.WaitGroup
+
+	total := float64(0)
+	n := 0
+
+	wg.Add(1)
+
+	go func() {
+		defer wg.Done()
+
+		for r := range results {
+			total += r.v
+			n += r.n
+		}
+	}()
+
+	if err := e.Run(d, 1024, 512); err != nil {
+		log.Fatal(err)
+	}
+
+	wg.Wait()
+
+	avg := total / float64(n)
+	ppl := math.Exp(avg)
+
+	fmt.Println(ppl)
+}
+
+func joined() dataset.Reader {
+	miniPile := data()
+
+	docs := make([]string, 0)
+
+	for d := range miniPile.Texts("text") {
+		docs = append(docs, d)
+	}
+
+	j := dataset.NewStringReader(strings.Join(docs, "\n\n"))
+
+	return j
+}