Fix perplexity calculation

* Track total evaluated tokens
* Use first token as context

1 files changed, 2 insertions, 2 deletions

diff --git a/llm/job.go b/llm/job.go
index 3e7c5c9..204d324 100644
--- a/llm/job.go
+++ b/llm/job.go
@@ -4,7 +4,7 @@ type job struct {
 	positions []int
 	tokens    [][]int64
 	seen      []int
-	results   []float64
+	results   []result
 	debug     [][2]int
 }
 
@@ -13,7 +13,7 @@ func newJob(batchSize int) *job {
 		positions: make([]int, 0, batchSize),
 		tokens:    make([][]int64, 0, batchSize),
 		seen:      make([]int, 0, batchSize),
-		results:   make([]float64, 0, batchSize),
+		results:   make([]result, 0, batchSize),
 		debug:     make([][2]int, 0),
 	}
 }