From e5d7c530ffd7ab88131988a05f66109ec880b809 Mon Sep 17 00:00:00 2001 From: Jonas Knobloch Date: Mon, 9 Mar 2026 19:14:50 +0100 Subject: Fix perplexity calculation * Track total evaluated tokens * Use first token as context --- llm/job.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'llm/job.go') diff --git a/llm/job.go b/llm/job.go index 3e7c5c9..204d324 100644 --- a/llm/job.go +++ b/llm/job.go @@ -4,7 +4,7 @@ type job struct { positions []int tokens [][]int64 seen []int - results []float64 + results []result debug [][2]int } @@ -13,7 +13,7 @@ func newJob(batchSize int) *job { positions: make([]int, 0, batchSize), tokens: make([][]int64, 0, batchSize), seen: make([]int, 0, batchSize), - results: make([]float64, 0, batchSize), + results: make([]result, 0, batchSize), debug: make([][2]int, 0), } } -- cgit v1.3.1