Add perplexity evaluation

1 files changed, 19 insertions, 0 deletions

diff --git a/llm/job.go b/llm/job.go
new file mode 100644
index 0000000..3e7c5c9
--- /dev/null
+++ b/llm/job.go
@@ -0,0 +1,19 @@
+package llm
+
+type job struct {
+	positions []int
+	tokens    [][]int64
+	seen      []int
+	results   []float64
+	debug     [][2]int
+}
+
+func newJob(batchSize int) *job {
+	return &job{
+		positions: make([]int, 0, batchSize),
+		tokens:    make([][]int64, 0, batchSize),
+		seen:      make([]int, 0, batchSize),
+		results:   make([]float64, 0, batchSize),
+		debug:     make([][2]int, 0),
+	}
+}