Add perplexity evaluation

1 files changed, 26 insertions, 0 deletions

diff --git a/llm/evaluator.go b/llm/evaluator.go
new file mode 100644
index 0000000..ccc554e
--- /dev/null
+++ b/llm/evaluator.go
@@ -0,0 +1,26 @@
+package llm
+
+import "sync"
+
+type Evaluator struct {
+	mutex     sync.RWMutex
+	models    []Causal
+	tokenizer Tokenizer
+	results   []float64
+	jobs      int
+}
+
+func NewEvaluator() *Evaluator {
+	return &Evaluator{
+		models:  make([]Causal, 0),
+		results: make([]float64, 0),
+	}
+}
+
+func (e *Evaluator) AddModel(model Causal) {
+	e.models = append(e.models, model)
+}
+
+func (e *Evaluator) SetTokenizer(tokenizer Tokenizer) {
+	e.tokenizer = tokenizer
+}