Add perplexity evaluation

1 files changed, 21 insertions, 0 deletions

diff --git a/llm/utility.go b/llm/utility.go
new file mode 100644
index 0000000..e27ea2e
--- /dev/null
+++ b/llm/utility.go
@@ -0,0 +1,21 @@
+package llm
+
+func toInt64(s []int) []int64 {
+	r := make([]int64, len(s))
+
+	for i, v := range s {
+		r[i] = int64(v)
+	}
+
+	return r
+}
+
+func toInt(s []int64) []int {
+	r := make([]int, len(s))
+
+	for i, v := range s {
+		r[i] = int(v)
+	}
+
+	return r
+}