From 7073b124c5eb31169442ef18d1627b7ed7401280 Mon Sep 17 00:00:00 2001 From: Jonas Knobloch Date: Thu, 13 Nov 2025 23:04:57 +0100 Subject: Scaffold byte-pair correction --- llm/causal.go | 5 +++++ llm/go.mod | 3 +++ llm/tokenizer.go | 5 +++++ 3 files changed, 13 insertions(+) create mode 100644 llm/causal.go create mode 100644 llm/go.mod create mode 100644 llm/tokenizer.go (limited to 'llm') diff --git a/llm/causal.go b/llm/causal.go new file mode 100644 index 0000000..296a744 --- /dev/null +++ b/llm/causal.go @@ -0,0 +1,5 @@ +package llm + +type Causal interface { + Generate(prompt []int64, steps int64, logits *[][]float32) ([]int64, error) +} diff --git a/llm/go.mod b/llm/go.mod new file mode 100644 index 0000000..9a09bef --- /dev/null +++ b/llm/go.mod @@ -0,0 +1,3 @@ +module llm + +go 1.24.7 diff --git a/llm/tokenizer.go b/llm/tokenizer.go new file mode 100644 index 0000000..c6ea3c1 --- /dev/null +++ b/llm/tokenizer.go @@ -0,0 +1,5 @@ +package llm + +type Tokenizer interface { + Tokenize(s string) []int +} -- cgit v1.3.1