From 7073b124c5eb31169442ef18d1627b7ed7401280 Mon Sep 17 00:00:00 2001 From: Jonas Knobloch Date: Thu, 13 Nov 2025 23:04:57 +0100 Subject: Scaffold byte-pair correction --- bpc/bpc.go | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 bpc/bpc.go (limited to 'bpc/bpc.go') diff --git a/bpc/bpc.go b/bpc/bpc.go new file mode 100644 index 0000000..67804d3 --- /dev/null +++ b/bpc/bpc.go @@ -0,0 +1,28 @@ +package bpc + +import ( + "fmt" + "llm" +) + +func Run(model llm.Causal, tokenizer llm.Tokenizer) { + t := tokenizer.Tokenize("The quick brown") + + logits := make([][]float32, 0) + + if _, err := model.Generate(toInt64(t), 0, &logits); err != nil { + // TODO handle + } + + fmt.Println(logits) +} + +func toInt64(s []int) []int64 { + r := make([]int64, len(s)) + + for i, v := range s { + r[i] = int64(v) + } + + return r +} -- cgit v1.3.1