From 3fe37224ccc2500e7e767e5fc00163ef61182828 Mon Sep 17 00:00:00 2001
From: Jonas Knobloch <jonas.knobloch@t-online.de>
Date: Fri, 17 Apr 2026 16:07:14 +0200
Subject: Add sander module

---
 research/sander/cmd/sander/main.go | 42 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 42 insertions(+)
 create mode 100644 research/sander/cmd/sander/main.go

(limited to 'research/sander/cmd')

diff --git a/research/sander/cmd/sander/main.go b/research/sander/cmd/sander/main.go
new file mode 100644
index 0000000..fd9ece3
--- /dev/null
+++ b/research/sander/cmd/sander/main.go
@@ -0,0 +1,42 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"path"
+
+	"go.jknobloc.com/x/research/sander"
+	"go.jknobloc.com/x/tokenizer/bpe"
+)
+
+func main() {
+	vocab := []string{"50256"}
+	alpha := []string{"m000", "m010", "m020", "m030", "m040", "m050", "m060", "m070", "m080", "m090", "m100"}
+
+	for _, v := range vocab {
+		for _, a := range alpha {
+			src := fmt.Sprintf("gpt2/models/mbpe/gpt2_%s_%s_babylm_v2", v, a)
+			dst := fmt.Sprintf("out/sander/mbpe/gpt2_%s_%s_babylm_v2", v, a)
+
+			if err := run(src, dst); err != nil {
+				log.Fatal(err)
+			}
+		}
+	}
+
+}
+
+func run(src, dst string) error {
+	t := must(bpe.NewTokenizerFromFiles(path.Join(src, "vocab.json"), path.Join(src, "merges.txt")))
+	e := must(sander.NewExperiment(dst, path.Join(src, "model.onnx"), sander.UnusedTokensMBPE(t)))
+
+	return e.Run()
+}
+
+func must[T any](v T, err error) T {
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	return v
+}
-- 
cgit v1.3.1