Add lesci module

14 files changed, 1110 insertions, 0 deletions

diff --git a/go.work b/go.work
index 61c84dd..d836d80 100644
--- a/go.work
+++ b/go.work
@@ -5,6 +5,7 @@ use (
 	./gpt2
 	./llm
 	./mbpe
+	./research/lesci
 	./tensor
 	./tokenizer
 	./tui
diff --git a/research/lesci/analyze.go b/research/lesci/analyze.go
new file mode 100644
index 0000000..7205ac1
--- /dev/null
+++ b/research/lesci/analyze.go
@@ -0,0 +1,106 @@
+package lesci
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+)
+
+var (
+	sqlPairs = `
+		CREATE TEMP TABLE pairs AS
+		SELECT uid, pos, token, logprob,
+		       LEAD(token, 1) OVER w AS next_tok,
+		       LAG(token, 1)  OVER w AS prev_tok
+		FROM context
+		WINDOW w AS (PARTITION BY uid ORDER BY pos)`
+
+	sqlMatched = `
+		CREATE TEMP TABLE matched AS
+		SELECT p.*, r1.c AS merged_as_first, r2.c AS merged_as_second
+		FROM pairs p
+		LEFT JOIN oov_rules r1 ON p.token = r1.a AND p.next_tok = r1.b
+		LEFT JOIN oov_rules r2 ON p.prev_tok = r2.a AND p.token = r2.b`
+
+	sqlSummedOOV = `
+		CREATE TEMP TABLE summed_oov AS
+		WITH processed AS (
+		    SELECT
+		        uid,
+		        logprob,
+		        COALESCE(merged_as_first, merged_as_second, token) AS tok,
+		        CASE
+		            WHEN merged_as_first  IS NOT NULL THEN pos
+		            WHEN merged_as_second IS NOT NULL THEN pos - 1
+		            ELSE pos
+		        END AS adjusted_pos
+		    FROM matched
+		)
+		SELECT
+		    uid,
+		    tok,
+		    adjusted_pos AS pos,
+		    SUM(logprob) AS token_logprob
+		FROM processed
+		WHERE tok >= ? AND tok < ?
+		GROUP BY uid, tok, adjusted_pos`
+
+	sqlResults = `
+		CREATE TABLE lesci_results AS
+		SELECT tok,
+		       QUANTILE_CONT(token_logprob, 0.5)  AS median,
+		       QUANTILE_CONT(token_logprob, 0.75) AS q75,
+		       QUANTILE_CONT(token_logprob, 0.25) AS q25,
+		       QUANTILE_CONT(token_logprob, 0.75)
+		         - QUANTILE_CONT(token_logprob, 0.25) AS iqr,
+		       AVG(token_logprob)        AS mean,
+		       STDDEV_SAMP(token_logprob) AS std,
+		       COUNT(*)                  AS num,
+		       tok < ?                   AS treat
+		FROM summed_oov
+		GROUP BY tok`
+)
+
+func (e *Experiment) Analyze(db *sql.DB) error {
+	ctx := context.Background()
+
+	var tx *sql.Tx
+
+	if t, err := db.BeginTx(ctx, nil); err != nil {
+		return err
+	} else {
+		tx = t
+	}
+
+	defer tx.Rollback()
+
+	if _, err := tx.ExecContext(ctx, `DROP TABLE IF EXISTS lesci_results`); err != nil {
+		return err
+	}
+
+	if _, err := tx.ExecContext(ctx, sqlPairs); err != nil {
+		return fmt.Errorf("pairs: %w", err)
+	}
+
+	if _, err := tx.ExecContext(ctx, sqlMatched); err != nil {
+		return fmt.Errorf("matched: %w", err)
+	}
+
+	low := e.cutoff - e.window
+	high := e.cutoff + e.window
+
+	if e.window == -1 {
+		low = 0
+		high = 10000000 // TODO use counterfactual
+	}
+
+	if _, err := tx.ExecContext(ctx, sqlSummedOOV, low, high); err != nil {
+		return fmt.Errorf("summed_oov: %w", err)
+	}
+
+	if _, err := tx.ExecContext(ctx, sqlResults, e.cutoff); err != nil {
+		return fmt.Errorf("lesci_results: %w", err)
+	}
+
+	return tx.Commit()
+}
diff --git a/research/lesci/append.go b/research/lesci/append.go
new file mode 100644
index 0000000..a3459ff
--- /dev/null
+++ b/research/lesci/append.go
@@ -0,0 +1,46 @@
+package lesci
+
+import (
+	"context"
+	"database/sql"
+	"database/sql/driver"
+
+	"github.com/duckdb/duckdb-go/v2"
+)
+
+type AppendFunc func([]driver.Value) error
+type AppendRowsFunc func(appendFunc AppendFunc) error
+
+func AppendRows(db *sql.DB, table string, appendRowsFunc AppendRowsFunc) error {
+	var conn *sql.Conn
+
+	if c, err := db.Conn(context.Background()); err != nil {
+		return err
+	} else {
+		conn = c
+	}
+
+	defer conn.Close()
+
+	return conn.Raw(func(driverConn any) error {
+		duckConn := driverConn.(*duckdb.Conn)
+
+		var appender *duckdb.Appender
+
+		if a, err := duckdb.NewAppenderFromConn(duckConn, "", table); err != nil {
+			return err
+		} else {
+			appender = a
+		}
+
+		defer appender.Close()
+
+		if err := appendRowsFunc(func(row []driver.Value) error {
+			return appender.AppendRow(row...)
+		}); err != nil {
+			return err
+		}
+
+		return appender.Flush()
+	})
+}
diff --git a/research/lesci/cmd/lesci/main.go b/research/lesci/cmd/lesci/main.go
new file mode 100644
index 0000000..b485c43
--- /dev/null
+++ b/research/lesci/cmd/lesci/main.go
@@ -0,0 +1,76 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"path"
+
+	"go.jknobloc.com/x/dataset"
+	"go.jknobloc.com/x/gpt2"
+	"go.jknobloc.com/x/research/lesci"
+	"go.jknobloc.com/x/tokenizer/bpe"
+)
+
+func main() {
+	if err := gpt2.InitializeEnvironment(); err != nil {
+		log.Fatal(err)
+	}
+
+	sizes := []int{8192, 16384, 32768, 50256, 100512}
+
+	for i := range len(sizes) - 1 {
+		e, m := setup(sizes[i], sizes[len(sizes)-1])
+
+		if err := m.Init(); err != nil {
+			log.Fatal(err)
+		}
+
+		if err := e.Run(); err != nil {
+			log.Fatal(err)
+		}
+
+		if err := m.Destroy(); err != nil {
+			log.Fatal(err)
+		}
+	}
+
+	if err := gpt2.DestroyEnvironment(); err != nil {
+		log.Fatal(err)
+	}
+}
+
+func setup(control, treatment int) (*lesci.Experiment, *gpt2.Model) {
+	a := fmt.Sprintf("gpt2/models/onnx_eval/gpt2_%d_m000_babylm_v2", control)
+	b := fmt.Sprintf("gpt2/models/onnx_eval/gpt2_%d_m000_babylm_v2", 100512)
+
+	m := must(model(path.Join(a, "model_eval.onnx"), "0", control))
+	t := must(bpe.NewTokenizerFromFiles(path.Join(a, "vocab.json"), path.Join(a, "merges.txt")))
+	c := must(bpe.NewTokenizerFromFiles(path.Join(b, "vocab.json"), path.Join(b, "merges.txt")))
+	d := must(dataset.NewFileReader("dataset/cmd/dataset/tmp/babylm/train_100M", "*.train"))
+
+	o := fmt.Sprintf("out/lesci/m000/babylm_%d_%d", control, treatment)
+
+	return must(lesci.NewExperiment(m, t, c, d, o, control, 5000)), m
+}
+
+func must[T any](v T, err error) T {
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	return v
+}
+
+func model(name, device string, vocabSize int) (*gpt2.Model, error) {
+	cfg := gpt2.DefaultConfig()
+
+	cfg.VocabSize = vocabSize + 1
+
+	m := gpt2.NewModel(name, device, cfg, gpt2.Options{
+		WithCache:    false,
+		WithLogits:   false,
+		WithLogProbs: true,
+	})
+
+	return m, nil
+}
diff --git a/research/lesci/context.go b/research/lesci/context.go
new file mode 100644
index 0000000..df9b059
--- /dev/null
+++ b/research/lesci/context.go
@@ -0,0 +1,56 @@
+package lesci
+
+import (
+	"database/sql"
+	"database/sql/driver"
+	"fmt"
+
+	"go.jknobloc.com/x/llm"
+)
+
+type logProb struct {
+	document int
+	token    int
+	value    float32
+	offset   int
+}
+
+func (e *Experiment) BuildContext(db *sql.DB) error {
+	eval := llm.NewEvaluator(e.model, e.tokenizer, func(job llm.Job, logProbs []float32, tokens []int) []logProb {
+		r := make([]logProb, len(tokens))
+
+		for i, token := range tokens {
+			r[i] = logProb{
+				document: job.Document,
+				token:    token,
+				value:    logProbs[i],
+				offset:   job.Position*512 + job.Seen + i, // TODO refactor
+			}
+		}
+
+		return r
+	}, llm.EvaluatorConfig{
+		BatchSize:  32,
+		NumWorkers: 64,
+	})
+
+	if ok, err := EnsureTable(db, "context", `CREATE TABLE context(uid INTEGER, token INTEGER, logprob FLOAT, pos INTEGER)`); err != nil {
+		return err
+	} else if !ok {
+		fmt.Println("context table not empty")
+
+		return nil
+	}
+
+	return AppendRows(db, "context", func(append AppendFunc) error {
+		return eval.RunAndCollect("Context", e.data, 1024, 512, func(r []logProb) error {
+			for _, l := range r {
+				if err := append([]driver.Value{l.document, l.token, l.value, l.offset}); err != nil {
+					return err
+				}
+			}
+
+			return nil
+		})
+	})
+}
diff --git a/research/lesci/database.go b/research/lesci/database.go
new file mode 100644
index 0000000..69c8d49
--- /dev/null
+++ b/research/lesci/database.go
@@ -0,0 +1,38 @@
+package lesci
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+)
+
+func EnsureTable(db *sql.DB, table string, query string) (bool, error) {
+	ctx := context.Background()
+
+	var exists bool
+
+	queryTableExists := `SELECT COUNT(*) > 0 FROM information_schema.tables WHERE table_name = ?`
+
+	if err := db.QueryRowContext(ctx, queryTableExists, table).Scan(&exists); err != nil {
+		return false, err
+	}
+
+	if !exists {
+		if _, err := db.ExecContext(ctx, query); err != nil {
+			return false, err
+		}
+
+		return true, nil
+	}
+
+	var count int64
+
+	queryNumRows := fmt.Sprintf(`SELECT COUNT(*) FROM "%s"`, table)
+
+	if err := db.QueryRowContext(ctx, queryNumRows).Scan(&count); err != nil {
+		return false, errors.New("table not empty")
+	}
+
+	return count == 0, nil
+}
diff --git a/research/lesci/experiment.go b/research/lesci/experiment.go
new file mode 100644
index 0000000..4184199
--- /dev/null
+++ b/research/lesci/experiment.go
@@ -0,0 +1,96 @@
+package lesci
+
+import (
+	"database/sql"
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+
+	_ "github.com/duckdb/duckdb-go/v2"
+
+	"go.jknobloc.com/x/dataset"
+	"go.jknobloc.com/x/llm"
+)
+
+type Experiment struct {
+	model          llm.Causal
+	tokenizer      llm.Tokenizer
+	counterfactual llm.Tokenizer
+	data           dataset.Reader
+	name           string
+	cutoff         int
+	window         int
+}
+
+func NewExperiment(model llm.Causal, tokenizer, counterfactual llm.Tokenizer, data dataset.Reader, name string, cutoff, window int) (*Experiment, error) {
+	e := &Experiment{
+		model:          model,
+		tokenizer:      tokenizer,
+		counterfactual: counterfactual,
+		data:           data,
+		name:           name,
+		cutoff:         cutoff,
+		window:         window,
+	}
+
+	return e, nil
+}
+
+func (e *Experiment) Run() error {
+	if err := os.MkdirAll(e.name, 0775); err != nil {
+		log.Fatal(err)
+	}
+
+	dsn := filepath.Join(e.name, "lesci.db")
+
+	var db *sql.DB
+
+	if database, err := initDatabase(dsn); err != nil {
+		return err
+	} else {
+		db = database
+	}
+
+	defer db.Close()
+
+	db.SetMaxOpenConns(1)
+
+	fmt.Println(e.name)
+
+	if err := e.BuildContext(db); err != nil {
+		return err
+	}
+
+	if err := e.ExtractData(db); err != nil {
+		return err
+	}
+
+	if err := e.Analyze(db); err != nil {
+		return err
+	}
+
+	if err := e.Plot(db); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func initDatabase(dsn string) (*sql.DB, error) {
+	var db *sql.DB
+
+	if database, err := sql.Open("duckdb", dsn); err != nil {
+		return nil, err
+	} else {
+		db = database
+	}
+
+	if err := db.Ping(); err != nil {
+		_ = db.Close()
+
+		return nil, err
+	}
+
+	return db, nil
+}
diff --git a/research/lesci/extract.go b/research/lesci/extract.go
new file mode 100644
index 0000000..fb24316
--- /dev/null
+++ b/research/lesci/extract.go
@@ -0,0 +1,74 @@
+package lesci
+
+import (
+	"context"
+	"database/sql"
+	"database/sql/driver"
+	"fmt"
+
+	"go.jknobloc.com/x/llm"
+	"go.jknobloc.com/x/tensor"
+	"go.jknobloc.com/x/tokenizer/bpe"
+)
+
+func (e *Experiment) ExtractData(db *sql.DB) error {
+	if ok, err := EnsureTable(db, "oov_rules", `CREATE TABLE oov_rules (a INTEGER, b INTEGER, c INTEGER)`); err != nil {
+		return err
+	} else if !ok {
+		fmt.Println("oov_rules table not empty")
+
+		if _, err := db.ExecContext(context.Background(), `DELETE FROM oov_rules`); err != nil {
+			return err
+		}
+	}
+
+	merges := bpe.Merges(e.counterfactual.(*bpe.Tokenizer))
+
+	rules, valid := Rules(e.counterfactual, merges)
+
+	mask := ExtractData(rules, valid, int64(e.cutoff), int64(e.window))
+
+	return AppendRows(db, "oov_rules", func(append AppendFunc) error {
+		for i, m := range mask {
+			if !m {
+				continue
+			}
+
+			if err := append([]driver.Value{
+				rules.At([]int{i, 0}),
+				rules.At([]int{i, 1}),
+				rules.At([]int{i, 2}),
+			}); err != nil {
+				return err
+			}
+		}
+
+		return nil
+	})
+}
+
+func Rules(tokenizer llm.Tokenizer, merges [][2]string) (tensor.Dense[int64], []bool) {
+	valid := bpe.ReachableMerges(tokenizer.(*bpe.Tokenizer), merges)
+
+	rules := tensor.NewDense[int64]([]int{len(merges), 3}, nil)
+
+	for i, merge := range merges {
+		if !valid[i] {
+			continue
+		}
+
+		a := tokenizer.Tokenize(merge[0])
+		b := tokenizer.Tokenize(merge[1])
+		c := tokenizer.Tokenize(merge[0] + merge[1])
+
+		if len(a) != 1 || len(b) != 1 || len(c) != 1 {
+			panic("unexpected token IDs")
+		}
+
+		rules.Set([]int{i, 0}, int64(a[0]))
+		rules.Set([]int{i, 1}, int64(b[0]))
+		rules.Set([]int{i, 2}, int64(c[0]))
+	}
+
+	return rules, valid
+}
diff --git a/research/lesci/go.mod b/research/lesci/go.mod
new file mode 100644
index 0000000..7d3ff30
--- /dev/null
+++ b/research/lesci/go.mod
@@ -0,0 +1,61 @@
+module go.jknobloc.com/x/research/lesci
+
+go 1.25.0
+
+require (
+	github.com/duckdb/duckdb-go/v2 v2.10500.0
+	go.jknobloc.com/x/dataset v0.0.0-20260410210408-456f8218a2d1
+	go.jknobloc.com/x/gpt2 v0.0.0-20260410210408-456f8218a2d1
+	go.jknobloc.com/x/llm v0.0.0-20260410210408-456f8218a2d1
+	go.jknobloc.com/x/tensor v0.0.0-20260317200409-2284e401d29f
+	go.jknobloc.com/x/tokenizer v0.0.0-20260410210408-456f8218a2d1
+	gonum.org/v1/gonum v0.17.0
+	gonum.org/v1/plot v0.16.0
+)
+
+require (
+	codeberg.org/go-fonts/liberation v0.5.0 // indirect
+	codeberg.org/go-latex/latex v0.2.0 // indirect
+	codeberg.org/go-pdf/fpdf v0.11.1 // indirect
+	git.sr.ht/~sbinet/gg v0.7.0 // indirect
+	github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b // indirect
+	github.com/andybalholm/brotli v1.2.0 // indirect
+	github.com/apache/arrow-go/v18 v18.5.1 // indirect
+	github.com/apache/thrift v0.22.0 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/duckdb/duckdb-go-bindings v0.10500.0 // indirect
+	github.com/duckdb/duckdb-go-bindings/lib/darwin-amd64 v0.10500.0 // indirect
+	github.com/duckdb/duckdb-go-bindings/lib/darwin-arm64 v0.10500.0 // indirect
+	github.com/duckdb/duckdb-go-bindings/lib/linux-amd64 v0.10500.0 // indirect
+	github.com/duckdb/duckdb-go-bindings/lib/linux-arm64 v0.10500.0 // indirect
+	github.com/duckdb/duckdb-go-bindings/lib/windows-amd64 v0.10500.0 // indirect
+	github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
+	github.com/goccy/go-json v0.10.5 // indirect
+	github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect
+	github.com/golang/snappy v1.0.0 // indirect
+	github.com/google/flatbuffers v25.12.19+incompatible // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/jonasknobloch/mbpe v0.1.1 // indirect
+	github.com/klauspost/asmfmt v1.3.2 // indirect
+	github.com/klauspost/compress v1.18.3 // indirect
+	github.com/klauspost/cpuid/v2 v2.3.0 // indirect
+	github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect
+	github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect
+	github.com/pierrec/lz4/v4 v4.1.25 // indirect
+	github.com/yalue/onnxruntime_go v1.25.0 // indirect
+	github.com/zeebo/xxh3 v1.1.0 // indirect
+	go.jknobloc.com/x/tui v0.0.0-20260324194423-87bbece7e040 // indirect
+	golang.org/x/exp v0.0.0-20260112195511-716be5621a96 // indirect
+	golang.org/x/image v0.37.0 // indirect
+	golang.org/x/mod v0.33.0 // indirect
+	golang.org/x/net v0.50.0 // indirect
+	golang.org/x/sync v0.20.0 // indirect
+	golang.org/x/sys v0.41.0 // indirect
+	golang.org/x/telemetry v0.0.0-20260209163413-e7419c687ee4 // indirect
+	golang.org/x/text v0.35.0 // indirect
+	golang.org/x/tools v0.42.0 // indirect
+	golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20251029180050-ab9386a59fda // indirect
+	google.golang.org/grpc v1.78.0 // indirect
+	google.golang.org/protobuf v1.36.11 // indirect
+)
diff --git a/research/lesci/go.sum b/research/lesci/go.sum
new file mode 100644
index 0000000..8221068
--- /dev/null
+++ b/research/lesci/go.sum
@@ -0,0 +1,175 @@
+codeberg.org/go-fonts/dejavu v0.4.0 h1:2yn58Vkh4CFK3ipacWUAIE3XVBGNa0y1bc95Bmfx91I=
+codeberg.org/go-fonts/dejavu v0.4.0/go.mod h1:abni088lmhQJvso2Lsb7azCKzwkfcnttl6tL1UTWKzg=
+codeberg.org/go-fonts/latin-modern v0.4.0 h1:vkRCc1y3whKA7iL9Ep0fSGVuJfqjix0ica9UflHORO8=
+codeberg.org/go-fonts/latin-modern v0.4.0/go.mod h1:BF68mZznJ9QHn+hic9ks2DaFl4sR5YhfM6xTYaP9vNw=
+codeberg.org/go-fonts/liberation v0.5.0 h1:SsKoMO1v1OZmzkG2DY+7ZkCL9U+rrWI09niOLfQ5Bo0=
+codeberg.org/go-fonts/liberation v0.5.0/go.mod h1:zS/2e1354/mJ4pGzIIaEtm/59VFCFnYC7YV6YdGl5GU=
+codeberg.org/go-latex/latex v0.2.0 h1:Ol/a6VHY06N+5gPfewswymoRb5ZcKDXWVaVegcx4hbI=
+codeberg.org/go-latex/latex v0.2.0/go.mod h1:VJAwQir7/T8LZxj7xAPivISKiVOwkMpQ8bTuPQ31X0Y=
+codeberg.org/go-pdf/fpdf v0.11.1 h1:U8+coOTDVLxHIXZgGvkfQEi/q0hYHYvEHFuGNX2GzGs=
+codeberg.org/go-pdf/fpdf v0.11.1/go.mod h1:Y0DGRAdZ0OmnZPvjbMp/1bYxmIPxm0ws4tfoPOc4LjU=
+git.sr.ht/~sbinet/cmpimg v0.1.0 h1:E0zPRk2muWuCqSKSVZIWsgtU9pjsw3eKHi8VmQeScxo=
+git.sr.ht/~sbinet/cmpimg v0.1.0/go.mod h1:FU12psLbF4TfNXkKH2ZZQ29crIqoiqTZmeQ7dkp/pxE=
+git.sr.ht/~sbinet/gg v0.7.0 h1:YmNf7YKd7diDMTPm86hZa1EM3pbkOyD/zzjl0LZUdNM=
+git.sr.ht/~sbinet/gg v0.7.0/go.mod h1:VYeli15tpMM4EvqlivlVbbyvWZlOU+EZn4XZmfBGUdM=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/ajstarks/deck v0.0.0-20200831202436-30c9fc6549a9/go.mod h1:JynElWSGnm/4RlzPXRlREEwqTHAN3T56Bv2ITsFT3gY=
+github.com/ajstarks/deck/generate v0.0.0-20210309230005-c3f852c02e19/go.mod h1:T13YZdzov6OU0A1+RfKZiZN9ca6VeKdBdyDV+BY97Tk=
+github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b h1:slYM766cy2nI3BwyRiyQj/Ud48djTMtMebDqepE95rw=
+github.com/ajstarks/svgo v0.0.0-20211024235047-1546f124cd8b/go.mod h1:1KcenG0jGWcpt8ov532z81sp/kMMUG485J2InIOyADM=
+github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ=
+github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
+github.com/apache/arrow-go/v18 v18.5.1 h1:yaQ6zxMGgf9YCYw4/oaeOU3AULySDlAYDOcnr4LdHdI=
+github.com/apache/arrow-go/v18 v18.5.1/go.mod h1:OCCJsmdq8AsRm8FkBSSmYTwL/s4zHW9CqxeBxEytkNE=
+github.com/apache/thrift v0.22.0 h1:r7mTJdj51TMDe6RtcmNdQxgn9XcyfGDOzegMDRg47uc=
+github.com/apache/thrift v0.22.0/go.mod h1:1e7J/O1Ae6ZQMTYdy9xa3w9k+XHWPfRvdPyJeynQ+/g=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ=
+github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
+github.com/duckdb/duckdb-go-bindings v0.10500.0 h1:xvYsvKVBPvl9rk1Z0W1LYHnZPGQsKxuNU2YUWlEHBJg=
+github.com/duckdb/duckdb-go-bindings v0.10500.0/go.mod h1:fjqSNEOmzfXbuszjb9tJBQjjC61KLCHM5Cy8tghENWc=
+github.com/duckdb/duckdb-go-bindings/lib/darwin-amd64 v0.10500.0 h1:ugUBH5e9c7F/3X14F2eXvaG9+hCdQtv2GZZ35AD+ai4=
+github.com/duckdb/duckdb-go-bindings/lib/darwin-amd64 v0.10500.0/go.mod h1:EnAvZh1kNJHp5yF+M1ZHNEvapnmt6anq1xXHVrAGqMo=
+github.com/duckdb/duckdb-go-bindings/lib/darwin-arm64 v0.10500.0 h1:8/J5noOOvIohkNaDFDMr5eYqk8bCRwkEUfGxi8QwsAg=
+github.com/duckdb/duckdb-go-bindings/lib/darwin-arm64 v0.10500.0/go.mod h1:IGLSeEcFhNeZF16aVjQCULD7TsFZKG5G7SyKJAXKp5c=
+github.com/duckdb/duckdb-go-bindings/lib/linux-amd64 v0.10500.0 h1:Wf2rKNI/fGY9nlqb6zPAeiv27DGxgC34RpglkMeQhis=
+github.com/duckdb/duckdb-go-bindings/lib/linux-amd64 v0.10500.0/go.mod h1:KAIynZ0GHCS7X5fRyuFnQMg/SZBPK/bS9OCOVojClxw=
+github.com/duckdb/duckdb-go-bindings/lib/linux-arm64 v0.10500.0 h1:Ktea0Z993M26LLfCIu+dQbPS+AnyvVSFxZpEm8UbNu4=
+github.com/duckdb/duckdb-go-bindings/lib/linux-arm64 v0.10500.0/go.mod h1:81SGOYoEUs8qaAfSk1wRfM5oobrIJ5KI7AzYhK6/bvQ=
+github.com/duckdb/duckdb-go-bindings/lib/windows-amd64 v0.10500.0 h1:6ZUdeqS5oWAJBELJbRMs1d6U5yXYmzH3vNkiH+qIU2U=
+github.com/duckdb/duckdb-go-bindings/lib/windows-amd64 v0.10500.0/go.mod h1:K25pJL26ARblGDeuAkrdblFvUen92+CwksLtPEHRqqQ=
+github.com/duckdb/duckdb-go/v2 v2.10500.0 h1:fINiqzIVyZYCuOeA1JZlT6xFXsDOM8DHMOXV+WEaYDY=
+github.com/duckdb/duckdb-go/v2 v2.10500.0/go.mod h1:0VI/sNb61DMuNLWR4kPh6AjPGnnhlKjuefcmrskXurY=
+github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
+github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro=
+github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
+github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4=
+github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
+github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g=
+github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
+github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
+github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
+github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs=
+github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/flatbuffers v25.12.19+incompatible h1:haMV2JRRJCe1998HeW/p0X9UaMTK6SDo0ffLn2+DbLs=
+github.com/google/flatbuffers v25.12.19+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
+github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/jonasknobloch/mbpe v0.1.1 h1:eXUrMdM7Wt6kPTA8cyX+t1PsInk3QIXpt430pE8CJEs=
+github.com/jonasknobloch/mbpe v0.1.1/go.mod h1:2qW/5BfAu7GKAXXNYJzMrIPYk/8z5+/TbabaAb+vPv8=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4=
+github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE=
+github.com/klauspost/compress v1.18.3 h1:9PJRvfbmTabkOX8moIpXPbMMbYN60bWImDDU7L+/6zw=
+github.com/klauspost/compress v1.18.3/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
+github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
+github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
+github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs=
+github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY=
+github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI=
+github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE=
+github.com/pierrec/lz4/v4 v4.1.25 h1:kocOqRffaIbU5djlIBr7Wh+cx82C0vtFb0fOurZHqD0=
+github.com/pierrec/lz4/v4 v4.1.25/go.mod h1:EoQMVJgeeEOMsCqCzqFm2O0cJvljX2nGZjcRIPL34O4=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
+github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
+github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
+github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
+github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
+github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
+github.com/yalue/onnxruntime_go v1.25.0 h1:nlhVau1BpLZ/BYr+WpPZCJRD/WES0qo6dK7aKyyAs3g=
+github.com/yalue/onnxruntime_go v1.25.0/go.mod h1:b4X26A8pekNb1ACJ58wAXgNKeUCGEAQ9dmACut9Sm/4=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
+github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
+github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
+github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
+go.jknobloc.com/x/dataset v0.0.0-20260410210408-456f8218a2d1 h1:W7C1aiXena5iyiUa4Oe+DVPcpfxgHhajOcALrEWUc+0=
+go.jknobloc.com/x/dataset v0.0.0-20260410210408-456f8218a2d1/go.mod h1:UZypBoGqi23LRt29l2z9NtVD2pKBHd4unV/sztHRz4o=
+go.jknobloc.com/x/gpt2 v0.0.0-20260410210408-456f8218a2d1 h1:UNifV0zjvuAXUzPXBvMGRiRmTwieyozXzmvbFfX6LBE=
+go.jknobloc.com/x/gpt2 v0.0.0-20260410210408-456f8218a2d1/go.mod h1:+bjSkUYyMsvlX1p6ln4zkZaZHF6z+XnCu/6mAMWRmIc=
+go.jknobloc.com/x/llm v0.0.0-20260410210408-456f8218a2d1 h1:NNYSIsyYcmBBtaYFqCjXk1hIg0KskXTqyFbM9kwksfc=
+go.jknobloc.com/x/llm v0.0.0-20260410210408-456f8218a2d1/go.mod h1:Ffq3FMFeZZyK3Xfx6eNYpBJZhzIBkytqmOEk+L7E6mE=
+go.jknobloc.com/x/tensor v0.0.0-20260317200409-2284e401d29f h1:coStpLBd6b1lGem4cmGaNSBeeeYaCQ9jRdzHI9651Zw=
+go.jknobloc.com/x/tensor v0.0.0-20260317200409-2284e401d29f/go.mod h1:Citqq246efr+IGX0RBARqDm4gXwvPA4aNAz3F8Il6kk=
+go.jknobloc.com/x/tokenizer v0.0.0-20260410210408-456f8218a2d1 h1:LmNp5OFgcm+4dsNrXPmMPUW/f0ZcvlwnRv5IzZAYY+8=
+go.jknobloc.com/x/tokenizer v0.0.0-20260410210408-456f8218a2d1/go.mod h1:A6usfKV6fYSVHGvEMO7xMMIshky4NRCBzLhs36W7KC8=
+go.jknobloc.com/x/tui v0.0.0-20260324194423-87bbece7e040 h1:7Ago/qaKuXIJ7/+/XDRO8Yb/4VWzBLKEcbAq2jD5k3Q=
+go.jknobloc.com/x/tui v0.0.0-20260324194423-87bbece7e040/go.mod h1:kblmBsWO7WlRCo4sWpstaVV9ukNfYP3PeAPFhcoJvKA=
+go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
+go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
+go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8=
+go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM=
+go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA=
+go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI=
+go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E=
+go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg=
+go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM=
+go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA=
+go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE=
+go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/exp v0.0.0-20260112195511-716be5621a96 h1:Z/6YuSHTLOHfNFdb8zVZomZr7cqNgTJvA8+Qz75D8gU=
+golang.org/x/exp v0.0.0-20260112195511-716be5621a96/go.mod h1:nzimsREAkjBCIEFtHiYkrJyT+2uy9YZJB7H1k68CXZU=
+golang.org/x/image v0.37.0 h1:ZiRjArKI8GwxZOoEtUfhrBtaCN+4b/7709dlT6SSnQA=
+golang.org/x/image v0.37.0/go.mod h1:/3f6vaXC+6CEanU4KJxbcUZyEePbyKbaLoDOe4ehFYY=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8=
+golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60=
+golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
+golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
+golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/telemetry v0.0.0-20260209163413-e7419c687ee4 h1:bTLqdHv7xrGlFbvf5/TXNxy/iUwwdkjhqQTJDjW7aj0=
+golang.org/x/telemetry v0.0.0-20260209163413-e7419c687ee4/go.mod h1:g5NllXBEermZrmR51cJDQxmJUHUOfRAaNyWBM+R+548=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8=
+golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
+golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k=
+golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY=
+golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
+gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
+gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
+gonum.org/v1/plot v0.16.0 h1:dK28Qx/Ky4VmPUN/2zeW0ELyM6ucDnBAj5yun7M9n1g=
+gonum.org/v1/plot v0.16.0/go.mod h1:Xz6U1yDMi6Ni6aaXILqmVIb6Vro8E+K7Q/GeeH+Pn0c=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20251029180050-ab9386a59fda h1:i/Q+bfisr7gq6feoJnS/DlpdwEL4ihp41fvRiM3Ork0=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20251029180050-ab9386a59fda/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk=
+google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc=
+google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U=
+google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
+google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las=
+rsc.io/pdf v0.1.1 h1:k1MczvYDUvJBe93bYd7wrZLLUEcLZAuF824/I4e5Xr4=
+rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
diff --git a/research/lesci/lesci.go b/research/lesci/lesci.go
new file mode 100644
index 0000000..4c3ad85
--- /dev/null
+++ b/research/lesci/lesci.go
@@ -0,0 +1,207 @@
+package lesci
+
+import (
+	"fmt"
+
+	"go.jknobloc.com/x/tensor"
+)
+
+// ExtractData
+//
+// https://github.com/pietrolesci/tokenisation-bias/blob/376abc0ed6924986cbaf696ea10fdda71e550e45/notebooks/01_extract_data.ipynb
+func ExtractData(rules tensor.Dense[int64], valid []bool, cutoff, window int64) []bool {
+	shape := rules.Shape()
+
+	if len(shape) != 2 || shape[0] != len(valid) || shape[1] != 3 {
+		panic("shape mismatch")
+	}
+
+	// clamped := Window(rules, valid, cutoff, window)
+
+	clamped := valid // collect everything for now
+
+	filtered := Filter(rules, clamped, cutoff)
+	oov := OutOfVocab(rules, filtered, cutoff)
+
+	num := func(mask []bool) int {
+		n := 0
+
+		for _, v := range mask {
+			if !v {
+				continue
+			}
+
+			n++
+		}
+
+		return n
+	}
+
+	fmt.Println("clamped:", num(clamped))
+	fmt.Println("filtered:", num(filtered))
+	fmt.Println("oov:", num(oov))
+
+	return oov
+}
+
+// Window
+//
+// # Filter merges based on the window size and vocab size
+// merges_df = (
+//
+//	merges_df.filter((pl.col("tok") < vocab_size + window_size) & (pl.col("tok") >= vocab_size - window_size))
+//	.sort("tok")
+//	.drop("count")
+//
+// )
+func Window(rules tensor.Dense[int64], mask []bool, cutoff, window int64) []bool {
+	shape := rules.Shape()
+
+	if len(shape) != 2 {
+		// TODO
+	}
+
+	rows := shape[0]
+	cols := shape[1]
+
+	if cols != 3 {
+		// TODO
+	}
+
+	windowed := make([]bool, rows)
+
+	for i, m := range mask {
+		if !m {
+			continue
+		}
+
+		token := rules.At([]int{i, 2})
+
+		if (token >= cutoff-window) && (token < cutoff+window) {
+			windowed[i] = true
+		}
+	}
+
+	return windowed
+}
+
+// Filter
+//
+// # Find tokens (in-vocab) that got merged into others, either as first or second part of the token
+// to_drop = pl.concat(
+//
+//	[
+//	    merges_df.filter(pl.col("tok") < vocab_size).join(
+//	        merges_df.select(["tok", col]), left_on="tok", right_on=col, how="inner", suffix="_new"
+//	    )
+//	    for col in ["tok_a", "tok_b"]
+//	]
+//
+// ).select(["tok", "tok_new"])
+// print(f"{len(to_drop)} tokens dropped because are part of other tokens in the window (window size: {window_size} * 2)")
+// merges_df = merges_df.filter(pl.col("tok").is_in(to_drop["tok"].implode()).not_())
+func Filter(rules tensor.Dense[int64], mask []bool, cutoff int64) []bool {
+	shape := rules.Shape()
+
+	if len(shape) != 2 {
+		// TODO
+	}
+
+	rows := shape[0]
+	cols := shape[1]
+
+	if len(mask) != rows {
+		panic("") // TODO
+	}
+
+	if cols != 3 {
+		panic("") // TODO
+	}
+
+	// drop all rules a + b -> c where c is a or b elsewhere in window
+
+	intermediate := make(map[int64]struct{})
+
+	for i, m := range mask {
+		if !m {
+			continue
+		}
+
+		var a, b int64
+
+		if row, ok := rules.Select(0, i).Contiguous().Data(); !ok {
+			panic("") // TODO
+		} else {
+			a, b = row[0], row[1]
+		}
+
+		if _, ok := intermediate[a]; !ok {
+			intermediate[a] = struct{}{}
+		}
+
+		if _, ok := intermediate[b]; !ok {
+			intermediate[b] = struct{}{}
+		}
+	}
+
+	filtered := make([]bool, rows)
+
+	for i, m := range mask {
+		if !m {
+			continue
+		}
+
+		c := rules.At([]int{i, 2})
+
+		if _, ok := intermediate[c]; ok {
+			if c < cutoff {
+				continue // only drop in-vocab tokens
+			}
+		}
+
+		filtered[i] = true
+	}
+
+	return filtered
+}
+
+// OutOfVocab
+//
+// # We only need this to get the tokens composing the OOV tokens
+// merges_df = merges_df.filter(pl.col("tok") >= vocab_size)  # notice the '='
+func OutOfVocab(rules tensor.Dense[int64], mask []bool, cutoff int64) []bool {
+	shape := rules.Shape()
+
+	if len(shape) != 2 {
+		// TODO
+	}
+
+	rows := shape[0]
+	cols := shape[1]
+
+	if len(mask) != rows {
+		panic("") // TODO
+	}
+
+	if cols != 3 {
+		panic("") // TODO
+	}
+
+	oov := make([]bool, rows)
+
+	for i, m := range mask {
+		if !m {
+			continue
+		}
+
+		c := rules.At([]int{i, 2})
+
+		if c < cutoff {
+			continue
+		}
+
+		oov[i] = true
+	}
+
+	return oov
+}
diff --git a/research/lesci/plot.go b/research/lesci/plot.go
new file mode 100644
index 0000000..07a0c20
--- /dev/null
+++ b/research/lesci/plot.go
@@ -0,0 +1,140 @@
+package lesci
+
+import (
+	"context"
+	"database/sql"
+	"image/color"
+	"path"
+	"slices"
+
+	"gonum.org/v1/gonum/stat"
+	"gonum.org/v1/plot"
+	"gonum.org/v1/plot/plotter"
+	"gonum.org/v1/plot/vg"
+	"gonum.org/v1/plot/vg/draw"
+)
+
+func (e *Experiment) Plot(db *sql.DB) error {
+	var rows *sql.Rows
+
+	if r, err := db.QueryContext(context.Background(), "SELECT tok, mean, treat FROM lesci_results"); err != nil {
+		return err
+	} else {
+		rows = r
+	}
+
+	defer rows.Close()
+
+	var pts plotter.XYs
+
+	var inX, inY, oovX, oovY []float64
+
+	for rows.Next() {
+		var tok, mean float64
+
+		var treat bool
+
+		if err := rows.Scan(&tok, &mean, &treat); err != nil {
+			return err
+		}
+
+		if mean < -20 || mean > 0 {
+			continue
+		}
+
+		pts = append(pts, plotter.XY{X: tok, Y: mean})
+
+		if treat {
+			inX = append(inX, tok)
+			inY = append(inY, mean)
+		} else {
+			oovX = append(oovX, tok)
+			oovY = append(oovY, mean)
+		}
+	}
+
+	if err := rows.Err(); err != nil {
+		return err
+	}
+
+	return render(pts, inX, inY, oovX, oovY, float64(e.cutoff), path.Join(e.name, "scatter.png"))
+}
+
+func render(pts plotter.XYs, inX, inY, oovX, oovY []float64, cutoff float64, out string) error {
+	p := plot.New()
+
+	p.Title.Text = "Original Data and Regression Fit for Mean"
+
+	p.X.Label.Text = "Token"
+	p.Y.Label.Text = "Mean"
+
+	p.Y.Min = -20
+	p.Y.Max = 0
+
+	p.Add(plotter.NewGrid())
+
+	if err := addScatter(p, pts); err != nil {
+		return err
+	}
+
+	orangeRed := color.RGBA{R: 255, G: 69, B: 0, A: 255}
+
+	aIn, bIn := stat.LinearRegression(inX, inY, nil, false)
+	aOov, bOov := stat.LinearRegression(oovX, oovY, nil, false)
+
+	addLine(p, aIn, bIn, slices.Min(inX), slices.Max(inX), orangeRed)
+	addLine(p, aOov, bOov, cutoff, slices.Max(oovX), orangeRed)
+
+	if err := addStep(p, aIn, bIn, aOov, bOov, cutoff, orangeRed); err != nil {
+		return err
+	}
+
+	return p.Save(10*vg.Inch, 6*vg.Inch, out)
+}
+
+func addScatter(p *plot.Plot, pts plotter.XYs) error {
+	s, err := plotter.NewScatter(pts)
+
+	if err != nil {
+		return err
+	}
+
+	s.Color = color.RGBA{A: 64}
+	s.Radius = vg.Points(2)
+	s.Shape = draw.CircleGlyph{}
+
+	p.Add(s)
+
+	return nil
+}
+
+func addLine(p *plot.Plot, alpha, beta, xMin, xMax float64, c color.Color) {
+	f := plotter.NewFunction(func(x float64) float64 {
+		return alpha + beta*x
+	})
+
+	f.Color = c
+	f.Width = vg.Points(1.5)
+	f.XMin = xMin
+	f.XMax = xMax
+
+	p.Add(f)
+}
+
+func addStep(p *plot.Plot, aIn, bIn, aOov, bOov, cutoff float64, c color.Color) error {
+	l, err := plotter.NewLine(plotter.XYs{
+		{X: cutoff, Y: aIn + bIn*cutoff},
+		{X: cutoff, Y: aOov + bOov*cutoff},
+	})
+
+	if err != nil {
+		return err
+	}
+
+	l.Color = c
+	l.Width = vg.Points(1.5)
+
+	p.Add(l)
+
+	return nil
+}
diff --git a/tokenizer/bpe/utility.go b/tokenizer/bpe/utility.go
index f47c073..fbe99fc 100644
--- a/tokenizer/bpe/utility.go
+++ b/tokenizer/bpe/utility.go
@@ -18,3 +18,23 @@ func NewTokenizerFromFiles(vocab, merges string) (*Tokenizer, error) {
 
 	return NewTokenizer(tokenizer), nil
 }
+
+func Vocab(t *Tokenizer) []string {
+	m, ok := t.mbpe.Model().(*mbpe.MBPE)
+
+	if !ok {
+		panic("unimplemented")
+	}
+
+	return m.Vocab()
+}
+
+func Merges(t *Tokenizer) [][2]string {
+	m, ok := t.mbpe.Model().(*mbpe.MBPE)
+
+	if !ok {
+		panic("unimplemented")
+	}
+
+	return m.Merges()
+}
diff --git a/tokenizer/bpe/validate.go b/tokenizer/bpe/validate.go
index f9bd265..84910cf 100644
--- a/tokenizer/bpe/validate.go
+++ b/tokenizer/bpe/validate.go
@@ -106,3 +106,17 @@ func ByteCoverage(t *Tokenizer) bool {
 
 	return covered
 }
+
+func ReachableMerges(t *Tokenizer, merges [][2]string) []bool {
+	mask := make([]bool, len(merges))
+
+	for i, merge := range merges {
+		a := t.Tokenize(merge[0])
+		b := t.Tokenize(merge[1])
+		c := t.Tokenize(merge[0] + merge[1])
+
+		mask[i] = len(a) == 1 && len(b) == 1 && len(c) == 1
+	}
+
+	return mask
+}