From 7ae0651a55ace3c5d1f651775c1160f6e0b72152 Mon Sep 17 00:00:00 2001 From: Jonas Knobloch Date: Sun, 7 Jun 2026 20:26:59 +0200 Subject: Add config to lesci experiment --- research/lesci/lesci.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'research/lesci/lesci.go') diff --git a/research/lesci/lesci.go b/research/lesci/lesci.go index 8bae6c6..871cf8f 100644 --- a/research/lesci/lesci.go +++ b/research/lesci/lesci.go @@ -9,16 +9,18 @@ import ( // ExtractData // // https://github.com/pietrolesci/tokenisation-bias/blob/376abc0ed6924986cbaf696ea10fdda71e550e45/notebooks/01_extract_data.ipynb -func ExtractData(rules tensor.Dense[int64], valid []bool, cutoff, window int64) []bool { +func ExtractData(rules tensor.Dense[int64], valid []bool, cutoff, window int64, clampRulesBeforeFilter bool) []bool { shape := rules.Shape() if len(shape) != 2 || shape[0] != len(valid) || shape[1] != 3 { panic("shape mismatch") } - clamped := Window(rules, valid, cutoff, window) + clamped := valid // collect everything for now - // clamped := valid // collect everything for now + if clampRulesBeforeFilter { + clamped = Window(rules, valid, cutoff, window) + } filtered := Filter(rules, clamped, cutoff) oov := OutOfVocab(rules, filtered, cutoff) -- cgit v1.3.1