From f998fee7427403698f9262d1e92d1a307c652094 Mon Sep 17 00:00:00 2001 From: Jonas Knobloch Date: Thu, 10 Sep 2026 01:59:34 +0200 Subject: WIP --- research/entropy/cmd/dict3/main.go | 159 ------------------------- research/entropy/cmd/main.go | 221 ----------------------------------- research/entropy/cmd/segment/main.go | 134 +++++++++++++++++++++ 3 files changed, 134 insertions(+), 380 deletions(-) delete mode 100644 research/entropy/cmd/dict3/main.go delete mode 100644 research/entropy/cmd/main.go create mode 100644 research/entropy/cmd/segment/main.go (limited to 'research/entropy/cmd') diff --git a/research/entropy/cmd/dict3/main.go b/research/entropy/cmd/dict3/main.go deleted file mode 100644 index a6b0211..0000000 --- a/research/entropy/cmd/dict3/main.go +++ /dev/null @@ -1,159 +0,0 @@ -package main - -import ( - "bufio" - "database/sql" - "fmt" - "io" - "log" - "os" - - _ "github.com/duckdb/duckdb-go/v2" - "github.com/jonasknobloch/mbpe" - "go.jknobloc.com/x/dict" - "go.jknobloc.com/x/research/entropy" - "go.jknobloc.com/x/tokenizer/bpe" - - "go.jknobloc.com/x/shelf" -) - -type NoPreTok struct{} - -func (p *NoPreTok) PreTokenize(s string) []string { - return []string{s} -} - -func main() { - o := shelf.Abs("results/runs-entropy/lesci_minipile_test_256_seed_default/gpt2_256_m000_minipile_seed/lesci.db") - - var db *sql.DB - - if database, err := initDatabase(o); err != nil { - log.Fatal(err) - } else { - db = database - } - - defer db.Close() - - db.SetMaxOpenConns(1) - - var file *os.File - - if f, err := os.Open(shelf.Abs("results/knobloch/minipile/dict.txt")); err != nil { - log.Fatal(err) - } else { - file = f - } - - tok := tokenizerMBPE() - - var dict *dict.Dict[*entropy.Entry] - - if d, err := loadDict(file, tok); err != nil { - log.Fatal(err) - } else { - dict = d - } - - if err := entropy.Run(db, dict); err != nil { - log.Fatal(err) - } -} - -func initDatabase(dsn string) (*sql.DB, error) { - var db *sql.DB - - if database, err := sql.Open("duckdb", dsn); err != nil { - return nil, err - } else { - db = database - } - - if err := db.Ping(); err != nil { - _ = db.Close() - - return nil, err - } - - return db, nil -} - -func loadDict(r io.Reader, tok *mbpe.Tokenizer) (*dict.Dict[*entropy.Entry], error) { - d := dict.NewDict[*entropy.Entry]() - - // TODO we could count non empty lines to pre alloc dict - - scanner := bufio.NewScanner(r) - - for scanner.Scan() { - line := scanner.Text() - - if err := scanner.Err(); err != nil { - return nil, err - } - - var s string - var n int - - if _, err := fmt.Sscanf(line, "%s %d", &s, &n); err != nil { - return nil, err - } - - // TODO see cmd/main.go we currently need mbpe tok ?? - - if n < 100 { - continue - } - - ids := tok.Tokenize(s) - - if len(ids) < 4 { - continue - } - - b := to[int, uint8](ids) // TODO that conversion is fucking ass - - // TODO vs decode each buffer suffix and use s as map key - - d.Set(string(b), &entropy.Entry{ - Encoded: s, // TODO encoded or decoded - TokenIDs: b, - LogProbs: make([]float32, len(ids)), - N: 0, - }) - } - - return d, nil -} - -func to[a bpe.Integer, b bpe.Integer](s []a) []b { - r := make([]b, len(s)) - - // TODO handle overflow - - for i, v := range s { - r[i] = b(v) - } - - return r -} - -func tokenizerMBPE() *mbpe.Tokenizer { - var tok *bpe.Tokenizer - - v := shelf.Abs("tokenizers/minipile/tokenizer_gpt2_256_m000_minipile/vocab.json") - m := shelf.Abs("tokenizers/minipile/tokenizer_gpt2_256_m000_minipile/merges.txt") - - if t, err := bpe.NewTokenizerFromFiles(v, m, bpe.DefaultConfig()); err != nil { - log.Fatal(err) - } else { - tok = t - } - - foo := bpe.MBPE(tok) - - foo.SetPreTokenizer(&NoPreTok{}) - - return foo -} diff --git a/research/entropy/cmd/main.go b/research/entropy/cmd/main.go deleted file mode 100644 index cba8c33..0000000 --- a/research/entropy/cmd/main.go +++ /dev/null @@ -1,221 +0,0 @@ -package main - -import ( - "database/sql" - "fmt" - "log" - - _ "github.com/duckdb/duckdb-go/v2" - "go.jknobloc.com/x/research/entropy" - - "github.com/jonasknobloch/mbpe" - "go.jknobloc.com/x/dataset" - "go.jknobloc.com/x/gpt2" - "go.jknobloc.com/x/shelf" - "go.jknobloc.com/x/tokenizer/bpe" -) - -type NoPreTok struct{} - -func (p *NoPreTok) PreTokenize(s string) []string { - return []string{s} -} - -const CorpusMean = -0.7749100363022372 - -const Threshold = -2.0 - -func main() { - dict := mbpe.NewDict() - - if err := dict.Load(shelf.Abs("results/knobloch/minipile/dict.txt")); err != nil { - log.Fatal(err) - } - - if err := gpt2.InitializeEnvironment(); err != nil { - log.Fatal(err) - } - - // m := model() - // t := tokenizer() - // d := data() - // - // if err := entropy.Context(m, t, d); err != nil { - // log.Fatal(err) - // } - - o := shelf.Abs("results/runs-entropy/lesci_minipile_test_256_seed_default/gpt2_256_m000_minipile_seed/lesci.db") - - var db *sql.DB - - if database, err := initDatabase(o); err != nil { - log.Fatal(err) - } else { - db = database - } - - defer db.Close() - - db.SetMaxOpenConns(1) - - foo := tokenizerMBPE() - - type candidate struct { - src string - ids []int - means []float64 - occurrences int64 - } - - candidates := make([]candidate, 0) - baseline := entropy.NewBaseline() - - for i, c := range dict.Items()[10000:] { - if i > 10 { - break - } - - s := c.Src() - - if len(s) < 10 { - continue - } - - ids := foo.Tokenize(s) - - means, occurrences, err := entropy.MeanLogProbs(ids, db) - - if err != nil { - log.Fatal(err) - } - - if occurrences == 0 { - continue - } - - { - occurrences, err := entropy.LogProbs(ids, db) - - if err != nil { - log.Fatal(err) - } - - fmt.Println(ids) - fmt.Println(occurrences) - } - - candidates = append(candidates, candidate{s, ids, means, occurrences}) - - baseline.Add(means) - } - - for _, c := range candidates { - fmt.Println(c.ids) - fmt.Println(foo.Decoder().Decode([]string{c.src})) - fmt.Println(c.means, c.occurrences) - - fmt.Println("excess ", entropy.Excess(c.means, Threshold)) - fmt.Println("spikes ", entropy.Spikes(c.means)) - fmt.Println("deviations", baseline.Deviations(c.means)) - - fmt.Println() - fmt.Println() - fmt.Println() - } - - if err := gpt2.DestroyEnvironment(); err != nil { - log.Fatal(err) - } -} - -// func must[T any](v T, err error) T { -// if err != nil { -// log.Fatal(err) -// } -// -// return v -// } - -func model() *gpt2.Model { - cfg := gpt2.ConfigDefault() - - cfg.VocabSize = 256 - - opts := gpt2.Options{ - WithCache: false, - WithLogits: false, - WithLogProbs: true, - } - - m := gpt2.NewModel(shelf.Abs("models/mbpe/minipile-byte/gpt2_256_m000_minipile_seed/model_eval.onnx"), cfg, opts) - - if err := m.Init(); err != nil { - log.Fatal(err) - } - - return m -} - -func tokenizer() *bpe.Tokenizer { - var tok *bpe.Tokenizer - - v := shelf.Abs("tokenizers/minipile/tokenizer_gpt2_256_m000_minipile/vocab.json") - m := shelf.Abs("tokenizers/minipile/tokenizer_gpt2_256_m000_minipile/merges.txt") - - if t, err := bpe.NewTokenizerFromFiles(v, m, bpe.DefaultConfig()); err != nil { - log.Fatal(err) - } else { - tok = t - } - - return tok -} - -func tokenizerMBPE() *mbpe.Tokenizer { - var tok *bpe.Tokenizer - - v := shelf.Abs("tokenizers/minipile/tokenizer_gpt2_256_m000_minipile/vocab.json") - m := shelf.Abs("tokenizers/minipile/tokenizer_gpt2_256_m000_minipile/merges.txt") - - if t, err := bpe.NewTokenizerFromFiles(v, m, bpe.DefaultConfig()); err != nil { - log.Fatal(err) - } else { - tok = t - } - - foo := bpe.MBPE(tok) - - foo.SetPreTokenizer(&NoPreTok{}) - - return foo -} - -func data() dataset.Reader { - var miniPile *dataset.ParquetReader - - if r, err := dataset.NewParquetReader(shelf.Abs("data/minipile/validation")); err != nil { - log.Fatal(err) - } else { - miniPile = r - } - - return dataset.NewClampedReader(miniPile, 10) -} - -func initDatabase(dsn string) (*sql.DB, error) { - var db *sql.DB - - if database, err := sql.Open("duckdb", dsn); err != nil { - return nil, err - } else { - db = database - } - - if err := db.Ping(); err != nil { - _ = db.Close() - - return nil, err - } - - return db, nil -} diff --git a/research/entropy/cmd/segment/main.go b/research/entropy/cmd/segment/main.go new file mode 100644 index 0000000..cc6aea4 --- /dev/null +++ b/research/entropy/cmd/segment/main.go @@ -0,0 +1,134 @@ +package main + +import ( + "database/sql" + "fmt" + "io" + "log" + "os" + + "github.com/jonasknobloch/mbpe" + + "go.jknobloc.com/x/dict" + "go.jknobloc.com/x/research/entropy" + "go.jknobloc.com/x/shelf" + "go.jknobloc.com/x/tokenizer/bpe" + + _ "github.com/duckdb/duckdb-go/v2" +) + +type NoPreTok struct{} + +func (p *NoPreTok) PreTokenize(s string) []string { + return []string{s} +} + +func main() { + o := shelf.Abs("results/runs-entropy/lesci_minipile_test_256_seed_default/gpt2_256_m000_minipile_seed/lesci.db") + + var db *sql.DB + + if database, err := initDatabase(o); err != nil { + log.Fatal(err) + } else { + db = database + } + + defer db.Close() + + db.SetMaxOpenConns(1) + + var file *os.File + + if f, err := os.Open(shelf.Abs("results/entropy/minipile/dict.txt")); err != nil { + log.Fatal(err) + } else { + file = f + } + + tok := tokenizerMBPE() + + var dict *dict.Dict[*entropy.Entry] + + if d, err := loadDict(file, tok); err != nil { + log.Fatal(err) + } else { + dict = d + } + + if err := entropy.Run(db, dict); err != nil { + log.Fatal(err) + } +} + +func initDatabase(dsn string) (*sql.DB, error) { + var db *sql.DB + + if database, err := sql.Open("duckdb", dsn); err != nil { + return nil, err + } else { + db = database + } + + if err := db.Ping(); err != nil { + _ = db.Close() + + return nil, err + } + + return db, nil +} + +func loadDict(r io.Reader, tok *mbpe.Tokenizer) (*dict.Dict[*entropy.Entry], error) { + return dict.LoadDict[*entropy.Entry](r, func(l string) (string, *entropy.Entry) { + var s string + var n int + + if _, err := fmt.Sscanf(l, "%s %d", &s, &n); err != nil { + panic(err) + } + + ids := tok.Tokenize(s) + + b := to[int, uint8](ids) // TODO that conversion is fucking ass + + return string(b), &entropy.Entry{ + Encoded: s, // TODO encoded or decoded + TokenIDs: b, + LogProbs: make([]float32, len(ids)), + Bounds: make([]bool, len(ids)+1), + N: 0, + } + }) +} + +func to[a bpe.Integer, b bpe.Integer](s []a) []b { + r := make([]b, len(s)) + + // TODO handle overflow + + for i, v := range s { + r[i] = b(v) + } + + return r +} + +func tokenizerMBPE() *mbpe.Tokenizer { + var tok *bpe.Tokenizer + + v := shelf.Abs("tokenizers/minipile/tokenizer_gpt2_256_m000_minipile/vocab.json") + m := shelf.Abs("tokenizers/minipile/tokenizer_gpt2_256_m000_minipile/merges.txt") + + if t, err := bpe.NewTokenizerFromFiles(v, m, bpe.DefaultConfig()); err != nil { + log.Fatal(err) + } else { + tok = t + } + + foo := bpe.MBPE(tok) + + foo.SetPreTokenizer(&NoPreTok{}) + + return foo +} -- cgit v1.3.1