summaryrefslogtreecommitdiff
path: root/research/entropy/cmd
diff options
context:
space:
mode:
Diffstat (limited to 'research/entropy/cmd')
-rw-r--r--research/entropy/cmd/main.go221
-rw-r--r--research/entropy/cmd/segment/main.go (renamed from research/entropy/cmd/dict3/main.go)47
2 files changed, 11 insertions, 257 deletions
diff --git a/research/entropy/cmd/main.go b/research/entropy/cmd/main.go
deleted file mode 100644
index cba8c33..0000000
--- a/research/entropy/cmd/main.go
+++ /dev/null
@@ -1,221 +0,0 @@
-package main
-
-import (
- "database/sql"
- "fmt"
- "log"
-
- _ "github.com/duckdb/duckdb-go/v2"
- "go.jknobloc.com/x/research/entropy"
-
- "github.com/jonasknobloch/mbpe"
- "go.jknobloc.com/x/dataset"
- "go.jknobloc.com/x/gpt2"
- "go.jknobloc.com/x/shelf"
- "go.jknobloc.com/x/tokenizer/bpe"
-)
-
-type NoPreTok struct{}
-
-func (p *NoPreTok) PreTokenize(s string) []string {
- return []string{s}
-}
-
-const CorpusMean = -0.7749100363022372
-
-const Threshold = -2.0
-
-func main() {
- dict := mbpe.NewDict()
-
- if err := dict.Load(shelf.Abs("results/knobloch/minipile/dict.txt")); err != nil {
- log.Fatal(err)
- }
-
- if err := gpt2.InitializeEnvironment(); err != nil {
- log.Fatal(err)
- }
-
- // m := model()
- // t := tokenizer()
- // d := data()
- //
- // if err := entropy.Context(m, t, d); err != nil {
- // log.Fatal(err)
- // }
-
- o := shelf.Abs("results/runs-entropy/lesci_minipile_test_256_seed_default/gpt2_256_m000_minipile_seed/lesci.db")
-
- var db *sql.DB
-
- if database, err := initDatabase(o); err != nil {
- log.Fatal(err)
- } else {
- db = database
- }
-
- defer db.Close()
-
- db.SetMaxOpenConns(1)
-
- foo := tokenizerMBPE()
-
- type candidate struct {
- src string
- ids []int
- means []float64
- occurrences int64
- }
-
- candidates := make([]candidate, 0)
- baseline := entropy.NewBaseline()
-
- for i, c := range dict.Items()[10000:] {
- if i > 10 {
- break
- }
-
- s := c.Src()
-
- if len(s) < 10 {
- continue
- }
-
- ids := foo.Tokenize(s)
-
- means, occurrences, err := entropy.MeanLogProbs(ids, db)
-
- if err != nil {
- log.Fatal(err)
- }
-
- if occurrences == 0 {
- continue
- }
-
- {
- occurrences, err := entropy.LogProbs(ids, db)
-
- if err != nil {
- log.Fatal(err)
- }
-
- fmt.Println(ids)
- fmt.Println(occurrences)
- }
-
- candidates = append(candidates, candidate{s, ids, means, occurrences})
-
- baseline.Add(means)
- }
-
- for _, c := range candidates {
- fmt.Println(c.ids)
- fmt.Println(foo.Decoder().Decode([]string{c.src}))
- fmt.Println(c.means, c.occurrences)
-
- fmt.Println("excess ", entropy.Excess(c.means, Threshold))
- fmt.Println("spikes ", entropy.Spikes(c.means))
- fmt.Println("deviations", baseline.Deviations(c.means))
-
- fmt.Println()
- fmt.Println()
- fmt.Println()
- }
-
- if err := gpt2.DestroyEnvironment(); err != nil {
- log.Fatal(err)
- }
-}
-
-// func must[T any](v T, err error) T {
-// if err != nil {
-// log.Fatal(err)
-// }
-//
-// return v
-// }
-
-func model() *gpt2.Model {
- cfg := gpt2.ConfigDefault()
-
- cfg.VocabSize = 256
-
- opts := gpt2.Options{
- WithCache: false,
- WithLogits: false,
- WithLogProbs: true,
- }
-
- m := gpt2.NewModel(shelf.Abs("models/mbpe/minipile-byte/gpt2_256_m000_minipile_seed/model_eval.onnx"), cfg, opts)
-
- if err := m.Init(); err != nil {
- log.Fatal(err)
- }
-
- return m
-}
-
-func tokenizer() *bpe.Tokenizer {
- var tok *bpe.Tokenizer
-
- v := shelf.Abs("tokenizers/minipile/tokenizer_gpt2_256_m000_minipile/vocab.json")
- m := shelf.Abs("tokenizers/minipile/tokenizer_gpt2_256_m000_minipile/merges.txt")
-
- if t, err := bpe.NewTokenizerFromFiles(v, m, bpe.DefaultConfig()); err != nil {
- log.Fatal(err)
- } else {
- tok = t
- }
-
- return tok
-}
-
-func tokenizerMBPE() *mbpe.Tokenizer {
- var tok *bpe.Tokenizer
-
- v := shelf.Abs("tokenizers/minipile/tokenizer_gpt2_256_m000_minipile/vocab.json")
- m := shelf.Abs("tokenizers/minipile/tokenizer_gpt2_256_m000_minipile/merges.txt")
-
- if t, err := bpe.NewTokenizerFromFiles(v, m, bpe.DefaultConfig()); err != nil {
- log.Fatal(err)
- } else {
- tok = t
- }
-
- foo := bpe.MBPE(tok)
-
- foo.SetPreTokenizer(&NoPreTok{})
-
- return foo
-}
-
-func data() dataset.Reader {
- var miniPile *dataset.ParquetReader
-
- if r, err := dataset.NewParquetReader(shelf.Abs("data/minipile/validation")); err != nil {
- log.Fatal(err)
- } else {
- miniPile = r
- }
-
- return dataset.NewClampedReader(miniPile, 10)
-}
-
-func initDatabase(dsn string) (*sql.DB, error) {
- var db *sql.DB
-
- if database, err := sql.Open("duckdb", dsn); err != nil {
- return nil, err
- } else {
- db = database
- }
-
- if err := db.Ping(); err != nil {
- _ = db.Close()
-
- return nil, err
- }
-
- return db, nil
-}
diff --git a/research/entropy/cmd/dict3/main.go b/research/entropy/cmd/segment/main.go
index a6b0211..cc6aea4 100644
--- a/research/entropy/cmd/dict3/main.go
+++ b/research/entropy/cmd/segment/main.go
@@ -1,20 +1,20 @@
package main
import (
- "bufio"
"database/sql"
"fmt"
"io"
"log"
"os"
- _ "github.com/duckdb/duckdb-go/v2"
"github.com/jonasknobloch/mbpe"
+
"go.jknobloc.com/x/dict"
"go.jknobloc.com/x/research/entropy"
+ "go.jknobloc.com/x/shelf"
"go.jknobloc.com/x/tokenizer/bpe"
- "go.jknobloc.com/x/shelf"
+ _ "github.com/duckdb/duckdb-go/v2"
)
type NoPreTok struct{}
@@ -40,7 +40,7 @@ func main() {
var file *os.File
- if f, err := os.Open(shelf.Abs("results/knobloch/minipile/dict.txt")); err != nil {
+ if f, err := os.Open(shelf.Abs("results/entropy/minipile/dict.txt")); err != nil {
log.Fatal(err)
} else {
file = f
@@ -80,51 +80,26 @@ func initDatabase(dsn string) (*sql.DB, error) {
}
func loadDict(r io.Reader, tok *mbpe.Tokenizer) (*dict.Dict[*entropy.Entry], error) {
- d := dict.NewDict[*entropy.Entry]()
-
- // TODO we could count non empty lines to pre alloc dict
-
- scanner := bufio.NewScanner(r)
-
- for scanner.Scan() {
- line := scanner.Text()
-
- if err := scanner.Err(); err != nil {
- return nil, err
- }
-
+ return dict.LoadDict[*entropy.Entry](r, func(l string) (string, *entropy.Entry) {
var s string
var n int
- if _, err := fmt.Sscanf(line, "%s %d", &s, &n); err != nil {
- return nil, err
- }
-
- // TODO see cmd/main.go we currently need mbpe tok ??
-
- if n < 100 {
- continue
+ if _, err := fmt.Sscanf(l, "%s %d", &s, &n); err != nil {
+ panic(err)
}
ids := tok.Tokenize(s)
- if len(ids) < 4 {
- continue
- }
-
b := to[int, uint8](ids) // TODO that conversion is fucking ass
- // TODO vs decode each buffer suffix and use s as map key
-
- d.Set(string(b), &entropy.Entry{
+ return string(b), &entropy.Entry{
Encoded: s, // TODO encoded or decoded
TokenIDs: b,
LogProbs: make([]float32, len(ids)),
+ Bounds: make([]bool, len(ids)+1),
N: 0,
- })
- }
-
- return d, nil
+ }
+ })
}
func to[a bpe.Integer, b bpe.Integer](s []a) []b {
generated by cgit v1.2.3 (git 2.25.1) at 2026-10-06 23:56:01 +0000