1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
|
package sander
import (
"database/sql"
"database/sql/driver"
"fmt"
"go.jknobloc.com/x/onnx"
"go.jknobloc.com/x/research/lesci"
"go.jknobloc.com/x/tensor"
)
func (e *Experiment) Extract(db *sql.DB) error {
var data []float32
var shape []int
var model *onnx.Model
if m, err := onnx.NewModel(e.model); err != nil {
model = m
} else {
return err
}
if d, s, err := model.ExtractInitializer("transformer.wte.weight"); err != nil {
return err
} else {
data = d
shape = s
}
if shape[1] != e.hiddenDim {
panic("shape mismatch")
}
t := tensor.NewDense[float32](shape, data)
if _, err := db.Exec(fmt.Sprintf(`
DROP TABLE IF EXISTS embeddings;
CREATE TABLE embeddings (token_id INTEGER, embedding FLOAT[%d], reference BOOLEAN);
`, e.hiddenDim)); err != nil {
return err
}
return lesci.AppendRows(db, "embeddings", func(appendFunc lesci.AppendFunc) error {
for i := range shape[0] {
embedding, ok := t.Select(0, i).Contiguous().Data()
if !ok {
panic("")
}
_, reference := e.reference[int64(i)]
if err := appendFunc([]driver.Value{int32(i), embedding, reference}); err != nil {
return err
}
}
return nil
})
}
|
