1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
|
package knobloch
import (
"encoding/gob"
"io"
"os"
"sync"
"go.jknobloc.com/x/shelf"
"go.jknobloc.com/x/tokenizer/bpe"
)
type segmentsEntry struct {
Value []string
OK bool
}
// SegmentsPath holds the morphological segmentation that decides which vocab
// tokens count as morphemes; resolved against the shelf root on first use.
var SegmentsPath shelf.Item = "results/knobloch/minipile_19_fix/segments.gob"
var morphemesCache struct {
once sync.Once
m map[string]int
err error
}
// morphemes caches the parsed segmentation; the gob is several hundred MB and
// every plot and stats pass needs the same map.
func morphemes() (map[string]int, error) {
morphemesCache.once.Do(func() {
morphemesCache.m, morphemesCache.err = Morphemes(shelf.Abs(SegmentsPath))
})
return morphemesCache.m, morphemesCache.err
}
// isMorpheme reports whether a vocab id spells out a known morpheme; the
// returned closure holds the id-to-token map so callers build it once.
func isMorpheme(t *bpe.Tokenizer) (func(id int) bool, error) {
m, err := morphemes()
if err != nil {
return nil, err
}
itoa := bpe.Itoa(t)
return func(id int) bool {
token, ok := itoa[int64(id)]
if !ok {
panic("unexpected token ID")
}
_, isMorph := m[token]
return isMorph
}, nil
}
func Morphemes(name string) (map[string]int, error) {
var file *os.File
if f, err := os.Open(name); err != nil {
return nil, err
} else {
file = f
defer file.Close()
}
var segments map[string]segmentsEntry
if m, err := loadSegments(file); err != nil {
return nil, err
} else {
segments = m
}
morphemes := make(map[string]int)
for _, v := range segments {
// the segmenter returns the unsegmented compound when it fails
if !v.OK || len(v.Value) == 1 {
continue
}
for _, s := range v.Value {
morphemes[s]++
}
}
return morphemes, nil
// r := make([]int64, 0)
// atoi := bpe.Atoi(t)
//
// for k, v := range morphemes {
// if v < 0 {
// continue
// }
//
// i, ok := atoi[k]
//
// if !ok {
// continue
// }
//
// r = append(r, i)
// }
//
// return r, nil
}
func loadSegments(r io.Reader) (map[string]segmentsEntry, error) {
m := make(map[string]segmentsEntry)
dec := gob.NewDecoder(r)
if err := dec.Decode(&m); err != nil {
return nil, err
}
return m, nil
}
|
