1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
|
package bpe
import (
"fmt"
"slices"
"github.com/jonasknobloch/mbpe"
)
func InitialAlphabet() []rune {
alphabet := make([]rune, 256)
bc := mbpe.BytesChar
for i := 0; i < 256; i++ {
b := uint8(i)
runes := []rune(bc[b])
if len(runes) != 1 {
panic("unexpected replacement")
}
alphabet[i] = runes[0]
}
slices.Sort(alphabet)
return alphabet
}
func UnknownRunes(t *Tokenizer, s string) []rune {
atoi := make(map[string]int)
vocab := t.mbpe.Model().(*mbpe.MBPE).Vocab()
for i, token := range vocab {
if _, ok := atoi[token]; ok {
continue
}
atoi[token] = i
}
unknown := make(map[rune]struct{})
chunks := t.mbpe.PreTokenizer().PreTokenize(s)
for _, chunk := range chunks {
for _, r := range chunk {
i, ok := atoi[string(r)]
if !ok {
fmt.Printf("%d %s %v not in vocabulary\n", i, string(r), []byte(string(r)))
if _, ok := unknown[r]; ok {
continue
}
unknown[r] = struct{}{}
}
}
}
result := make([]rune, 0, len(unknown))
for r := range unknown {
result = append(result, r)
}
slices.Sort(result)
return result
}
func ByteCoverage(t *Tokenizer) bool {
atoi := make(map[string]int)
vocab := t.mbpe.Model().(*mbpe.MBPE).Vocab()
for i, token := range vocab {
if _, ok := atoi[token]; ok {
continue
}
atoi[token] = i
}
bc := mbpe.BytesChar
covered := true
for i := 0; i < 256; i++ {
c, ok := bc[byte(i)]
if !ok {
panic("not in replacement table")
}
if _, ok := atoi[c]; !ok {
fmt.Printf("%d %s %v not in vocabulary\n", i, c, []byte(c))
covered = false
}
}
return covered
}
|
