blob: d6aa46604e828460cf01c57b267780141b37a7b3 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
|
package sander
import (
"go.jknobloc.com/x/tokenizer/bpe"
)
func UnusedTokensGPT2(tokenizer *bpe.Tokenizer) map[int64]struct{} {
unused := map[int64]struct{}{
177: {},
178: {},
179: {},
180: {},
181: {},
182: {},
183: {},
184: {},
185: {},
186: {},
187: {},
}
alphabet := bpe.InitialAlphabet()
itoa := bpe.Itoa(tokenizer)
for id := range unused {
token, ok := itoa[id]
if !ok {
panic("unknown token id")
}
if token != string(alphabet[id]) {
panic("unexpected token")
}
}
return unused
}
func UnusedTokensMBPE(tokenizer *bpe.Tokenizer) map[int64]struct{} {
unused := make(map[int64]struct{})
vocab := bpe.Vocab(tokenizer)
mask := bpe.ReachableTokens(tokenizer, vocab)
for i := range vocab {
if mask[i] {
continue
}
unused[int64(i)] = struct{}{}
}
return unused
}
// TODO we could just filter some input data
|
