1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
|
package main
import (
"fmt"
"log"
"go.jknobloc.com/x/research/knobloch"
"go.jknobloc.com/x/shelf"
"go.jknobloc.com/x/tokenizer/bpe"
)
const modelDir = "models/mbpe/minipile"
// the two families are swept separately, so a model is only ever compared
// against its own kind and the shared/unshared split never mixes m with mi
var families = []string{"m", "mi"}
func models(family string) []string {
var r []string
for alpha := 0; alpha <= 100; alpha += 10 {
r = append(r, fmt.Sprintf("gpt2_50256_%s%03d_minipile", family, alpha))
}
return r
}
func main() {
dict := shelf.Abs("results/knobloch/minipile/dict.txt")
cfg := bpe.Config{
Recover: false,
}
for _, family := range families {
names := models(family)
knobloch.SharedVocabs = nil
for _, model := range names {
knobloch.SharedVocabs = append(knobloch.SharedVocabs, shelf.Item(modelDir+"/"+model+"/vocab.json"))
}
for _, model := range names {
dir := shelf.Item(modelDir + "/" + model)
t, err := bpe.NewTokenizerFromFiles(shelf.Abs(dir+"/vocab.json"), shelf.Abs(dir+"/merges.txt"), cfg)
if err != nil {
log.Fatal(err)
}
if err := knobloch.TokenFrequencies(t, dict, model); err != nil {
log.Fatal(err)
}
}
}
}
|
