blob: 622644ae35cd37b88cfab535dd05c97b92a12e77 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
|
package main
import (
"fmt"
"log"
"github.com/jonasknobloch/mbpe"
"github.com/jonasknobloch/x/dataset"
"github.com/jonasknobloch/x/gpt2"
"github.com/jonasknobloch/x/llm"
)
func main() {
d := data()
m := model()
t := tokenizer()
e := llm.NewEvaluator()
e.SetTokenizer(t)
e.AddModel(m)
ppl, err := e.Perplexity(d)
if err != nil {
log.Fatal(err)
}
fmt.Println(ppl)
}
func data() *dataset.Reader {
var miniPile *dataset.Reader
if r, err := dataset.NewReader("dataset/cmd/dataset/tmp/minipile/validation"); err != nil {
log.Fatal(err)
} else {
miniPile = r
}
return miniPile
}
func model() *gpt2.Model {
m := gpt2.NewModel("gpt2/models/base/model.onnx", "0")
if err := m.Init(); err != nil {
log.Fatal(err)
}
return m
}
func tokenizer() *mbpe.Tokenizer {
m := mbpe.NewMBPE()
if err := m.Load("gpt2/models/base/vocab.json", "gpt2/models/base/merges.txt"); err != nil {
log.Fatal(err)
}
t := mbpe.NewTokenizer(m)
byteLevel := mbpe.NewByteLevel(false)
t.SetPreTokenizer(byteLevel)
t.SetDecoder(byteLevel)
return t
}
|
