1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
|
package main
import (
"flag"
"log"
"os"
"runtime"
"runtime/pprof"
"sync/atomic"
"time"
"go.jknobloc.com/x/dataset"
"go.jknobloc.com/x/llm"
"go.jknobloc.com/x/tokenizer/bpe"
"go.jknobloc.com/x/tui"
)
var cpuprofile = flag.String("cpuprofile", "", "write cpu profile to `file`")
var memprofile = flag.String("memprofile", "", "write memory profile to `file`")
func main() {
flag.Parse()
if *cpuprofile != "" {
var file *os.File
if f, err := os.Create(*cpuprofile); err != nil {
log.Fatal("could not create CPU profile: ", err)
} else {
file = f
defer file.Close()
}
if err := pprof.StartCPUProfile(file); err != nil {
log.Fatal("could not start CPU profile: ", err)
}
defer pprof.StopCPUProfile()
}
reader := data()
t := tokenizer()
pb := tui.NewProgressBar("Tokenize", 20, 1000, time.Now())
var processed atomic.Int64
pb.Start(1*time.Second, func() int {
return int(processed.Load())
})
defer pb.Close()
for n, d := range reader.Texts() {
if n >= 1000 {
break
}
tokens := t.Tokenize(d)
_ = tokens
processed.Add(1)
}
if *memprofile != "" {
var file *os.File
if f, err := os.Create(*memprofile); err != nil {
log.Fatal("could not create memory profile: ", err)
} else {
file = f
defer file.Close()
}
runtime.GC()
if err := pprof.Lookup("allocs").WriteTo(file, 0); err != nil {
log.Fatal("could not write memory profile: ", err)
}
}
}
func data() *dataset.ParquetReader {
var simple *dataset.ParquetReader
if r, err := dataset.NewParquetReader("dataset/cmd/dataset/tmp/wikipedia/simple/train"); err != nil {
log.Fatal(err)
} else {
simple = r
}
return simple
}
func tokenizer() llm.Tokenizer {
var tok *bpe.Tokenizer
if t, err := bpe.NewTokenizerFromFiles("gpt2/models/base/vocab.json", "gpt2/models/base/merges.txt"); err != nil {
log.Fatal(err)
} else {
tok = t
}
_ = bpe.ByteCoverage(tok)
return tok
}
|
