blob: f8c81c1ea3bfd4174c09f369f61d203e866efb1d (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
|
package main
import (
"log"
"sync/atomic"
"time"
"go.jknobloc.com/x/dataset"
"go.jknobloc.com/x/llm"
"go.jknobloc.com/x/profile"
"go.jknobloc.com/x/shelf"
"go.jknobloc.com/x/tokenizer/bpe"
"go.jknobloc.com/x/tui"
)
func main() {
stop := profile.CPU()
reader := data()
t := tokenizer()
pb := tui.NewProgressBar("Tokenize", 20, 1000, time.Now())
var processed atomic.Int64
pb.Start(1*time.Second, func() int {
return int(processed.Load())
})
defer pb.Close()
for n, d := range reader.Texts() {
if n >= 1000 {
break
}
tokens := t.Tokenize(d)
_ = tokens
processed.Add(1)
}
profile.Mem()
stop()
}
func data() *dataset.ParquetReader {
var simple *dataset.ParquetReader
if r, err := dataset.NewParquetReader(shelf.Abs("data/wikipedia/20231101/simple/train")); err != nil {
log.Fatal(err)
} else {
simple = r
}
return simple
}
func tokenizer() llm.Tokenizer {
var tok *bpe.Tokenizer
v := shelf.Abs("models/gpt2/vocab.json")
m := shelf.Abs("models/gpt2/merges.txt")
if t, err := bpe.NewTokenizerFromFiles(v, m, bpe.DefaultConfig()); err != nil {
log.Fatal(err)
} else {
tok = t
}
_ = bpe.ByteCoverage(tok)
return tok
}
|
