blob: aaf10e760be687703f8064829123409f3a58fc8a (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
|
package main
import (
"log"
"sync/atomic"
"time"
"go.jknobloc.com/x/dataset"
"go.jknobloc.com/x/llm"
"go.jknobloc.com/x/tokenizer/bpe"
"go.jknobloc.com/x/tui"
)
func main() {
reader := data()
t := tokenizer()
pb := tui.NewProgressBar("Tokenize", 20, 1000, time.Now())
var processed atomic.Int64
pb.Start(1*time.Second, func() int {
return int(processed.Load())
})
defer pb.Close()
n := 0
for d := range reader.Texts("text") {
if n >= 1000 {
break
}
tokens := t.Tokenize(d)
_ = tokens
processed.Add(1)
n++
}
}
func data() *dataset.ParquetReader {
var simple *dataset.ParquetReader
if r, err := dataset.NewParquetReader("dataset/cmd/dataset/tmp/wikipedia/simple/train"); err != nil {
log.Fatal(err)
} else {
simple = r
}
return simple
}
func tokenizer() llm.Tokenizer {
var tok *bpe.Tokenizer
if t, err := bpe.NewTokenizerFromFiles("gpt2/models/base/vocab.json", "gpt2/models/base/merges.txt"); err != nil {
log.Fatal(err)
} else {
tok = t
}
return tok
}
|
