5 files changed, 216 insertions, 0 deletions
diff --git a/llmc/cmd/data/fineweb.go b/llmc/cmd/data/fineweb.go
new file mode 100644
index 0000000..cabfa74
--- /dev/null
+++ b/llmc/cmd/data/fineweb.go
@@ -0,0 +1,43 @@
+package main
+
+import (
+	"log"
+
+	"go.jknobloc.com/x/dataset"
+	"go.jknobloc.com/x/llm"
+	"go.jknobloc.com/x/llmc"
+	"go.jknobloc.com/x/tokenizer/bpe"
+)
+
+// classic + 10B: fineweb10B; sample-10BT
+// classic + 100B: fineweb100B; sample-100BT
+// edu + 10B: edu_fineweb10B; sample-10BT
+// edu + 100B: edu_fineweb100B; sample-100BT
+
+func fineWeb() {
+	var reader dataset.Reader
+
+	if r, err := dataset.NewParquetReader("artifacts/data/raw/fineweb-edu/sample-100BT/train"); err != nil {
+		log.Fatal(err)
+	} else {
+		reader = r
+	}
+
+	var tokenizer llm.Tokenizer
+
+	if t, err := bpe.NewTokenizerFromFiles("gpt2/models/base/vocab.json", "gpt2/models/base/merges.txt"); err != nil {
+		log.Fatal(err)
+	} else {
+		tokenizer = t
+	}
+
+	docs := llmc.TokenizeAll(reader, tokenizer, 50256)
+
+	if err := llmc.WriteShards("artifacts/data/llmc/edu_fineweb100B", "edu_fineweb", 100_000_000, docs); err != nil {
+		log.Fatal(err)
+	}
+
+	if err := reader.Err(); err != nil {
+		log.Fatal(err)
+	}
+}
diff --git a/llmc/cmd/data/main.go b/llmc/cmd/data/main.go
new file mode 100644
index 0000000..b12d124
--- /dev/null
+++ b/llmc/cmd/data/main.go
@@ -0,0 +1,6 @@
+package main
+
+func main() {
+	// tinyShakespeare()
+	// fineWeb()
+}
diff --git a/llmc/cmd/data/tinyshakespeare.go b/llmc/cmd/data/tinyshakespeare.go
new file mode 100644
index 0000000..b2d062c
--- /dev/null
+++ b/llmc/cmd/data/tinyshakespeare.go
@@ -0,0 +1,66 @@
+package main
+
+import (
+	"fmt"
+	"log"
+
+	"go.jknobloc.com/x/dataset"
+	"go.jknobloc.com/x/llmc"
+	"go.jknobloc.com/x/tokenizer/bpe"
+)
+
+func tinyShakespeare() {
+	var reader *dataset.FileReader
+
+	if r, err := dataset.NewFileReader("artifacts/data/raw/tinyshakespeare/input.txt", "*.txt"); err != nil {
+		log.Fatal(err)
+	} else {
+		reader = r
+	}
+
+	reader.SetDelimiters("\n\n")
+
+	var tokenizer *bpe.Tokenizer
+
+	if t, err := bpe.NewTokenizerFromFiles("gpt2/models/base/vocab.json", "gpt2/models/base/merges.txt"); err != nil {
+		log.Fatal(err)
+	} else {
+		tokenizer = t
+	}
+
+	result := make([]int64, 0)
+
+	for _, doc := range reader.Texts() {
+		ids := tokenizer.Tokenize(doc)
+
+		toks := make([]int64, len(ids)+1)
+
+		toks[0] = 50256 // end of text
+
+		for i, id := range ids {
+			toks[i+1] = int64(id)
+		}
+
+		result = append(result, toks...)
+	}
+
+	val := llmc.DataFile[int64]{
+		Model:  llmc.GPT2,
+		Tokens: result[:32768],
+	}
+
+	train := llmc.DataFile[int64]{
+		Model:  llmc.GPT2,
+		Tokens: result[32768:],
+	}
+
+	if _, err := llmc.Serialize(&val, "artifacts/data/llmc/tinyshakespeare/tiny_shakespeare_val.bin"); err != nil {
+		log.Fatal(err)
+	}
+
+	if _, err := llmc.Serialize(&train, "artifacts/data/llmc/tinyshakespeare/tiny_shakespeare_train.bin"); err != nil {
+		log.Fatal(err)
+	}
+
+	fmt.Println(val)
+}
diff --git a/llmc/cmd/debug/debug.go b/llmc/cmd/debug/debug.go
new file mode 100644
index 0000000..289e55f
--- /dev/null
+++ b/llmc/cmd/debug/debug.go
@@ -0,0 +1,49 @@
+package main
+
+import (
+	"fmt"
+	"log"
+
+	"go.jknobloc.com/x/llmc"
+	"go.jknobloc.com/x/tokenizer/bpe"
+)
+
+func main() {
+	var data llmc.DataFile[uint16]
+	var gold llmc.DataFile[uint16]
+
+	_ = must(llmc.Deserialize("artifacts/data/llmc/tinyshakespeare/tiny_shakespeare_train.bin", &data))
+	_ = must(llmc.Deserialize("artifacts/test/llmc/tinyshakespeare/tiny_shakespeare_train.bin", &gold))
+
+	t := must(bpe.NewTokenizerFromFiles("gpt2/models/base/vocab.json", "gpt2/models/base/merges.txt"))
+
+	itoa := bpe.Itoa(t)
+
+	fmt.Println(len(data.Tokens))
+	fmt.Println(len(gold.Tokens))
+
+	if len(data.Tokens) != len(gold.Tokens) {
+		log.Fatal("length mismatch")
+	}
+
+	for i, a := range data.Tokens {
+		b := gold.Tokens[i]
+
+		if a != b {
+			fmt.Printf("token mismatch: %d %d at index %d\n", a, b, i)
+
+			fmt.Println("data", itoa[int64(a)])
+			fmt.Println("gold", itoa[int64(b)])
+
+			break
+		}
+	}
+}
+
+func must[T any](v T, err error) T {
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	return v
+}
diff --git a/llmc/cmd/peek/main.go b/llmc/cmd/peek/main.go
new file mode 100644
index 0000000..caacb31
--- /dev/null
+++ b/llmc/cmd/peek/main.go
@@ -0,0 +1,52 @@
+package main
+
+import (
+	"fmt"
+	"log"
+
+	"golang.org/x/exp/constraints"
+
+	"go.jknobloc.com/x/llmc"
+	"go.jknobloc.com/x/tokenizer/bpe"
+)
+
+func main() {
+	var a llmc.DataFile[uint16]
+	var b llmc.DataFile[uint16]
+
+	_ = must(llmc.Deserialize("artifacts/data/llmc/edu_fineweb100B/edu_fineweb_val_000000.bin", &a))
+	_ = must(llmc.Deserialize("artifacts/test/llmc/edu_fineweb100B/edu_fineweb_val_000000.bin", &b))
+
+	t := must(bpe.NewTokenizerFromFiles("gpt2/models/base/vocab.json", "gpt2/models/base/merges.txt"))
+
+	s := decode(&a, 1024, t)
+	k := decode(&b, 1024, t)
+
+	fmt.Println(s)
+
+	fmt.Println()
+
+	fmt.Println(k)
+}
+
+func must[T any](v T, err error) T {
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	return v
+}
+
+func decode[T constraints.Integer](src *llmc.DataFile[T], n int, t *bpe.Tokenizer) string {
+	ids := make([]int, n)
+
+	if len(src.Tokens) < n {
+		panic("not enough tokens")
+	}
+
+	for i := range n {
+		ids[i] = int(src.Tokens[i])
+	}
+
+	return t.Decode(ids)
+}