Recover tokenization based on config

4 files changed, 16 insertions, 4 deletions

diff --git a/llmc/cmd/data/fineweb.go b/llmc/cmd/data/fineweb.go
index 3d3d51b..a1bf7df 100644
--- a/llmc/cmd/data/fineweb.go
+++ b/llmc/cmd/data/fineweb.go
@@ -26,7 +26,10 @@ func fineWeb() {
 
 	var tokenizer llm.Tokenizer
 
-	if t, err := bpe.NewTokenizerFromFiles(shelf.Abs("models/gpt2/vocab.json"), shelf.Abs("models/gpt2/merges.txt")); err != nil {
+	v := shelf.Abs("models/gpt2/vocab.json")
+	m := shelf.Abs("models/gpt2/merges.txt")
+
+	if t, err := bpe.NewTokenizerFromFiles(v, m, bpe.DefaultConfig()); err != nil {
 		log.Fatal(err)
 	} else {
 		tokenizer = t
diff --git a/llmc/cmd/data/tinyshakespeare.go b/llmc/cmd/data/tinyshakespeare.go
index 70d4fd4..d71151b 100644
--- a/llmc/cmd/data/tinyshakespeare.go
+++ b/llmc/cmd/data/tinyshakespeare.go
@@ -23,7 +23,10 @@ func tinyShakespeare() {
 
 	var tokenizer *bpe.Tokenizer
 
-	if t, err := bpe.NewTokenizerFromFiles(shelf.Abs("models/gpt2/vocab.json"), shelf.Abs("models/gpt2/merges.txt")); err != nil {
+	v := shelf.Abs("models/gpt2/vocab.json")
+	m := shelf.Abs("models/gpt2/merges.txt")
+
+	if t, err := bpe.NewTokenizerFromFiles(v, m, bpe.DefaultConfig()); err != nil {
 		log.Fatal(err)
 	} else {
 		tokenizer = t
diff --git a/llmc/cmd/debug/debug.go b/llmc/cmd/debug/debug.go
index d1a621a..f076949 100644
--- a/llmc/cmd/debug/debug.go
+++ b/llmc/cmd/debug/debug.go
@@ -16,7 +16,10 @@ func main() {
 	_ = must(llmc.Deserialize(shelf.Abs("llmc/tinyshakespeare/tiny_shakespeare_train.bin"), &data))
 	_ = must(llmc.Deserialize(shelf.Abs("test/llmc/tinyshakespeare/tiny_shakespeare_train.bin"), &gold))
 
-	t := must(bpe.NewTokenizerFromFiles(shelf.Abs("models/gpt2/vocab.json"), shelf.Abs("models/gpt2/merges.txt")))
+	v := shelf.Abs("models/gpt2/vocab.json")
+	m := shelf.Abs("models/gpt2/merges.txt")
+
+	t := must(bpe.NewTokenizerFromFiles(v, m, bpe.DefaultConfig()))
 
 	itoa := bpe.Itoa(t)
 
diff --git a/llmc/cmd/peek/main.go b/llmc/cmd/peek/main.go
index 0a5eeab..1bfcca8 100644
--- a/llmc/cmd/peek/main.go
+++ b/llmc/cmd/peek/main.go
@@ -18,7 +18,10 @@ func main() {
 	_ = must(llmc.Deserialize(shelf.Abs("llmc/edu_fineweb100B/edu_fineweb_val_000000.bin"), &a))
 	_ = must(llmc.Deserialize(shelf.Abs("test/llmc/edu_fineweb100B/edu_fineweb_val_000000.bin"), &b))
 
-	t := must(bpe.NewTokenizerFromFiles(shelf.Abs("models/gpt2/vocab.json"), shelf.Abs("models/gpt2/merges.txt")))
+	v := shelf.Abs("models/gpt2/vocab.json")
+	m := shelf.Abs("models/gpt2/merges.txt")
+
+	t := must(bpe.NewTokenizerFromFiles(v, m, bpe.DefaultConfig()))
 
 	s := decode(&a, 1024, t)
 	k := decode(&b, 1024, t)