From 0c03645aca88ed546121d5eef44cd25a200df305 Mon Sep 17 00:00:00 2001 From: Jonas Knobloch Date: Mon, 27 Apr 2026 22:32:57 +0200 Subject: Limit splitting to single delimiter --- llmc/cmd/data/tinyshakespeare.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'llmc/cmd/data') diff --git a/llmc/cmd/data/tinyshakespeare.go b/llmc/cmd/data/tinyshakespeare.go index d71151b..960dca6 100644 --- a/llmc/cmd/data/tinyshakespeare.go +++ b/llmc/cmd/data/tinyshakespeare.go @@ -19,7 +19,7 @@ func tinyShakespeare() { reader = r } - reader.SetDelimiters("\n\n") + reader.SetDelimiter("\n\n") var tokenizer *bpe.Tokenizer -- cgit v1.3.1