diff options
| author | Jonas Knobloch <jonas.knobloch@t-online.de> | 2026-04-27 22:32:57 +0200 |
|---|---|---|
| committer | Jonas Knobloch <jonas.knobloch@t-online.de> | 2026-04-27 23:00:53 +0200 |
| commit | 0c03645aca88ed546121d5eef44cd25a200df305 (patch) | |
| tree | 7f6f8fd50a16154ae0885fad52342ccbefd107fb /llmc/cmd | |
| parent | dca63cee471976f44bea351f28448d34b7481201 (diff) | |
Limit splitting to single delimiter
Diffstat (limited to 'llmc/cmd')
| -rw-r--r-- | llmc/cmd/data/tinyshakespeare.go | 2 |
1 files changed, 1 insertions, 1 deletions
diff --git a/llmc/cmd/data/tinyshakespeare.go b/llmc/cmd/data/tinyshakespeare.go index d71151b..960dca6 100644 --- a/llmc/cmd/data/tinyshakespeare.go +++ b/llmc/cmd/data/tinyshakespeare.go @@ -19,7 +19,7 @@ func tinyShakespeare() { reader = r } - reader.SetDelimiters("\n\n") + reader.SetDelimiter("\n\n") var tokenizer *bpe.Tokenizer |