From ce9c65cb1e13f7ae6eed4b419232a4da9dd63fdb Mon Sep 17 00:00:00 2001
From: Jonas Knobloch <jonas.knobloch@t-online.de>
Date: Tue, 31 Mar 2026 16:21:28 +0200
Subject: Handle unknown runes

---
 tokenizer/bpe/cmd/tokenize/main.go | 2 ++
 1 file changed, 2 insertions(+)

(limited to 'tokenizer/bpe/cmd/tokenize/main.go')

diff --git a/tokenizer/bpe/cmd/tokenize/main.go b/tokenizer/bpe/cmd/tokenize/main.go
index 1b0b868..1dbfaae 100644
--- a/tokenizer/bpe/cmd/tokenize/main.go
+++ b/tokenizer/bpe/cmd/tokenize/main.go
@@ -109,5 +109,7 @@ func tokenizer() llm.Tokenizer {
 		tok = t
 	}
 
+	_ = bpe.ByteCoverage(tok)
+
 	return tok
 }
-- 
cgit v1.3.1