From 87bbece7e0400af29c52e34ebb794d74498d13a9 Mon Sep 17 00:00:00 2001 From: Jonas Knobloch Date: Tue, 24 Mar 2026 20:44:23 +0100 Subject: Preserve per-byte error replacements --- tokenizer/bpe/split/fsa.go | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'tokenizer/bpe/split/fsa.go') diff --git a/tokenizer/bpe/split/fsa.go b/tokenizer/bpe/split/fsa.go index a0c4cd1..b85254b 100644 --- a/tokenizer/bpe/split/fsa.go +++ b/tokenizer/bpe/split/fsa.go @@ -155,6 +155,13 @@ func (f *FSA) Read(next rune) bool { } func (f *FSA) FindAll(s string) []string { + if !utf8.ValidString(s) { + // s = strings.ToValidUTF8(s, string(utf8.RuneError)) replaces with a single utf8.RuneError + // s = strings.Map(func(r rune) rune { return r }, s) should be identical to s = string([]rune(s)) + + s = string([]rune(s)) // replaces each invalid byte with utf8.RuneError during decoding + } + var findAll func(start int, matches []string) []string findAll = func(start int, matches []string) []string { -- cgit v1.3.1