Preserve per-byte error replacements

1 files changed, 7 insertions, 0 deletions

diff --git a/tokenizer/bpe/split/fsa.go b/tokenizer/bpe/split/fsa.go
index a0c4cd1..b85254b 100644
--- a/tokenizer/bpe/split/fsa.go
+++ b/tokenizer/bpe/split/fsa.go
@@ -155,6 +155,13 @@ func (f *FSA) Read(next rune) bool {
 }
 
 func (f *FSA) FindAll(s string) []string {
+	if !utf8.ValidString(s) {
+		// s = strings.ToValidUTF8(s, string(utf8.RuneError)) replaces with a single utf8.RuneError
+		// s = strings.Map(func(r rune) rune { return r }, s) should be identical to s = string([]rune(s))
+
+		s = string([]rune(s)) // replaces each invalid byte with utf8.RuneError during decoding
+	}
+
 	var findAll func(start int, matches []string) []string
 
 	findAll = func(start int, matches []string) []string {