Preserve per-byte error replacements

1 files changed, 27 insertions, 0 deletions

diff --git a/tokenizer/bpe/split/fsa_test.go b/tokenizer/bpe/split/fsa_test.go
index be11e79..9188d99 100644
--- a/tokenizer/bpe/split/fsa_test.go
+++ b/tokenizer/bpe/split/fsa_test.go
@@ -38,3 +38,30 @@ func BenchmarkFSA_FindAllReference(b *testing.B) {
 		_ = f.FindAll(s)
 	}
 }
+
+func FuzzFSA_FindAll(f *testing.F) {
+	fsa := NewFSA()
+
+	ref := mbpe.NewFSA()
+
+	f.Add("foo")
+	f.Add(" ")
+	f.Add("\n")
+	f.Add("   bar")
+
+	f.Fuzz(func(t *testing.T, s string) {
+		out := fsa.FindAll(s)
+
+		expected := ref.FindAll(s)
+
+		if len(out) != len(expected) {
+			t.Fatalf("expected %s but got %s", expected, out)
+		}
+
+		for i, m := range out {
+			if m != expected[i] {
+				t.Errorf("expected [%s] but got [%s]", expected[i], m)
+			}
+		}
+	})
+}