From 87bbece7e0400af29c52e34ebb794d74498d13a9 Mon Sep 17 00:00:00 2001 From: Jonas Knobloch Date: Tue, 24 Mar 2026 20:44:23 +0100 Subject: Preserve per-byte error replacements --- tokenizer/bpe/split/fsa_test.go | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) (limited to 'tokenizer/bpe/split/fsa_test.go') diff --git a/tokenizer/bpe/split/fsa_test.go b/tokenizer/bpe/split/fsa_test.go index be11e79..9188d99 100644 --- a/tokenizer/bpe/split/fsa_test.go +++ b/tokenizer/bpe/split/fsa_test.go @@ -38,3 +38,30 @@ func BenchmarkFSA_FindAllReference(b *testing.B) { _ = f.FindAll(s) } } + +func FuzzFSA_FindAll(f *testing.F) { + fsa := NewFSA() + + ref := mbpe.NewFSA() + + f.Add("foo") + f.Add(" ") + f.Add("\n") + f.Add(" bar") + + f.Fuzz(func(t *testing.T, s string) { + out := fsa.FindAll(s) + + expected := ref.FindAll(s) + + if len(out) != len(expected) { + t.Fatalf("expected %s but got %s", expected, out) + } + + for i, m := range out { + if m != expected[i] { + t.Errorf("expected [%s] but got [%s]", expected[i], m) + } + } + }) +} -- cgit v1.3.1