summaryrefslogtreecommitdiff
path: root/tokenizer/bpe/split/fsa_test.go
blob: 9188d994e8341fc2858bf6ee01991e9088789bf0 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
package split

import (
	"slices"
	"testing"

	"github.com/jonasknobloch/mbpe"
)

var s = "The quick brown fox jumps over the lazy dog's back."

func TestFSA_FindAll(t *testing.T) {
	a, b := mbpe.NewFSA(), NewFSA()

	x, y := a.FindAll(s), b.FindAll(s)

	if !slices.Equal(x, y) {
		t.Errorf("expected %v\nbut got %v\n", x, y)
	}
}

func BenchmarkFSA_FindAll(b *testing.B) {
	f := NewFSA()

	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		_ = f.FindAll(s)
	}
}

func BenchmarkFSA_FindAllReference(b *testing.B) {
	f := mbpe.NewFSA()

	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		_ = f.FindAll(s)
	}
}

func FuzzFSA_FindAll(f *testing.F) {
	fsa := NewFSA()

	ref := mbpe.NewFSA()

	f.Add("foo")
	f.Add(" ")
	f.Add("\n")
	f.Add("   bar")

	f.Fuzz(func(t *testing.T, s string) {
		out := fsa.FindAll(s)

		expected := ref.FindAll(s)

		if len(out) != len(expected) {
			t.Fatalf("expected %s but got %s", expected, out)
		}

		for i, m := range out {
			if m != expected[i] {
				t.Errorf("expected [%s] but got [%s]", expected[i], m)
			}
		}
	})
}