summaryrefslogtreecommitdiff
path: root/tokenizer/bpe/split/fsa_test.go
blob: be11e79cc4488b0f6a56dbaf771532348e0a78ff (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
package split

import (
	"slices"
	"testing"

	"github.com/jonasknobloch/mbpe"
)

var s = "The quick brown fox jumps over the lazy dog's back."

func TestFSA_FindAll(t *testing.T) {
	a, b := mbpe.NewFSA(), NewFSA()

	x, y := a.FindAll(s), b.FindAll(s)

	if !slices.Equal(x, y) {
		t.Errorf("expected %v\nbut got %v\n", x, y)
	}
}

func BenchmarkFSA_FindAll(b *testing.B) {
	f := NewFSA()

	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		_ = f.FindAll(s)
	}
}

func BenchmarkFSA_FindAllReference(b *testing.B) {
	f := mbpe.NewFSA()

	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		_ = f.FindAll(s)
	}
}