From f28f740f3385c3bd9f1620776e7ca09d21ea03c4 Mon Sep 17 00:00:00 2001 From: Jonas Knobloch Date: Mon, 20 Apr 2026 14:53:13 +0200 Subject: Fix FSA statics * Add missing 've match --- tokenizer/bpe/split/fsa.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tokenizer') diff --git a/tokenizer/bpe/split/fsa.go b/tokenizer/bpe/split/fsa.go index b85254b..885aa5e 100644 --- a/tokenizer/bpe/split/fsa.go +++ b/tokenizer/bpe/split/fsa.go @@ -29,7 +29,7 @@ type FSA struct { func NewFSA() *FSA { return &FSA{ state: StateInitial, - static: []string{"'s", "'t", "'re", "'m", "'ll", "'d"}, + static: []string{"'s", "'t", "'re", "'ve", "'m", "'ll", "'d"}, } } -- cgit v1.3.1