diff options
| author | Jonas Knobloch <jonas.knobloch@t-online.de> | 2026-04-20 14:53:13 +0200 |
|---|---|---|
| committer | Jonas Knobloch <jonas.knobloch@t-online.de> | 2026-04-22 12:51:53 +0200 |
| commit | f28f740f3385c3bd9f1620776e7ca09d21ea03c4 (patch) | |
| tree | 44491433fca804fa9becdf906cddb4892a177377 /tokenizer | |
| parent | 3fe37224ccc2500e7e767e5fc00163ef61182828 (diff) | |
Fix FSA statics
* Add missing 've match
Diffstat (limited to 'tokenizer')
| -rw-r--r-- | tokenizer/bpe/split/fsa.go | 2 |
1 files changed, 1 insertions, 1 deletions
diff --git a/tokenizer/bpe/split/fsa.go b/tokenizer/bpe/split/fsa.go index b85254b..885aa5e 100644 --- a/tokenizer/bpe/split/fsa.go +++ b/tokenizer/bpe/split/fsa.go @@ -29,7 +29,7 @@ type FSA struct { func NewFSA() *FSA { return &FSA{ state: StateInitial, - static: []string{"'s", "'t", "'re", "'m", "'ll", "'d"}, + static: []string{"'s", "'t", "'re", "'ve", "'m", "'ll", "'d"}, } } |