diff options
Diffstat (limited to 'tokenizer/bpe/split/fsa.go')
| -rw-r--r-- | tokenizer/bpe/split/fsa.go | 2 |
1 files changed, 1 insertions, 1 deletions
diff --git a/tokenizer/bpe/split/fsa.go b/tokenizer/bpe/split/fsa.go index b85254b..885aa5e 100644 --- a/tokenizer/bpe/split/fsa.go +++ b/tokenizer/bpe/split/fsa.go @@ -29,7 +29,7 @@ type FSA struct { func NewFSA() *FSA { return &FSA{ state: StateInitial, - static: []string{"'s", "'t", "'re", "'m", "'ll", "'d"}, + static: []string{"'s", "'t", "'re", "'ve", "'m", "'ll", "'d"}, } } |