diff options
| m--------- | mbpe | 0 | ||||
| -rw-r--r-- | tokenizer/bpe/split/fsa.go | 2 |
2 files changed, 1 insertions, 1 deletions
diff --git a/mbpe b/mbpe -Subproject afd29810a3f59dabad1560c299b0607473ea702 +Subproject d6c99eeabc16e7c58dcc179cf4cbf6e72d556db diff --git a/tokenizer/bpe/split/fsa.go b/tokenizer/bpe/split/fsa.go index b85254b..885aa5e 100644 --- a/tokenizer/bpe/split/fsa.go +++ b/tokenizer/bpe/split/fsa.go @@ -29,7 +29,7 @@ type FSA struct { func NewFSA() *FSA { return &FSA{ state: StateInitial, - static: []string{"'s", "'t", "'re", "'m", "'ll", "'d"}, + static: []string{"'s", "'t", "'re", "'ve", "'m", "'ll", "'d"}, } } |