From f28f740f3385c3bd9f1620776e7ca09d21ea03c4 Mon Sep 17 00:00:00 2001 From: Jonas Knobloch Date: Mon, 20 Apr 2026 14:53:13 +0200 Subject: Fix FSA statics * Add missing 've match --- mbpe | 2 +- tokenizer/bpe/split/fsa.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mbpe b/mbpe index afd2981..d6c99ee 160000 --- a/mbpe +++ b/mbpe @@ -1 +1 @@ -Subproject commit afd29810a3f59dabad1560c299b0607473ea7024 +Subproject commit d6c99eeabc16e7c58dcc179cf4cbf6e72d556db3 diff --git a/tokenizer/bpe/split/fsa.go b/tokenizer/bpe/split/fsa.go index b85254b..885aa5e 100644 --- a/tokenizer/bpe/split/fsa.go +++ b/tokenizer/bpe/split/fsa.go @@ -29,7 +29,7 @@ type FSA struct { func NewFSA() *FSA { return &FSA{ state: StateInitial, - static: []string{"'s", "'t", "'re", "'m", "'ll", "'d"}, + static: []string{"'s", "'t", "'re", "'ve", "'m", "'ll", "'d"}, } } -- cgit v1.3.1