diff options
| author | Jonas Knobloch <jonas.knobloch@t-online.de> | 2026-09-11 18:38:02 +0200 |
|---|---|---|
| committer | Jonas Knobloch <jonas.knobloch@t-online.de> | 2026-09-11 18:38:02 +0200 |
| commit | 330c6387962b197c5ca7a8051d4864feaecc0f25 (patch) | |
| tree | f13c82306ab432de6a610b1006d7cc10373dc7f6 /tokenizer/bpe/utility.go | |
| parent | 6e6a94b12d6d43c1ce4b8e587742627af6d86a43 (diff) | |
WIP
Diffstat (limited to 'tokenizer/bpe/utility.go')
| -rw-r--r-- | tokenizer/bpe/utility.go | 10 |
1 files changed, 10 insertions, 0 deletions
diff --git a/tokenizer/bpe/utility.go b/tokenizer/bpe/utility.go index 79e6faf..cb79d76 100644 --- a/tokenizer/bpe/utility.go +++ b/tokenizer/bpe/utility.go @@ -42,3 +42,13 @@ func Merges(t *Tokenizer) [][2]string { return m.Merges() } + +func AddToken(t *Tokenizer, token string) { + m, ok := t.mbpe.Model().(*mbpe.MBPE) + + if !ok { + panic("unimplemented") + } + + m.AddToken(token) +} |