blob: 405199f9eddb5b67138511b1747974d0fdfd5b2b (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
|
package bpe
// Add helper to ensure special token
// wip-special-tokens
func EnsureSpecialToken(t *Tokenizer, token string) int {
vocab := Vocab(t)
for i, v := range vocab {
if v == token {
return i
}
}
AddToken(t, token)
return len(vocab) - 1
}
|
