package bpe

// Add helper to ensure special token
// wip-special-tokens

func EnsureSpecialToken(t *Tokenizer, token string) int {
	vocab := Vocab(t)

	for i, v := range vocab {
		if v == token {
			return i
		}
	}

	AddToken(t, token)

	return len(vocab) - 1
}