diff options
Diffstat (limited to 'research/lesci/extract.go')
| -rw-r--r-- | research/lesci/extract.go | 12 |
1 files changed, 1 insertions, 11 deletions
diff --git a/research/lesci/extract.go b/research/lesci/extract.go index c7c85e8..659cdc3 100644 --- a/research/lesci/extract.go +++ b/research/lesci/extract.go @@ -52,17 +52,7 @@ func Rules(tokenizer llm.Tokenizer, merges [][2]string) (tensor.Dense[int64], [] rules := tensor.NewDense[int64]([]int{len(merges), 3}, nil) - atoi := make(map[string]int) - - vocab := bpe.Vocab(tokenizer.(*bpe.Tokenizer)) - - for i, token := range vocab { - if _, ok := atoi[token]; ok { - continue - } - - atoi[token] = i - } + atoi := bpe.Atoi(tokenizer.(*bpe.Tokenizer)) for i, merge := range merges { if !valid[i] { |