diff options
Diffstat (limited to 'tokenizer/bpe')
| -rw-r--r-- | tokenizer/bpe/cmd/tokenize/main.go | 6 |
1 files changed, 1 insertions, 5 deletions
diff --git a/tokenizer/bpe/cmd/tokenize/main.go b/tokenizer/bpe/cmd/tokenize/main.go index a0519cf..08d88aa 100644 --- a/tokenizer/bpe/cmd/tokenize/main.go +++ b/tokenizer/bpe/cmd/tokenize/main.go @@ -53,9 +53,7 @@ func main() { defer pb.Close() - n := 0 - - for d := range reader.Texts() { + for n, d := range reader.Texts() { if n >= 1000 { break } @@ -65,8 +63,6 @@ func main() { _ = tokens processed.Add(1) - - n++ } if *memprofile != "" { |