diff options
| author | Jonas Knobloch <jonas.knobloch@t-online.de> | 2026-04-03 19:20:57 +0200 |
|---|---|---|
| committer | Jonas Knobloch <jonas.knobloch@t-online.de> | 2026-04-03 20:06:42 +0200 |
| commit | fda1267d7d6600c04a6cd7eb2a818ebefb8999e4 (patch) | |
| tree | cbfab8e35e396a8b5dc4e9d5ed765ad699dac1db /tokenizer | |
| parent | c1644da32ecb9856f15e2c919a84d2666d5ef29c (diff) | |
Return document index in text iterator
Diffstat (limited to 'tokenizer')
| -rw-r--r-- | tokenizer/bpe/cmd/tokenize/main.go | 6 |
1 files changed, 1 insertions, 5 deletions
diff --git a/tokenizer/bpe/cmd/tokenize/main.go b/tokenizer/bpe/cmd/tokenize/main.go index a0519cf..08d88aa 100644 --- a/tokenizer/bpe/cmd/tokenize/main.go +++ b/tokenizer/bpe/cmd/tokenize/main.go @@ -53,9 +53,7 @@ func main() { defer pb.Close() - n := 0 - - for d := range reader.Texts() { + for n, d := range reader.Texts() { if n >= 1000 { break } @@ -65,8 +63,6 @@ func main() { _ = tokens processed.Add(1) - - n++ } if *memprofile != "" { |