From fda1267d7d6600c04a6cd7eb2a818ebefb8999e4 Mon Sep 17 00:00:00 2001
From: Jonas Knobloch <jonas.knobloch@t-online.de>
Date: Fri, 3 Apr 2026 19:20:57 +0200
Subject: Return document index in text iterator

---
 tokenizer/bpe/cmd/tokenize/main.go | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

(limited to 'tokenizer')

diff --git a/tokenizer/bpe/cmd/tokenize/main.go b/tokenizer/bpe/cmd/tokenize/main.go
index a0519cf..08d88aa 100644
--- a/tokenizer/bpe/cmd/tokenize/main.go
+++ b/tokenizer/bpe/cmd/tokenize/main.go
@@ -53,9 +53,7 @@ func main() {
 
 	defer pb.Close()
 
-	n := 0
-
-	for d := range reader.Texts() {
+	for n, d := range reader.Texts() {
 		if n >= 1000 {
 			break
 		}
@@ -65,8 +63,6 @@ func main() {
 		_ = tokens
 
 		processed.Add(1)
-
-		n++
 	}
 
 	if *memprofile != "" {
-- 
cgit v1.2.3