From cc7aa68fa63cb69cb3a565364ee0bee1f482f6f5 Mon Sep 17 00:00:00 2001
From: Jonas Knobloch <jonas.knobloch@t-online.de>
Date: Thu, 30 Apr 2026 13:46:00 +0200
Subject: Index delimited texts instead of shards

---
 dataset/file.go | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/dataset/file.go b/dataset/file.go
index 3cd48a7..83ff16f 100644
--- a/dataset/file.go
+++ b/dataset/file.go
@@ -47,7 +47,7 @@ func (f *FileReader) StripDelimiter(stripDelimiter bool) {
 }
 
 func (f *FileReader) Num() (int, error) {
-	return len(f.shards), nil
+	return countLinesAll(f.shards, []byte(f.delimiter))
 }
 
 func (f *FileReader) Err() error {
@@ -56,15 +56,19 @@ func (f *FileReader) Err() error {
 
 func (f *FileReader) Texts() iter.Seq2[int, string] {
 	return func(yield func(int, string) bool) {
-		for n, name := range f.shards {
-			for text := range f.read(name) {
-				if f.err != nil {
-					return
-				}
+		n := 0
 
+		for _, name := range f.shards {
+			for text := range f.read(name) {
 				if !yield(n, text) {
 					return
 				}
+
+				n++
+			}
+
+			if f.err != nil {
+				return
 			}
 		}
 	}
-- 
cgit v1.2.3