diff options
| -rw-r--r-- | dataset/parquet.go | 10 |
1 files changed, 8 insertions, 2 deletions
diff --git a/dataset/parquet.go b/dataset/parquet.go index 81454f8..1ba2a9b 100644 --- a/dataset/parquet.go +++ b/dataset/parquet.go @@ -7,6 +7,7 @@ import ( "iter" "path/filepath" "slices" + "strings" "github.com/apache/arrow-go/v18/arrow" "github.com/apache/arrow-go/v18/arrow/array" @@ -23,12 +24,17 @@ type ParquetReader struct { } func NewParquetReader(name string) (*ParquetReader, error) { - var shards []string + shards := make([]string, 0) if matches, err := filepath.Glob(filepath.Join(name, "*.parquet")); err != nil { return nil, err } else { - shards = matches + // skip AppleDouble metadata files + for _, m := range matches { + if !strings.HasPrefix(filepath.Base(m), "._") { + shards = append(shards, m) + } + } } if len(shards) == 0 { |