Tests and test data

Author: irees

dbf.go

@@ -171,11 +171,7 @@ func ReadDBF(r io.Reader, _ int64, options *ReadDBFOptions) (*DBF, error) {
 				fieldData := recordData[offset : offset+fieldDescriptor.Length]
 				offset += fieldDescriptor.Length
 				field, err := fieldDescriptor.ParseRecord(fieldData, decoder)
-				if options.SkipBrokenFields && err != nil {
-					field = nil
-					err = nil
-				}
-				if err != nil {
+				if err != nil && !options.SkipBrokenFields {
 					return nil, fmt.Errorf("field %s: %w", fieldDescriptor.Name, err)
 				}
 				record = append(record, field)

scanner.go

@@ -758,11 +758,7 @@ func (s *ScannerDBF) Scan() (DBFRecord, error) {
 			fieldData := recordData[offset : offset+fieldDescriptor.Length]
 			offset += fieldDescriptor.Length
 			field, err := fieldDescriptor.ParseRecord(fieldData, s.decoder)
-			if s.options.SkipBrokenFields {
-				field = nil
-				err = nil
-			}
-			if err != nil {
+			if err != nil && !s.options.SkipBrokenFields {
 				s.err = fmt.Errorf("field %s: %w", fieldDescriptor.Name, err)
 				return nil, s.err
 			}

scanner_test.go

@@ -189,12 +189,14 @@ func TestReadScannerFSAndZipFile(t *testing.T) {
 	for _, tc := range []struct {
 		filename                 string
 		basename                 string
+		options                  *ReadShapefileOptions
 		expectedShapeType        ShapeType
 		expectedBounds           *geom.Bounds
 		expectedRecordsLen       int
 		expectedDBFRecord0Fields map[string]any
 		expectedSHPRecord0       *SHPRecord
 		expectedExport           any
+		expectedErr              string
 	}{
 		{
 			filename:          "testdata/110m-admin-0-countries.zip",
@@ -242,6 +244,23 @@ func TestReadScannerFSAndZipFile(t *testing.T) {
 				ShapeType:     ShapeTypePolygon,
 			},
 		},
+		{
+			filename:           "testdata/10m_populated_places_simple.zip",
+			expectedRecordsLen: 177,
+			expectedErr:        `scanning DBF: field labelrank: "**": invalid numeric: strconv.ParseInt: parsing "**": invalid syntax`,
+		},
+		{
+			filename: "testdata/10m_populated_places_simple.zip",
+			options: &ReadShapefileOptions{
+				DBF: &ReadDBFOptions{SkipBrokenFields: true},
+			},
+			expectedShapeType:  ShapeTypePoint,
+			expectedRecordsLen: 7342,
+			expectedBounds: geom.NewBounds(geom.XY).Set(
+				-179.5899789, -89.9999998,
+				179.3833036, 82.4833232,
+			),
+		},
 		{
 			filename:          "testdata/Luftfahrthindernisse.zip",
 			basename:          "Luftfahrthindernisse",
@@ -384,10 +403,14 @@ func TestReadScannerFSAndZipFile(t *testing.T) {
 			}
 
 			t.Run("ReadZipFile", func(t *testing.T) {
-				scanner, err := NewScannerFromZipFile(tc.filename, nil)
+				scanner, err := NewScannerFromZipFile(tc.filename, tc.options)
 				assert.NoError(t, err)
 				assert.NotZero(t, scanner)
 				shapefile, err := ReadScanner(scanner)
+				if tc.expectedErr != "" {
+					assert.Error(t, err)
+					return
+				}
 				assert.NoError(t, err)
 				assert.NotZero(t, scanner)
 				testShapefile(t, shapefile)

shapefile_test.go

@@ -386,16 +386,50 @@ func TestReadFSAndZipFile(t *testing.T) {
 }
 
 func TestShapefileRecords(t *testing.T) {
-	s, err := ReadZipFile("testdata/110m-admin-0-countries.zip", nil)
-	assert.NoError(t, err)
-	fieldss := make([]map[string]any, 0, s.NumRecords())
-	geoms := make([]geom.T, 0, s.NumRecords())
-	for fields, g := range s.Records() {
-		fieldss = append(fieldss, fields)
-		geoms = append(geoms, g)
+	for _, tc := range []struct {
+		filename           string
+		expectedRecordsLen int
+		expectedGeomsLen   int
+		expectedErr        string
+		ReadDBFOptions     *ReadDBFOptions
+	}{
+		{
+			filename:           "testdata/110m-admin-0-countries.zip",
+			expectedRecordsLen: 177,
+			expectedGeomsLen:   177,
+		},
+		{
+			filename:    "testdata/10m_populated_places_simple.zip",
+			expectedErr: "failed to parse DBF field",
+		},
+		{
+			filename:           "testdata/10m_populated_places_simple.zip",
+			expectedRecordsLen: 7342,
+			expectedGeomsLen:   7342,
+			ReadDBFOptions: &ReadDBFOptions{
+				SkipBrokenFields: true,
+			},
+		},
+	} {
+		t.Run(tc.filename, func(t *testing.T) {
+			s, err := ReadZipFile(tc.filename, &ReadShapefileOptions{
+				DBF: tc.ReadDBFOptions,
+			})
+			if tc.expectedErr != "" {
+				assert.Error(t, err, tc.expectedErr)
+				return
+			}
+			assert.NoError(t, err)
+			fieldss := make([]map[string]any, 0, s.NumRecords())
+			geoms := make([]geom.T, 0, s.NumRecords())
+			for fields, g := range s.Records() {
+				fieldss = append(fieldss, fields)
+				geoms = append(geoms, g)
+			}
+			assert.Equal(t, tc.expectedRecordsLen, len(fieldss))
+			assert.Equal(t, tc.expectedGeomsLen, len(geoms))
+		})
 	}
-	assert.Equal(t, 177, len(fieldss))
-	assert.Equal(t, 177, len(geoms))
 }
 
 func addFuzzDataFromFS(f *testing.F, fsys fs.FS, root, ext string) error {