-
Notifications
You must be signed in to change notification settings - Fork 0
Open
Description
Missing Fields Exception (MissingFieldsException) and other exceptions in idigbio_ingestion/lib/delimited.py are not increasing the exceptions count during db-check.
For example, processing of recordset 7c1a1d78-aeaa-4501-87e1-83eceb8ca8ea contains MissingFieldsException for every row, but the summary and metadata json files only have zeroes (other than csv_line_count).
$ cat /mnt/data/ingestion/7c1a1d78-aeaa-4501-87e1-83eceb8ca8ea.metadata.json
{
"paused": false,
"commited": true,
"name": "7c1a1d78-aeaa-4501-87e1-83eceb8ca8ea",
"total_processing_time": 498.359323,
"processing_start_datetime": "2022-02-07T08:46:03.925453",
"counts": {
"7c1a1d78-aeaa-4501-87e1-83eceb8ca8ea_extracted/identifications.csv": {
"resurrections": 0,
"dberrors": 0,
"processed_line_count": 0,
"total_line_count": 502875,
"deleted": 0,
"record_exceptions": 0,
"create": 0,
"no_recordid_count": 0,
"update": 0,
"processing_time": 284.93712,
"duplicate_record_count": 0,
"assertions": 0,
"ingestions": 0,
"exceptions": 0,
"to_undelete": 0,
"duplicate_id_count": 0,
"type": "dwc:Identification",
"delete": 0
},
"7c1a1d78-aeaa-4501-87e1-83eceb8ca8ea_extracted/images.csv": {
"resurrections": 0,
"dberrors": 0,
"processed_line_count": 0,
"total_line_count": 63001,
"deleted": 0,
"record_exceptions": 0,
"create": 0,
"no_recordid_count": 0,
"update": 0,
"processing_time": 37.730087,
"duplicate_record_count": 0,
"assertions": 0,
"ingestions": 0,
"exceptions": 0,
"to_undelete": 0,
"duplicate_id_count": 0,
"type": "dwc:Multimedia",
"delete": 0
},
"7c1a1d78-aeaa-4501-87e1-83eceb8ca8ea_extracted/occurrences.csv": {
"resurrections": 0,
"dberrors": 0,
"processed_line_count": 0,
"total_line_count": 213018,
"deleted": 0,
"record_exceptions": 0,
"create": 0,
"no_recordid_count": 0,
"update": 0,
"processing_time": 171.584769,
"duplicate_record_count": 0,
"assertions": 0,
"ingestions": 0,
"exceptions": 0,
"to_undelete": 0,
"duplicate_id_count": 0,
"type": "dwc:Occurrence",
"delete": 0
}
},
"filemd5": "46cbdf40136276883676ae3393d2aac0",
"recordset_id": "7c1a1d78-aeaa-4501-87e1-83eceb8ca8ea"
}
Metadata
Metadata
Assignees
Labels
No labels