Make CSV column headers case insensitive

src/extractors/CSVAdverseEventExtractor.js

@@ -10,8 +10,23 @@ function formatData(adverseEventData, patientId) {
   logger.debug('Reformatting adverse event data from CSV into template format');
   return adverseEventData.map((data) => {
     const {
-      adverseEventId, adverseEventCode, adverseEventCodeSystem, adverseEventDisplayText, suspectedCauseId, suspectedCauseType, seriousness, seriousnessCodeSystem, seriousnessDisplayText,
-      category, categoryCodeSystem, categoryDisplayText, severity, actuality, studyId, effectiveDate, recordedDate,
+      adverseeventid: adverseEventId,
+      adverseeventcode: adverseEventCode,
+      adverseeventcodesystem: adverseEventCodeSystem,
+      adverseeventdisplaytext: adverseEventDisplayText,
+      suspectedcauseid: suspectedCauseId,
+      suspectedcausetype: suspectedCauseType,
+      seriousness,
+      seriousnesscodesystem: seriousnessCodeSystem,
+      seriousnessdisplaytext: seriousnessDisplayText,
+      category,
+      categorycodesystem: categoryCodeSystem,
+      categorydisplaytext: categoryDisplayText,
+      severity,
+      actuality,
+      studyid: studyId,
+      effectivedate: effectiveDate,
+      recordeddate: recordedDate,
     } = data;
 
     if (!(adverseEventCode && effectiveDate)) {

src/extractors/CSVCancerDiseaseStatusExtractor.js

@@ -16,31 +16,40 @@ class CSVCancerDiseaseStatusExtractor extends BaseCSVExtractor {
   joinAndReformatData(arrOfDiseaseStatusData, patientId) {
     logger.debug('Reformatting disease status data from CSV into template format');
     // Check the shape of the data
-    arrOfDiseaseStatusData.forEach((record) => {
-      if (!record.conditionId || !record.diseaseStatusCode || !record.dateOfObservation) {
+    return arrOfDiseaseStatusData.map((record) => {
+      const {
+        conditionid: conditionId,
+        diseasestatuscode: diseaseStatusCode,
+        diseasestatustext: diseaseStatusText,
+        dateofobservation: dateOfObservation,
+        observationstatus: observationStatus,
+        evidence,
+      } = record;
+
+      if (!conditionId || !diseaseStatusCode || !dateOfObservation) {
         throw new Error('DiseaseStatusData missing an expected property: conditionId, diseaseStatusCode, and dateOfObservation are required.');
       }
+
+      return {
+        status: observationStatus || 'final',
+        value: {
+          code: diseaseStatusCode,
+          system: 'http://snomed.info/sct',
+          display: diseaseStatusText || getDiseaseStatusDisplay(diseaseStatusCode, this.implementation),
+        },
+        subject: {
+          id: patientId,
+        },
+        condition: {
+          id: conditionId,
+        },
+        effectiveDateTime: formatDateTime(dateOfObservation),
+        evidence: !evidence ? null : evidence.split('|').map((evidenceCode) => ({
+          code: evidenceCode,
+          display: getDiseaseStatusEvidenceDisplay(evidenceCode),
+        })),
+      };
     });
-    const evidenceDelimiter = '|';
-    return arrOfDiseaseStatusData.map((record) => ({
-      status: record.observationStatus || 'final',
-      value: {
-        code: record.diseaseStatusCode,
-        system: 'http://snomed.info/sct',
-        display: record.diseaseStatusText ? record.diseaseStatusText : getDiseaseStatusDisplay(record.diseaseStatusCode, this.implementation),
-      },
-      subject: {
-        id: patientId,
-      },
-      condition: {
-        id: record.conditionId,
-      },
-      effectiveDateTime: formatDateTime(record.dateOfObservation),
-      evidence: !record.evidence ? null : record.evidence.split(evidenceDelimiter).map((evidenceCode) => ({
-        code: evidenceCode,
-        display: getDiseaseStatusEvidenceDisplay(evidenceCode),
-      })),
-    }));
   }
 
   async getDiseaseStatusData(mrn, fromDate, toDate) {

src/extractors/CSVCancerRelatedMedicationExtractor.js

@@ -11,7 +11,17 @@ function formatData(medicationData, patientId) {
 
   return medicationData.map((medication) => {
     const {
-      medicationId, code, codeSystem, displayText, startDate, endDate, treatmentReasonCode, treatmentReasonCodeSystem, treatmentReasonDisplayText, treatmentIntent, status,
+      medicationid: medicationId,
+      code,
+      codesystem: codeSystem,
+      displaytext: displayText,
+      startdate: startDate,
+      enddate: endDate,
+      treatmentreasoncode: treatmentReasonCode,
+      treatmentreasoncodesystem: treatmentReasonCodeSystem,
+      treatmentreasondisplaytext: treatmentReasonDisplayText,
+      treatmentintent: treatmentIntent,
+      status,
     } = medication;
 
     if (!(code && codeSystem && status)) {

src/extractors/CSVClinicalTrialInformationExtractor.js

@@ -18,7 +18,11 @@ class CSVClinicalTrialInformationExtractor extends BaseCSVExtractor {
   joinClinicalTrialData(clinicalTrialData, patientId) {
     logger.debug('Reformatting clinical trial data from CSV into template format');
     const {
-      trialSubjectID, enrollmentStatus, trialResearchID, trialStatus, trialResearchSystem,
+      trialsubjectid: trialSubjectID,
+      enrollmentstatus: enrollmentStatus,
+      trialresearchid: trialResearchID,
+      trialstatus: trialStatus,
+      trialresearchsystem: trialResearchSystem,
     } = clinicalTrialData;
     const { clinicalSiteID, clinicalSiteSystem } = this;
 

src/extractors/CSVConditionExtractor.js

@@ -11,7 +11,17 @@ function formatData(conditionData, patientId) {
   logger.debug('Reformatting condition data from CSV into template format');
   return conditionData.map((data) => {
     const {
-      conditionId, codeSystem, code, displayName, category, dateOfDiagnosis, clinicalStatus, verificationStatus, bodySite, laterality, histology,
+      conditionid: conditionId,
+      codesystem: codeSystem,
+      code,
+      displayname: displayName,
+      category,
+      dateofdiagnosis: dateOfDiagnosis,
+      clinicalstatus: clinicalStatus,
+      verificationstatus: verificationStatus,
+      bodysite: bodySite,
+      laterality,
+      histology,
     } = data;
 
     if (!(conditionId && codeSystem && code && category)) {

src/extractors/CSVObservationExtractor.js

@@ -9,7 +9,16 @@ function formatData(observationData, patientId) {
   logger.debug('Reformatting observation data from CSV into template format');
   return observationData.map((data) => {
     const {
-      observationId, status, code, codeSystem, displayName, value, valueCodeSystem, effectiveDate, bodySite, laterality,
+      observationid: observationId,
+      status,
+      code,
+      codesystem: codeSystem,
+      displayname: displayName,
+      value,
+      valuecodesystem: valueCodeSystem,
+      effectivedate: effectiveDate,
+      bodysite: bodySite,
+      laterality,
     } = data;
 
     if (!observationId || !status || !code || !codeSystem || !value || !effectiveDate) {

src/extractors/CSVPatientExtractor.js

@@ -13,7 +13,19 @@ function joinAndReformatData(patientData) {
   logger.debug('Reformatting patient data from CSV into template format');
   // No join needed, just a reformatting
   const {
-    mrn, familyName, givenName, gender, birthsex, dateOfBirth, race, ethnicity, language, addressLine, city, state, zip,
+    mrn,
+    familyname: familyName,
+    givenname: givenName,
+    gender,
+    birthsex,
+    dateofbirth: dateOfBirth,
+    race,
+    ethnicity,
+    language,
+    addressline: addressLine,
+    city,
+    state,
+    zip,
   } = patientData;
 
   if (!mrn) {

src/extractors/CSVProcedureExtractor.js

@@ -10,7 +10,19 @@ function formatData(procedureData, patientId) {
   logger.debug('Reformatting procedure data from CSV into template format');
   return procedureData.map((data) => {
     const {
-      procedureId, conditionId, status, code, codeSystem, displayName, reasonCode, reasonCodeSystem, reasonDisplayName, bodySite, laterality, effectiveDate, treatmentIntent,
+      procedureid: procedureId,
+      conditionid: conditionId,
+      status,
+      code,
+      codesystem: codeSystem,
+      displayname: displayName,
+      reasoncode: reasonCode,
+      reasoncodesystem: reasonCodeSystem,
+      reasondisplayname: reasonDisplayName,
+      bodysite: bodySite,
+      laterality,
+      effectivedate: effectiveDate,
+      treatmentintent: treatmentIntent,
     } = data;
 
     if (!(procedureId && status && code && codeSystem && effectiveDate)) {

src/extractors/CSVStagingExtractor.js

@@ -9,7 +9,14 @@ function formatTNMCategoryData(stagingData, patientId) {
   logger.debug('Reformatting TNM Category data into template format');
   const formattedData = [];
   const {
-    conditionId, t, n, m, type, stagingSystem, stagingCodeSystem, effectiveDate,
+    conditionid: conditionId,
+    t,
+    n,
+    m,
+    type,
+    stagingsystem: stagingSystem,
+    stagingcodesystem: stagingCodeSystem,
+    effectivedate: effectiveDate,
   } = stagingData;
 
   if (!conditionId || !effectiveDate) {
@@ -35,7 +42,12 @@ function formatTNMCategoryData(stagingData, patientId) {
 
 function formatStagingData(stagingData, categoryIds, patientId) {
   const {
-    conditionId, type, stageGroup, stagingSystem, stagingCodeSystem, effectiveDate,
+    conditionid: conditionId,
+    type,
+    stagegroup: stageGroup,
+    stagingsystem: stagingSystem,
+    stagingcodesystem: stagingCodeSystem,
+    effectivedate: effectiveDate,
   } = stagingData;
 
   return {

src/extractors/CSVTreatmentPlanChangeExtractor.js

@@ -21,7 +21,10 @@ function formatData(tpcData, patientId) {
 
   // If there are multiple entries, combine them into one object with multiple reviews
   const combinedData = _.reduce(tpcData, (res, currentDataEntry) => {
-    const { dateOfCarePlan, changed, reasonCode, reasonDisplayText } = currentDataEntry;
+    const { dateofcareplan: dateOfCarePlan,
+      changed,
+      reasoncode: reasonCode,
+      reasondisplaytext: reasonDisplayText } = currentDataEntry;
 
     if (!dateOfCarePlan || !changed) {
       throw new Error('Treatment Plan Change Data missing an expected property: dateOfCarePlan, changed are required');

src/helpers/csvValidator.js

@@ -5,17 +5,17 @@ function validateCSV(pathToCSVFile, csvSchema, csvData) {
   let isValid = true;
 
   // Check headers
-  const headers = Object.keys(csvData[0]);
-  const schemaDiff = _.difference(csvSchema.headers.map((h) => h.name), headers);
-  const fileDiff = _.difference(headers, csvSchema.headers.map((h) => h.name));
+  const headers = Object.keys(csvData[0]).map((h) => h.toLowerCase());
+  const schemaDiff = _.difference(csvSchema.headers.map((h) => h.name.toLowerCase()), headers);
+  const fileDiff = _.difference(headers, csvSchema.headers.map((h) => h.name.toLowerCase()));
 
   if (fileDiff.length > 0) {
     logger.warn(`Found extra column(s) in CSV ${pathToCSVFile}: "${fileDiff.join(',')}"`);
   }
 
   if (schemaDiff.length > 0) {
     schemaDiff.forEach((sd) => {
-      const headerSchema = csvSchema.headers.find((h) => h.name === sd);
+      const headerSchema = csvSchema.headers.find((h) => h.name.toLowerCase() === sd);
       if (headerSchema.required) {
         logger.error(`Column ${sd} is marked as required but is missing in CSV ${pathToCSVFile}`);
         isValid = false;

src/modules/CSVModule.js

@@ -5,22 +5,22 @@ const logger = require('../helpers/logger');
 
 class CSVModule {
   constructor(csvFilePath) {
-    this.data = parse(fs.readFileSync(csvFilePath), { columns: true, bom: true });
+    this.data = parse(fs.readFileSync(csvFilePath), { columns: (header) => header.map((column) => column.toLowerCase()), bom: true });
   }
 
   async get(key, value, fromDate, toDate) {
     logger.debug(`Get csvModule info by key '${key}'`);
     // return all rows if key and value aren't provided
     if (!key && !value) return this.data;
-    let result = this.data.filter((d) => d[key] === value);
+    let result = this.data.filter((d) => d[key.toLowerCase()] === value);
     if (result.length === 0) {
       logger.warn(`CSV Record with provided key '${key}' and value was not found`);
       return result;
     }
 
     // If fromDate and toDate is provided, filter out all results that fall outside that timespan
-    if (fromDate && moment(fromDate).isValid()) result = result.filter((r) => !(r.dateRecorded && moment(fromDate).isAfter(r.dateRecorded)));
-    if (toDate && moment(toDate).isValid()) result = result.filter((r) => !(r.dateRecorded && moment(toDate).isBefore(r.dateRecorded)));
+    if (fromDate && moment(fromDate).isValid()) result = result.filter((r) => !(r.daterecorded && moment(fromDate).isAfter(r.daterecorded)));
+    if (toDate && moment(toDate).isValid()) result = result.filter((r) => !(r.daterecorded && moment(toDate).isBefore(r.daterecorded)));
     if (result.length === 0) logger.warn('No data for patient within specified time range');
     return result;
   }

test/extractors/CSVAdverseEventExtractor.test.js

@@ -51,17 +51,17 @@ describe('CSVAdverseEventExtractor', () => {
       expect(() => formatData(localData, patientId)).toThrow(new Error(expectedCategoryErrorString));
 
       // Test that adding another category but adding a corresponding categoryCodeSystem and categoryDisplayText works fine
-      localData[0].categoryCodeSystem = 'http://terminology.hl7.org/CodeSystem/adverse-event-category|http://snomed.info/sct';
-      localData[0].categoryDisplayText = 'Product Use Error|Product Problem';
+      localData[0].categorycodesystem = 'http://terminology.hl7.org/CodeSystem/adverse-event-category|http://snomed.info/sct';
+      localData[0].categorydisplaytext = 'Product Use Error|Product Problem';
       expect(formatData(localData, patientId)).toEqual(expect.anything());
 
       // Test that adding another category but including syntax for default categoryCodeSystem and categoryDisplayText values works fine
-      localData[0].categoryCodeSystem = 'http://terminology.hl7.org/CodeSystem/adverse-event-category|';
-      localData[0].categoryDisplayText = 'Product Use Error|';
+      localData[0].categorycodesystem = 'http://terminology.hl7.org/CodeSystem/adverse-event-category|';
+      localData[0].categorydisplaytext = 'Product Use Error|';
       expect(formatData(localData, patientId)).toEqual(expect.anything());
 
       // Test that deleting a mandatory value throws an error
-      delete localData[0].adverseEventCode;
+      delete localData[0].adverseeventcode;
       expect(() => formatData(localData, patientId)).toThrow(new Error(expectedErrorString));
     });
   });

test/extractors/CSVCancerDiseaseStatusExtractor.test.js

@@ -35,12 +35,12 @@ describe('CSVCancerDiseaseStatusExtractor', () => {
       expect(csvCancerDiseaseStatusExtractor.joinAndReformatData(exampleCSVDiseaseStatusModuleResponse, patientId)).toEqual(expect.anything());
 
       localData[0].evidence = ''; // Evidence is not required and will not throw an error
-      localData[0].observationStatus = ''; // Observation Status is not required and will not throw an error
+      localData[0].observationstatus = ''; // Observation Status is not required and will not throw an error
 
       // Only including required properties is valid
       expect(csvCancerDiseaseStatusExtractor.joinAndReformatData(localData, patientId)).toEqual(expect.anything());
 
-      const requiredProperties = ['conditionId', 'diseaseStatusCode', 'dateOfObservation'];
+      const requiredProperties = ['conditionid', 'diseasestatuscode', 'dateofobservation'];
 
       // Removing each required property should throw an error
       requiredProperties.forEach((key) => {

test/extractors/CSVClinicalTrialInformationExtractor.test.js

@@ -39,26 +39,26 @@ describe('CSVClinicalTrialInformationExtractor', () => {
         const clonedData = _.cloneDeep(firstClinicalTrialInfoResponse);
         expect(csvClinicalTrialInformationExtractor.joinClinicalTrialData(clonedData, patientId)).toEqual(expect.anything());
         if (key === 'patientId') return; // MRN is optional
-        if (key === 'trialResearchSystem') return; // trialResearchSystem is optional
+        if (key === 'trialresearchsystem') return; // trialResearchSystem is optional
         delete clonedData[key];
         expect(() => csvClinicalTrialInformationExtractor.joinClinicalTrialData(clonedData, patientId)).toThrow(new Error(expectedErrorString));
       });
 
       // joinClinicalTrialData should return correct format
       expect(csvClinicalTrialInformationExtractor.joinClinicalTrialData(firstClinicalTrialInfoResponse, patientId)).toEqual({
         formattedDataSubject: {
-          enrollmentStatus: firstClinicalTrialInfoResponse.enrollmentStatus,
-          trialSubjectID: firstClinicalTrialInfoResponse.trialSubjectID,
-          trialResearchID: firstClinicalTrialInfoResponse.trialResearchID,
+          enrollmentStatus: firstClinicalTrialInfoResponse.enrollmentstatus,
+          trialSubjectID: firstClinicalTrialInfoResponse.trialsubjectid,
+          trialResearchID: firstClinicalTrialInfoResponse.trialresearchid,
           patientId,
-          trialResearchSystem: firstClinicalTrialInfoResponse.trialResearchSystem,
+          trialResearchSystem: firstClinicalTrialInfoResponse.trialresearchsystem,
         },
         formattedDataStudy: {
-          trialStatus: firstClinicalTrialInfoResponse.trialStatus,
-          trialResearchID: firstClinicalTrialInfoResponse.trialResearchID,
+          trialStatus: firstClinicalTrialInfoResponse.trialstatus,
+          trialResearchID: firstClinicalTrialInfoResponse.trialresearchid,
           clinicalSiteID: MOCK_CLINICAL_SITE_ID,
           clinicalSiteSystem: MOCK_CLINICAL_SITE_SYSTEM,
-          trialResearchSystem: firstClinicalTrialInfoResponse.trialResearchSystem,
+          trialResearchSystem: firstClinicalTrialInfoResponse.trialresearchsystem,
         },
       });
     });

test/extractors/CSVObservationExtractor.test.js

@@ -38,7 +38,7 @@ describe('CSVObservationExtractor', () => {
       expect(formatData(exampleCSVObservationModuleResponse, patientId)).toEqual(expect.anything());
 
       // Test that deleting an optional value works fine
-      delete localData[0].bodySite;
+      delete localData[0].bodysite;
       expect(formatData(exampleCSVObservationModuleResponse, patientId)).toEqual(expect.anything());
 
       // Test that deleting a mandatory value throws an error

test/extractors/CSVProcedureExtractor.test.js

@@ -37,11 +37,11 @@ describe('CSVProcedureExtractor', () => {
       expect(formatData(localData, patientId)).toEqual(expect.anything());
 
       // Test that removing an optional value works
-      delete localData[0].bodySite;
+      delete localData[0].bodysite;
       expect(formatData(localData, patientId)).toEqual(expect.anything());
 
       // Test that removing a required value throws
-      delete localData[0].procedureId;
+      delete localData[0].procedureid;
       expect(() => formatData(localData, patientId)).toThrow(new Error(expectedErrorString));
     });
   });