Merge pull request #140 from mcode/csv-read-from-url

Csv read from url

Author: dmendelowitz

package-lock.json

@@ -23,6 +23,7 @@
   "dependencies": {
     "ajv": "^6.12.6",
     "antlr4": "4.8.0",
+    "axios": "^0.21.1",
     "commander": "^6.2.0",
     "csv-parse": "^4.8.8",
     "fhir-crud-client": "^1.2.2",

package.json

@@ -25,32 +25,37 @@ class BaseClient {
   }
 
   // Given an extractor configuration, initialize all the necessary extractors
-  initializeExtractors(extractorConfig, commonExtractorArgs) {
-    let allExtractorsValid = true;
-
+  async initializeExtractors(extractorConfig, commonExtractorArgs) {
+    // Loop to initialize the extractors
     extractorConfig.forEach((curExtractorConfig) => {
       const { label, type, constructorArgs } = curExtractorConfig;
       logger.debug(`Initializing ${label} extractor with type ${type}`);
       const ExtractorClass = this.extractorClasses[type];
-
       try {
         const newExtractor = new ExtractorClass({ ...commonExtractorArgs, ...constructorArgs });
-
-        if (newExtractor.validate) {
-          const isExtractorValid = newExtractor.validate();
-          allExtractorsValid = (allExtractorsValid && isExtractorValid);
-          if (isExtractorValid) {
-            logger.debug(`Extractor ${label} PASSED CSV validation`);
-          } else {
-            logger.debug(`Extractor ${label} FAILED CSV validation`);
-          }
-        }
-
         this.extractors.push(newExtractor);
       } catch (e) {
         throw new Error(`Unable to initialize ${label} extractor with type ${type}: ${e.message}`);
       }
     });
+    // For validation, we are looping over extractors and performing an async operation on each.
+    // We need to loop without forEach (since forEach is sequential).
+    // Using Reduce to compute the validity of all extractors
+    const allExtractorsValid = await this.extractors.reduce(async (curExtractorsValid, curExtractor) => {
+      const { name } = curExtractor.constructor;
+
+      if (curExtractor.validate) {
+        logger.debug(`Validating ${name}`);
+        const isExtractorValid = await curExtractor.validate();
+        if (isExtractorValid) {
+          logger.debug(`Extractor ${name} PASSED CSV validation`);
+        } else {
+          logger.warn(`Extractor ${name} FAILED CSV validation`);
+        }
+        return (curExtractorsValid && isExtractorValid);
+      }
+      return curExtractorsValid;
+    }, true);
 
     if (allExtractorsValid) {
       logger.info('Validation succeeded');

src/client/BaseClient.js

@@ -1,25 +1,24 @@
-const path = require('path');
 const { Extractor } = require('./Extractor');
-const { CSVModule } = require('../modules');
-const { validateCSV } = require('../helpers/csvValidator');
-const logger = require('../helpers/logger');
+const { CSVFileModule, CSVURLModule } = require('../modules');
 
 class BaseCSVExtractor extends Extractor {
-  constructor({ filePath, csvSchema, unalterableColumns }) {
+  constructor({ filePath, url, csvSchema, unalterableColumns }) {
     super();
     this.unalterableColumns = unalterableColumns || [];
     this.csvSchema = csvSchema;
-    this.filePath = path.resolve(filePath);
-    this.csvModule = new CSVModule(this.filePath, this.unalterableColumns);
+    if (url) {
+      this.url = url;
+      this.csvModule = new CSVURLModule(this.url, this.unalterableColumns);
+    } else if (filePath) {
+      this.filePath = filePath;
+      this.csvModule = new CSVFileModule(this.filePath, this.unalterableColumns);
+    } else {
+      throw new Error('Trying to instantiate a CSVExtractor without a filePath or url');
+    }
   }
 
-  validate() {
-    if (this.csvSchema) {
-      logger.info(`Validating CSV file for ${this.filePath}`);
-      return validateCSV(this.filePath, this.csvSchema, this.csvModule.data);
-    }
-    logger.warn(`No CSV schema provided for ${this.filePath}`);
-    return true;
+  async validate() {
+    return this.csvModule.validate(this.csvSchema);
   }
 }
 

src/extractors/BaseCSVExtractor.js

@@ -69,8 +69,8 @@ function formatData(adverseEventData, patientId) {
 }
 
 class CSVAdverseEventExtractor extends BaseCSVExtractor {
-  constructor({ filePath }) {
-    super({ filePath });
+  constructor({ filePath, url }) {
+    super({ filePath, url });
   }
 
   async getAdverseEventData(mrn) {

src/extractors/CSVAdverseEventExtractor.js

@@ -8,8 +8,8 @@ const logger = require('../helpers/logger');
 const { CSVCancerDiseaseStatusSchema } = require('../helpers/schemas/csv');
 
 class CSVCancerDiseaseStatusExtractor extends BaseCSVExtractor {
-  constructor({ filePath, implementation }) {
-    super({ filePath, csvSchema: CSVCancerDiseaseStatusSchema });
+  constructor({ filePath, url, implementation }) {
+    super({ filePath, url, csvSchema: CSVCancerDiseaseStatusSchema });
     this.implementation = implementation;
   }
 

src/extractors/CSVCancerDiseaseStatusExtractor.js

@@ -46,8 +46,8 @@ function formatData(medicationData, patientId) {
 }
 
 class CSVCancerRelatedMedicationExtractor extends BaseCSVExtractor {
-  constructor({ filePath }) {
-    super({ filePath });
+  constructor({ filePath, url }) {
+    super({ filePath, url });
   }
 
   async getMedicationData(mrn) {

src/extractors/CSVCancerRelatedMedicationExtractor.js

@@ -8,8 +8,8 @@ const { CSVClinicalTrialInformationSchema } = require('../helpers/schemas/csv');
 
 
 class CSVClinicalTrialInformationExtractor extends BaseCSVExtractor {
-  constructor({ filePath, clinicalSiteID, clinicalSiteSystem }) {
-    super({ filePath, csvSchema: CSVClinicalTrialInformationSchema });
+  constructor({ filePath, url, clinicalSiteID, clinicalSiteSystem }) {
+    super({ filePath, url, csvSchema: CSVClinicalTrialInformationSchema });
     if (!clinicalSiteID) logger.warn(`${this.constructor.name} expects a value for clinicalSiteID but got ${clinicalSiteID}`);
     this.clinicalSiteID = clinicalSiteID;
     this.clinicalSiteSystem = clinicalSiteSystem;

src/extractors/CSVClinicalTrialInformationExtractor.js

@@ -49,8 +49,8 @@ function formatData(conditionData, patientId) {
 }
 
 class CSVConditionExtractor extends BaseCSVExtractor {
-  constructor({ filePath }) {
-    super({ filePath, csvSchema: CSVConditionSchema });
+  constructor({ filePath, url }) {
+    super({ filePath, url, csvSchema: CSVConditionSchema });
   }
 
   async getConditionData(mrn) {

src/extractors/CSVConditionExtractor.js

@@ -42,8 +42,8 @@ function formatData(observationData, patientId) {
 }
 
 class CSVObservationExtractor extends BaseCSVExtractor {
-  constructor({ filePath }) {
-    super({ filePath });
+  constructor({ filePath, url }) {
+    super({ filePath, url });
   }
 
   async getObservationData(mrn) {