Merge pull request #641 from jkbschmid/jkbschmid-licensee-parallel

Make licensee's max degree of parallism configurable

Author: elrayle

config/cdConfig.js

@@ -21,6 +21,11 @@ const maxRequeueAttemptCount = config.get('CRAWLER_MAX_REQUEUE_ATTEMPTS') || 5
 const fetchedCacheTtlSeconds = config.get('CRAWLER_FETCHED_CACHE_TTL_SECONDS') || 60 * 60 * 8 //8 hours
 const azqueueVisibilityTimeoutSeconds = parseInt(config.get('CRAWLER_HARVESTS_QUEUE_VISIBILITY_TIMEOUT_SECONDS'))
 
+function getPositiveNum(configName, defaultValue) {
+  const num = Number(config.get(configName))
+  return num > 0 ? num : defaultValue
+}
+
 module.exports = {
   provider: 'memory', // change this to redis if/when we want distributed config
   searchPath: [module],
@@ -68,7 +73,9 @@ module.exports = {
     },
     gem: { githubToken },
     go: { githubToken },
-    licensee: {},
+    licensee: {
+      processes: getPositiveNum('CRAWLER_LICENSEE_PARALLELISM', 10)
+    },
     maven: { githubToken },
     npm: { githubToken },
     nuget: { githubToken },

providers/process/licensee.js

@@ -50,9 +50,10 @@ class LicenseeProcessor extends AbstractProcessor {
     const root = request.document.location
     const subfolders = await this.getFolders(root, ['/.git'])
     const paths = ['', ...trimAllParents(subfolders, root)]
+    const { processes } = this.options
     try {
       const results = (
-        await Promise.all(paths.map(throat(10, path => this._runOnFolder(path, root, parameters))))
+        await Promise.all(paths.map(throat(processes, path => this._runOnFolder(path, root, parameters))))
       ).filter(x => x)
       const licenses = uniqBy(flatten(results.map(result => result.licenses)), 'spdx_id')
       const matched_files = flatten(results.map(result => result.matched_files))

test/unit/providers/process/licenseeTests.js

@@ -69,7 +69,7 @@ describe('Licensee process', () => {
 })
 
 function setup(fixture, error, versionError) {
-  const options = { logger: { log: sinon.stub() } }
+  const options = { logger: { log: sinon.stub() }, processes: 10 }
   const testRequest = new request('npm', 'cd:/npm/npmjs/-/test/1.1')
   testRequest.document = { _metadata: { links: {} }, location: path.resolve(`test/fixtures/licensee/${fixture}`) }
   testRequest.crawler = { storeDeadletter: sinon.stub() }