Merge pull request #633 from clearlydefined/seperate-crawler-queue-from-harvest

Separate crawler queue connection from harvest

Author: RomanIakovlev

config/cdConfig.js

@@ -140,8 +140,9 @@ module.exports = {
       attenuation: {
         ttl: 3000
       },
-      spnAuth: config.get('CRAWLER_HARVESTS_QUEUE_SPN_AUTH'),
-      account: cd_azblob.account
+      spnAuth: config.get('CRAWLER_QUEUE_AZURE_SPN_AUTH') || cd_azblob.spnAuth,
+      account: config.get('CRAWLER_QUEUE_AZURE_ACCOUNT_NAME') || cd_azblob.account,
+      isSpnAuth: config.get('CRAWLER_QUEUE_AZURE_IS_SPN_AUTH') || false
     },
     appVersion: config.get('APP_VERSION'),
     buildsha: config.get('BUILD_SHA')

ghcrawler/providers/queuing/storageQueueManager.js

@@ -18,19 +18,28 @@ class StorageQueueManager {
         retryPolicyType: StorageRetryPolicyType.EXPONENTIAL
       }
     }
+
+    const { account, spnAuth, isSpnAuth } = options
+    if (isSpnAuth) {
+      const authParsed = JSON.parse(spnAuth)
+      this.client = new QueueServiceClient(
+        `https://${account}.queue.core.windows.net`,
+        new ClientSecretCredential(authParsed.tenantId, authParsed.clientId, authParsed.clientSecret),
+        pipelineOptions
+      )
+      return
+    }
+
     if (connectionString) {
       this.client = QueueServiceClient.fromConnectionString(connectionString, pipelineOptions)
-    } else {
-      const { account, spnAuth } = options
-      let credential
-      if (spnAuth) {
-        const authParsed = JSON.parse(spnAuth)
-        credential = new ClientSecretCredential(authParsed.tenantId, authParsed.clientId, authParsed.clientSecret)
-      } else {
-        credential = new DefaultAzureCredential()
-      }
-      this.client = new QueueServiceClient(`https://${account}.queue.core.windows.net`, credential, pipelineOptions)
+      return
     }
+
+    this.client = new QueueServiceClient(
+      `https://${account}.queue.core.windows.net`,
+      new DefaultAzureCredential(),
+      pipelineOptions
+    )
   }
 
   createQueueClient(name, formatter, options) {