Modify ordered auth selection in azureBlobFactory and azureQueueStore

This is to make it consistent with the storageQueueManager.

Author: RomanIakovlev

config/cdConfig.js

@@ -7,7 +7,8 @@ const cd_azblob = {
   connection: config.get('CRAWLER_AZBLOB_CONNECTION_STRING'),
   container: config.get('CRAWLER_AZBLOB_CONTAINER_NAME'),
   account: config.get('CRAWLER_AZBLOB_ACCOUNT_NAME'),
-  spnAuth: config.get('CRAWLER_AZBLOB_SPN_AUTH')
+  spnAuth: config.get('CRAWLER_AZBLOB_SPN_AUTH'),
+  isSpnAuth: config.get('CRAWLER_AZBLOB_IS_SPN_AUTH') || false
 }
 
 const githubToken = config.get('CRAWLER_GITHUB_TOKEN')
@@ -115,7 +116,8 @@ module.exports = {
       connectionString: cd_azblob.connection,
       account: cd_azblob.account,
       queueName: config.get('CRAWLER_HARVESTS_QUEUE_NAME') || 'harvests',
-      spnAuth: config.get('CRAWLER_HARVESTS_QUEUE_SPN_AUTH')
+      spnAuth: config.get('CRAWLER_HARVESTS_QUEUE_SPN_AUTH'),
+      isSpnAuth: config.get('CRAWLER_HARVESTS_QUEUE_IS_SPN_AUTH') || false
     },
     'cd(azblob)': cd_azblob,
     'cd(file)': cd_file

ghcrawler/providers/queuing/storageQueueManager.js

@@ -21,6 +21,7 @@ class StorageQueueManager {
 
     const { account, spnAuth, isSpnAuth } = options
     if (isSpnAuth) {
+      options.logger.info('using service principal credentials in storageQueueManager')
       const authParsed = JSON.parse(spnAuth)
       this.client = new QueueServiceClient(
         `https://${account}.queue.core.windows.net`,
@@ -31,10 +32,12 @@ class StorageQueueManager {
     }
 
     if (connectionString) {
+      options.logger.info('using connection string in storageQueueManager')
       this.client = QueueServiceClient.fromConnectionString(connectionString, pipelineOptions)
       return
     }
 
+    options.logger.info('using default credentials in storageQueueManager')
     this.client = new QueueServiceClient(
       `https://${account}.queue.core.windows.net`,
       new DefaultAzureCredential(),

ghcrawler/providers/storage/azureBlobFactory.js

@@ -13,10 +13,11 @@ const { DefaultAzureCredential, ClientSecretCredential } = require('@azure/ident
  * @param {string} options.container
  * @param {object} options.logger
  * @param {object} options.spnAuth
+ * @param {object} options.isSpnAuth
  */
 module.exports = options => {
   options.logger.info('creating azure storage store')
-  const { account, connection, container, spnAuth } = options
+  const { account, connection, container, spnAuth, isSpnAuth } = options
 
   const pipelineOptions = {
     retryOptions: {
@@ -27,22 +28,28 @@ module.exports = options => {
       retryPolicyType: StorageRetryPolicyType.EXPONENTIAL
     }
   }
-
   let blobServiceClient
-  if (connection) {
-    options.logger.info('using connection string')
-    blobServiceClient = BlobServiceClient.fromConnectionString(connection, pipelineOptions)
+
+  if (isSpnAuth) {
+    options.logger.info('using service principal credentials in azureBlobFactory')
+    const authParsed = JSON.parse(spnAuth)
+    blobServiceClient = new BlobServiceClient(
+      `https://${account}.queue.core.windows.net`,
+      new ClientSecretCredential(authParsed.tenantId, authParsed.clientId, authParsed.clientSecret),
+      pipelineOptions
+    )
   } else {
-    let credential
-    if (spnAuth) {
-      const authParsed = JSON.parse(spnAuth)
-      credential = new ClientSecretCredential(authParsed.tenantId, authParsed.clientId, authParsed.clientSecret)
-      options.logger.info('using service principal credentials')
+    if (connection) {
+      options.logger.info('using connection string in azureBlobFactory')
+      blobServiceClient = BlobServiceClient.fromConnectionString(connection, pipelineOptions)
     } else {
-      credential = new DefaultAzureCredential()
-      options.logger.info('using default credentials')
+      options.logger.info('using default credentials in azureBlobFactory')
+      blobServiceClient = new BlobServiceClient(
+        `https://${account}.queue.core.windows.net`,
+        new DefaultAzureCredential(),
+        pipelineOptions
+      )
     }
-    blobServiceClient = new BlobServiceClient(`https://${account}.blob.core.windows.net`, credential, pipelineOptions)
   }
 
   const containerClient = blobServiceClient.getContainerClient(container)

providers/store/azureQueueStore.js

@@ -10,7 +10,7 @@ class AzureStorageQueue {
     this.queueName = options.queueName
     this.logger = options.logger
 
-    const { connectionString, account, spnAuth } = options
+    const { connectionString, account, spnAuth, isSpnAuth } = options
 
     const pipelineOptions = {
       retryOptions: {
@@ -21,18 +21,30 @@ class AzureStorageQueue {
         retryPolicyType: StorageRetryPolicyType.FIXED
       }
     }
+
+    if (isSpnAuth) {
+      options.logger.info('using service principal credentials in azureQueueStore')
+      const authParsed = JSON.parse(spnAuth)
+      this.client = new QueueServiceClient(
+        `https://${account}.queue.core.windows.net`,
+        new ClientSecretCredential(authParsed.tenantId, authParsed.clientId, authParsed.clientSecret),
+        pipelineOptions
+      )
+      return
+    }
+
     if (connectionString) {
+      options.logger.info('using connection string in azureQueueStore')
       this.client = QueueServiceClient.fromConnectionString(connectionString, pipelineOptions)
-    } else {
-      let credential
-      if (spnAuth) {
-        const authParsed = JSON.parse(spnAuth)
-        credential = new ClientSecretCredential(authParsed.tenantId, authParsed.clientId, authParsed.clientSecret)
-      } else {
-        credential = new DefaultAzureCredential()
-      }
-      this.client = new QueueServiceClient(`https://${account}.queue.core.windows.net`, credential, pipelineOptions)
+      return
     }
+
+    options.logger.info('using default credentials in azureQueueStore')
+    this.client = new QueueServiceClient(
+      `https://${account}.queue.core.windows.net`,
+      new DefaultAzureCredential(),
+      pipelineOptions
+    )
   }
 
   async connect() {