diff --git a/config/cdConfig.js b/config/cdConfig.js index 05b5a524..c1a6a1be 100644 --- a/config/cdConfig.js +++ b/config/cdConfig.js @@ -5,7 +5,10 @@ const config = require('painless-config') const cd_azblob = { connection: config.get('CRAWLER_AZBLOB_CONNECTION_STRING'), - container: config.get('CRAWLER_AZBLOB_CONTAINER_NAME') + container: config.get('CRAWLER_AZBLOB_CONTAINER_NAME'), + account: config.get('CRAWLER_AZBLOB_ACCOUNT_NAME'), + spnAuth: config.get('CRAWLER_AZBLOB_SPN_AUTH'), + isSpnAuth: config.get('CRAWLER_AZBLOB_IS_SPN_AUTH') || false } const githubToken = config.get('CRAWLER_GITHUB_TOKEN') @@ -111,7 +114,10 @@ module.exports = { }, azqueue: { connectionString: cd_azblob.connection, - queueName: config.get('CRAWLER_HARVESTS_QUEUE_NAME') || 'harvests' + account: cd_azblob.account, + queueName: config.get('CRAWLER_HARVESTS_QUEUE_NAME') || 'harvests', + spnAuth: config.get('CRAWLER_HARVESTS_QUEUE_SPN_AUTH'), + isSpnAuth: config.get('CRAWLER_HARVESTS_QUEUE_IS_SPN_AUTH') || false }, 'cd(azblob)': cd_azblob, 'cd(file)': cd_file @@ -135,7 +141,10 @@ module.exports = { maxDequeueCount: 5, attenuation: { ttl: 3000 - } + }, + spnAuth: config.get('CRAWLER_QUEUE_AZURE_SPN_AUTH') || cd_azblob.spnAuth, + account: config.get('CRAWLER_QUEUE_AZURE_ACCOUNT_NAME') || cd_azblob.account, + isSpnAuth: config.get('CRAWLER_QUEUE_AZURE_IS_SPN_AUTH') || false }, appVersion: config.get('APP_VERSION'), buildsha: config.get('BUILD_SHA') diff --git a/ghcrawler/lib/crawler.js b/ghcrawler/lib/crawler.js index 69553802..f494493c 100644 --- a/ghcrawler/lib/crawler.js +++ b/ghcrawler/lib/crawler.js @@ -638,7 +638,7 @@ class Crawler { metadata.errorMessage = request._error.message metadata.errorStack = request._error.stack } - metadata.version = 1 + metadata.version = '1' metadata.meta = request.meta metadata.type = 'deadletter' metadata.url = request.url.replace('//', '//deadletter.') diff --git a/ghcrawler/providers/queuing/storageQueue.js b/ghcrawler/providers/queuing/storageQueue.js index c52e1257..a2ae9f9f 100644 --- a/ghcrawler/providers/queuing/storageQueue.js +++ b/ghcrawler/providers/queuing/storageQueue.js @@ -1,50 +1,47 @@ // Copyright (c) Microsoft Corporation and others. Made available under the MIT license. // SPDX-License-Identifier: MIT +// eslint-disable-next-line no-unused-vars +const { QueueServiceClient } = require('@azure/storage-queue') const qlimit = require('qlimit') const { cloneDeep } = require('lodash') class StorageQueue { + /** + * @param {QueueServiceClient} client + * @param {string} name + * @param {string} queueName + * @param {object} formatter + * @param {object} options + */ constructor(client, name, queueName, formatter, options) { - this.client = client this.name = name this.queueName = queueName this.messageFormatter = formatter this.options = options this.logger = options.logger + this.queueClient = client.getQueueClient(this.queueName) } async subscribe() { - return new Promise((resolve, reject) => { - this.client.createQueueIfNotExists(this.queueName, error => { - if (error) { - return reject(error) - } - this.logger.info(`Subscribed to ${this.queueName} using Queue Storage`) - resolve() - }) - }) + await this.queueClient.createIfNotExists() + this.logger.info(`Subscribed to ${this.queueName} using Queue Storage`) } async unsubscribe() { - return + // No specific unsubscribe logic for Azure Queue Storage } - async push(requests, option) { + async push(requests) { requests = Array.isArray(requests) ? requests : [requests] return Promise.all( requests.map( - qlimit(this.options.parallelPush || 1)(request => { + qlimit(this.options.parallelPush || 1)(async request => { const body = JSON.stringify(request) - return new Promise((resolve, reject) => { - this.client.createMessage(this.queueName, body, option, (error, queueMessageResult) => { - if (error) { - return reject(error) - } - this._log('Queued', request) - resolve(this._buildMessageReceipt(queueMessageResult, request)) - }) - }) + const encoded = this._encodeXMLSafe(body) + const queueMessageResult = await this.queueClient.sendMessage(encoded) + this._log('Queued', request) + return this._buildMessageReceipt(queueMessageResult, request) }) ) ) @@ -56,47 +53,46 @@ class StorageQueue { } async pop() { - const msgOptions = { numOfMessages: 1, visibilityTimeout: this.options.visibilityTimeout || 60 * 60 } - return new Promise((resolve, reject) => { - this.client.getMessages(this.queueName, msgOptions, (error, result) => { - if (error) { - return reject(error) - } - const message = result[0] - if (!message) { - this.logger.verbose('No messages to receive') - return resolve(null) - } - if (this.options.maxDequeueCount && message.dequeueCount > this.options.maxDequeueCount) { - this.logger.verbose('maxDequeueCount exceeded') - this.client.deleteMessage(this.queueName, message.messageId, message.popReceipt, error => { - if (error) return reject(error) - resolve(null) - }) - } else { - message.body = JSON.parse(message.messageText) - const request = this.messageFormatter(message) - request._message = message - this._log('Popped', message.body) - resolve(request) - } - }) - }) + const msgOptions = { numberOfMessages: 1, visibilityTimeout: this.options.visibilityTimeout || 60 * 60 } + const response = await this.queueClient.receiveMessages(msgOptions) + const message = response.receivedMessageItems[0] + if (!message) { + this.logger.verbose('No messages to receive') + return null + } + if (this.options.maxDequeueCount && message.dequeueCount > this.options.maxDequeueCount) { + this.logger.verbose('maxDequeueCount exceeded') + try { + await this.queueClient.deleteMessage(message.messageId, message.popReceipt) + } catch (error) { + this.logger.error(`Failed to delete message ${message.messageId} in storageQueue, error: ${error.message}`) + throw error + } + return null + } else { + try { + const decodedText = this._decodeXMLSafe(message.messageText) + message.body = JSON.parse(decodedText) + } catch (error) { + this.logger.error(`Failed to parse message ${message.messageId}:`) + this.logger.error(`Raw message: ${message.messageText}`) + this.logger.error(`Parse error: ${error.message}`) + await this.queueClient.deleteMessage(message.messageId, message.popReceipt) + return null + } + const request = this.messageFormatter(message) + request._message = message + this._log('Popped', message.body) + return request + } } async done(request) { if (!request || !request._message) { return } - return new Promise((resolve, reject) => { - this.client.deleteMessage(this.queueName, request._message.messageId, request._message.popReceipt, error => { - if (error) { - return reject(error) - } - this._log('ACKed', request._message.body) - resolve() - }) - }) + await this.queueClient.deleteMessage(request._message.messageId, request._message.popReceipt) + this._log('ACKed', request._message.body) } async defer(request) { @@ -110,47 +106,30 @@ class StorageQueue { await this.updateVisibilityTimeout(request) } - updateVisibilityTimeout(request, visibilityTimeout = 0) { - return new Promise((resolve, reject) => { - // visibilityTimeout is updated to 0 to unlock/unlease the message - this.client.updateMessage( - this.queueName, - request._message.messageId, - request._message.popReceipt, - visibilityTimeout, - (error, result) => { - if (error) { - return reject(error) - } - this._log('NAKed', request._message.body) - resolve(this._buildMessageReceipt(result, request._message.body)) - } - ) - }) + async updateVisibilityTimeout(request, visibilityTimeout = 0) { + const response = await this.queueClient.updateMessage( + request._message.messageId, + request._message.popReceipt, + undefined, + visibilityTimeout + ) + this._log('NAKed', request._message.body) + return this._buildMessageReceipt({ messageId: request._message.messageId, ...response }, request) } async flush() { - return new Promise((resolve, reject) => { - this.client.deleteQueue(this.queueName, error => { - if (error) return reject(error) - this.client.createQueueIfNotExists(this.queueName, error => { - if (error) return reject(error) - resolve() - }) - }) - }) + await this.queueClient.clearMessages() + this.logger.info(`Flushed all messages from ${this.queueName}`) } async getInfo() { - return new Promise(resolve => { - this.client.getQueueMetadata(this.queueName, (result, error) => { - if (error) { - this.logger.error(error) - resolve(null) - } - resolve({ count: result[0].approximateMessageCount }) - }) - }) + try { + const properties = await this.queueClient.getProperties() + return { count: properties.approximateMessagesCount } + } catch (error) { + this.logger.error(error) + return null + } } getName() { @@ -164,6 +143,35 @@ class StorageQueue { isMessageNotFound(error) { return error?.code === 'MessageNotFound' } + + _encodeXMLSafe(text) { + if (typeof text !== 'string') return text + + return ( + text + // Handle & first to prevent double-encoding + .replace(/&/g, '&') + .replace(/"/g, '"') + .replace(/'/g, ''') + .replace(//g, '>') + ) + } + + _decodeXMLSafe(text) { + if (typeof text !== 'string') return text + + return ( + text + // Handle both XML and HTML encodings for quotes and apostrophes + .replace(/'|'|'/g, "'") + .replace(/"|"|"/g, '"') + // Handle basic XML entities + .replace(/<|<|[Cc];/g, '<') + .replace(/>|>|[Ee];/g, '>') + .replace(/&|&|&/g, '&') // Must be after other & entities + ) + } } module.exports = StorageQueue diff --git a/ghcrawler/providers/queuing/storageQueueManager.js b/ghcrawler/providers/queuing/storageQueueManager.js index 2f23a7c9..2e662978 100644 --- a/ghcrawler/providers/queuing/storageQueueManager.js +++ b/ghcrawler/providers/queuing/storageQueueManager.js @@ -2,14 +2,47 @@ // SPDX-License-Identifier: MIT const AttenuatedQueue = require('./attenuatedQueue') -const AzureStorage = require('azure-storage') +const { QueueServiceClient, StorageRetryPolicyType } = require('@azure/storage-queue') const Request = require('../../lib/request') const StorageQueue = require('./storageQueue') +const { DefaultAzureCredential, ClientSecretCredential } = require('@azure/identity') class StorageQueueManager { - constructor(connectionString) { - const retryOperations = new AzureStorage.ExponentialRetryPolicyFilter() - this.client = AzureStorage.createQueueService(connectionString).withFilter(retryOperations) + constructor(connectionString, options) { + const pipelineOptions = { + retryOptions: { + maxTries: 3, + retryDelayInMs: 1000, + maxRetryDelayInMs: 120 * 1000, + tryTimeoutInMs: 30000, + retryPolicyType: StorageRetryPolicyType.EXPONENTIAL + } + } + + const { account, spnAuth, isSpnAuth } = options + if (isSpnAuth) { + options.logger.info('using service principal credentials in storageQueueManager') + const authParsed = JSON.parse(spnAuth) + this.client = new QueueServiceClient( + `https://${account}.queue.core.windows.net`, + new ClientSecretCredential(authParsed.tenantId, authParsed.clientId, authParsed.clientSecret), + pipelineOptions + ) + return + } + + if (connectionString) { + options.logger.info('using connection string in storageQueueManager') + this.client = QueueServiceClient.fromConnectionString(connectionString, pipelineOptions) + return + } + + options.logger.info('using default credentials in storageQueueManager') + this.client = new QueueServiceClient( + `https://${account}.queue.core.windows.net`, + new DefaultAzureCredential(), + pipelineOptions + ) } createQueueClient(name, formatter, options) { diff --git a/ghcrawler/providers/storage/azureBlobFactory.js b/ghcrawler/providers/storage/azureBlobFactory.js index 2d2d5eb5..5416ed5e 100644 --- a/ghcrawler/providers/storage/azureBlobFactory.js +++ b/ghcrawler/providers/storage/azureBlobFactory.js @@ -1,15 +1,58 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // SPDX-License-Identifier: MIT -const AzureStorage = require('azure-storage') +// @ts-check +const { BlobServiceClient, StorageRetryPolicyType } = require('@azure/storage-blob') const AzureStorageDocStore = require('./storageDocStore') +const { DefaultAzureCredential, ClientSecretCredential } = require('@azure/identity') +/** + * @param {object} options + * @param {string} options.account + * @param {string} options.connection + * @param {string} options.container + * @param {object} options.logger + * @param {object} options.spnAuth + * @param {object} options.isSpnAuth + */ module.exports = options => { options.logger.info('creating azure storage store') - const { account, key, connection, container } = options - const retryOperations = new AzureStorage.ExponentialRetryPolicyFilter() - const blobService = connection - ? AzureStorage.createBlobService(connection).withFilter(retryOperations) - : AzureStorage.createBlobService(account, key).withFilter(retryOperations) - return new AzureStorageDocStore(blobService, container, options) + const { account, connection, container, spnAuth, isSpnAuth } = options + + const pipelineOptions = { + retryOptions: { + maxTries: 3, + retryDelayInMs: 1000, + maxRetryDelayInMs: 120 * 1000, + tryTimeoutInMs: 30000, + retryPolicyType: StorageRetryPolicyType.EXPONENTIAL + } + } + let blobServiceClient + + if (isSpnAuth) { + options.logger.info('using service principal credentials in azureBlobFactory') + const authParsed = JSON.parse(spnAuth) + blobServiceClient = new BlobServiceClient( + `https://${account}.queue.core.windows.net`, + new ClientSecretCredential(authParsed.tenantId, authParsed.clientId, authParsed.clientSecret), + pipelineOptions + ) + } else { + if (connection) { + options.logger.info('using connection string in azureBlobFactory') + blobServiceClient = BlobServiceClient.fromConnectionString(connection, pipelineOptions) + } else { + options.logger.info('using default credentials in azureBlobFactory') + blobServiceClient = new BlobServiceClient( + `https://${account}.queue.core.windows.net`, + new DefaultAzureCredential(), + pipelineOptions + ) + } + } + + const containerClient = blobServiceClient.getContainerClient(container) + + return new AzureStorageDocStore(containerClient, options) } diff --git a/ghcrawler/providers/storage/storageDocStore.js b/ghcrawler/providers/storage/storageDocStore.js index 8131dfa5..a531269c 100644 --- a/ghcrawler/providers/storage/storageDocStore.js +++ b/ghcrawler/providers/storage/storageDocStore.js @@ -1,32 +1,29 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -const azure = require('azure-storage') +// eslint-disable-next-line no-unused-vars +const { ContainerClient } = require('@azure/storage-blob') const memoryCache = require('memory-cache') const { Readable } = require('stream') const URL = require('url') class AzureStorageDocStore { - constructor(blobService, name, options) { - this.service = blobService - this.name = name + /** + * @param {ContainerClient} containerClient + * @param {object} options + */ + constructor(containerClient, options) { + this.containerClient = containerClient this.options = options this._getBlobNameFromKey = this.options.blobKey === 'url' ? this._getBlobNameFromUrl : this._getBlobNameFromUrn } async connect() { - return this._createContainer(this.name) + await this._createContainer(this.containerClient) } - async _createContainer(name) { - return new Promise((resolve, reject) => { - this.service.createContainerIfNotExists(name, error => { - if (error) { - return reject(error) - } - resolve(this.service) - }) - }) + async _createContainer(containerClient) { + await containerClient.createIfNotExists() } async upsert(document) { @@ -43,85 +40,47 @@ class AzureStorageDocStore { if (document._metadata.extra) { blobMetadata.extra = JSON.stringify(document._metadata.extra) } - const options = { metadata: blobMetadata, contentSettings: { contentType: 'application/json' } } + const options = { metadata: blobMetadata, blobHTTPHeaders: { blobContentType: 'application/json' } } const dataStream = new Readable() dataStream.push(JSON.stringify(document)) dataStream.push(null) - return new Promise((resolve, reject) => { - dataStream - .pipe(this.service.createWriteStreamToBlockBlob(this.name, blobName, options)) - .on('error', error => { - return reject(error) - }) - .on('finish', () => { - resolve(blobName) - }) - }) + const blockBlobClient = this.containerClient.getBlockBlobClient(blobName) + await blockBlobClient.uploadStream(dataStream, 8 << 20, 5, options) + return blobName } - // TODO: Consistency on whether key is a URL or URN async get(type, key) { const blobName = this._getBlobNameFromKey(type, key) - return new Promise((resolve, reject) => { - this.service.getBlobToText(this.name, blobName, (error, text) => { - if (error) { - return reject(error) - } - const result = JSON.parse(text) - resolve(result) - }) - }) + const blockBlobClient = this.containerClient.getBlockBlobClient(blobName) + const downloadBlockBlobResponse = await blockBlobClient.download(0) + const downloaded = await this._streamToString(downloadBlockBlobResponse.readableStreamBody) + return JSON.parse(downloaded) } - // TODO: Consistency on whether key is a URL or URN async etag(type, key) { const blobName = this._getBlobNameFromKey(type, key) - return new Promise(resolve => { - this.service.getBlobMetadata(this.name, blobName, (error, blob) => { - resolve(error ? null : blob.metadata.etag) - }) - }) + const blockBlobClient = this.containerClient.getBlockBlobClient(blobName) + const properties = await blockBlobClient.getProperties() + return properties.etag } // This API can only be used for the 'deadletter' store because we cannot look up documents by type performantly async list(type) { this._ensureDeadletter(type) let entries = [] - let continuationToken = null - do { - const result = await new Promise((resolve, reject) => { - this.service.listBlobsSegmented( - this.name, - continuationToken, - { - include: azure.BlobUtilities.BlobListingDetails.METADATA, - location: azure.StorageUtilities.LocationMode.PRIMARY_THEN_SECONDARY - }, - (error, response) => { - if (error) { - continuationToken = null - reject(error) - } - return resolve(response) - } - ) + for await (const blob of this.containerClient.listBlobsFlat({ includeMetadata: true })) { + const blobMetadata = blob.metadata + entries.push({ + version: blobMetadata.version, + etag: blobMetadata.etag, + type: blobMetadata.type, + url: blobMetadata.url, + urn: blobMetadata.urn, + fetchedAt: blobMetadata.fetchedat, + processedAt: blobMetadata.processedat, + extra: blobMetadata.extra ? JSON.parse(blobMetadata.extra) : undefined }) - entries = entries.concat( - result.entries.map(entry => { - const blobMetadata = entry.metadata - return { - version: blobMetadata.version, - etag: blobMetadata.etag, - type: blobMetadata.type, - url: blobMetadata.url, - urn: blobMetadata.urn, - fetchedAt: blobMetadata.fetchedat, - processedAt: blobMetadata.processedat, - extra: blobMetadata.extra ? JSON.parse(blobMetadata.extra) : undefined - } - }) - ) - } while (continuationToken && entries.length < 10000) + } return entries } @@ -129,20 +88,14 @@ class AzureStorageDocStore { async delete(type, key) { this._ensureDeadletter(type) const blobName = this._getBlobNameFromKey(type, key) - return new Promise((resolve, reject) => { - this.service.deleteBlob(this.name, blobName, error => { - if (error) { - return reject(error) - } - resolve(true) - }) - }) + const blockBlobClient = this.containerClient.getBlockBlobClient(blobName) + await blockBlobClient.delete() } // This API can only be used for the 'deadletter' store because we cannot look up documents by type performantly async count(type, force = false) { this._ensureDeadletter(type) - const key = `${this.name}:count:${type || ''}` + const key = `${this.options.container}:count:${type || ''}` if (!force) { const cachedCount = memoryCache.get(key) if (cachedCount) { @@ -150,24 +103,11 @@ class AzureStorageDocStore { } } let entryCount = 0 - let continuationToken = null - do { - const result = await new Promise((resolve, reject) => { - this.service.listBlobsSegmented( - this.name, - continuationToken, - { location: azure.StorageUtilities.LocationMode.PRIMARY_THEN_SECONDARY }, - (error, response) => { - if (error) { - continuationToken = null - reject(error) - } - return resolve(response) - } - ) - }) - entryCount += result.entries.length - } while (continuationToken) + for await (const page of this.containerClient.listBlobsFlat().byPage({ maxPageSize: 1000 })) { + if (page.segment.blobItems) { + entryCount += page.segment.blobItems.length() + } + } memoryCache.put(key, entryCount, 60000) return entryCount } @@ -216,6 +156,19 @@ class AzureStorageDocStore { } return `${this._getBlobPathFromUrn(type, urn)}.json` } + + async _streamToString(readableStream) { + return new Promise((resolve, reject) => { + const chunks = [] + readableStream.on('data', data => { + chunks.push(data.toString()) + }) + readableStream.on('end', () => { + resolve(chunks.join('')) + }) + readableStream.on('error', reject) + }) + } } module.exports = AzureStorageDocStore diff --git a/package-lock.json b/package-lock.json index 59e09015..1e7d385c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,13 +10,15 @@ "hasInstallScript": true, "license": "MIT", "dependencies": { + "@azure/identity": "^4.5.0", + "@azure/storage-blob": "^12.26.0", + "@azure/storage-queue": "^12.25.0", "@clearlydefined/spdx": "github:clearlydefined/spdx#v0.1.9", "@microsoft/refreshing-config": "^0.1.3", "applicationinsights": "^1.5.0", "ar-async": "^0.1.4", "axios": "^1.7.4", "axios-retry": "^3.2.5", - "azure-storage": "^2.10.3", "body-parser": "^1.19.0", "debug": "^4.3.4", "decompress": "^4.2.1", @@ -58,6 +60,7 @@ "xml2js": "^0.5.0" }, "devDependencies": { + "@types/node": "^22.10.1", "chai": "^4.2.0", "chai-as-promised": "^7.1.1", "chai-spies": "^1.0.0", @@ -69,7 +72,8 @@ "prettier": "3.2.4", "proxyquire": "^2.1.3", "request": "^2.88.2", - "sinon": "^5.0.0" + "sinon": "^5.0.0", + "typescript": "^5.7.2" } }, "node_modules/@aashutoshrathi/word-wrap": { @@ -81,6 +85,288 @@ "node": ">=0.10.0" } }, + "node_modules/@azure/abort-controller": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.1.2.tgz", + "integrity": "sha512-nBrLsEWm4J2u5LpAPjxADTlq3trDgVZZXHNKabeXZtpq3d3AbN/KGO82R87rdDz5/lYB024rtEf10/q0urNgsA==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@azure/core-auth": { + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@azure/core-auth/-/core-auth-1.9.0.tgz", + "integrity": "sha512-FPwHpZywuyasDSLMqJ6fhbOK3TqUdviZNF8OqRGA4W5Ewib2lEEZ+pBsYcBa88B2NGO/SEnYPGhyBqNlE8ilSw==", + "dependencies": { + "@azure/abort-controller": "^2.0.0", + "@azure/core-util": "^1.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@azure/core-client": { + "version": "1.9.2", + "resolved": "https://registry.npmjs.org/@azure/core-client/-/core-client-1.9.2.tgz", + "integrity": "sha512-kRdry/rav3fUKHl/aDLd/pDLcB+4pOFwPPTVEExuMyaI5r+JBbMWqRbCY1pn5BniDaU3lRxO9eaQ1AmSMehl/w==", + "dependencies": { + "@azure/abort-controller": "^2.0.0", + "@azure/core-auth": "^1.4.0", + "@azure/core-rest-pipeline": "^1.9.1", + "@azure/core-tracing": "^1.0.0", + "@azure/core-util": "^1.6.1", + "@azure/logger": "^1.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@azure/core-http-compat": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/@azure/core-http-compat/-/core-http-compat-2.1.2.tgz", + "integrity": "sha512-5MnV1yqzZwgNLLjlizsU3QqOeQChkIXw781Fwh1xdAqJR5AA32IUaq6xv1BICJvfbHoa+JYcaij2HFkhLbNTJQ==", + "dependencies": { + "@azure/abort-controller": "^2.0.0", + "@azure/core-client": "^1.3.0", + "@azure/core-rest-pipeline": "^1.3.0" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@azure/core-lro": { + "version": "2.7.2", + "resolved": "https://registry.npmjs.org/@azure/core-lro/-/core-lro-2.7.2.tgz", + "integrity": "sha512-0YIpccoX8m/k00O7mDDMdJpbr6mf1yWo2dfmxt5A8XVZVVMz2SSKaEbMCeJRvgQ0IaSlqhjT47p4hVIRRy90xw==", + "dependencies": { + "@azure/abort-controller": "^2.0.0", + "@azure/core-util": "^1.2.0", + "@azure/logger": "^1.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@azure/core-paging": { + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/@azure/core-paging/-/core-paging-1.6.2.tgz", + "integrity": "sha512-YKWi9YuCU04B55h25cnOYZHxXYtEvQEbKST5vqRga7hWY9ydd3FZHdeQF8pyh+acWZvppw13M/LMGx0LABUVMA==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@azure/core-rest-pipeline": { + "version": "1.18.1", + "resolved": "https://registry.npmjs.org/@azure/core-rest-pipeline/-/core-rest-pipeline-1.18.1.tgz", + "integrity": "sha512-/wS73UEDrxroUEVywEm7J0p2c+IIiVxyfigCGfsKvCxxCET4V/Hef2aURqltrXMRjNmdmt5IuOgIpl8f6xdO5A==", + "dependencies": { + "@azure/abort-controller": "^2.0.0", + "@azure/core-auth": "^1.8.0", + "@azure/core-tracing": "^1.0.1", + "@azure/core-util": "^1.11.0", + "@azure/logger": "^1.0.0", + "http-proxy-agent": "^7.0.0", + "https-proxy-agent": "^7.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@azure/core-rest-pipeline/node_modules/agent-base": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.1.tgz", + "integrity": "sha512-H0TSyFNDMomMNJQBn8wFV5YC/2eJ+VXECwOadZJT554xP6cODZHPX3H9QMQECxvrgiSOP1pHjy1sMWQVYJOUOA==", + "dependencies": { + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/@azure/core-rest-pipeline/node_modules/https-proxy-agent": { + "version": "7.0.5", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.5.tgz", + "integrity": "sha512-1e4Wqeblerz+tMKPIq2EMGiiWW1dIjZOksyHWSUm1rmuvw/how9hBHZ38lAGj5ID4Ik6EdkOw7NmWPy6LAwalw==", + "dependencies": { + "agent-base": "^7.0.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/@azure/core-tracing": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@azure/core-tracing/-/core-tracing-1.2.0.tgz", + "integrity": "sha512-UKTiEJPkWcESPYJz3X5uKRYyOcJD+4nYph+KpfdPRnQJVrZfk0KJgdnaAWKfhsBBtAf/D58Az4AvCJEmWgIBAg==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@azure/core-util": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/@azure/core-util/-/core-util-1.11.0.tgz", + "integrity": "sha512-DxOSLua+NdpWoSqULhjDyAZTXFdP/LKkqtYuxxz1SCN289zk3OG8UOpnCQAz/tygyACBtWp/BoO72ptK7msY8g==", + "dependencies": { + "@azure/abort-controller": "^2.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@azure/core-xml": { + "version": "1.4.4", + "resolved": "https://registry.npmjs.org/@azure/core-xml/-/core-xml-1.4.4.tgz", + "integrity": "sha512-J4FYAqakGXcbfeZjwjMzjNcpcH4E+JtEBv+xcV1yL0Ydn/6wbQfeFKTCHh9wttAi0lmajHw7yBbHPRG+YHckZQ==", + "dependencies": { + "fast-xml-parser": "^4.4.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@azure/identity": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/@azure/identity/-/identity-4.5.0.tgz", + "integrity": "sha512-EknvVmtBuSIic47xkOqyNabAme0RYTw52BTMz8eBgU1ysTyMrD1uOoM+JdS0J/4Yfp98IBT3osqq3BfwSaNaGQ==", + "dependencies": { + "@azure/abort-controller": "^2.0.0", + "@azure/core-auth": "^1.9.0", + "@azure/core-client": "^1.9.2", + "@azure/core-rest-pipeline": "^1.17.0", + "@azure/core-tracing": "^1.0.0", + "@azure/core-util": "^1.11.0", + "@azure/logger": "^1.0.0", + "@azure/msal-browser": "^3.26.1", + "@azure/msal-node": "^2.15.0", + "events": "^3.0.0", + "jws": "^4.0.0", + "open": "^8.0.0", + "stoppable": "^1.1.0", + "tslib": "^2.2.0" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@azure/identity/node_modules/open": { + "version": "8.4.2", + "resolved": "https://registry.npmjs.org/open/-/open-8.4.2.tgz", + "integrity": "sha512-7x81NCL719oNbsq/3mh+hVrAWmFuEYUqrq/Iw3kUzH8ReypT9QQ0BLoJS7/G9k6N81XjW4qHWtjWwe/9eLy1EQ==", + "dependencies": { + "define-lazy-prop": "^2.0.0", + "is-docker": "^2.1.1", + "is-wsl": "^2.2.0" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@azure/logger": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/@azure/logger/-/logger-1.1.4.tgz", + "integrity": "sha512-4IXXzcCdLdlXuCG+8UKEwLA1T1NHqUfanhXYHiQTn+6sfWCZXduqbtXDGceg3Ce5QxTGo7EqmbV6Bi+aqKuClQ==", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@azure/msal-browser": { + "version": "3.28.0", + "resolved": "https://registry.npmjs.org/@azure/msal-browser/-/msal-browser-3.28.0.tgz", + "integrity": "sha512-1c1qUF6vB52mWlyoMem4xR1gdwiQWYEQB2uhDkbAL4wVJr8WmAcXybc1Qs33y19N4BdPI8/DHI7rPE8L5jMtWw==", + "dependencies": { + "@azure/msal-common": "14.16.0" + }, + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/@azure/msal-common": { + "version": "14.16.0", + "resolved": "https://registry.npmjs.org/@azure/msal-common/-/msal-common-14.16.0.tgz", + "integrity": "sha512-1KOZj9IpcDSwpNiQNjt0jDYZpQvNZay7QAEi/5DLubay40iGYtLzya/jbjRPLyOTZhEKyL1MzPuw2HqBCjceYA==", + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/@azure/msal-node": { + "version": "2.16.2", + "resolved": "https://registry.npmjs.org/@azure/msal-node/-/msal-node-2.16.2.tgz", + "integrity": "sha512-An7l1hEr0w1HMMh1LU+rtDtqL7/jw74ORlc9Wnh06v7TU/xpG39/Zdr1ZJu3QpjUfKJ+E0/OXMW8DRSWTlh7qQ==", + "dependencies": { + "@azure/msal-common": "14.16.0", + "jsonwebtoken": "^9.0.0", + "uuid": "^8.3.0" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/@azure/storage-blob": { + "version": "12.26.0", + "resolved": "https://registry.npmjs.org/@azure/storage-blob/-/storage-blob-12.26.0.tgz", + "integrity": "sha512-SriLPKezypIsiZ+TtlFfE46uuBIap2HeaQVS78e1P7rz5OSbq0rsd52WE1mC5f7vAeLiXqv7I7oRhL3WFZEw3Q==", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-auth": "^1.4.0", + "@azure/core-client": "^1.6.2", + "@azure/core-http-compat": "^2.0.0", + "@azure/core-lro": "^2.2.0", + "@azure/core-paging": "^1.1.1", + "@azure/core-rest-pipeline": "^1.10.1", + "@azure/core-tracing": "^1.1.2", + "@azure/core-util": "^1.6.1", + "@azure/core-xml": "^1.4.3", + "@azure/logger": "^1.0.0", + "events": "^3.0.0", + "tslib": "^2.2.0" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@azure/storage-queue": { + "version": "12.25.0", + "resolved": "https://registry.npmjs.org/@azure/storage-queue/-/storage-queue-12.25.0.tgz", + "integrity": "sha512-uoobHFbH/o7wIul/sCm32X2YFq6zb1XpNdpKIms9I60mwG3BBaOpEs5pgQV5a5ONG5WMSHlo8E1dNFB5ZZIa1g==", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-auth": "^1.4.0", + "@azure/core-client": "^1.6.2", + "@azure/core-http-compat": "^2.0.0", + "@azure/core-paging": "^1.1.1", + "@azure/core-rest-pipeline": "^1.10.1", + "@azure/core-tracing": "^1.1.2", + "@azure/core-util": "^1.6.1", + "@azure/core-xml": "^1.4.3", + "@azure/logger": "^1.0.0", + "tslib": "^2.2.0" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@babel/code-frame": { "version": "7.12.11", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.12.11.tgz", @@ -905,12 +1191,12 @@ } }, "node_modules/@types/node": { - "version": "20.14.10", - "resolved": "https://registry.npmjs.org/@types/node/-/node-20.14.10.tgz", - "integrity": "sha512-MdiXf+nDuMvY0gJKxyfZ7/6UFsETO7mGKF54MVD/ekJS6HdFtpZFBgrh6Pseu64XTb2MLyFPlbW6hj8HYRQNOQ==", + "version": "22.10.1", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.10.1.tgz", + "integrity": "sha512-qKgsUwfHZV2WCWLAnVP1JqnpE6Im6h3Y0+fYgMTasNQ7V++CBX5OT1as0g0f+OyubbFqhf6XVNIsmN4IIhEgGQ==", "dev": true, "dependencies": { - "undici-types": "~5.26.4" + "undici-types": "~6.20.0" } }, "node_modules/@types/ws": { @@ -1282,46 +1568,6 @@ "node": ">= 6" } }, - "node_modules/azure-storage": { - "version": "2.10.7", - "resolved": "https://registry.npmjs.org/azure-storage/-/azure-storage-2.10.7.tgz", - "integrity": "sha512-4oeFGtn3Ziw/fGs/zkoIpKKtygnCVIcZwzJ7UQzKTxhkGQqVCByOFbYqMGYR3L+wOsunX9lNfD0jc51SQuKSSA==", - "deprecated": "Please note: newer packages @azure/storage-blob, @azure/storage-queue and @azure/storage-file are available as of November 2019 and @azure/data-tables is available as of June 2021. While the legacy azure-storage package will continue to receive critical bug fixes, we strongly encourage you to upgrade. Migration guide can be found: https://github.com/Azure/azure-sdk-for-js/blob/main/sdk/storage/MigrationGuide.md", - "dependencies": { - "browserify-mime": "^1.2.9", - "extend": "^3.0.2", - "json-edm-parser": "~0.1.2", - "json-schema": "~0.4.0", - "md5.js": "^1.3.4", - "readable-stream": "^2.0.0", - "request": "^2.86.0", - "underscore": "^1.12.1", - "uuid": "^3.0.0", - "validator": "^13.7.0", - "xml2js": "~0.2.8", - "xmlbuilder": "^9.0.7" - }, - "engines": { - "node": ">= 0.8.26" - } - }, - "node_modules/azure-storage/node_modules/uuid": { - "version": "3.3.2", - "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", - "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==", - "deprecated": "Please upgrade to version 7 or higher. Older versions may use Math.random() in certain circumstances, which is known to be problematic. See https://v8.dev/blog/math-random for details.", - "bin": { - "uuid": "bin/uuid" - } - }, - "node_modules/azure-storage/node_modules/xml2js": { - "version": "0.2.8", - "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.2.8.tgz", - "integrity": "sha512-ZHZBIAO55GHCn2jBYByVPHvHS+o3j8/a/qmpEe6kxO3cTnTCWC3Htq9RYJ5G4XMwMMClD2QkXA9SNdPadLyn3Q==", - "dependencies": { - "sax": "0.5.x" - } - }, "node_modules/backo2": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/backo2/-/backo2-1.0.2.tgz", @@ -1499,11 +1745,6 @@ "integrity": "sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw==", "dev": true }, - "node_modules/browserify-mime": { - "version": "1.2.9", - "resolved": "https://registry.npmjs.org/browserify-mime/-/browserify-mime-1.2.9.tgz", - "integrity": "sha1-rrGvKN5sDXpqLOQK22j/GEIq8x8=" - }, "node_modules/buffer": { "version": "5.4.0", "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.4.0.tgz", @@ -1535,6 +1776,11 @@ "node": "*" } }, + "node_modules/buffer-equal-constant-time": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", + "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==" + }, "node_modules/buffer-fill": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/buffer-fill/-/buffer-fill-1.0.0.tgz", @@ -2364,6 +2610,14 @@ "abstract-leveldown": "~2.6.0" } }, + "node_modules/define-lazy-prop": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/define-lazy-prop/-/define-lazy-prop-2.0.0.tgz", + "integrity": "sha512-Ds09qNh8yw3khSjiJjiUInaGX9xlqZDY7JVryGxdxV7NPeuqQfplOpQ66yJFZut3jLa5zOwkXw1g9EI2uKh4Og==", + "engines": { + "node": ">=8" + } + }, "node_modules/degenerator": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/degenerator/-/degenerator-5.0.1.tgz", @@ -2559,6 +2813,14 @@ "safer-buffer": "^2.1.0" } }, + "node_modules/ecdsa-sig-formatter": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", + "dependencies": { + "safe-buffer": "^5.0.1" + } + }, "node_modules/ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", @@ -3186,6 +3448,14 @@ "integrity": "sha512-tvtQIeLVHjDkJYnzf2dgVMxfuSGJeM/7UCG17TT4EumTfNtF+0nebF/4zWOIkCreAbtNqhGEboB6BWrwqNaw4Q==", "dev": true }, + "node_modules/events": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", + "engines": { + "node": ">=0.8.x" + } + }, "node_modules/express": { "version": "4.19.2", "resolved": "https://registry.npmjs.org/express/-/express-4.19.2.tgz", @@ -3405,6 +3675,27 @@ "integrity": "sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc=", "dev": true }, + "node_modules/fast-xml-parser": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-4.5.0.tgz", + "integrity": "sha512-/PlTQCI96+fZMAOLMZK4CWG1ItCbfZ/0jx7UIJFChPNrx7tcEgerUgWbeieCM9MfHInUDyK8DWYZ+YrywDJuTg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + }, + { + "type": "paypal", + "url": "https://paypal.me/naturalintelligence" + } + ], + "dependencies": { + "strnum": "^1.0.5" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, "node_modules/fastq": { "version": "1.16.0", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.16.0.tgz", @@ -4166,18 +4457,6 @@ "resolved": "https://registry.npmjs.org/has-unicode/-/has-unicode-2.0.1.tgz", "integrity": "sha1-4Ob+aijPUROIVeCG0Wkedx3iqLk=" }, - "node_modules/hash-base": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/hash-base/-/hash-base-3.0.4.tgz", - "integrity": "sha1-X8hoaEfs1zSZQDMZprCj8/auSRg=", - "dependencies": { - "inherits": "^2.0.1", - "safe-buffer": "^5.0.1" - }, - "engines": { - "node": ">=4" - } - }, "node_modules/hasha": { "version": "5.2.2", "resolved": "https://registry.npmjs.org/hasha/-/hasha-5.2.2.tgz", @@ -4272,7 +4551,6 @@ "version": "7.0.2", "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", - "dev": true, "dependencies": { "agent-base": "^7.1.0", "debug": "^4.3.4" @@ -4285,7 +4563,6 @@ "version": "7.1.1", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.1.tgz", "integrity": "sha512-H0TSyFNDMomMNJQBn8wFV5YC/2eJ+VXECwOadZJT554xP6cODZHPX3H9QMQECxvrgiSOP1pHjy1sMWQVYJOUOA==", - "dev": true, "dependencies": { "debug": "^4.3.4" }, @@ -4938,14 +5215,6 @@ "node": ">=4" } }, - "node_modules/json-edm-parser": { - "version": "0.1.2", - "resolved": "https://registry.npmjs.org/json-edm-parser/-/json-edm-parser-0.1.2.tgz", - "integrity": "sha1-HmCw/vG8CvZ7wNFG393lSGzWFbQ=", - "dependencies": { - "jsonparse": "~1.2.0" - } - }, "node_modules/json-schema": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.4.0.tgz", @@ -4990,13 +5259,45 @@ "graceful-fs": "^4.1.6" } }, - "node_modules/jsonparse": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/jsonparse/-/jsonparse-1.2.0.tgz", - "integrity": "sha1-XAxWhRBxYOcv50ib3eoLRMK8Z70=", - "engines": [ - "node >= 0.2.0" - ] + "node_modules/jsonwebtoken": { + "version": "9.0.2", + "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-9.0.2.tgz", + "integrity": "sha512-PRp66vJ865SSqOlgqS8hujT5U4AOgMfhrwYIuIhfKaoSCZcirrmASQr8CX7cUg+RMih+hgznrjp99o+W4pJLHQ==", + "dependencies": { + "jws": "^3.2.2", + "lodash.includes": "^4.3.0", + "lodash.isboolean": "^3.0.3", + "lodash.isinteger": "^4.0.4", + "lodash.isnumber": "^3.0.3", + "lodash.isplainobject": "^4.0.6", + "lodash.isstring": "^4.0.1", + "lodash.once": "^4.0.0", + "ms": "^2.1.1", + "semver": "^7.5.4" + }, + "engines": { + "node": ">=12", + "npm": ">=6" + } + }, + "node_modules/jsonwebtoken/node_modules/jwa": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-1.4.1.tgz", + "integrity": "sha512-qiLX/xhEEFKUAJ6FiBMbes3w9ATzyk5W7Hvzpa/SLYdxNtng+gcurvrI7TbACjIXlsJyr05/S1oUhZrc63evQA==", + "dependencies": { + "buffer-equal-constant-time": "1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/jsonwebtoken/node_modules/jws": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/jws/-/jws-3.2.2.tgz", + "integrity": "sha512-YHlZCB6lMTllWDtSPHz/ZXTsi8S00usEV6v1tjq8tOUZzw7DpSDWVXjXDre6ed1w/pd495ODpHZYSdkRTsa0HA==", + "dependencies": { + "jwa": "^1.4.1", + "safe-buffer": "^5.0.1" + } }, "node_modules/jsprim": { "version": "1.4.2", @@ -5018,6 +5319,25 @@ "integrity": "sha512-g3UB796vUFIY90VIv/WX3L2c8CS2MdWUww3CNrYmqza1Fg0DURc2K/O4YrnklBdQarSJ/y8JnJYDGc+1iumQjg==", "dev": true }, + "node_modules/jwa": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz", + "integrity": "sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==", + "dependencies": { + "buffer-equal-constant-time": "1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/jws": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz", + "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==", + "dependencies": { + "jwa": "^2.0.0", + "safe-buffer": "^5.0.1" + } + }, "node_modules/klaw-sync": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/klaw-sync/-/klaw-sync-6.0.0.tgz", @@ -5122,12 +5442,47 @@ "integrity": "sha1-LRd/ZS+jHpObRDjVNBSZ36OCXpk=", "dev": true }, + "node_modules/lodash.includes": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz", + "integrity": "sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==" + }, + "node_modules/lodash.isboolean": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz", + "integrity": "sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==" + }, + "node_modules/lodash.isinteger": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/lodash.isinteger/-/lodash.isinteger-4.0.4.tgz", + "integrity": "sha512-DBwtEWN2caHQ9/imiNeEA5ys1JoRtRfY3d7V9wkqtbycnAmTvRRmbHKDV4a0EYc678/dia0jrte4tjYwVBaZUA==" + }, + "node_modules/lodash.isnumber": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/lodash.isnumber/-/lodash.isnumber-3.0.3.tgz", + "integrity": "sha512-QYqzpfwO3/CWf3XP+Z+tkQsfaLL/EnUlXWVkIk5FUPc4sBdTehEqZONuyRt2P67PXAk+NXmTBcc97zw9t1FQrw==" + }, + "node_modules/lodash.isplainobject": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", + "integrity": "sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==" + }, + "node_modules/lodash.isstring": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/lodash.isstring/-/lodash.isstring-4.0.1.tgz", + "integrity": "sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw==" + }, "node_modules/lodash.merge": { "version": "4.6.2", "resolved": "https://registry.npmjs.org/lodash.merge/-/lodash.merge-4.6.2.tgz", "integrity": "sha512-0KpjqXRVvrYyCsX1swR/XTK0va6VQkQM6MNo7PqW77ByjAhoARA8EfrP1N4+KlKj8YS0ZUCtRT/YUuhyYDujIQ==", "dev": true }, + "node_modules/lodash.once": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/lodash.once/-/lodash.once-4.1.1.tgz", + "integrity": "sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==" + }, "node_modules/log-symbols": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.0.0.tgz", @@ -5262,15 +5617,6 @@ "resolved": "https://registry.npmjs.org/map-async/-/map-async-0.1.1.tgz", "integrity": "sha1-yJfARJ+Fhkx0taPxlu20IVZDF0U=" }, - "node_modules/md5.js": { - "version": "1.3.4", - "resolved": "https://registry.npmjs.org/md5.js/-/md5.js-1.3.4.tgz", - "integrity": "sha1-6b296UogpawYsENA/Fdk1bCdkB0=", - "dependencies": { - "hash-base": "^3.0.0", - "inherits": "^2.0.1" - } - }, "node_modules/media-typer": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", @@ -7183,11 +7529,6 @@ "deprecated": "This package has been deprecated in favour of @sinonjs/samsam", "dev": true }, - "node_modules/sax": { - "version": "0.5.8", - "resolved": "https://registry.npmjs.org/sax/-/sax-0.5.8.tgz", - "integrity": "sha1-1HLbIo6zMcJQaw6MFVJK25OdEsE=" - }, "node_modules/seek-bzip": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/seek-bzip/-/seek-bzip-1.0.5.tgz", @@ -7613,6 +7954,15 @@ "node": ">= 0.8" } }, + "node_modules/stoppable": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/stoppable/-/stoppable-1.1.0.tgz", + "integrity": "sha512-KXDYZ9dszj6bzvnEMRYvxgeTHU74QBFL54XKtP3nyMuJ81CFYtABZ3bAzL2EdFUaEwJOBOgENyFj3R7oTzDyyw==", + "engines": { + "node": ">=4", + "npm": ">=6" + } + }, "node_modules/stream-shift": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.3.tgz", @@ -7672,6 +8022,11 @@ "node": ">=0.10.0" } }, + "node_modules/strnum": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-1.0.5.tgz", + "integrity": "sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==" + }, "node_modules/supports-color": { "version": "5.5.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", @@ -7852,8 +8207,7 @@ "node_modules/tslib": { "version": "2.6.3", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.6.3.tgz", - "integrity": "sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ==", - "dev": true + "integrity": "sha512-xNvxJEOUiWPGhUuUdQgAJPKOOJfGnIyKySOc09XkKsgdUV/3E2zvwZYdejjmRgPCgcym1juLH3226yA7sEFJKQ==" }, "node_modules/tunnel-agent": { "version": "0.6.0", @@ -7925,6 +8279,19 @@ "is-typedarray": "^1.0.0" } }, + "node_modules/typescript": { + "version": "5.7.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.7.2.tgz", + "integrity": "sha512-i5t66RHxDvVN40HfDd1PsEThGNnlMCMT3jMUuoh9/0TaqWevNontacunWyN02LA9/fIbEWlcHZcgTKb9QoaLfg==", + "dev": true, + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, "node_modules/unbzip2-stream": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/unbzip2-stream/-/unbzip2-stream-1.3.3.tgz", @@ -7934,15 +8301,10 @@ "through": "^2.3.8" } }, - "node_modules/underscore": { - "version": "1.13.6", - "resolved": "https://registry.npmjs.org/underscore/-/underscore-1.13.6.tgz", - "integrity": "sha512-+A5Sja4HP1M08MaXya7p5LvjuM7K6q/2EaC0+iovj/wOcMsTzMvDFbasi/oSapiwOlt252IqsKqPjCl7huKS0A==" - }, "node_modules/undici-types": { - "version": "5.26.5", - "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", - "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", + "version": "6.20.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.20.0.tgz", + "integrity": "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==", "dev": true }, "node_modules/universalify": { @@ -7992,19 +8354,10 @@ "version": "8.3.2", "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", - "dev": true, "bin": { "uuid": "dist/bin/uuid" } }, - "node_modules/validator": { - "version": "13.11.0", - "resolved": "https://registry.npmjs.org/validator/-/validator-13.11.0.tgz", - "integrity": "sha512-Ii+sehpSfZy+At5nPdnyMhx78fEoPDkR2XW/zimHEL3MyGJQOCQ7WeP20jPYRz7ZCpcKLB21NxuXHF3bxjStBQ==", - "engines": { - "node": ">= 0.10" - } - }, "node_modules/value-or-promise": { "version": "1.0.11", "resolved": "https://registry.npmjs.org/value-or-promise/-/value-or-promise-1.0.11.tgz", @@ -8284,14 +8637,6 @@ "node": ">=4.0" } }, - "node_modules/xmlbuilder": { - "version": "9.0.7", - "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz", - "integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=", - "engines": { - "node": ">=4.0" - } - }, "node_modules/xtend": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.1.tgz", diff --git a/package.json b/package.json index 2fe6b1cd..e9e81518 100644 --- a/package.json +++ b/package.json @@ -34,13 +34,15 @@ "url": "https://github.com/clearlydefined/crawler" }, "dependencies": { + "@azure/identity": "^4.5.0", + "@azure/storage-blob": "^12.26.0", + "@azure/storage-queue": "^12.25.0", "@clearlydefined/spdx": "github:clearlydefined/spdx#v0.1.9", "@microsoft/refreshing-config": "^0.1.3", "applicationinsights": "^1.5.0", "ar-async": "^0.1.4", "axios": "^1.7.4", "axios-retry": "^3.2.5", - "azure-storage": "^2.10.3", "body-parser": "^1.19.0", "debug": "^4.3.4", "decompress": "^4.2.1", @@ -93,6 +95,8 @@ "prettier": "3.2.4", "proxyquire": "^2.1.3", "request": "^2.88.2", - "sinon": "^5.0.0" + "sinon": "^5.0.0", + "@types/node": "^22.10.1", + "typescript": "^5.7.2" } } diff --git a/providers/store/azureQueueStore.js b/providers/store/azureQueueStore.js index b1e0e461..9ad21bb3 100644 --- a/providers/store/azureQueueStore.js +++ b/providers/store/azureQueueStore.js @@ -1,25 +1,60 @@ // Copyright (c) Microsoft Corporation and others. Licensed under the MIT license. // SPDX-License-Identifier: MIT -const azure = require('azure-storage') -const { promisify } = require('util') +const { DefaultAzureCredential, ClientSecretCredential } = require('@azure/identity') +const { QueueServiceClient, StorageRetryPolicyType } = require('@azure/storage-queue') class AzureStorageQueue { constructor(options) { this.options = options + this.queueName = options.queueName this.logger = options.logger + + const { connectionString, account, spnAuth, isSpnAuth } = options + + const pipelineOptions = { + retryOptions: { + maxTries: 3, + retryDelayInMs: 1000, + maxRetryDelayInMs: 120 * 1000, + tryTimeoutInMs: 30000, + retryPolicyType: StorageRetryPolicyType.FIXED + } + } + + if (isSpnAuth) { + options.logger.info('using service principal credentials in azureQueueStore') + const authParsed = JSON.parse(spnAuth) + this.client = new QueueServiceClient( + `https://${account}.queue.core.windows.net`, + new ClientSecretCredential(authParsed.tenantId, authParsed.clientId, authParsed.clientSecret), + pipelineOptions + ) + return + } + + if (connectionString) { + options.logger.info('using connection string in azureQueueStore') + this.client = QueueServiceClient.fromConnectionString(connectionString, pipelineOptions) + return + } + + options.logger.info('using default credentials in azureQueueStore') + this.client = new QueueServiceClient( + `https://${account}.queue.core.windows.net`, + new DefaultAzureCredential(), + pipelineOptions + ) } async connect() { - this.queueService = azure - .createQueueService(this.options.connectionString) - .withFilter(new azure.LinearRetryPolicyFilter()) - await promisify(this.queueService.createQueueIfNotExists).bind(this.queueService)(this.options.queueName) + this.queueService = this.client.getQueueClient(this.queueName) + this.queueService.createIfNotExists() } async upsert(document) { const message = Buffer.from(JSON.stringify({ _metadata: document._metadata })).toString('base64') - await promisify(this.queueService.createMessage).bind(this.queueService)(this.options.queueName, message) + return await this.queueService.sendMessage(message) } get() {