From 90be4a628ec999e396e3ea2d6a13a6ecc0fe4eb4 Mon Sep 17 00:00:00 2001 From: MicroFish91 <40250218+MicroFish91@users.noreply.github.com> Date: Thu, 18 Dec 2025 10:56:46 -0800 Subject: [PATCH 01/27] WIP --- .vscode/launch.json | 4 +- test/global.test.ts | 5 +- test/nightly/scenarios/durableDTS/dts.test.ts | 50 +++++++++++++++++ .../scenarios/durableDTS/durableDTS.md | 0 test/nightly/scenarios/durableDTS/geagea | 54 +++++++++++++++++++ test/test.code-workspace | 4 ++ 6 files changed, 112 insertions(+), 5 deletions(-) create mode 100644 test/nightly/scenarios/durableDTS/dts.test.ts create mode 100644 test/nightly/scenarios/durableDTS/durableDTS.md create mode 100644 test/nightly/scenarios/durableDTS/geagea diff --git a/.vscode/launch.json b/.vscode/launch.json index 402650c8d..2a435fd16 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -72,11 +72,11 @@ "env": { "MOCHA_grep": "", // RegExp of tests to run (empty for all) "MOCHA_timeout": "0", // Disable time-outs - "DEBUGTELEMETRY": "v", + "DEBUGTELEMETRY": "", "NODE_DEBUG": "", "FUNC_PATH": "func", "AZFUNC_UPDATE_BACKUP_TEMPLATES": "", - "AzCode_EnableLongRunningTestsLocal": "", + "AzCode_EnableLongRunningTestsLocal": "true", } }, { diff --git a/test/global.test.ts b/test/global.test.ts index 304462e5d..4dbc84b18 100644 --- a/test/global.test.ts +++ b/test/global.test.ts @@ -5,7 +5,6 @@ import { createTestActionContext, runWithTestActionContext, TestOutputChannel, TestUserInput } from '@microsoft/vscode-azext-dev'; import { AzExtFsExtra } from '@microsoft/vscode-azext-utils'; -import * as assert from 'assert'; import * as os from 'os'; import * as path from 'path'; import * as vscode from 'vscode'; @@ -170,8 +169,8 @@ async function initTestWorkspaceFolders(): Promise { const folders: string[] = []; for (let i = 0; i < workspaceFolders.length; i++) { const workspacePath: string = workspaceFolders[i].uri.fsPath; - const folderName = path.basename(workspacePath); - assert.equal(folderName, String(i), `Unexpected workspace folder name "${folderName}".`); + // const folderName = path.basename(workspacePath); + // assert.equal(folderName, String(i), `Unexpected workspace folder name "${folderName}".`); await AzExtFsExtra.ensureDir(workspacePath); await AzExtFsExtra.emptyDir(workspacePath); folders.push(workspacePath); diff --git a/test/nightly/scenarios/durableDTS/dts.test.ts b/test/nightly/scenarios/durableDTS/dts.test.ts new file mode 100644 index 000000000..9c4d189e1 --- /dev/null +++ b/test/nightly/scenarios/durableDTS/dts.test.ts @@ -0,0 +1,50 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { AzExtFsExtra } from '@microsoft/vscode-azext-utils'; +import * as assert from 'assert'; +import { workspace, type Uri, type WorkspaceFolder } from 'vscode'; +import { updateGlobalSetting } from '../../../../extension.bundle'; +import { longRunningTestsEnabled } from '../../../global.test'; + +suite.only('Durable Task Scheduler (DTS) Scenarios', function (this: Mocha.Suite): void { + this.timeout(7 * 60 * 1000); + + suiteSetup(async function (this: Mocha.Context): Promise { + if (!longRunningTestsEnabled) { + this.skip(); + } + + const workspaceFolderUri: Uri = getWorkspaceFolderUri('scenario-durable-dts'); + const rootFolder: WorkspaceFolder | undefined = workspace.getWorkspaceFolder(workspaceFolderUri); + assert.ok(rootFolder, 'Could not retrieve root workspace folder.'); + + cleanTestFolder(rootFolder); + await updateGlobalSetting('groupBy', 'resourceType', 'azureResourceGroups'); + }); + + test('Create function should bring down template files and make the necessary workspace configurations', async () => { + console.log("hello"); + }); +}); + +async function cleanTestFolder(testFolder: WorkspaceFolder) { + await AzExtFsExtra.emptyDir(testFolder.uri.fsPath); +} + +export function getWorkspaceFolderUri(folderName: string): Uri { + const workspaceFolders: readonly WorkspaceFolder[] | undefined = workspace.workspaceFolders; + if (!workspaceFolders || workspaceFolders.length === 0) { + throw new Error('No workspace is open'); + } else { + for (const workspaceFolder of workspaceFolders) { + if (workspaceFolder.name === folderName) { + return workspaceFolder.uri; + } + } + } + + throw new Error(`Unable to find workspace folder "${folderName}"`); +} diff --git a/test/nightly/scenarios/durableDTS/durableDTS.md b/test/nightly/scenarios/durableDTS/durableDTS.md new file mode 100644 index 000000000..e69de29bb diff --git a/test/nightly/scenarios/durableDTS/geagea b/test/nightly/scenarios/durableDTS/geagea new file mode 100644 index 000000000..60ed9cae9 --- /dev/null +++ b/test/nightly/scenarios/durableDTS/geagea @@ -0,0 +1,54 @@ +// const workspaceFolderUri: Uri = getWorkspaceFolderUri(scenario.folderName); +// const rootFolder: WorkspaceFolder | undefined = workspace.getWorkspaceFolder(workspaceFolderUri); +// assert.ok(rootFolder, 'Could not retrieve root workspace folder.'); + +// await cleanWorkspaceFolderSettings(rootFolder); + +// for (const testCase of scenario.testCases) { +// ext.outputChannel.appendLog(`[[[ *** ${scenario.label} - ${testCase.label} *** ]]]`); +// await runWithTestActionContext('deployWorkspaceProject', async context => { +// await context.ui.runWithInputs(testCase.inputs, async () => { +// let results: DeployWorkspaceProjectResults; +// let perr: IParsedError | undefined; +// try { +// results = await deployWorkspaceProject(context); +// } catch (e) { +// results = {}; + +// perr = parseError(e); +// console.log(perr); +// } + +// if (testCase.resourceGroupToDelete) { +// resourceGroupsToDelete.add(testCase.resourceGroupToDelete); +// } + +// // Verify 'expectedErrMsg' +// if (perr || testCase.expectedErrMsg) { +// if (testCase.expectedErrMsg instanceof RegExp) { +// assert.match(perr?.message ?? "", testCase.expectedErrMsg, 'DeployWorkspaceProject thrown and expected error message did not match.'); +// } else { +// assert.strictEqual(perr?.message ?? "", testCase.expectedErrMsg, 'DeployWorkspaceProject thrown and expected error message did not match.'); +// } +// } + +// // Verify 'expectedResults' +// assertStringPropsMatch(results as Partial>, (testCase.expectedResults ?? {}) as Record, 'DeployWorkspaceProject results mismatch.'); + +// // Verify 'expectedVSCodeSettings' +// const deploymentConfigurationsV2: DeploymentConfigurationSettings[] = await dwpSettingUtilsV2.getWorkspaceDeploymentConfigurations(rootFolder) ?? []; +// const expectedDeploymentConfigurations = testCase.expectedVSCodeSettings?.deploymentConfigurations ?? []; +// assert.strictEqual(deploymentConfigurationsV2.length, expectedDeploymentConfigurations.length, 'DeployWorkspaceProject ".vscode" saved settings mismatch.'); + +// for (const [i, expectedDeploymentConfiguration] of expectedDeploymentConfigurations.entries()) { +// const deploymentConfiguration: DeploymentConfigurationSettings = deploymentConfigurationsV2[i] ?? {}; +// assertStringPropsMatch(deploymentConfiguration as Partial>, expectedDeploymentConfiguration, 'DeployWorkspaceProject ".vscode" saved settings mismatch.'); +// } + +// // Verify 'postTestAssertion' +// await testCase.postTestAssertion?.(context, results, 'DeployWorkspaceProject resource settings mismatch.'); +// }); +// }); +// } + +// await cleanWorkspaceFolderSettings(rootFolder); diff --git a/test/test.code-workspace b/test/test.code-workspace index 26a370e25..13736a4cf 100644 --- a/test/test.code-workspace +++ b/test/test.code-workspace @@ -29,6 +29,10 @@ }, { "path": "../testWorkspace/9" + }, + { + "name": "scenario-durable-dts", + "path": "../testWorkspace/scenarios/durable-dts" } ], "settings": { From 3ff5184ba6db4d3aee4dcb327910b82e0fd9ee05 Mon Sep 17 00:00:00 2001 From: MicroFish91 <40250218+MicroFish91@users.noreply.github.com> Date: Fri, 19 Dec 2025 00:30:19 -0800 Subject: [PATCH 02/27] WIP: Create New Project done --- src/commands/createFunction/createFunction.ts | 2 + .../scenarios/durableDTS/TestCombinations.md | 119 ++++++++++++++++++ test/nightly/scenarios/durableDTS/dts.test.ts | 33 +++-- .../scenarios/durableDTS/durableDTS.md | 54 ++++++++ test/nightly/scenarios/durableDTS/geagea | 54 -------- test/test.code-workspace | 16 ++- 6 files changed, 215 insertions(+), 63 deletions(-) create mode 100644 test/nightly/scenarios/durableDTS/TestCombinations.md delete mode 100644 test/nightly/scenarios/durableDTS/geagea diff --git a/src/commands/createFunction/createFunction.ts b/src/commands/createFunction/createFunction.ts index 27e62aae7..2de2ef4c1 100644 --- a/src/commands/createFunction/createFunction.ts +++ b/src/commands/createFunction/createFunction.ts @@ -90,4 +90,6 @@ export async function createFunctionInternal(context: IActionContext, options: a }); await wizard.prompt(); await wizard.execute(); + + console.log("test") } diff --git a/test/nightly/scenarios/durableDTS/TestCombinations.md b/test/nightly/scenarios/durableDTS/TestCombinations.md new file mode 100644 index 000000000..d73eb7e96 --- /dev/null +++ b/test/nightly/scenarios/durableDTS/TestCombinations.md @@ -0,0 +1,119 @@ +# Test Combinations + +## I. Create New Project / Create Function + +### Workspace Project Test Matrix: + +| No. | Language | Runtime | Programming Model | Comment | +|-----|----------|---------|-------------------|---------| +| I | TS | Node | v3 | Skip unless special requirements | +| II | TS | Node | v4 | | +| III | Python | Python | v1 | Skip unless special requirements | +| IV | Python | Python | v2 | | +| V | C# | .NET | isolated | | +| VI | C# | .NET | in-proc | | + +## II. Debug +TBD + +## III. Create Function App / Deployment + +### Create / Deploy Test Matrix: + +| No. | Workspace Project | Connection Type | Operating System | Plan Type | Comment | +|-----|-------------------|-------------------|------------------|------------------------|---------| +| 1 | I | Managed Identity | Linux | Flex Consumption | Skip | +| 2 | I | Managed Identity | Windows | Flex Consumption | Skip | +| 4 | I | Managed Identity | Windows | Premium | Skip | +| 5 | I | Managed Identity | Linux | Consumption (Legacy) | Skip | +| 6 | I | Managed Identity | Windows | Consumption (Legacy) | Skip | +| 7 | I | Managed Identity | Linux | App Service | Skip | +| 8 | I | Managed Identity | Windows | App Service | Skip | +| 9 | I | Secrets | Linux | Flex Consumption | Skip | +| 10 | I | Secrets | Windows | Flex Consumption | Skip | +| 11 | I | Secrets | Linux | Premium | Skip | +| 12 | I | Secrets | Windows | Premium | Skip | +| 13 | I | Secrets | Linux | Consumption (Legacy) | Skip | +| 14 | I | Secrets | Windows | Consumption (Legacy) | Skip | +| 15 | I | Secrets | Linux | App Service | Skip | +| 16 | I | Secrets | Windows | App Service | Skip | +| 17 | II | Managed Identity | Linux | Flex Consumption | | +| 18 | II | Managed Identity | Windows | Flex Consumption | | +| 19 | II | Managed Identity | Linux | Premium | | +| 20 | II | Managed Identity | Windows | Premium | | +| 21 | II | Managed Identity | Linux | Consumption (Legacy) | | +| 22 | II | Managed Identity | Windows | Consumption (Legacy) | | +| 23 | II | Managed Identity | Linux | App Service | | +| 24 | II | Managed Identity | Windows | App Service | | +| 25 | II | Secrets | Linux | Flex Consumption | | +| 26 | II | Secrets | Windows | Flex Consumption | | +| 27 | II | Secrets | Linux | Premium | | +| 28 | II | Secrets | Windows | Premium | | +| 29 | II | Secrets | Linux | Consumption (Legacy) | | +| 30 | II | Secrets | Windows | Consumption (Legacy) | | +| 31 | II | Secrets | Linux | App Service | | +| 32 | II | Secrets | Windows | App Service | | +| 33 | III | Managed Identity | Linux | Flex Consumption | Skip | +| 34 | III | Managed Identity | Windows | Flex Consumption | Skip | +| 35 | III | Managed Identity | Linux | Premium | Skip | +| 36 | III | Managed Identity | Windows | Premium | Skip | +| 37 | III | Managed Identity | Linux | Consumption (Legacy) | Skip | +| 38 | III | Managed Identity | Windows | Consumption (Legacy) | Skip | +| 39 | III | Managed Identity | Linux | App Service | Skip | +| 40 | III | Managed Identity | Windows | App Service | Skip | +| 41 | III | Secrets | Linux | Flex Consumption | Skip | +| 42 | III | Secrets | Windows | Flex Consumption | Skip | +| 43 | III | Secrets | Linux | Premium | Skip | +| 44 | III | Secrets | Windows | Premium | Skip | +| 45 | III | Secrets | Linux | Consumption (Legacy) | Skip | +| 46 | III | Secrets | Windows | Consumption (Legacy) | Skip | +| 47 | III | Secrets | Linux | App Service | Skip | +| 48 | III | Secrets | Windows | App Service | Skip | +| 49 | IV | Managed Identity | Linux | Flex Consumption | | +| 50 | IV | Managed Identity | Windows | Flex Consumption | | +| 51 | IV | Managed Identity | Linux | Premium | | +| 52 | IV | Managed Identity | Windows | Premium | | +| 53 | IV | Managed Identity | Linux | Consumption (Legacy) | | +| 54 | IV | Managed Identity | Windows | Consumption (Legacy) | | +| 55 | IV | Managed Identity | Linux | App Service | | +| 56 | IV | Managed Identity | Windows | App Service | | +| 57 | IV | Secrets | Linux | Flex Consumption | | +| 58 | IV | Secrets | Windows | Flex Consumption | | +| 59 | IV | Secrets | Linux | Premium | | +| 60 | IV | Secrets | Windows | Premium | | +| 61 | IV | Secrets | Linux | Consumption (Legacy) | | +| 62 | IV | Secrets | Windows | Consumption (Legacy) | | +| 63 | IV | Secrets | Linux | App Service | | +| 64 | IV | Secrets | Windows | App Service | | +| 65 | V | Managed Identity | Linux | Flex Consumption | | +| 66 | V | Managed Identity | Windows | Flex Consumption | | +| 67 | V | Managed Identity | Linux | Premium | | +| 68 | V | Managed Identity | Windows | Premium | | +| 69 | V | Managed Identity | Linux | Consumption (Legacy) | | +| 70 | V | Managed Identity | Windows | Consumption (Legacy) | | +| 71 | V | Managed Identity | Linux | App Service | | +| 72 | V | Managed Identity | Windows | App Service | | +| 73 | V | Secrets | Linux | Flex Consumption | | +| 74 | V | Secrets | Windows | Flex Consumption | | +| 75 | V | Secrets | Linux | Premium | | +| 76 | V | Secrets | Windows | Premium | | +| 77 | V | Secrets | Linux | Consumption (Legacy) | | +| 78 | V | Secrets | Windows | Consumption (Legacy) | | +| 79 | V | Secrets | Linux | App Service | | +| 80 | V | Secrets | Windows | App Service | | +| 81 | VI | Managed Identity | Linux | Flex Consumption | | +| 82 | VI | Managed Identity | Windows | Flex Consumption | | +| 83 | VI | Managed Identity | Linux | Premium | | +| 84 | VI | Managed Identity | Windows | Premium | | +| 85 | VI | Managed Identity | Linux | Consumption (Legacy) | | +| 86 | VI | Managed Identity | Windows | Consumption (Legacy) | | +| 87 | VI | Managed Identity | Linux | App Service | | +| 88 | VI | Managed Identity | Windows | App Service | | +| 89 | VI | Secrets | Linux | Flex Consumption | | +| 90 | VI | Secrets | Windows | Flex Consumption | | +| 91 | VI | Secrets | Linux | Premium | | +| 92 | VI | Secrets | Windows | Premium | | +| 93 | VI | Secrets | Linux | Consumption (Legacy) | | +| 94 | VI | Secrets | Windows | Consumption (Legacy) | | +| 95 | VI | Secrets | Linux | App Service | | +| 96 | VI | Secrets | Windows | App Service | | diff --git a/test/nightly/scenarios/durableDTS/dts.test.ts b/test/nightly/scenarios/durableDTS/dts.test.ts index 9c4d189e1..51f14a7c8 100644 --- a/test/nightly/scenarios/durableDTS/dts.test.ts +++ b/test/nightly/scenarios/durableDTS/dts.test.ts @@ -3,30 +3,49 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ +import { runWithTestActionContext } from '@microsoft/vscode-azext-dev'; import { AzExtFsExtra } from '@microsoft/vscode-azext-utils'; import * as assert from 'assert'; import { workspace, type Uri, type WorkspaceFolder } from 'vscode'; -import { updateGlobalSetting } from '../../../../extension.bundle'; +import { createNewProjectInternal, updateGlobalSetting } from '../../../../extension.bundle'; import { longRunningTestsEnabled } from '../../../global.test'; -suite.only('Durable Task Scheduler (DTS) Scenarios', function (this: Mocha.Suite): void { - this.timeout(7 * 60 * 1000); +let rootFolder: WorkspaceFolder | undefined; +suite.only('Durable Task Scheduler (DTS) Scenarios', function (this: Mocha.Suite): void { suiteSetup(async function (this: Mocha.Context): Promise { if (!longRunningTestsEnabled) { this.skip(); } - const workspaceFolderUri: Uri = getWorkspaceFolderUri('scenario-durable-dts'); - const rootFolder: WorkspaceFolder | undefined = workspace.getWorkspaceFolder(workspaceFolderUri); + const workspaceFolderUri: Uri = getWorkspaceFolderUri('scenarios-dts-tsnode'); + rootFolder = workspace.getWorkspaceFolder(workspaceFolderUri); assert.ok(rootFolder, 'Could not retrieve root workspace folder.'); cleanTestFolder(rootFolder); await updateGlobalSetting('groupBy', 'resourceType', 'azureResourceGroups'); }); - test('Create function should bring down template files and make the necessary workspace configurations', async () => { - console.log("hello"); + test('Create function should bring down template files and make the necessary workspace changes', async () => { + const createNewProjectInputs = [ + 'TypeScript', + /v4/i, + /durable functions orchestrator/i, + 'Durable Task Scheduler', + 'dtsFunctionName', + ]; + + await runWithTestActionContext('scenarios.createNewProject', async context => { + await context.ui.runWithInputs(createNewProjectInputs, async () => { + await createNewProjectInternal(context, { folderPath: rootFolder?.uri.fsPath }); + }); + }); + + // Verify that the right project settings were set + + // Create Function App + + // Deploy }); }); diff --git a/test/nightly/scenarios/durableDTS/durableDTS.md b/test/nightly/scenarios/durableDTS/durableDTS.md index e69de29bb..28f16bdec 100644 --- a/test/nightly/scenarios/durableDTS/durableDTS.md +++ b/test/nightly/scenarios/durableDTS/durableDTS.md @@ -0,0 +1,54 @@ +# Durable Task Scheduler (DTS) Test Scenarios + +## I. Create New Project / Create Function + +### Workspace Project Test Matrix: + +| No. | Language | Runtime | Programming Model | Comment | +|-----|----------|---------|-------------------|---------| +| I | TS | Node | v4 | | +| II | Python | Python | v2 | | +| III | C# | .NET | isolated | | +| IV | C# | .NET | in-proc | | + +## II. Debug +TBD + +## III. Create Function App / Deployment + +### Create / Deploy Test Matrix: + +| Workspace Project | Connection Type | Operating System | Plan Type | Comment | +|-------------------|-------------------|------------------|-------------------|---------| +| I | Managed Identity | Linux | Flex Consumption | | +| I | Managed Identity | Windows | Flex Consumption | | +| I | Managed Identity | Linux | Premium | | +| I | Managed Identity | Windows | Premium | | +| I | Secrets | Linux | Flex Consumption | | +| I | Secrets | Windows | Flex Consumption | | +| I | Secrets | Linux | Premium | | +| I | Secrets | Windows | Premium | | +| II | Managed Identity | Linux | Flex Consumption | | +| II | Managed Identity | Windows | Flex Consumption | | +| II | Managed Identity | Linux | Premium | | +| II | Managed Identity | Windows | Premium | | +| II | Secrets | Linux | Flex Consumption | | +| II | Secrets | Windows | Flex Consumption | | +| II | Secrets | Linux | Premium | | +| II | Secrets | Windows | Premium | | +| III | Managed Identity | Linux | Flex Consumption | | +| III | Managed Identity | Windows | Flex Consumption | | +| III | Managed Identity | Linux | Premium | | +| III | Managed Identity | Windows | Premium | | +| III | Secrets | Linux | Flex Consumption | | +| III | Secrets | Windows | Flex Consumption | | +| III | Secrets | Linux | Premium | | +| III | Secrets | Windows | Premium | | +| IV | Managed Identity | Linux | Flex Consumption | | +| IV | Managed Identity | Windows | Flex Consumption | | +| IV | Managed Identity | Linux | Premium | | +| IV | Managed Identity | Windows | Premium | | +| IV | Secrets | Linux | Flex Consumption | | +| IV | Secrets | Windows | Flex Consumption | | +| IV | Secrets | Linux | Premium | | +| IV | Secrets | Windows | Premium | | diff --git a/test/nightly/scenarios/durableDTS/geagea b/test/nightly/scenarios/durableDTS/geagea deleted file mode 100644 index 60ed9cae9..000000000 --- a/test/nightly/scenarios/durableDTS/geagea +++ /dev/null @@ -1,54 +0,0 @@ -// const workspaceFolderUri: Uri = getWorkspaceFolderUri(scenario.folderName); -// const rootFolder: WorkspaceFolder | undefined = workspace.getWorkspaceFolder(workspaceFolderUri); -// assert.ok(rootFolder, 'Could not retrieve root workspace folder.'); - -// await cleanWorkspaceFolderSettings(rootFolder); - -// for (const testCase of scenario.testCases) { -// ext.outputChannel.appendLog(`[[[ *** ${scenario.label} - ${testCase.label} *** ]]]`); -// await runWithTestActionContext('deployWorkspaceProject', async context => { -// await context.ui.runWithInputs(testCase.inputs, async () => { -// let results: DeployWorkspaceProjectResults; -// let perr: IParsedError | undefined; -// try { -// results = await deployWorkspaceProject(context); -// } catch (e) { -// results = {}; - -// perr = parseError(e); -// console.log(perr); -// } - -// if (testCase.resourceGroupToDelete) { -// resourceGroupsToDelete.add(testCase.resourceGroupToDelete); -// } - -// // Verify 'expectedErrMsg' -// if (perr || testCase.expectedErrMsg) { -// if (testCase.expectedErrMsg instanceof RegExp) { -// assert.match(perr?.message ?? "", testCase.expectedErrMsg, 'DeployWorkspaceProject thrown and expected error message did not match.'); -// } else { -// assert.strictEqual(perr?.message ?? "", testCase.expectedErrMsg, 'DeployWorkspaceProject thrown and expected error message did not match.'); -// } -// } - -// // Verify 'expectedResults' -// assertStringPropsMatch(results as Partial>, (testCase.expectedResults ?? {}) as Record, 'DeployWorkspaceProject results mismatch.'); - -// // Verify 'expectedVSCodeSettings' -// const deploymentConfigurationsV2: DeploymentConfigurationSettings[] = await dwpSettingUtilsV2.getWorkspaceDeploymentConfigurations(rootFolder) ?? []; -// const expectedDeploymentConfigurations = testCase.expectedVSCodeSettings?.deploymentConfigurations ?? []; -// assert.strictEqual(deploymentConfigurationsV2.length, expectedDeploymentConfigurations.length, 'DeployWorkspaceProject ".vscode" saved settings mismatch.'); - -// for (const [i, expectedDeploymentConfiguration] of expectedDeploymentConfigurations.entries()) { -// const deploymentConfiguration: DeploymentConfigurationSettings = deploymentConfigurationsV2[i] ?? {}; -// assertStringPropsMatch(deploymentConfiguration as Partial>, expectedDeploymentConfiguration, 'DeployWorkspaceProject ".vscode" saved settings mismatch.'); -// } - -// // Verify 'postTestAssertion' -// await testCase.postTestAssertion?.(context, results, 'DeployWorkspaceProject resource settings mismatch.'); -// }); -// }); -// } - -// await cleanWorkspaceFolderSettings(rootFolder); diff --git a/test/test.code-workspace b/test/test.code-workspace index 13736a4cf..1dc010f0e 100644 --- a/test/test.code-workspace +++ b/test/test.code-workspace @@ -31,8 +31,20 @@ "path": "../testWorkspace/9" }, { - "name": "scenario-durable-dts", - "path": "../testWorkspace/scenarios/durable-dts" + "name": "scenarios-dts-tsnode", + "path": "../testWorkspace/scenarios/durable-dts/tsnode" + }, + { + "name": "scenarios-dts-python", + "path": "../testWorkspace/scenarios/durable-dts/python" + }, + { + "name": "scenarios-dts-dotnet-isolated", + "path": "../testWorkspace/scenarios/durable-dts/dotnet-isolated" + }, + { + "name": "scenarios-dts-dotnet-inproc", + "path": "../testWorkspace/scenarios/durable-dts/dotnet-inproc" } ], "settings": { From ef087854de53a10de2e27e83744e7809ae95e38d Mon Sep 17 00:00:00 2001 From: MicroFish91 <40250218+MicroFish91@users.noreply.github.com> Date: Fri, 19 Dec 2025 15:14:19 -0800 Subject: [PATCH 03/27] WIP: Everything creates properly --- .../localSettings/getLocalSettingsFile.ts | 2 +- src/commands/createFunction/createFunction.ts | 2 +- .../containerImage/detectDockerfile.ts | 2 +- .../createFunctionApp/createFunctionApp.ts | 7 +-- src/commands/deploy/deploy.ts | 4 +- src/utils/workspace.ts | 4 +- test/nightly/scenarios/durableDTS/dts.test.ts | 52 ++++++++++++++++--- 7 files changed, 55 insertions(+), 18 deletions(-) diff --git a/src/commands/appSettings/localSettings/getLocalSettingsFile.ts b/src/commands/appSettings/localSettings/getLocalSettingsFile.ts index 77428080e..d44a7206f 100644 --- a/src/commands/appSettings/localSettings/getLocalSettingsFile.ts +++ b/src/commands/appSettings/localSettings/getLocalSettingsFile.ts @@ -15,7 +15,7 @@ import { tryGetFunctionProjectRoot } from '../../createNewProject/verifyIsProjec * Otherwise, prompt */ export async function getLocalSettingsFile(context: IActionContext, message: string, workspaceFolder?: vscode.WorkspaceFolder): Promise { - workspaceFolder ||= await getRootWorkspaceFolder(); + workspaceFolder ||= await getRootWorkspaceFolder(context); if (workspaceFolder) { const projectPath: string | undefined = await tryGetFunctionProjectRoot(context, workspaceFolder); if (projectPath) { diff --git a/src/commands/createFunction/createFunction.ts b/src/commands/createFunction/createFunction.ts index 2de2ef4c1..86c3144f6 100644 --- a/src/commands/createFunction/createFunction.ts +++ b/src/commands/createFunction/createFunction.ts @@ -54,7 +54,7 @@ export async function createFunctionInternal(context: IActionContext, options: a let workspacePath: string | undefined = options.folderPath; if (workspacePath === undefined) { - workspaceFolder = await getRootWorkspaceFolder(); + workspaceFolder = await getRootWorkspaceFolder(context); workspacePath = workspaceFolder?.uri.fsPath; } else { workspaceFolder = getContainingWorkspace(workspacePath); diff --git a/src/commands/createFunctionApp/containerImage/detectDockerfile.ts b/src/commands/createFunctionApp/containerImage/detectDockerfile.ts index a3b253a17..dcbfab1ab 100644 --- a/src/commands/createFunctionApp/containerImage/detectDockerfile.ts +++ b/src/commands/createFunctionApp/containerImage/detectDockerfile.ts @@ -17,7 +17,7 @@ export async function detectDockerfile(context: ICreateFunctionAppContext): Prom return undefined; } - context.workspaceFolder ??= await getRootWorkspaceFolder() as WorkspaceFolder; + context.workspaceFolder ??= await getRootWorkspaceFolder(context) as WorkspaceFolder; context.rootPath ??= await tryGetFunctionProjectRoot(context, context.workspaceFolder, 'prompt') ?? context.workspaceFolder.uri.fsPath; const pattern: RelativePattern = new RelativePattern(context.rootPath, `**/${dockerfileGlobPattern}`); diff --git a/src/commands/createFunctionApp/createFunctionApp.ts b/src/commands/createFunctionApp/createFunctionApp.ts index 762bd83fd..3b97b98c0 100644 --- a/src/commands/createFunctionApp/createFunctionApp.ts +++ b/src/commands/createFunctionApp/createFunctionApp.ts @@ -3,7 +3,7 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { type AzExtParentTreeItem, type IActionContext } from '@microsoft/vscode-azext-utils'; +import { nonNullValueAndProp, type AzExtParentTreeItem, type IActionContext } from '@microsoft/vscode-azext-utils'; import { ext } from '../../extensionVariables'; import { localize } from '../../localize'; import { type SlotTreeItem } from '../../tree/SlotTreeItem'; @@ -38,12 +38,13 @@ export async function createFunctionApp(context: IActionContext & Partial { diff --git a/src/commands/deploy/deploy.ts b/src/commands/deploy/deploy.ts index 029e173ae..a251230db 100644 --- a/src/commands/deploy/deploy.ts +++ b/src/commands/deploy/deploy.ts @@ -52,8 +52,8 @@ export async function deployProductionSlot(context: IActionContext, target?: vsc await deploy(context, target, undefined); } -export async function deployProductionSlotByFunctionAppId(context: IActionContext, functionAppId?: string | {}): Promise { - await deploy(context, undefined, functionAppId); +export async function deployProductionSlotByFunctionAppId(context: IActionContext, functionAppId?: string | {}, target?: vscode.Uri | string | SlotTreeItem): Promise { + await deploy(context, target, functionAppId); } export async function deploySlot(context: IActionContext, target?: vscode.Uri | string | SlotTreeItem, functionAppId?: string | {}): Promise { diff --git a/src/utils/workspace.ts b/src/utils/workspace.ts index 787dad9b0..2688eb657 100644 --- a/src/utils/workspace.ts +++ b/src/utils/workspace.ts @@ -20,14 +20,14 @@ export function isMultiRootWorkspace(): boolean { * Use sparingly. Prefer storing and passing 'projectPaths' instead. * Over-reliance on this function may result in excessive prompting when a user employs a multi-root workspace. */ -export async function getRootWorkspaceFolder(): Promise { +export async function getRootWorkspaceFolder(context: IActionContext): Promise { if (!vscode.workspace.workspaceFolders || vscode.workspace.workspaceFolders.length === 0) { return undefined; } else if (vscode.workspace.workspaceFolders.length === 1) { return vscode.workspace.workspaceFolders[0]; } else { const placeHolder: string = localize('selectRootWorkspace', 'Select the folder containing your function project'); - const folder = await vscode.window.showWorkspaceFolderPick({ placeHolder }); + const folder = await context.ui.showWorkspaceFolderPick({ placeHolder }); if (!folder) { throw new UserCancelledError('selectRootWorkspace'); } diff --git a/test/nightly/scenarios/durableDTS/dts.test.ts b/test/nightly/scenarios/durableDTS/dts.test.ts index 51f14a7c8..e230ac2d9 100644 --- a/test/nightly/scenarios/durableDTS/dts.test.ts +++ b/test/nightly/scenarios/durableDTS/dts.test.ts @@ -4,10 +4,10 @@ *--------------------------------------------------------------------------------------------*/ import { runWithTestActionContext } from '@microsoft/vscode-azext-dev'; -import { AzExtFsExtra } from '@microsoft/vscode-azext-utils'; +import { AzExtFsExtra, nonNullValue } from '@microsoft/vscode-azext-utils'; import * as assert from 'assert'; import { workspace, type Uri, type WorkspaceFolder } from 'vscode'; -import { createNewProjectInternal, updateGlobalSetting } from '../../../../extension.bundle'; +import { createFunctionApp, createNewProjectInternal, deployProductionSlotByFunctionAppId, getRandomAlphanumericString, updateGlobalSetting } from '../../../../extension.bundle'; import { longRunningTestsEnabled } from '../../../global.test'; let rootFolder: WorkspaceFolder | undefined; @@ -27,12 +27,13 @@ suite.only('Durable Task Scheduler (DTS) Scenarios', function (this: Mocha.Suite }); test('Create function should bring down template files and make the necessary workspace changes', async () => { + const appName: string = getRandomAlphanumericString(); const createNewProjectInputs = [ - 'TypeScript', + /TypeScript/i, /v4/i, - /durable functions orchestrator/i, - 'Durable Task Scheduler', - 'dtsFunctionName', + /Durable Functions Orchestrator/i, + /Durable Task Scheduler/i, + 'durableHello1', ]; await runWithTestActionContext('scenarios.createNewProject', async context => { @@ -43,9 +44,44 @@ suite.only('Durable Task Scheduler (DTS) Scenarios', function (this: Mocha.Suite // Verify that the right project settings were set - // Create Function App + // For Loop - Tests for this project: - // Deploy + // For flex test basic, else test adv? + const createFunctionAppBasicInputs = [ + nonNullValue(rootFolder?.name), + appName, + /West US 2/i, + /Node\.js 22/i, + /Managed Identity/i, + ]; + + let functionAppId: string | undefined; + await runWithTestActionContext('scenarios.createFunctionApp', async context => { + await context.ui.runWithInputs(createFunctionAppBasicInputs, async () => { + functionAppId = await createFunctionApp(context); + }); + }); + assert.ok(functionAppId, 'Failed to create function app.'); + + const deployFunctionAppInputs = [ + // Todo: Expand regexp capability for context.ui.showWarningMessage + 'Connect Durable Task Scheduler', + /Create New Durable Task Scheduler/i, + appName, + /Create New Durable Task Hub/i, + appName, + /Assign New User[- ]Assigned Identity/i, + /Create New User[- ]Assigned Identity/i, + // Todo: Here too + 'Deploy', + ]; + await runWithTestActionContext('scenarios.deploy', async context => { + await context.ui.runWithInputs(deployFunctionAppInputs, async () => { + await deployProductionSlotByFunctionAppId(context, functionAppId, rootFolder?.uri); + }); + }); + + // Post deploy check }); }); From f7dd79c741daedfab4fecc5e916806adb4475171 Mon Sep 17 00:00:00 2001 From: MicroFish91 <40250218+MicroFish91@users.noreply.github.com> Date: Wed, 24 Dec 2025 01:26:17 -0800 Subject: [PATCH 04/27] WIP: More updates --- test/nightly/scenarios/durableDTS/dts.test.ts | 105 ----------- test/nightly/scenarios/parallelScenarios.ts | 96 ++++++++++ test/nightly/scenarios/scenarios.test.ts | 32 ++++ .../AvailableTestCombinations.md} | 17 +- .../AzExtFunctionsTestScenario.ts | 50 +++++ .../dtsScenarios/DurableDTS.md} | 0 .../dtsScenarios/dtsScenarios.ts | 13 ++ .../testScenarios/dtsScenarios/dtsTSNode.ts | 172 ++++++++++++++++++ .../scenarios/testScenarios/testScenarios.ts | 15 ++ test/utils/createFunctionAppUtils.ts | 0 10 files changed, 387 insertions(+), 113 deletions(-) delete mode 100644 test/nightly/scenarios/durableDTS/dts.test.ts create mode 100644 test/nightly/scenarios/parallelScenarios.ts create mode 100644 test/nightly/scenarios/scenarios.test.ts rename test/nightly/scenarios/{durableDTS/TestCombinations.md => testScenarios/AvailableTestCombinations.md} (95%) create mode 100644 test/nightly/scenarios/testScenarios/AzExtFunctionsTestScenario.ts rename test/nightly/scenarios/{durableDTS/durableDTS.md => testScenarios/dtsScenarios/DurableDTS.md} (100%) create mode 100644 test/nightly/scenarios/testScenarios/dtsScenarios/dtsScenarios.ts create mode 100644 test/nightly/scenarios/testScenarios/dtsScenarios/dtsTSNode.ts create mode 100644 test/nightly/scenarios/testScenarios/testScenarios.ts create mode 100644 test/utils/createFunctionAppUtils.ts diff --git a/test/nightly/scenarios/durableDTS/dts.test.ts b/test/nightly/scenarios/durableDTS/dts.test.ts deleted file mode 100644 index e230ac2d9..000000000 --- a/test/nightly/scenarios/durableDTS/dts.test.ts +++ /dev/null @@ -1,105 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { runWithTestActionContext } from '@microsoft/vscode-azext-dev'; -import { AzExtFsExtra, nonNullValue } from '@microsoft/vscode-azext-utils'; -import * as assert from 'assert'; -import { workspace, type Uri, type WorkspaceFolder } from 'vscode'; -import { createFunctionApp, createNewProjectInternal, deployProductionSlotByFunctionAppId, getRandomAlphanumericString, updateGlobalSetting } from '../../../../extension.bundle'; -import { longRunningTestsEnabled } from '../../../global.test'; - -let rootFolder: WorkspaceFolder | undefined; - -suite.only('Durable Task Scheduler (DTS) Scenarios', function (this: Mocha.Suite): void { - suiteSetup(async function (this: Mocha.Context): Promise { - if (!longRunningTestsEnabled) { - this.skip(); - } - - const workspaceFolderUri: Uri = getWorkspaceFolderUri('scenarios-dts-tsnode'); - rootFolder = workspace.getWorkspaceFolder(workspaceFolderUri); - assert.ok(rootFolder, 'Could not retrieve root workspace folder.'); - - cleanTestFolder(rootFolder); - await updateGlobalSetting('groupBy', 'resourceType', 'azureResourceGroups'); - }); - - test('Create function should bring down template files and make the necessary workspace changes', async () => { - const appName: string = getRandomAlphanumericString(); - const createNewProjectInputs = [ - /TypeScript/i, - /v4/i, - /Durable Functions Orchestrator/i, - /Durable Task Scheduler/i, - 'durableHello1', - ]; - - await runWithTestActionContext('scenarios.createNewProject', async context => { - await context.ui.runWithInputs(createNewProjectInputs, async () => { - await createNewProjectInternal(context, { folderPath: rootFolder?.uri.fsPath }); - }); - }); - - // Verify that the right project settings were set - - // For Loop - Tests for this project: - - // For flex test basic, else test adv? - const createFunctionAppBasicInputs = [ - nonNullValue(rootFolder?.name), - appName, - /West US 2/i, - /Node\.js 22/i, - /Managed Identity/i, - ]; - - let functionAppId: string | undefined; - await runWithTestActionContext('scenarios.createFunctionApp', async context => { - await context.ui.runWithInputs(createFunctionAppBasicInputs, async () => { - functionAppId = await createFunctionApp(context); - }); - }); - assert.ok(functionAppId, 'Failed to create function app.'); - - const deployFunctionAppInputs = [ - // Todo: Expand regexp capability for context.ui.showWarningMessage - 'Connect Durable Task Scheduler', - /Create New Durable Task Scheduler/i, - appName, - /Create New Durable Task Hub/i, - appName, - /Assign New User[- ]Assigned Identity/i, - /Create New User[- ]Assigned Identity/i, - // Todo: Here too - 'Deploy', - ]; - await runWithTestActionContext('scenarios.deploy', async context => { - await context.ui.runWithInputs(deployFunctionAppInputs, async () => { - await deployProductionSlotByFunctionAppId(context, functionAppId, rootFolder?.uri); - }); - }); - - // Post deploy check - }); -}); - -async function cleanTestFolder(testFolder: WorkspaceFolder) { - await AzExtFsExtra.emptyDir(testFolder.uri.fsPath); -} - -export function getWorkspaceFolderUri(folderName: string): Uri { - const workspaceFolders: readonly WorkspaceFolder[] | undefined = workspace.workspaceFolders; - if (!workspaceFolders || workspaceFolders.length === 0) { - throw new Error('No workspace is open'); - } else { - for (const workspaceFolder of workspaceFolders) { - if (workspaceFolder.name === folderName) { - return workspaceFolder.uri; - } - } - } - - throw new Error(`Unable to find workspace folder "${folderName}"`); -} diff --git a/test/nightly/scenarios/parallelScenarios.ts b/test/nightly/scenarios/parallelScenarios.ts new file mode 100644 index 000000000..5409cabe4 --- /dev/null +++ b/test/nightly/scenarios/parallelScenarios.ts @@ -0,0 +1,96 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { runWithTestActionContext } from '@microsoft/vscode-azext-dev'; +import { AzExtFsExtra } from '@microsoft/vscode-azext-utils'; +import * as assert from 'assert'; +import { workspace, type Uri, type WorkspaceFolder } from 'vscode'; +import { createFunctionApp, createNewProjectInternal, deployProductionSlotByFunctionAppId, ext } from '../../../extension.bundle'; +import { type AzExtFunctionsTestScenario, type CreateAndDeployTestCase } from './testScenarios/AzExtFunctionsTestScenario'; +import { generateTestScenarios } from './testScenarios/testScenarios'; + +export interface AzExtFunctionsParallelTestScenario { + title: string; + scenario?: Promise; + runScenario(): Promise; +} + +export function generateParallelScenarios(): AzExtFunctionsParallelTestScenario[] { + return generateTestScenarios().map(scenario => { + return { + title: scenario.label, + runScenario: runTestScenario(scenario), + }; + }); +} + +function runTestScenario(scenario: AzExtFunctionsTestScenario): AzExtFunctionsParallelTestScenario['runScenario'] { + return async () => { + const workspaceFolderUri: Uri = getWorkspaceFolderUri(scenario.folderName); + const rootFolder = workspace.getWorkspaceFolder(workspaceFolderUri); + assert.ok(rootFolder, `Failed to retrieve root workspace folder for scenario ${scenario.label}.`); + + await cleanTestFolder(rootFolder); + + // 1. Create shared workspace project + ext.outputChannel.appendLog(`[[[ *** ${scenario.label} - ${scenario.createNewProjectTest.label} *** ]]]`); + + await runWithTestActionContext('scenario.createNewProject', async context => { + await context.ui.runWithInputs(scenario.createNewProjectTest.inputs, async () => { + await createNewProjectInternal(context, { folderPath: rootFolder.uri.fsPath }); + await scenario.createNewProjectTest.postTestAssertion?.(context, rootFolder.uri.fsPath, ''); + }); + }); + + // 2. Immediately spin off all the create and deploy tests + const createAndDeployTests: Promise[] = scenario.createAndDeployTests.map(test => startCreateAndDeployTest(scenario.label, rootFolder, test)); + await Promise.allSettled(createAndDeployTests); + + await cleanTestFolder(rootFolder); + } +} + +async function startCreateAndDeployTest(scenarioLabel: string, rootFolder: WorkspaceFolder, test: CreateAndDeployTestCase): Promise { + // 3. Create function app + ext.outputChannel.appendLog(`[[[ *** ${scenarioLabel} - ${test.createFunctionApp.label} *** ]]]`); + + let functionAppId: string; + await runWithTestActionContext('scenario.createFunctionApp', async context => { + await context.ui.runWithInputs(test.createFunctionApp.inputs, async () => { + functionAppId = await createFunctionApp(context); + assert.ok(functionAppId, 'Failed to create function app.'); + test.createFunctionApp.postTest?.(context, functionAppId, ''); + }); + }); + + // 4. Deploy function app + ext.outputChannel.appendLog(`[[[ *** ${scenarioLabel} - ${test.deployFunctionApp.label} *** ]]]`); + + await runWithTestActionContext('scenario.deploy', async context => { + await context.ui.runWithInputs(test.deployFunctionApp.inputs, async () => { + await deployProductionSlotByFunctionAppId(context, functionAppId, rootFolder?.uri); + await test.deployFunctionApp.postTest?.(context, functionAppId, ''); + }); + }); +} + +async function cleanTestFolder(testFolder: WorkspaceFolder) { + await AzExtFsExtra.emptyDir(testFolder.uri.fsPath); +} + +export function getWorkspaceFolderUri(folderName: string): Uri { + const workspaceFolders: readonly WorkspaceFolder[] | undefined = workspace.workspaceFolders; + if (!workspaceFolders || workspaceFolders.length === 0) { + throw new Error('No workspace is open'); + } else { + for (const workspaceFolder of workspaceFolders) { + if (workspaceFolder.name === folderName) { + return workspaceFolder.uri; + } + } + } + + throw new Error(`Unable to find workspace folder "${folderName}"`); +} diff --git a/test/nightly/scenarios/scenarios.test.ts b/test/nightly/scenarios/scenarios.test.ts new file mode 100644 index 000000000..3d3b1ef10 --- /dev/null +++ b/test/nightly/scenarios/scenarios.test.ts @@ -0,0 +1,32 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { nonNullProp, updateGlobalSetting } from '../../../extension.bundle'; +import { longRunningTestsEnabled } from '../../global.test'; +import { generateParallelScenarios, type AzExtFunctionsParallelTestScenario } from './parallelScenarios'; + +const testScenarios: AzExtFunctionsParallelTestScenario[] = generateParallelScenarios(); + +suite.only('Scenarios', async function (this: Mocha.Suite) { + this.timeout(15 * 60 * 1000); + + suiteSetup(async function (this: Mocha.Context) { + if (!longRunningTestsEnabled) { + this.skip(); + } + // Todo: This should probably happen elsewhere? + await updateGlobalSetting('groupBy', 'resourceType', 'azureResourceGroups'); + + for (const s of testScenarios) { + s.scenario = s.runScenario(); + } + }); + + for (const s of testScenarios) { + test(s.title, async function () { + await nonNullProp(s, 'scenario'); + }); + } +}); diff --git a/test/nightly/scenarios/durableDTS/TestCombinations.md b/test/nightly/scenarios/testScenarios/AvailableTestCombinations.md similarity index 95% rename from test/nightly/scenarios/durableDTS/TestCombinations.md rename to test/nightly/scenarios/testScenarios/AvailableTestCombinations.md index d73eb7e96..4c262618a 100644 --- a/test/nightly/scenarios/durableDTS/TestCombinations.md +++ b/test/nightly/scenarios/testScenarios/AvailableTestCombinations.md @@ -24,6 +24,7 @@ TBD |-----|-------------------|-------------------|------------------|------------------------|---------| | 1 | I | Managed Identity | Linux | Flex Consumption | Skip | | 2 | I | Managed Identity | Windows | Flex Consumption | Skip | +| 3 | I | Managed Identity | Linux | Premium | Skip | | 4 | I | Managed Identity | Windows | Premium | Skip | | 5 | I | Managed Identity | Linux | Consumption (Legacy) | Skip | | 6 | I | Managed Identity | Windows | Consumption (Legacy) | Skip | @@ -38,7 +39,7 @@ TBD | 15 | I | Secrets | Linux | App Service | Skip | | 16 | I | Secrets | Windows | App Service | Skip | | 17 | II | Managed Identity | Linux | Flex Consumption | | -| 18 | II | Managed Identity | Windows | Flex Consumption | | +| 18 | II | Managed Identity | Windows | Flex Consumption | Not Offered | | 19 | II | Managed Identity | Linux | Premium | | | 20 | II | Managed Identity | Windows | Premium | | | 21 | II | Managed Identity | Linux | Consumption (Legacy) | | @@ -46,7 +47,7 @@ TBD | 23 | II | Managed Identity | Linux | App Service | | | 24 | II | Managed Identity | Windows | App Service | | | 25 | II | Secrets | Linux | Flex Consumption | | -| 26 | II | Secrets | Windows | Flex Consumption | | +| 26 | II | Secrets | Windows | Flex Consumption | Not Offered | | 27 | II | Secrets | Linux | Premium | | | 28 | II | Secrets | Windows | Premium | | | 29 | II | Secrets | Linux | Consumption (Legacy) | | @@ -70,7 +71,7 @@ TBD | 47 | III | Secrets | Linux | App Service | Skip | | 48 | III | Secrets | Windows | App Service | Skip | | 49 | IV | Managed Identity | Linux | Flex Consumption | | -| 50 | IV | Managed Identity | Windows | Flex Consumption | | +| 50 | IV | Managed Identity | Windows | Flex Consumption | Not Offered | | 51 | IV | Managed Identity | Linux | Premium | | | 52 | IV | Managed Identity | Windows | Premium | | | 53 | IV | Managed Identity | Linux | Consumption (Legacy) | | @@ -78,7 +79,7 @@ TBD | 55 | IV | Managed Identity | Linux | App Service | | | 56 | IV | Managed Identity | Windows | App Service | | | 57 | IV | Secrets | Linux | Flex Consumption | | -| 58 | IV | Secrets | Windows | Flex Consumption | | +| 58 | IV | Secrets | Windows | Flex Consumption | Not Offered | | 59 | IV | Secrets | Linux | Premium | | | 60 | IV | Secrets | Windows | Premium | | | 61 | IV | Secrets | Linux | Consumption (Legacy) | | @@ -86,7 +87,7 @@ TBD | 63 | IV | Secrets | Linux | App Service | | | 64 | IV | Secrets | Windows | App Service | | | 65 | V | Managed Identity | Linux | Flex Consumption | | -| 66 | V | Managed Identity | Windows | Flex Consumption | | +| 66 | V | Managed Identity | Windows | Flex Consumption | Not Offered | | 67 | V | Managed Identity | Linux | Premium | | | 68 | V | Managed Identity | Windows | Premium | | | 69 | V | Managed Identity | Linux | Consumption (Legacy) | | @@ -94,7 +95,7 @@ TBD | 71 | V | Managed Identity | Linux | App Service | | | 72 | V | Managed Identity | Windows | App Service | | | 73 | V | Secrets | Linux | Flex Consumption | | -| 74 | V | Secrets | Windows | Flex Consumption | | +| 74 | V | Secrets | Windows | Flex Consumption | Not Offered | | 75 | V | Secrets | Linux | Premium | | | 76 | V | Secrets | Windows | Premium | | | 77 | V | Secrets | Linux | Consumption (Legacy) | | @@ -102,7 +103,7 @@ TBD | 79 | V | Secrets | Linux | App Service | | | 80 | V | Secrets | Windows | App Service | | | 81 | VI | Managed Identity | Linux | Flex Consumption | | -| 82 | VI | Managed Identity | Windows | Flex Consumption | | +| 82 | VI | Managed Identity | Windows | Flex Consumption | Not Offered | | 83 | VI | Managed Identity | Linux | Premium | | | 84 | VI | Managed Identity | Windows | Premium | | | 85 | VI | Managed Identity | Linux | Consumption (Legacy) | | @@ -110,7 +111,7 @@ TBD | 87 | VI | Managed Identity | Linux | App Service | | | 88 | VI | Managed Identity | Windows | App Service | | | 89 | VI | Secrets | Linux | Flex Consumption | | -| 90 | VI | Secrets | Windows | Flex Consumption | | +| 90 | VI | Secrets | Windows | Flex Consumption | Not Offered | | 91 | VI | Secrets | Linux | Premium | | | 92 | VI | Secrets | Windows | Premium | | | 93 | VI | Secrets | Linux | Consumption (Legacy) | | diff --git a/test/nightly/scenarios/testScenarios/AzExtFunctionsTestScenario.ts b/test/nightly/scenarios/testScenarios/AzExtFunctionsTestScenario.ts new file mode 100644 index 000000000..d52308127 --- /dev/null +++ b/test/nightly/scenarios/testScenarios/AzExtFunctionsTestScenario.ts @@ -0,0 +1,50 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { type IActionContext } from "../../../../extension.bundle"; + +export interface AzExtFunctionsTestScenario { + label: string; + folderName: string; + createNewProjectTest: CreateNewProjectTestCase; + createAndDeployTests: CreateAndDeployTestCase[]; + + /** + * Indicates this scenario should be executed exclusively. This should only be used during local development. + */ + only?: boolean; +} + +export interface CreateNewProjectTestCase { + label: string; + inputs: (string | RegExp)[]; + postTestAssertion?: (context: IActionContext, workspaceFolderPath: string, errMsg?: string) => void | Promise; + resourceGroupToDelete?: string; + + /** + * Indicates this test case should be executed exclusively. This should only be used during local development. + */ + only?: boolean; +} + +export interface CreateAndDeployTestCase { + createFunctionApp: { + label: string; + inputs: (string | RegExp)[]; + postTest?: (context: IActionContext, functionAppId: string, errMsg?: string) => void | Promise; + }; + deployFunctionApp: { + label: string; + inputs: (string | RegExp)[]; + preTest?: (context: IActionContext, functionAppId: string, errMsg?: string) => void | Promise; + postTest?: (context: IActionContext, functionAppId: string, errMsg?: string) => void | Promise; + }; + resourceGroupToDelete?: string; + + /** + * Indicates this test case should be executed exclusively. This should only be used during local development. + */ + only?: boolean; +} diff --git a/test/nightly/scenarios/durableDTS/durableDTS.md b/test/nightly/scenarios/testScenarios/dtsScenarios/DurableDTS.md similarity index 100% rename from test/nightly/scenarios/durableDTS/durableDTS.md rename to test/nightly/scenarios/testScenarios/dtsScenarios/DurableDTS.md diff --git a/test/nightly/scenarios/testScenarios/dtsScenarios/dtsScenarios.ts b/test/nightly/scenarios/testScenarios/dtsScenarios/dtsScenarios.ts new file mode 100644 index 000000000..c0d6533fd --- /dev/null +++ b/test/nightly/scenarios/testScenarios/dtsScenarios/dtsScenarios.ts @@ -0,0 +1,13 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { type AzExtFunctionsTestScenario } from '../AzExtFunctionsTestScenario'; +import { generateTSNodeScenario } from './dtsTSNode'; + +export function generateDTSScenarios(): AzExtFunctionsTestScenario[] { + return [ + generateTSNodeScenario(), + ]; +} diff --git a/test/nightly/scenarios/testScenarios/dtsScenarios/dtsTSNode.ts b/test/nightly/scenarios/testScenarios/dtsScenarios/dtsTSNode.ts new file mode 100644 index 000000000..9af6b6327 --- /dev/null +++ b/test/nightly/scenarios/testScenarios/dtsScenarios/dtsTSNode.ts @@ -0,0 +1,172 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { getRandomAlphanumericString } from "../../../../../extension.bundle"; +import { type AzExtFunctionsTestScenario, type CreateAndDeployTestCase } from "../AzExtFunctionsTestScenario"; + +export enum ConnectionType { + Secrets = 'Secrets', + ManagedIdentity = 'Managed Identity', +} + +export enum OperatingSystem { + Linux = 'Linux', + Windows = 'Windows', +} + +export enum PlanType { + FlexConsumption = 'Flex Consumption', + Premium = 'Premium', + LegacyConsumption = 'Legacy', + AppService = 'App Service Plan', +} + +export function generateTSNodeScenario(): AzExtFunctionsTestScenario { + const appName: string = getRandomAlphanumericString(); + const folderName: string = 'scenarios-dts-tsnode'; + + return { + label: folderName, + folderName, + createNewProjectTest: { + label: '', + inputs: [ + /TypeScript/i, + /v4/i, + /Durable Functions Orchestrator/i, + /Durable Task Scheduler/i, + 'durableHello1', + ], + resourceGroupToDelete: appName, + }, + createAndDeployTests: [ + /** 1. */ generateCreateAndDeployTest(appName, folderName, ConnectionType.ManagedIdentity, OperatingSystem.Linux, PlanType.FlexConsumption), + /** 2. */ generateCreateAndDeployTest(appName, folderName, ConnectionType.ManagedIdentity, OperatingSystem.Windows, PlanType.FlexConsumption), + /** 3. */ generateCreateAndDeployTest(appName, folderName, ConnectionType.ManagedIdentity, OperatingSystem.Linux, PlanType.Premium), + /** 4. */ generateCreateAndDeployTest(appName, folderName, ConnectionType.ManagedIdentity, OperatingSystem.Windows, PlanType.Premium), + /** 5. */ generateCreateAndDeployTest(appName, folderName, ConnectionType.Secrets, OperatingSystem.Linux, PlanType.FlexConsumption), + /** 6. */ generateCreateAndDeployTest(appName, folderName, ConnectionType.Secrets, OperatingSystem.Windows, PlanType.FlexConsumption), + /** 7. */ generateCreateAndDeployTest(appName, folderName, ConnectionType.Secrets, OperatingSystem.Linux, PlanType.Premium), + /** 8. */ generateCreateAndDeployTest(appName, folderName, ConnectionType.Secrets, OperatingSystem.Windows, PlanType.Premium), + ], + } +} + +function generateCreateAndDeployTest(appName: string, folderName: string, connection: ConnectionType, os: OperatingSystem, plan: PlanType): CreateAndDeployTestCase { + return { + createFunctionApp: { + label: 'Create Function App', + inputs: plan === PlanType.FlexConsumption ? + generateCreateFunctionAppBasicInputs(appName, folderName, connection) : + generateCreateFunctionAppAdvancedInputs(appName, folderName, connection, os, plan), + }, + deployFunctionApp: { + label: 'Deploy Function App', + inputs: [ + // Todo: Expand regexp capability for context.ui.showWarningMessage + 'Connect Durable Task Scheduler', + /Create New Durable Task Scheduler/i, + appName, + /Create New Durable Task Hub/i, + appName, + /Assign New User[- ]Assigned Identity/i, + /Create New User[- ]Assigned Identity/i, + // Todo: Here too + 'Deploy', + ], + }, + resourceGroupToDelete: appName, + }; +} + +function generateCreateFunctionAppBasicInputs(appName: string, folderName: string, connection: ConnectionType): (string | RegExp)[] { + return [ + folderName, + appName, + /West US 2/i, + /Node\.js 22/i, + new RegExp(connection, 'i'), + ]; +} + +function generateCreateFunctionAppAdvancedInputs(appName: string, folderName: string, connection: ConnectionType, os: OperatingSystem, plan: PlanType): (string | RegExp)[] { + switch (plan) { + case PlanType.FlexConsumption: + return [ + folderName, + appName, + new RegExp(plan, 'i'), + /West US 2/i, + /Node\.js 22/i, + '2048', + '100', + /Create new resource group/i, + appName, + ...getConnectionTypeInputs(connection), + /Create new storage account/i, + appName, + /Create new application insights/i, + appName, + ]; + case PlanType.Premium: + return [ + folderName, + appName, + new RegExp(plan, 'i'), + /West US 2/i, + /Node\.js 22/i, + new RegExp(os, 'i'), + /Create new app service plan/i, + appName, + /EP1/i, + /Create new resource group/i, + appName, + ...getConnectionTypeInputs(connection), + /Create new storage account/i, + appName, + /Create new application insights/i, + appName, + ]; + case PlanType.LegacyConsumption: + return [ + folderName, + appName, + new RegExp(plan, 'i'), + /West US 2/i, + /Node\.js 22/i, + new RegExp(os, 'i'), + /Create new resource group/i, + appName, + ...getConnectionTypeInputs(connection), + /Create new storage account/i, + appName, + /Create new application insights/i, + appName, + ]; + case PlanType.AppService: + return [ + folderName, + appName, + new RegExp(plan, 'i'), + /West US 2/i, + /Node\.js 22/i, + new RegExp(os, 'i'), + /Create new app service plan/i, + appName, + /S1/i, + /Create new resource group/i, + appName, + ...getConnectionTypeInputs(connection), + /Create new storage account/i, + appName, + /Create new application insights/i, + appName, + ]; + } +} + +function getConnectionTypeInputs(connection: ConnectionType): (string | RegExp)[] { + return connection === ConnectionType.ManagedIdentity ? [new RegExp(connection, 'i'), /Create new user[- ]assigned identity/i] : [new RegExp(connection, 'i')]; +} diff --git a/test/nightly/scenarios/testScenarios/testScenarios.ts b/test/nightly/scenarios/testScenarios/testScenarios.ts new file mode 100644 index 000000000..0da5da2e2 --- /dev/null +++ b/test/nightly/scenarios/testScenarios/testScenarios.ts @@ -0,0 +1,15 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See LICENSE.md in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { type AzExtFunctionsTestScenario } from "./AzExtFunctionsTestScenario"; +import { generateDTSScenarios } from "./dtsScenarios/dtsScenarios"; + +export function generateTestScenarios(): AzExtFunctionsTestScenario[] { + const testScenarios: AzExtFunctionsTestScenario[] = [ + ...generateDTSScenarios(), + ]; + return testScenarios; +} + diff --git a/test/utils/createFunctionAppUtils.ts b/test/utils/createFunctionAppUtils.ts new file mode 100644 index 000000000..e69de29bb From 7607ee69d9106fc611f54179ed760491bda7fd1f Mon Sep 17 00:00:00 2001 From: MicroFish91 <40250218+MicroFish91@users.noreply.github.com> Date: Wed, 24 Dec 2025 21:22:55 -0800 Subject: [PATCH 05/27] Working for one section --- .../createFunctionApp/createFunctionApp.ts | 2 +- test/nightly/scenarios/parallelScenarios.ts | 21 ++- test/nightly/scenarios/scenarios.test.ts | 12 +- .../AzExtFunctionsTestScenario.ts | 5 +- .../testScenarios/dtsScenarios/DurableDTS.md | 2 - .../testScenarios/dtsScenarios/dtsTSNode.ts | 161 +++--------------- .../scenarios/testScenarios/testScenarios.ts | 1 - test/utils/createFunctionAppUtils.ts | 120 +++++++++++++ test/utils/deployFunctionAppUtils.ts | 33 ++++ 9 files changed, 203 insertions(+), 154 deletions(-) create mode 100644 test/utils/deployFunctionAppUtils.ts diff --git a/src/commands/createFunctionApp/createFunctionApp.ts b/src/commands/createFunctionApp/createFunctionApp.ts index 3b97b98c0..cb03824da 100644 --- a/src/commands/createFunctionApp/createFunctionApp.ts +++ b/src/commands/createFunctionApp/createFunctionApp.ts @@ -47,6 +47,6 @@ export async function createFunctionApp(context: IActionContext & Partial { +export async function createFunctionAppAdvanced(context: IActionContext, subscription?: AzExtParentTreeItem | string, nodesOrNewResourceGroupName?: string | (string | AzExtParentTreeItem)[]): Promise { return await createFunctionApp({ ...context, advancedCreation: true }, subscription, nodesOrNewResourceGroupName); } diff --git a/test/nightly/scenarios/parallelScenarios.ts b/test/nightly/scenarios/parallelScenarios.ts index 5409cabe4..59a8ef78e 100644 --- a/test/nightly/scenarios/parallelScenarios.ts +++ b/test/nightly/scenarios/parallelScenarios.ts @@ -7,7 +7,8 @@ import { runWithTestActionContext } from '@microsoft/vscode-azext-dev'; import { AzExtFsExtra } from '@microsoft/vscode-azext-utils'; import * as assert from 'assert'; import { workspace, type Uri, type WorkspaceFolder } from 'vscode'; -import { createFunctionApp, createNewProjectInternal, deployProductionSlotByFunctionAppId, ext } from '../../../extension.bundle'; +import { createFunctionApp, createFunctionAppAdvanced, createNewProjectInternal, deployProductionSlotByFunctionAppId, ext } from '../../../extension.bundle'; +import { CreateMode } from '../../utils/createFunctionAppUtils'; import { type AzExtFunctionsTestScenario, type CreateAndDeployTestCase } from './testScenarios/AzExtFunctionsTestScenario'; import { generateTestScenarios } from './testScenarios/testScenarios'; @@ -40,13 +41,18 @@ function runTestScenario(scenario: AzExtFunctionsTestScenario): AzExtFunctionsPa await runWithTestActionContext('scenario.createNewProject', async context => { await context.ui.runWithInputs(scenario.createNewProjectTest.inputs, async () => { await createNewProjectInternal(context, { folderPath: rootFolder.uri.fsPath }); - await scenario.createNewProjectTest.postTestAssertion?.(context, rootFolder.uri.fsPath, ''); + await scenario.createNewProjectTest.postTest?.(context, rootFolder.uri.fsPath, ''); }); }); // 2. Immediately spin off all the create and deploy tests - const createAndDeployTests: Promise[] = scenario.createAndDeployTests.map(test => startCreateAndDeployTest(scenario.label, rootFolder, test)); - await Promise.allSettled(createAndDeployTests); + const onlyTestCase: CreateAndDeployTestCase | undefined = scenario.createAndDeployTests.find(test => test.only); + if (onlyTestCase) { + await startCreateAndDeployTest(scenario.label, rootFolder, onlyTestCase); + } else { + const createAndDeployTests: Promise[] = scenario.createAndDeployTests.map(test => startCreateAndDeployTest(scenario.label, rootFolder, test)); + await Promise.allSettled(createAndDeployTests); + } await cleanTestFolder(rootFolder); } @@ -59,7 +65,12 @@ async function startCreateAndDeployTest(scenarioLabel: string, rootFolder: Works let functionAppId: string; await runWithTestActionContext('scenario.createFunctionApp', async context => { await context.ui.runWithInputs(test.createFunctionApp.inputs, async () => { - functionAppId = await createFunctionApp(context); + if (test.createFunctionApp.mode === CreateMode.Basic) { + functionAppId = await createFunctionApp(context); + } else { + functionAppId = await createFunctionAppAdvanced(context); + } + assert.ok(functionAppId, 'Failed to create function app.'); test.createFunctionApp.postTest?.(context, functionAppId, ''); }); diff --git a/test/nightly/scenarios/scenarios.test.ts b/test/nightly/scenarios/scenarios.test.ts index 3d3b1ef10..a9c763b79 100644 --- a/test/nightly/scenarios/scenarios.test.ts +++ b/test/nightly/scenarios/scenarios.test.ts @@ -3,14 +3,15 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { nonNullProp, updateGlobalSetting } from '../../../extension.bundle'; +import { updateGlobalSetting } from '../../../extension.bundle'; import { longRunningTestsEnabled } from '../../global.test'; import { generateParallelScenarios, type AzExtFunctionsParallelTestScenario } from './parallelScenarios'; const testScenarios: AzExtFunctionsParallelTestScenario[] = generateParallelScenarios(); suite.only('Scenarios', async function (this: Mocha.Suite) { - this.timeout(15 * 60 * 1000); + // Unfortunately, durable task schedulers sometimes take ~30m to provision + this.timeout(60 * 60 * 1000); suiteSetup(async function (this: Mocha.Context) { if (!longRunningTestsEnabled) { @@ -25,8 +26,11 @@ suite.only('Scenarios', async function (this: Mocha.Suite) { }); for (const s of testScenarios) { - test(s.title, async function () { - await nonNullProp(s, 'scenario'); + test(s.title, async function (this: Mocha.Context) { + if (!s.scenario) { + this.skip(); + } + await s.scenario; }); } }); diff --git a/test/nightly/scenarios/testScenarios/AzExtFunctionsTestScenario.ts b/test/nightly/scenarios/testScenarios/AzExtFunctionsTestScenario.ts index d52308127..f0a881072 100644 --- a/test/nightly/scenarios/testScenarios/AzExtFunctionsTestScenario.ts +++ b/test/nightly/scenarios/testScenarios/AzExtFunctionsTestScenario.ts @@ -4,6 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import { type IActionContext } from "../../../../extension.bundle"; +import { type CreateMode } from "../../../utils/createFunctionAppUtils"; export interface AzExtFunctionsTestScenario { label: string; @@ -20,8 +21,7 @@ export interface AzExtFunctionsTestScenario { export interface CreateNewProjectTestCase { label: string; inputs: (string | RegExp)[]; - postTestAssertion?: (context: IActionContext, workspaceFolderPath: string, errMsg?: string) => void | Promise; - resourceGroupToDelete?: string; + postTest?: (context: IActionContext, workspaceFolderPath: string, errMsg?: string) => void | Promise; /** * Indicates this test case should be executed exclusively. This should only be used during local development. @@ -32,6 +32,7 @@ export interface CreateNewProjectTestCase { export interface CreateAndDeployTestCase { createFunctionApp: { label: string; + mode: CreateMode; inputs: (string | RegExp)[]; postTest?: (context: IActionContext, functionAppId: string, errMsg?: string) => void | Promise; }; diff --git a/test/nightly/scenarios/testScenarios/dtsScenarios/DurableDTS.md b/test/nightly/scenarios/testScenarios/dtsScenarios/DurableDTS.md index 28f16bdec..1d4e149dd 100644 --- a/test/nightly/scenarios/testScenarios/dtsScenarios/DurableDTS.md +++ b/test/nightly/scenarios/testScenarios/dtsScenarios/DurableDTS.md @@ -21,11 +21,9 @@ TBD | Workspace Project | Connection Type | Operating System | Plan Type | Comment | |-------------------|-------------------|------------------|-------------------|---------| | I | Managed Identity | Linux | Flex Consumption | | -| I | Managed Identity | Windows | Flex Consumption | | | I | Managed Identity | Linux | Premium | | | I | Managed Identity | Windows | Premium | | | I | Secrets | Linux | Flex Consumption | | -| I | Secrets | Windows | Flex Consumption | | | I | Secrets | Linux | Premium | | | I | Secrets | Windows | Premium | | | II | Managed Identity | Linux | Flex Consumption | | diff --git a/test/nightly/scenarios/testScenarios/dtsScenarios/dtsTSNode.ts b/test/nightly/scenarios/testScenarios/dtsScenarios/dtsTSNode.ts index 9af6b6327..33beea022 100644 --- a/test/nightly/scenarios/testScenarios/dtsScenarios/dtsTSNode.ts +++ b/test/nightly/scenarios/testScenarios/dtsScenarios/dtsTSNode.ts @@ -3,35 +3,19 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { getRandomAlphanumericString } from "../../../../../extension.bundle"; +import { DurableBackend, getRandomAlphanumericString } from "../../../../../extension.bundle"; +import { ConnectionType, createFunctionAppUtils, CreateMode, OperatingSystem, PlanType } from "../../../../utils/createFunctionAppUtils"; +import { deployFunctionAppUtils } from "../../../../utils/deployFunctionAppUtils"; import { type AzExtFunctionsTestScenario, type CreateAndDeployTestCase } from "../AzExtFunctionsTestScenario"; -export enum ConnectionType { - Secrets = 'Secrets', - ManagedIdentity = 'Managed Identity', -} - -export enum OperatingSystem { - Linux = 'Linux', - Windows = 'Windows', -} - -export enum PlanType { - FlexConsumption = 'Flex Consumption', - Premium = 'Premium', - LegacyConsumption = 'Legacy', - AppService = 'App Service Plan', -} - export function generateTSNodeScenario(): AzExtFunctionsTestScenario { - const appName: string = getRandomAlphanumericString(); const folderName: string = 'scenarios-dts-tsnode'; return { - label: folderName, + label: 'dts-tsnode', folderName, createNewProjectTest: { - label: '', + label: 'create-new-project', inputs: [ /TypeScript/i, /v4/i, @@ -39,134 +23,33 @@ export function generateTSNodeScenario(): AzExtFunctionsTestScenario { /Durable Task Scheduler/i, 'durableHello1', ], - resourceGroupToDelete: appName, }, createAndDeployTests: [ - /** 1. */ generateCreateAndDeployTest(appName, folderName, ConnectionType.ManagedIdentity, OperatingSystem.Linux, PlanType.FlexConsumption), - /** 2. */ generateCreateAndDeployTest(appName, folderName, ConnectionType.ManagedIdentity, OperatingSystem.Windows, PlanType.FlexConsumption), - /** 3. */ generateCreateAndDeployTest(appName, folderName, ConnectionType.ManagedIdentity, OperatingSystem.Linux, PlanType.Premium), - /** 4. */ generateCreateAndDeployTest(appName, folderName, ConnectionType.ManagedIdentity, OperatingSystem.Windows, PlanType.Premium), - /** 5. */ generateCreateAndDeployTest(appName, folderName, ConnectionType.Secrets, OperatingSystem.Linux, PlanType.FlexConsumption), - /** 6. */ generateCreateAndDeployTest(appName, folderName, ConnectionType.Secrets, OperatingSystem.Windows, PlanType.FlexConsumption), - /** 7. */ generateCreateAndDeployTest(appName, folderName, ConnectionType.Secrets, OperatingSystem.Linux, PlanType.Premium), - /** 8. */ generateCreateAndDeployTest(appName, folderName, ConnectionType.Secrets, OperatingSystem.Windows, PlanType.Premium), + /** 1. */ generateCreateAndDeployTest(folderName, CreateMode.Basic, ConnectionType.ManagedIdentity, OperatingSystem.Linux, PlanType.FlexConsumption), + /** 2. */ generateCreateAndDeployTest(folderName, CreateMode.Advanced, ConnectionType.ManagedIdentity, OperatingSystem.Linux, PlanType.Premium), + /** 3. */ generateCreateAndDeployTest(folderName, CreateMode.Advanced, ConnectionType.ManagedIdentity, OperatingSystem.Windows, PlanType.Premium), + /** 4. */ generateCreateAndDeployTest(folderName, CreateMode.Advanced, ConnectionType.Secrets, OperatingSystem.Linux, PlanType.FlexConsumption), + /** 5. */ generateCreateAndDeployTest(folderName, CreateMode.Advanced, ConnectionType.Secrets, OperatingSystem.Linux, PlanType.Premium), + /** 6. */ generateCreateAndDeployTest(folderName, CreateMode.Advanced, ConnectionType.Secrets, OperatingSystem.Windows, PlanType.Premium), ], - } + }; } -function generateCreateAndDeployTest(appName: string, folderName: string, connection: ConnectionType, os: OperatingSystem, plan: PlanType): CreateAndDeployTestCase { +function generateCreateAndDeployTest(folderName: string, createMode: CreateMode, connection: ConnectionType, os: OperatingSystem, plan: PlanType): CreateAndDeployTestCase { + const appName: string = getRandomAlphanumericString(); + return { createFunctionApp: { - label: 'Create Function App', - inputs: plan === PlanType.FlexConsumption ? - generateCreateFunctionAppBasicInputs(appName, folderName, connection) : - generateCreateFunctionAppAdvancedInputs(appName, folderName, connection, os, plan), + label: 'create-function-app', + mode: createMode, + inputs: createMode === CreateMode.Basic ? + createFunctionAppUtils.generateBasicCreateInputs(appName, folderName, connection) : + createFunctionAppUtils.generateAdvancedCreateInputs(appName, folderName, connection, os, plan), }, deployFunctionApp: { - label: 'Deploy Function App', - inputs: [ - // Todo: Expand regexp capability for context.ui.showWarningMessage - 'Connect Durable Task Scheduler', - /Create New Durable Task Scheduler/i, - appName, - /Create New Durable Task Hub/i, - appName, - /Assign New User[- ]Assigned Identity/i, - /Create New User[- ]Assigned Identity/i, - // Todo: Here too - 'Deploy', - ], + label: 'deploy-function-app', + inputs: deployFunctionAppUtils.generateDurableDeployInputs(appName, DurableBackend.DTS), }, resourceGroupToDelete: appName, }; } - -function generateCreateFunctionAppBasicInputs(appName: string, folderName: string, connection: ConnectionType): (string | RegExp)[] { - return [ - folderName, - appName, - /West US 2/i, - /Node\.js 22/i, - new RegExp(connection, 'i'), - ]; -} - -function generateCreateFunctionAppAdvancedInputs(appName: string, folderName: string, connection: ConnectionType, os: OperatingSystem, plan: PlanType): (string | RegExp)[] { - switch (plan) { - case PlanType.FlexConsumption: - return [ - folderName, - appName, - new RegExp(plan, 'i'), - /West US 2/i, - /Node\.js 22/i, - '2048', - '100', - /Create new resource group/i, - appName, - ...getConnectionTypeInputs(connection), - /Create new storage account/i, - appName, - /Create new application insights/i, - appName, - ]; - case PlanType.Premium: - return [ - folderName, - appName, - new RegExp(plan, 'i'), - /West US 2/i, - /Node\.js 22/i, - new RegExp(os, 'i'), - /Create new app service plan/i, - appName, - /EP1/i, - /Create new resource group/i, - appName, - ...getConnectionTypeInputs(connection), - /Create new storage account/i, - appName, - /Create new application insights/i, - appName, - ]; - case PlanType.LegacyConsumption: - return [ - folderName, - appName, - new RegExp(plan, 'i'), - /West US 2/i, - /Node\.js 22/i, - new RegExp(os, 'i'), - /Create new resource group/i, - appName, - ...getConnectionTypeInputs(connection), - /Create new storage account/i, - appName, - /Create new application insights/i, - appName, - ]; - case PlanType.AppService: - return [ - folderName, - appName, - new RegExp(plan, 'i'), - /West US 2/i, - /Node\.js 22/i, - new RegExp(os, 'i'), - /Create new app service plan/i, - appName, - /S1/i, - /Create new resource group/i, - appName, - ...getConnectionTypeInputs(connection), - /Create new storage account/i, - appName, - /Create new application insights/i, - appName, - ]; - } -} - -function getConnectionTypeInputs(connection: ConnectionType): (string | RegExp)[] { - return connection === ConnectionType.ManagedIdentity ? [new RegExp(connection, 'i'), /Create new user[- ]assigned identity/i] : [new RegExp(connection, 'i')]; -} diff --git a/test/nightly/scenarios/testScenarios/testScenarios.ts b/test/nightly/scenarios/testScenarios/testScenarios.ts index 0da5da2e2..ee57f30f7 100644 --- a/test/nightly/scenarios/testScenarios/testScenarios.ts +++ b/test/nightly/scenarios/testScenarios/testScenarios.ts @@ -12,4 +12,3 @@ export function generateTestScenarios(): AzExtFunctionsTestScenario[] { ]; return testScenarios; } - diff --git a/test/utils/createFunctionAppUtils.ts b/test/utils/createFunctionAppUtils.ts index e69de29bb..a44f4f079 100644 --- a/test/utils/createFunctionAppUtils.ts +++ b/test/utils/createFunctionAppUtils.ts @@ -0,0 +1,120 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +export namespace createFunctionAppUtils { + export function generateBasicCreateInputs(appName: string, folderName: string, connection: ConnectionType): (string | RegExp)[] { + return [ + folderName, + appName, + /West US 2/i, + /Node\.js 22/i, + new RegExp(connection, 'i'), + ]; + } + + export function generateAdvancedCreateInputs(appName: string, folderName: string, connection: ConnectionType, os: OperatingSystem, plan: PlanType): (string | RegExp)[] { + switch (plan) { + case PlanType.FlexConsumption: + return [ + folderName, + appName, + new RegExp(plan, 'i'), + /West US 2/i, + /Node\.js 22/i, + '2048', + '100', + /Create new resource group/i, + appName, + ...getConnectionTypeInputs(connection), + /Create new storage account/i, + appName, + /Create new application insights/i, + appName, + ]; + case PlanType.Premium: + return [ + folderName, + appName, + new RegExp(plan, 'i'), + /West US 2/i, + /Node\.js 22/i, + new RegExp(os, 'i'), + /Create new app service plan/i, + appName, + /EP1/i, + /Create new resource group/i, + appName, + ...getConnectionTypeInputs(connection), + /Create new storage account/i, + appName, + /Create new application insights/i, + appName, + ]; + case PlanType.LegacyConsumption: + return [ + folderName, + appName, + new RegExp(plan, 'i'), + /West US 2/i, + /Node\.js 22/i, + new RegExp(os, 'i'), + /Create new resource group/i, + appName, + ...getConnectionTypeInputs(connection), + /Create new storage account/i, + appName, + /Create new application insights/i, + appName, + ]; + case PlanType.AppService: + return [ + folderName, + appName, + new RegExp(plan, 'i'), + /West US 2/i, + /Node\.js 22/i, + new RegExp(os, 'i'), + /Create new app service plan/i, + appName, + /S1/i, + /Create new resource group/i, + appName, + ...getConnectionTypeInputs(connection), + /Create new storage account/i, + appName, + /Create new application insights/i, + appName, + ]; + } + } + + function getConnectionTypeInputs(connection: ConnectionType): (string | RegExp)[] { + return connection === ConnectionType.ManagedIdentity ? + [new RegExp(connection, 'i'), /Create new user[- ]assigned identity/i] : + [new RegExp(connection, 'i')]; + } +} + +export enum ConnectionType { + Secrets = 'Secrets', + ManagedIdentity = 'Managed Identity', +} + +export enum OperatingSystem { + Linux = 'Linux', + Windows = 'Windows', +} + +export enum PlanType { + FlexConsumption = 'Flex Consumption', + Premium = 'Premium', + LegacyConsumption = 'Legacy', + AppService = 'App Service Plan', +} + +export enum CreateMode { + Basic = 'Basic', + Advanced = 'Advanced', +} diff --git a/test/utils/deployFunctionAppUtils.ts b/test/utils/deployFunctionAppUtils.ts new file mode 100644 index 000000000..1f72029ba --- /dev/null +++ b/test/utils/deployFunctionAppUtils.ts @@ -0,0 +1,33 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { DurableBackend } from "../../extension.bundle"; + +export namespace deployFunctionAppUtils { + export function generateDurableDeployInputs(appName: string, storageType?: DurableBackend): (string | RegExp)[] { + switch (storageType) { + case DurableBackend.DTS: + return [ + // Todo: Expand regexp capability for context.ui.showWarningMessage + 'Connect Durable Task Scheduler', + /Create New Durable Task Scheduler/i, + appName, + /Create New Durable Task Hub/i, + appName, + /Assign New User[- ]Assigned Identity/i, + /Create New User[- ]Assigned Identity/i, + // Todo: Here too + 'Deploy', + ]; + case DurableBackend.Netherite: + return []; + case DurableBackend.SQL: + return []; + case DurableBackend.Storage: + default: + return []; + } + } +} From 5c6ba287a1edef674c8fe4eb22da2a9d715e77cf Mon Sep 17 00:00:00 2001 From: MicroFish91 <40250218+MicroFish91@users.noreply.github.com> Date: Fri, 26 Dec 2025 17:16:39 -0800 Subject: [PATCH 06/27] Set up some custom coding agents for planning and reviewing --- .github/agents/test-scenario-builder.agent.md | 0 .github/agents/test-scenario-planner.agent.md | 37 ++++++++++++++ .../agents/test-scenario-reviewer.agent.md | 22 +++++++++ .github/copilot-instructions.md | 28 +++++++++++ .../AvailableTestCombinations.md | 0 .../azureStorage/DurableAzureStorage.plan.md | 48 +++++++++++++++++++ .../dts/DurableDTS.plan.md} | 0 .../dts}/dtsScenarios.ts | 4 +- .../dts/dtsTSNodeScenario.ts} | 13 ++--- .../scenarios/testScenarios/testScenarios.ts | 2 +- 10 files changed, 145 insertions(+), 9 deletions(-) create mode 100644 .github/agents/test-scenario-builder.agent.md create mode 100644 .github/agents/test-scenario-planner.agent.md create mode 100644 .github/agents/test-scenario-reviewer.agent.md rename test/nightly/scenarios/{testScenarios => docs}/AvailableTestCombinations.md (100%) create mode 100644 test/nightly/scenarios/testScenarios/durable/azureStorage/DurableAzureStorage.plan.md rename test/nightly/scenarios/testScenarios/{dtsScenarios/DurableDTS.md => durable/dts/DurableDTS.plan.md} (100%) rename test/nightly/scenarios/testScenarios/{dtsScenarios => durable/dts}/dtsScenarios.ts (76%) rename test/nightly/scenarios/testScenarios/{dtsScenarios/dtsTSNode.ts => durable/dts/dtsTSNodeScenario.ts} (86%) diff --git a/.github/agents/test-scenario-builder.agent.md b/.github/agents/test-scenario-builder.agent.md new file mode 100644 index 000000000..e69de29bb diff --git a/.github/agents/test-scenario-planner.agent.md b/.github/agents/test-scenario-planner.agent.md new file mode 100644 index 000000000..6c2b7825a --- /dev/null +++ b/.github/agents/test-scenario-planner.agent.md @@ -0,0 +1,37 @@ +--- +name: AzExtFunc - Test Planner +description: A tool to help Azure Functions extension internal developers plan test scenarios for new features. +model: GPT-4o (copilot) +tools: ['execute/runInTerminal', 'read', 'edit', 'search'] +handoffs: + - label: Review the plan + agent: AzExtFunc - Test Reviewer + prompt: Review and clean up the test plan. + send: true +--- +## Planning Instructions + +You are in planning mode. Your task is to generate a test plan for a new feature. +Don't make any code edits, just focus on generating a good plan. + +The plan consists of a Markdown document showing the test matrices for different Azure Functions commands through the create and deploy lifecycle. + +All possible test combinations are shown here. This is what should be initially offered to the user unless they specify otherwise: +`vscode-azurefunctions/test/nightly/scenarios/testScenarios/AvailableTestCombinations.md` + +The final target will eventually be a stripped-down version that looks like this. This should always be your example reference target when making future changes: +`vscode-azurefunctions/test/nightly/scenarios/testScenarios/durable/dts/DurableDTS.plan.md` + +## Steps + +1. Start by having the user provide the empty directory to place their new plan (do not ask for the name). This directory should be somewhere within the `testScenarios` folder. +1. Using the `cp` command in the terminal, copy the markdown for all possible test combinations into this directory. Change the title and file name to reflect the new test plan we are creating. Inspect the file path to inform your naming decisions. +1. Next, ask the user to fill in "Skip" for any rows that the user would like to remove from the test. + - If the user asks you to help with this directly through chat, you can help them do this. +1. Once the user confirms that all changes have been made, handoff to the reviewer agent. + +## Additional Instructions + +1. Keep the information simple for the user. Give them only next step options, do not enumerate all the steps nor should you overload the user with information unless asked. +2. Do not copy over sections that do not exist in the example target provided. +3. Always only give only one next step for the user, keep the info direct and short. diff --git a/.github/agents/test-scenario-reviewer.agent.md b/.github/agents/test-scenario-reviewer.agent.md new file mode 100644 index 000000000..9e71144fa --- /dev/null +++ b/.github/agents/test-scenario-reviewer.agent.md @@ -0,0 +1,22 @@ +--- +name: AzExtFunc - Test Reviewer +description: A tool to help Azure Functions internal developers review their test scenarios before requesting Copilot to implement them. +model: GPT-5 mini (copilot) +tools: ['read', 'edit'] +--- +# Review instructions + +You are reviewing a test scenario plan document previously generated with an Azure Functions extension test agent. +You will ensure the test matrices in the provided file are trimmed down and organized to only the relevant tests. + +Here is an example reference target: +`vscode-azurefunctions/test/nightly/scenarios/testScenarios/durable/dts/DurableDTS.plan.md` + +## Steps + +1. Ask the user for a test plan. +1. Check that the plan title matches the directory for the plan, use the example reference target for formatting. +1. Ensure the file name matches the example target reference as well. The name should end with `.plan.md`. +1. Ask the user if you have permission to remove any test matrix rows that have comments in them. If the comment doesn't sound like it's meant for the row to be removed, then ask for additional details before deciding to remove. +1. Since some rows were removed, renumber everything starting from 1 (or I). +1. Suggest the changes and then declare when complete. diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 01b21c4d4..7f49a155e 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -1,3 +1,31 @@ # Coding Instructions for GitHub Copilot - Never commit or suggest changes to `main.js`. + +## Code Review +The following instructions are only to be applied when performing a code review. + +### Prompt file guide + +**Only apply to files that end in `.prompt.md`** + +- [ ] The prompt has markdown front matter. +- [ ] The prompt has a `mode` field specified of either `agent` or `ask`. +- [ ] The prompt has a `description` field. +- [ ] The `description` field is not empty. +- [ ] The `description` field value is wrapped in single quotes. +- [ ] The file name is lower case, with words separated by hyphens. +- [ ] Encourage the use of `tools`, but it's not required. +- [ ] Strongly encourage the use of `model` to specify the model that the prompt is optimised for. + +### Chat Mode file guide + +**Only apply to files that end in `.agent.md`** + +- [ ] The chat mode has markdown front matter. +- [ ] The chat mode has a `description` field. +- [ ] The `description` field is not empty. +- [ ] The `description` field value is wrapped in single quotes. +- [ ] The file name is lower case, with words separated by hyphens. +- [ ] Encourage the use of `tools`, but it's not required. +- [ ] Strongly encourage the use of `model` to specify the model that the chat mode is optimised for. diff --git a/test/nightly/scenarios/testScenarios/AvailableTestCombinations.md b/test/nightly/scenarios/docs/AvailableTestCombinations.md similarity index 100% rename from test/nightly/scenarios/testScenarios/AvailableTestCombinations.md rename to test/nightly/scenarios/docs/AvailableTestCombinations.md diff --git a/test/nightly/scenarios/testScenarios/durable/azureStorage/DurableAzureStorage.plan.md b/test/nightly/scenarios/testScenarios/durable/azureStorage/DurableAzureStorage.plan.md new file mode 100644 index 000000000..6dbf1ff85 --- /dev/null +++ b/test/nightly/scenarios/testScenarios/durable/azureStorage/DurableAzureStorage.plan.md @@ -0,0 +1,48 @@ +# Durable Azure Storage — Test Plan + +## I. Create New Project / Create Function + +### Workspace Project Test Matrix: + +| No. | Language | Runtime | Programming Model | Comment | +|-----|----------|---------|-------------------|---------| +| 1 | TS | Node | v4 | | +| 2 | Python | Python | v2 | | + +## II. Debug +TBD + +## III. Create Function App / Deployment + +### Create / Deploy Test Matrix: + +| No. | Workspace Project | Connection Type | Operating System | Plan Type | Comment | +|-----|-------------------|-------------------|------------------|------------------------|---------| +| 1 | 1 | Managed Identity | Linux | Flex Consumption | | +| 2 | 1 | Managed Identity | Linux | Premium | | +| 3 | 1 | Managed Identity | Windows | Premium | | +| 4 | 1 | Managed Identity | Linux | Consumption (Legacy) | | +| 5 | 1 | Managed Identity | Windows | Consumption (Legacy) | | +| 6 | 1 | Managed Identity | Linux | App Service | | +| 7 | 1 | Managed Identity | Windows | App Service | | +| 8 | 1 | Secrets | Linux | Flex Consumption | | +| 9 | 1 | Secrets | Linux | Premium | | +| 10 | 1 | Secrets | Windows | Premium | | +| 11 | 1 | Secrets | Linux | Consumption (Legacy) | | +| 12 | 1 | Secrets | Windows | Consumption (Legacy) | | +| 13 | 1 | Secrets | Linux | App Service | | +| 14 | 1 | Secrets | Windows | App Service | | +| 15 | 2 | Managed Identity | Linux | Flex Consumption | | +| 16 | 2 | Managed Identity | Linux | Premium | | +| 17 | 2 | Managed Identity | Windows | Premium | | +| 18 | 2 | Managed Identity | Linux | Consumption (Legacy) | | +| 19 | 2 | Managed Identity | Windows | Consumption (Legacy) | | +| 20 | 2 | Managed Identity | Linux | App Service | | +| 21 | 2 | Managed Identity | Windows | App Service | | +| 22 | 2 | Secrets | Linux | Flex Consumption | | +| 23 | 2 | Secrets | Linux | Premium | | +| 24 | 2 | Secrets | Windows | Premium | | +| 25 | 2 | Secrets | Linux | Consumption (Legacy) | | +| 26 | 2 | Secrets | Windows | Consumption (Legacy) | | +| 27 | 2 | Secrets | Linux | App Service | | +| 28 | 2 | Secrets | Windows | App Service | | diff --git a/test/nightly/scenarios/testScenarios/dtsScenarios/DurableDTS.md b/test/nightly/scenarios/testScenarios/durable/dts/DurableDTS.plan.md similarity index 100% rename from test/nightly/scenarios/testScenarios/dtsScenarios/DurableDTS.md rename to test/nightly/scenarios/testScenarios/durable/dts/DurableDTS.plan.md diff --git a/test/nightly/scenarios/testScenarios/dtsScenarios/dtsScenarios.ts b/test/nightly/scenarios/testScenarios/durable/dts/dtsScenarios.ts similarity index 76% rename from test/nightly/scenarios/testScenarios/dtsScenarios/dtsScenarios.ts rename to test/nightly/scenarios/testScenarios/durable/dts/dtsScenarios.ts index c0d6533fd..67842b776 100644 --- a/test/nightly/scenarios/testScenarios/dtsScenarios/dtsScenarios.ts +++ b/test/nightly/scenarios/testScenarios/durable/dts/dtsScenarios.ts @@ -3,8 +3,8 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { type AzExtFunctionsTestScenario } from '../AzExtFunctionsTestScenario'; -import { generateTSNodeScenario } from './dtsTSNode'; +import { type AzExtFunctionsTestScenario } from '../../AzExtFunctionsTestScenario'; +import { generateTSNodeScenario } from './dtsTSNodeScenario'; export function generateDTSScenarios(): AzExtFunctionsTestScenario[] { return [ diff --git a/test/nightly/scenarios/testScenarios/dtsScenarios/dtsTSNode.ts b/test/nightly/scenarios/testScenarios/durable/dts/dtsTSNodeScenario.ts similarity index 86% rename from test/nightly/scenarios/testScenarios/dtsScenarios/dtsTSNode.ts rename to test/nightly/scenarios/testScenarios/durable/dts/dtsTSNodeScenario.ts index 33beea022..4d0564b02 100644 --- a/test/nightly/scenarios/testScenarios/dtsScenarios/dtsTSNode.ts +++ b/test/nightly/scenarios/testScenarios/durable/dts/dtsTSNodeScenario.ts @@ -3,10 +3,10 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { DurableBackend, getRandomAlphanumericString } from "../../../../../extension.bundle"; -import { ConnectionType, createFunctionAppUtils, CreateMode, OperatingSystem, PlanType } from "../../../../utils/createFunctionAppUtils"; -import { deployFunctionAppUtils } from "../../../../utils/deployFunctionAppUtils"; -import { type AzExtFunctionsTestScenario, type CreateAndDeployTestCase } from "../AzExtFunctionsTestScenario"; +import { DurableBackend, getRandomAlphanumericString } from "../../../../../../extension.bundle"; +import { ConnectionType, createFunctionAppUtils, CreateMode, OperatingSystem, PlanType } from "../../../../../utils/createFunctionAppUtils"; +import { deployFunctionAppUtils } from "../../../../../utils/deployFunctionAppUtils"; +import { type AzExtFunctionsTestScenario, type CreateAndDeployTestCase } from "../../AzExtFunctionsTestScenario"; export function generateTSNodeScenario(): AzExtFunctionsTestScenario { const folderName: string = 'scenarios-dts-tsnode'; @@ -37,17 +37,18 @@ export function generateTSNodeScenario(): AzExtFunctionsTestScenario { function generateCreateAndDeployTest(folderName: string, createMode: CreateMode, connection: ConnectionType, os: OperatingSystem, plan: PlanType): CreateAndDeployTestCase { const appName: string = getRandomAlphanumericString(); + const description: string = `${createMode}-${connection}-${os}-${plan}`; return { createFunctionApp: { - label: 'create-function-app', + label: `create-function-app-${description}`, mode: createMode, inputs: createMode === CreateMode.Basic ? createFunctionAppUtils.generateBasicCreateInputs(appName, folderName, connection) : createFunctionAppUtils.generateAdvancedCreateInputs(appName, folderName, connection, os, plan), }, deployFunctionApp: { - label: 'deploy-function-app', + label: `deploy-function-app-${description}`, inputs: deployFunctionAppUtils.generateDurableDeployInputs(appName, DurableBackend.DTS), }, resourceGroupToDelete: appName, diff --git a/test/nightly/scenarios/testScenarios/testScenarios.ts b/test/nightly/scenarios/testScenarios/testScenarios.ts index ee57f30f7..ea14ca24b 100644 --- a/test/nightly/scenarios/testScenarios/testScenarios.ts +++ b/test/nightly/scenarios/testScenarios/testScenarios.ts @@ -4,7 +4,7 @@ *--------------------------------------------------------------------------------------------*/ import { type AzExtFunctionsTestScenario } from "./AzExtFunctionsTestScenario"; -import { generateDTSScenarios } from "./dtsScenarios/dtsScenarios"; +import { generateDTSScenarios } from "./durable/dts/dtsScenarios"; export function generateTestScenarios(): AzExtFunctionsTestScenario[] { const testScenarios: AzExtFunctionsTestScenario[] = [ From ce6701d3bbd263da8778a737fb0dd9f2d81ff056 Mon Sep 17 00:00:00 2001 From: MicroFish91 <40250218+MicroFish91@users.noreply.github.com> Date: Fri, 26 Dec 2025 23:13:49 -0800 Subject: [PATCH 07/27] Update lots of copilotey things --- .github/agents/test-scenario-builder.agent.md | 0 .github/agents/test-scenario-planner.agent.md | 18 ++- .../agents/test-scenario-reviewer.agent.md | 17 ++- .../docs/AvailableTestCombinations.md | 120 ------------------ .../docs/ScenarioTestCombinations.md | 120 ++++++++++++++++++ .../azureStorage/DurableAzureStorage.plan.md | 74 +++++------ .../durable/dts/DurableDTS.plan.md | 70 +++++----- 7 files changed, 215 insertions(+), 204 deletions(-) delete mode 100644 .github/agents/test-scenario-builder.agent.md delete mode 100644 test/nightly/scenarios/docs/AvailableTestCombinations.md create mode 100644 test/nightly/scenarios/docs/ScenarioTestCombinations.md diff --git a/.github/agents/test-scenario-builder.agent.md b/.github/agents/test-scenario-builder.agent.md deleted file mode 100644 index e69de29bb..000000000 diff --git a/.github/agents/test-scenario-planner.agent.md b/.github/agents/test-scenario-planner.agent.md index 6c2b7825a..dcb973745 100644 --- a/.github/agents/test-scenario-planner.agent.md +++ b/.github/agents/test-scenario-planner.agent.md @@ -12,21 +12,25 @@ handoffs: ## Planning Instructions You are in planning mode. Your task is to generate a test plan for a new feature. -Don't make any code edits, just focus on generating a good plan. +Don't make any code edits, just focus on generating a the plan. -The plan consists of a Markdown document showing the test matrices for different Azure Functions commands through the create and deploy lifecycle. +The plan should take the form of a Markdown document, with test matrices for major functions commands. -All possible test combinations are shown here. This is what should be initially offered to the user unless they specify otherwise: -`vscode-azurefunctions/test/nightly/scenarios/testScenarios/AvailableTestCombinations.md` -The final target will eventually be a stripped-down version that looks like this. This should always be your example reference target when making future changes: +## References + +- All Scenario Test Combinations: This is what should be initially offered to the user unless specified otherwise: +`vscode-azurefunctions/test/nightly/scenarios/docs/ScenarioTestCombinations.md` + +- Plan Example: The final target will eventually be a stripped-down version that looks like the document that follows. This should always be your example reference target when making future changes: `vscode-azurefunctions/test/nightly/scenarios/testScenarios/durable/dts/DurableDTS.plan.md` ## Steps 1. Start by having the user provide the empty directory to place their new plan (do not ask for the name). This directory should be somewhere within the `testScenarios` folder. -1. Using the `cp` command in the terminal, copy the markdown for all possible test combinations into this directory. Change the title and file name to reflect the new test plan we are creating. Inspect the file path to inform your naming decisions. -1. Next, ask the user to fill in "Skip" for any rows that the user would like to remove from the test. +1. Next run the `cp` command in the terminal to directly copy the markdown for all scenario test combinations into this directory. If you can't find the file or fail to copy it in any way, please say so to the user. +1. Change the title and file name to reflect the new test plan we are creating. Do not create a separate file, always change the existing file so we don't end up with duplicates. Inspect the file path to inform your file and title naming decisions. +1. Next, ask the user to mark rows that should be tested under the `Selected` column. - If the user asks you to help with this directly through chat, you can help them do this. 1. Once the user confirms that all changes have been made, handoff to the reviewer agent. diff --git a/.github/agents/test-scenario-reviewer.agent.md b/.github/agents/test-scenario-reviewer.agent.md index 9e71144fa..f43f75bbe 100644 --- a/.github/agents/test-scenario-reviewer.agent.md +++ b/.github/agents/test-scenario-reviewer.agent.md @@ -9,14 +9,21 @@ tools: ['read', 'edit'] You are reviewing a test scenario plan document previously generated with an Azure Functions extension test agent. You will ensure the test matrices in the provided file are trimmed down and organized to only the relevant tests. -Here is an example reference target: +## References +- Plan Example: The final target will eventually be a stripped-down version that looks like the document that follows. This should always be your example reference target when making future changes: `vscode-azurefunctions/test/nightly/scenarios/testScenarios/durable/dts/DurableDTS.plan.md` ## Steps -1. Ask the user for a test plan. +1. Ask the user for a test plan if one was not provided. 1. Check that the plan title matches the directory for the plan, use the example reference target for formatting. 1. Ensure the file name matches the example target reference as well. The name should end with `.plan.md`. -1. Ask the user if you have permission to remove any test matrix rows that have comments in them. If the comment doesn't sound like it's meant for the row to be removed, then ask for additional details before deciding to remove. -1. Since some rows were removed, renumber everything starting from 1 (or I). -1. Suggest the changes and then declare when complete. +1. If there are any non-selected rows, ask to delete them. +1. If the selected column is still there, remove it. +1. Renumber the rows if needed. +1. Warn if there are any workspace projects in the test matrix that don't show up in the bottom test matrix. + +## Additional Instructions + +- Only perform one step at a time, ask where appropriate. +- Keep text information minimal, don't overload the user with context. diff --git a/test/nightly/scenarios/docs/AvailableTestCombinations.md b/test/nightly/scenarios/docs/AvailableTestCombinations.md deleted file mode 100644 index 4c262618a..000000000 --- a/test/nightly/scenarios/docs/AvailableTestCombinations.md +++ /dev/null @@ -1,120 +0,0 @@ -# Test Combinations - -## I. Create New Project / Create Function - -### Workspace Project Test Matrix: - -| No. | Language | Runtime | Programming Model | Comment | -|-----|----------|---------|-------------------|---------| -| I | TS | Node | v3 | Skip unless special requirements | -| II | TS | Node | v4 | | -| III | Python | Python | v1 | Skip unless special requirements | -| IV | Python | Python | v2 | | -| V | C# | .NET | isolated | | -| VI | C# | .NET | in-proc | | - -## II. Debug -TBD - -## III. Create Function App / Deployment - -### Create / Deploy Test Matrix: - -| No. | Workspace Project | Connection Type | Operating System | Plan Type | Comment | -|-----|-------------------|-------------------|------------------|------------------------|---------| -| 1 | I | Managed Identity | Linux | Flex Consumption | Skip | -| 2 | I | Managed Identity | Windows | Flex Consumption | Skip | -| 3 | I | Managed Identity | Linux | Premium | Skip | -| 4 | I | Managed Identity | Windows | Premium | Skip | -| 5 | I | Managed Identity | Linux | Consumption (Legacy) | Skip | -| 6 | I | Managed Identity | Windows | Consumption (Legacy) | Skip | -| 7 | I | Managed Identity | Linux | App Service | Skip | -| 8 | I | Managed Identity | Windows | App Service | Skip | -| 9 | I | Secrets | Linux | Flex Consumption | Skip | -| 10 | I | Secrets | Windows | Flex Consumption | Skip | -| 11 | I | Secrets | Linux | Premium | Skip | -| 12 | I | Secrets | Windows | Premium | Skip | -| 13 | I | Secrets | Linux | Consumption (Legacy) | Skip | -| 14 | I | Secrets | Windows | Consumption (Legacy) | Skip | -| 15 | I | Secrets | Linux | App Service | Skip | -| 16 | I | Secrets | Windows | App Service | Skip | -| 17 | II | Managed Identity | Linux | Flex Consumption | | -| 18 | II | Managed Identity | Windows | Flex Consumption | Not Offered | -| 19 | II | Managed Identity | Linux | Premium | | -| 20 | II | Managed Identity | Windows | Premium | | -| 21 | II | Managed Identity | Linux | Consumption (Legacy) | | -| 22 | II | Managed Identity | Windows | Consumption (Legacy) | | -| 23 | II | Managed Identity | Linux | App Service | | -| 24 | II | Managed Identity | Windows | App Service | | -| 25 | II | Secrets | Linux | Flex Consumption | | -| 26 | II | Secrets | Windows | Flex Consumption | Not Offered | -| 27 | II | Secrets | Linux | Premium | | -| 28 | II | Secrets | Windows | Premium | | -| 29 | II | Secrets | Linux | Consumption (Legacy) | | -| 30 | II | Secrets | Windows | Consumption (Legacy) | | -| 31 | II | Secrets | Linux | App Service | | -| 32 | II | Secrets | Windows | App Service | | -| 33 | III | Managed Identity | Linux | Flex Consumption | Skip | -| 34 | III | Managed Identity | Windows | Flex Consumption | Skip | -| 35 | III | Managed Identity | Linux | Premium | Skip | -| 36 | III | Managed Identity | Windows | Premium | Skip | -| 37 | III | Managed Identity | Linux | Consumption (Legacy) | Skip | -| 38 | III | Managed Identity | Windows | Consumption (Legacy) | Skip | -| 39 | III | Managed Identity | Linux | App Service | Skip | -| 40 | III | Managed Identity | Windows | App Service | Skip | -| 41 | III | Secrets | Linux | Flex Consumption | Skip | -| 42 | III | Secrets | Windows | Flex Consumption | Skip | -| 43 | III | Secrets | Linux | Premium | Skip | -| 44 | III | Secrets | Windows | Premium | Skip | -| 45 | III | Secrets | Linux | Consumption (Legacy) | Skip | -| 46 | III | Secrets | Windows | Consumption (Legacy) | Skip | -| 47 | III | Secrets | Linux | App Service | Skip | -| 48 | III | Secrets | Windows | App Service | Skip | -| 49 | IV | Managed Identity | Linux | Flex Consumption | | -| 50 | IV | Managed Identity | Windows | Flex Consumption | Not Offered | -| 51 | IV | Managed Identity | Linux | Premium | | -| 52 | IV | Managed Identity | Windows | Premium | | -| 53 | IV | Managed Identity | Linux | Consumption (Legacy) | | -| 54 | IV | Managed Identity | Windows | Consumption (Legacy) | | -| 55 | IV | Managed Identity | Linux | App Service | | -| 56 | IV | Managed Identity | Windows | App Service | | -| 57 | IV | Secrets | Linux | Flex Consumption | | -| 58 | IV | Secrets | Windows | Flex Consumption | Not Offered | -| 59 | IV | Secrets | Linux | Premium | | -| 60 | IV | Secrets | Windows | Premium | | -| 61 | IV | Secrets | Linux | Consumption (Legacy) | | -| 62 | IV | Secrets | Windows | Consumption (Legacy) | | -| 63 | IV | Secrets | Linux | App Service | | -| 64 | IV | Secrets | Windows | App Service | | -| 65 | V | Managed Identity | Linux | Flex Consumption | | -| 66 | V | Managed Identity | Windows | Flex Consumption | Not Offered | -| 67 | V | Managed Identity | Linux | Premium | | -| 68 | V | Managed Identity | Windows | Premium | | -| 69 | V | Managed Identity | Linux | Consumption (Legacy) | | -| 70 | V | Managed Identity | Windows | Consumption (Legacy) | | -| 71 | V | Managed Identity | Linux | App Service | | -| 72 | V | Managed Identity | Windows | App Service | | -| 73 | V | Secrets | Linux | Flex Consumption | | -| 74 | V | Secrets | Windows | Flex Consumption | Not Offered | -| 75 | V | Secrets | Linux | Premium | | -| 76 | V | Secrets | Windows | Premium | | -| 77 | V | Secrets | Linux | Consumption (Legacy) | | -| 78 | V | Secrets | Windows | Consumption (Legacy) | | -| 79 | V | Secrets | Linux | App Service | | -| 80 | V | Secrets | Windows | App Service | | -| 81 | VI | Managed Identity | Linux | Flex Consumption | | -| 82 | VI | Managed Identity | Windows | Flex Consumption | Not Offered | -| 83 | VI | Managed Identity | Linux | Premium | | -| 84 | VI | Managed Identity | Windows | Premium | | -| 85 | VI | Managed Identity | Linux | Consumption (Legacy) | | -| 86 | VI | Managed Identity | Windows | Consumption (Legacy) | | -| 87 | VI | Managed Identity | Linux | App Service | | -| 88 | VI | Managed Identity | Windows | App Service | | -| 89 | VI | Secrets | Linux | Flex Consumption | | -| 90 | VI | Secrets | Windows | Flex Consumption | Not Offered | -| 91 | VI | Secrets | Linux | Premium | | -| 92 | VI | Secrets | Windows | Premium | | -| 93 | VI | Secrets | Linux | Consumption (Legacy) | | -| 94 | VI | Secrets | Windows | Consumption (Legacy) | | -| 95 | VI | Secrets | Linux | App Service | | -| 96 | VI | Secrets | Windows | App Service | | diff --git a/test/nightly/scenarios/docs/ScenarioTestCombinations.md b/test/nightly/scenarios/docs/ScenarioTestCombinations.md new file mode 100644 index 000000000..d497b22b6 --- /dev/null +++ b/test/nightly/scenarios/docs/ScenarioTestCombinations.md @@ -0,0 +1,120 @@ +# Scenario Test Combinations + +## I. Create New Project / Create Function + +### Workspace Project Test Matrix + +| No. | Language | Runtime | Programming Model | Comment | Selected | +|-----|----------|---------|-------------------|------------------------------------|-------------| +| 1 | TS | Node | v3 | Skip unless special requirements | | +| 2 | TS | Node | v4 | | | +| 3 | Python | Python | v1 | Skip unless special requirements | | +| 4 | Python | Python | v2 | | | +| 5 | C# | .NET | isolated | | | +| 6 | C# | .NET | in-proc | | | + +## II. Debug +TBD + +## III. Create Function App / Deployment + +### Create / Deploy Test Matrix + +| No. | Workspace Project | Connection Type | Operating System | Plan Type | Comment | Selected | +|-----|-------------------|-------------------|------------------|------------------------|-------------|-------------| +| 1 | 1 | Managed Identity | Linux | Flex Consumption | Skip | | +| 2 | 1 | Managed Identity | Windows | Flex Consumption | Skip | | +| 3 | 1 | Managed Identity | Linux | Premium | Skip | | +| 4 | 1 | Managed Identity | Windows | Premium | Skip | | +| 5 | 1 | Managed Identity | Linux | Consumption (Legacy) | Skip | | +| 6 | 1 | Managed Identity | Windows | Consumption (Legacy) | Skip | | +| 7 | 1 | Managed Identity | Linux | App Service | Skip | | +| 8 | 1 | Managed Identity | Windows | App Service | Skip | | +| 9 | 1 | Secrets | Linux | Flex Consumption | Skip | | +| 10 | 1 | Secrets | Windows | Flex Consumption | Skip | | +| 11 | 1 | Secrets | Linux | Premium | Skip | | +| 12 | 1 | Secrets | Windows | Premium | Skip | | +| 13 | 1 | Secrets | Linux | Consumption (Legacy) | Skip | | +| 14 | 1 | Secrets | Windows | Consumption (Legacy) | Skip | | +| 15 | 1 | Secrets | Linux | App Service | Skip | | +| 16 | 1 | Secrets | Windows | App Service | Skip | | +| 17 | 2 | Managed Identity | Linux | Flex Consumption | | | +| 18 | 2 | Managed Identity | Windows | Flex Consumption | Not Offered | | +| 19 | 2 | Managed Identity | Linux | Premium | | | +| 20 | 2 | Managed Identity | Windows | Premium | | | +| 21 | 2 | Managed Identity | Linux | Consumption (Legacy) | | | +| 22 | 2 | Managed Identity | Windows | Consumption (Legacy) | | | +| 23 | 2 | Managed Identity | Linux | App Service | | | +| 24 | 2 | Managed Identity | Windows | App Service | | | +| 25 | 2 | Secrets | Linux | Flex Consumption | | | +| 26 | 2 | Secrets | Windows | Flex Consumption | Not Offered | | +| 27 | 2 | Secrets | Linux | Premium | | | +| 28 | 2 | Secrets | Windows | Premium | | | +| 29 | 2 | Secrets | Linux | Consumption (Legacy) | | | +| 30 | 2 | Secrets | Windows | Consumption (Legacy) | | | +| 31 | 2 | Secrets | Linux | App Service | | | +| 32 | 2 | Secrets | Windows | App Service | | | +| 33 | 3 | Managed Identity | Linux | Flex Consumption | Skip | | +| 34 | 3 | Managed Identity | Windows | Flex Consumption | Skip | | +| 35 | 3 | Managed Identity | Linux | Premium | Skip | | +| 36 | 3 | Managed Identity | Windows | Premium | Skip | | +| 37 | 3 | Managed Identity | Linux | Consumption (Legacy) | Skip | | +| 38 | 3 | Managed Identity | Windows | Consumption (Legacy) | Skip | | +| 39 | 3 | Managed Identity | Linux | App Service | Skip | | +| 40 | 3 | Managed Identity | Windows | App Service | Skip | | +| 41 | 3 | Secrets | Linux | Flex Consumption | Skip | | +| 42 | 3 | Secrets | Windows | Flex Consumption | Skip | | +| 43 | 3 | Secrets | Linux | Premium | Skip | | +| 44 | 3 | Secrets | Windows | Premium | Skip | | +| 45 | 3 | Secrets | Linux | Consumption (Legacy) | Skip | | +| 46 | 3 | Secrets | Windows | Consumption (Legacy) | Skip | | +| 47 | 3 | Secrets | Linux | App Service | Skip | | +| 48 | 3 | Secrets | Windows | App Service | Skip | | +| 49 | 4 | Managed Identity | Linux | Flex Consumption | | | +| 50 | 4 | Managed Identity | Windows | Flex Consumption | Not Offered | | +| 51 | 4 | Managed Identity | Linux | Premium | | | +| 52 | 4 | Managed Identity | Windows | Premium | | | +| 53 | 4 | Managed Identity | Linux | Consumption (Legacy) | | | +| 54 | 4 | Managed Identity | Windows | Consumption (Legacy) | | | +| 55 | 4 | Managed Identity | Linux | App Service | | | +| 56 | 4 | Managed Identity | Windows | App Service | | | +| 57 | 4 | Secrets | Linux | Flex Consumption | | | +| 58 | 4 | Secrets | Windows | Flex Consumption | Not Offered | | +| 59 | 4 | Secrets | Linux | Premium | | | +| 60 | 4 | Secrets | Windows | Premium | | | +| 61 | 4 | Secrets | Linux | Consumption (Legacy) | | | +| 62 | 4 | Secrets | Windows | Consumption (Legacy) | | | +| 63 | 4 | Secrets | Linux | App Service | | | +| 64 | 4 | Secrets | Windows | App Service | | | +| 65 | 5 | Managed Identity | Linux | Flex Consumption | | | +| 66 | 5 | Managed Identity | Windows | Flex Consumption | Not Offered | | +| 67 | 5 | Managed Identity | Linux | Premium | | | +| 68 | 5 | Managed Identity | Windows | Premium | | | +| 69 | 5 | Managed Identity | Linux | Consumption (Legacy) | | | +| 70 | 5 | Managed Identity | Windows | Consumption (Legacy) | | | +| 71 | 5 | Managed Identity | Linux | App Service | | | +| 72 | 5 | Managed Identity | Windows | App Service | | | +| 73 | 5 | Secrets | Linux | Flex Consumption | | | +| 74 | 5 | Secrets | Windows | Flex Consumption | Not Offered | | +| 75 | 5 | Secrets | Linux | Premium | | | +| 76 | 5 | Secrets | Windows | Premium | | | +| 77 | 5 | Secrets | Linux | Consumption (Legacy) | | | +| 78 | 5 | Secrets | Windows | Consumption (Legacy) | | | +| 79 | 5 | Secrets | Linux | App Service | | | +| 80 | 5 | Secrets | Windows | App Service | | | +| 81 | 6 | Managed Identity | Linux | Flex Consumption | | | +| 82 | 6 | Managed Identity | Windows | Flex Consumption | Not Offered | | +| 83 | 6 | Managed Identity | Linux | Premium | | | +| 84 | 6 | Managed Identity | Windows | Premium | | | +| 85 | 6 | Managed Identity | Linux | Consumption (Legacy) | | | +| 86 | 6 | Managed Identity | Windows | Consumption (Legacy) | | | +| 87 | 6 | Managed Identity | Linux | App Service | | | +| 88 | 6 | Managed Identity | Windows | App Service | | | +| 89 | 6 | Secrets | Linux | Flex Consumption | | | +| 90 | 6 | Secrets | Windows | Flex Consumption | Not Offered | | +| 91 | 6 | Secrets | Linux | Premium | | | +| 92 | 6 | Secrets | Windows | Premium | | | +| 93 | 6 | Secrets | Linux | Consumption (Legacy) | | | +| 94 | 6 | Secrets | Windows | Consumption (Legacy) | | | +| 95 | 6 | Secrets | Linux | App Service | | | +| 96 | 6 | Secrets | Windows | App Service | | | diff --git a/test/nightly/scenarios/testScenarios/durable/azureStorage/DurableAzureStorage.plan.md b/test/nightly/scenarios/testScenarios/durable/azureStorage/DurableAzureStorage.plan.md index 6dbf1ff85..272c47b1f 100644 --- a/test/nightly/scenarios/testScenarios/durable/azureStorage/DurableAzureStorage.plan.md +++ b/test/nightly/scenarios/testScenarios/durable/azureStorage/DurableAzureStorage.plan.md @@ -1,48 +1,48 @@ -# Durable Azure Storage — Test Plan +# Durable Azure Storage Test Plan ## I. Create New Project / Create Function -### Workspace Project Test Matrix: +### Workspace Project Test Matrix -| No. | Language | Runtime | Programming Model | Comment | -|-----|----------|---------|-------------------|---------| -| 1 | TS | Node | v4 | | -| 2 | Python | Python | v2 | | +| No. | Language | Runtime | Programming Model | Comment | +|-----|----------|---------|-------------------|------------------------------------| +| 1 | TS | Node | v4 | | +| 2 | Python | Python | v2 | | +| 3 | C# | .NET | isolated | | +| 4 | C# | .NET | in-proc | | ## II. Debug TBD ## III. Create Function App / Deployment -### Create / Deploy Test Matrix: +### Create / Deploy Test Matrix -| No. | Workspace Project | Connection Type | Operating System | Plan Type | Comment | -|-----|-------------------|-------------------|------------------|------------------------|---------| -| 1 | 1 | Managed Identity | Linux | Flex Consumption | | -| 2 | 1 | Managed Identity | Linux | Premium | | -| 3 | 1 | Managed Identity | Windows | Premium | | -| 4 | 1 | Managed Identity | Linux | Consumption (Legacy) | | -| 5 | 1 | Managed Identity | Windows | Consumption (Legacy) | | -| 6 | 1 | Managed Identity | Linux | App Service | | -| 7 | 1 | Managed Identity | Windows | App Service | | -| 8 | 1 | Secrets | Linux | Flex Consumption | | -| 9 | 1 | Secrets | Linux | Premium | | -| 10 | 1 | Secrets | Windows | Premium | | -| 11 | 1 | Secrets | Linux | Consumption (Legacy) | | -| 12 | 1 | Secrets | Windows | Consumption (Legacy) | | -| 13 | 1 | Secrets | Linux | App Service | | -| 14 | 1 | Secrets | Windows | App Service | | -| 15 | 2 | Managed Identity | Linux | Flex Consumption | | -| 16 | 2 | Managed Identity | Linux | Premium | | -| 17 | 2 | Managed Identity | Windows | Premium | | -| 18 | 2 | Managed Identity | Linux | Consumption (Legacy) | | -| 19 | 2 | Managed Identity | Windows | Consumption (Legacy) | | -| 20 | 2 | Managed Identity | Linux | App Service | | -| 21 | 2 | Managed Identity | Windows | App Service | | -| 22 | 2 | Secrets | Linux | Flex Consumption | | -| 23 | 2 | Secrets | Linux | Premium | | -| 24 | 2 | Secrets | Windows | Premium | | -| 25 | 2 | Secrets | Linux | Consumption (Legacy) | | -| 26 | 2 | Secrets | Windows | Consumption (Legacy) | | -| 27 | 2 | Secrets | Linux | App Service | | -| 28 | 2 | Secrets | Windows | App Service | | +| No. | Workspace Project | Connection Type | Operating System | Plan Type | Comment | +|-----|-------------------|-------------------|------------------|------------------------|-------------| +| 1 | 2 | Managed Identity | Linux | Flex Consumption | | +| 2 | 2 | Managed Identity | Linux | Premium | | +| 3 | 2 | Managed Identity | Windows | Premium | | +| 4 | 2 | Managed Identity | Linux | Consumption (Legacy) | | +| 5 | 2 | Managed Identity | Windows | Consumption (Legacy) | | +| 6 | 2 | Managed Identity | Linux | App Service | | +| 7 | 2 | Managed Identity | Windows | App Service | | +| 8 | 2 | Secrets | Linux | Flex Consumption | | +| 9 | 2 | Secrets | Linux | Premium | | +| 10 | 2 | Secrets | Windows | Premium | | +| 11 | 2 | Secrets | Linux | Consumption (Legacy) | | +| 12 | 2 | Secrets | Windows | Consumption (Legacy) | | +| 13 | 2 | Secrets | Linux | App Service | | +| 14 | 2 | Secrets | Windows | App Service | | +| 15 | 4 | Managed Identity | Linux | Flex Consumption | | +| 16 | 4 | Managed Identity | Linux | Premium | | +| 17 | 4 | Secrets | Linux | Flex Consumption | | +| 18 | 4 | Secrets | Linux | Premium | | +| 19 | 5 | Managed Identity | Linux | Flex Consumption | | +| 20 | 5 | Managed Identity | Linux | Premium | | +| 21 | 5 | Secrets | Linux | Flex Consumption | | +| 22 | 5 | Secrets | Linux | Premium | | +| 23 | 6 | Managed Identity | Linux | Flex Consumption | | +| 24 | 6 | Managed Identity | Linux | Premium | | +| 25 | 6 | Secrets | Linux | Flex Consumption | | +| 26 | 6 | Secrets | Linux | Premium | | diff --git a/test/nightly/scenarios/testScenarios/durable/dts/DurableDTS.plan.md b/test/nightly/scenarios/testScenarios/durable/dts/DurableDTS.plan.md index 1d4e149dd..a54c60687 100644 --- a/test/nightly/scenarios/testScenarios/durable/dts/DurableDTS.plan.md +++ b/test/nightly/scenarios/testScenarios/durable/dts/DurableDTS.plan.md @@ -1,4 +1,4 @@ -# Durable Task Scheduler (DTS) Test Scenarios +# Durable Task Scheduler - Test Plan ## I. Create New Project / Create Function @@ -6,10 +6,10 @@ | No. | Language | Runtime | Programming Model | Comment | |-----|----------|---------|-------------------|---------| -| I | TS | Node | v4 | | -| II | Python | Python | v2 | | -| III | C# | .NET | isolated | | -| IV | C# | .NET | in-proc | | +| 1 | TS | Node | v4 | | +| 2 | Python | Python | v2 | | +| 3 | C# | .NET | isolated | | +| 4 | C# | .NET | in-proc | | ## II. Debug TBD @@ -20,33 +20,33 @@ TBD | Workspace Project | Connection Type | Operating System | Plan Type | Comment | |-------------------|-------------------|------------------|-------------------|---------| -| I | Managed Identity | Linux | Flex Consumption | | -| I | Managed Identity | Linux | Premium | | -| I | Managed Identity | Windows | Premium | | -| I | Secrets | Linux | Flex Consumption | | -| I | Secrets | Linux | Premium | | -| I | Secrets | Windows | Premium | | -| II | Managed Identity | Linux | Flex Consumption | | -| II | Managed Identity | Windows | Flex Consumption | | -| II | Managed Identity | Linux | Premium | | -| II | Managed Identity | Windows | Premium | | -| II | Secrets | Linux | Flex Consumption | | -| II | Secrets | Windows | Flex Consumption | | -| II | Secrets | Linux | Premium | | -| II | Secrets | Windows | Premium | | -| III | Managed Identity | Linux | Flex Consumption | | -| III | Managed Identity | Windows | Flex Consumption | | -| III | Managed Identity | Linux | Premium | | -| III | Managed Identity | Windows | Premium | | -| III | Secrets | Linux | Flex Consumption | | -| III | Secrets | Windows | Flex Consumption | | -| III | Secrets | Linux | Premium | | -| III | Secrets | Windows | Premium | | -| IV | Managed Identity | Linux | Flex Consumption | | -| IV | Managed Identity | Windows | Flex Consumption | | -| IV | Managed Identity | Linux | Premium | | -| IV | Managed Identity | Windows | Premium | | -| IV | Secrets | Linux | Flex Consumption | | -| IV | Secrets | Windows | Flex Consumption | | -| IV | Secrets | Linux | Premium | | -| IV | Secrets | Windows | Premium | | +| 1 | Managed Identity | Linux | Flex Consumption | | +| 1 | Managed Identity | Linux | Premium | | +| 1 | Managed Identity | Windows | Premium | | +| 1 | Secrets | Linux | Flex Consumption | | +| 1 | Secrets | Linux | Premium | | +| 1 | Secrets | Windows | Premium | | +| 2 | Managed Identity | Linux | Flex Consumption | | +| 2 | Managed Identity | Windows | Flex Consumption | | +| 2 | Managed Identity | Linux | Premium | | +| 2 | Managed Identity | Windows | Premium | | +| 2 | Secrets | Linux | Flex Consumption | | +| 2 | Secrets | Windows | Flex Consumption | | +| 2 | Secrets | Linux | Premium | | +| 2 | Secrets | Windows | Premium | | +| 3 | Managed Identity | Linux | Flex Consumption | | +| 3 | Managed Identity | Windows | Flex Consumption | | +| 3 | Managed Identity | Linux | Premium | | +| 3 | Managed Identity | Windows | Premium | | +| 3 | Secrets | Linux | Flex Consumption | | +| 3 | Secrets | Windows | Flex Consumption | | +| 3 | Secrets | Linux | Premium | | +| 3 | Secrets | Windows | Premium | | +| 4 | Managed Identity | Linux | Flex Consumption | | +| 4 | Managed Identity | Windows | Flex Consumption | | +| 4 | Managed Identity | Linux | Premium | | +| 4 | Managed Identity | Windows | Premium | | +| 4 | Secrets | Linux | Flex Consumption | | +| 4 | Secrets | Windows | Flex Consumption | | +| 4 | Secrets | Linux | Premium | | +| 4 | Secrets | Windows | Premium | | From ee4386cd93f1187d0bf5ef93a7442818a3f6704b Mon Sep 17 00:00:00 2001 From: MicroFish91 <40250218+MicroFish91@users.noreply.github.com> Date: Mon, 29 Dec 2025 20:33:11 -0800 Subject: [PATCH 08/27] Another set of scenarios working --- .github/agents/test-scenario-planner.agent.md | 1 + .../agents/test-scenario-reviewer.agent.md | 20 +++--- .github/copilot-instructions.md | 28 -------- test/global.test.ts | 8 ++- test/nightly/scenarios/docs/TestScenarios.md | 0 test/nightly/scenarios/parallelScenarios.ts | 2 + test/nightly/scenarios/scenarios.test.ts | 9 ++- .../AzExtFunctionsTestScenario.ts | 7 +- .../azureStorage/DurableAzureStorage.plan.md | 52 +++++++-------- .../azureStorage/azureStorageScenarios.ts | 13 ++++ .../azureStorageTSNodeScenario.ts | 65 +++++++++++++++++++ .../durable/dts/DurableDTS.plan.md | 2 +- .../testScenarios/durable/dts/dtsScenarios.ts | 2 +- .../durable/dts/dtsTSNodeScenario.ts | 16 ++--- .../scenarios/testScenarios/testScenarios.ts | 6 +- test/test.code-workspace | 12 ++-- test/utils/deployFunctionAppUtils.ts | 4 +- tsconfig.json | 3 +- 18 files changed, 163 insertions(+), 87 deletions(-) create mode 100644 test/nightly/scenarios/docs/TestScenarios.md create mode 100644 test/nightly/scenarios/testScenarios/durable/azureStorage/azureStorageScenarios.ts create mode 100644 test/nightly/scenarios/testScenarios/durable/azureStorage/azureStorageTSNodeScenario.ts diff --git a/.github/agents/test-scenario-planner.agent.md b/.github/agents/test-scenario-planner.agent.md index dcb973745..0cd7b12e5 100644 --- a/.github/agents/test-scenario-planner.agent.md +++ b/.github/agents/test-scenario-planner.agent.md @@ -9,6 +9,7 @@ handoffs: prompt: Review and clean up the test plan. send: true --- + ## Planning Instructions You are in planning mode. Your task is to generate a test plan for a new feature. diff --git a/.github/agents/test-scenario-reviewer.agent.md b/.github/agents/test-scenario-reviewer.agent.md index f43f75bbe..188ad1e85 100644 --- a/.github/agents/test-scenario-reviewer.agent.md +++ b/.github/agents/test-scenario-reviewer.agent.md @@ -4,26 +4,28 @@ description: A tool to help Azure Functions internal developers review their tes model: GPT-5 mini (copilot) tools: ['read', 'edit'] --- -# Review instructions -You are reviewing a test scenario plan document previously generated with an Azure Functions extension test agent. +## Review Instructions + +You are an agent specialized in reviewing test scenario plans to ensure they are ready to be built. Test plan docs should end in `.plan.md`. You will ensure the test matrices in the provided file are trimmed down and organized to only the relevant tests. ## References -- Plan Example: The final target will eventually be a stripped-down version that looks like the document that follows. This should always be your example reference target when making future changes: + +- Example Plan: The final target will eventually be a stripped-down version that looks like the document that follows. This should always be your example reference target when making future changes: `vscode-azurefunctions/test/nightly/scenarios/testScenarios/durable/dts/DurableDTS.plan.md` ## Steps 1. Ask the user for a test plan if one was not provided. -1. Check that the plan title matches the directory for the plan, use the example reference target for formatting. -1. Ensure the file name matches the example target reference as well. The name should end with `.plan.md`. -1. If there are any non-selected rows, ask to delete them. +1. Check that the plan title matches the containing directory, use the example plan for formatting examples. +1. Ensure the file name matches the example plan reference as well. The name should end with `.plan.md`. +1. If there are any rows that are not selected, confirm and delete them. 1. If the selected column is still there, remove it. -1. Renumber the rows if needed. -1. Warn if there are any workspace projects in the test matrix that don't show up in the bottom test matrix. +1. Re-number the rows if needed. +1. Warn if not all listed workspace projects are referenced in the final test matrix. The final list should only show what is used. ## Additional Instructions - Only perform one step at a time, ask where appropriate. -- Keep text information minimal, don't overload the user with context. +- Keep provided information minimal, don't overload the user with context. diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 7f49a155e..01b21c4d4 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -1,31 +1,3 @@ # Coding Instructions for GitHub Copilot - Never commit or suggest changes to `main.js`. - -## Code Review -The following instructions are only to be applied when performing a code review. - -### Prompt file guide - -**Only apply to files that end in `.prompt.md`** - -- [ ] The prompt has markdown front matter. -- [ ] The prompt has a `mode` field specified of either `agent` or `ask`. -- [ ] The prompt has a `description` field. -- [ ] The `description` field is not empty. -- [ ] The `description` field value is wrapped in single quotes. -- [ ] The file name is lower case, with words separated by hyphens. -- [ ] Encourage the use of `tools`, but it's not required. -- [ ] Strongly encourage the use of `model` to specify the model that the prompt is optimised for. - -### Chat Mode file guide - -**Only apply to files that end in `.agent.md`** - -- [ ] The chat mode has markdown front matter. -- [ ] The chat mode has a `description` field. -- [ ] The `description` field is not empty. -- [ ] The `description` field value is wrapped in single quotes. -- [ ] The file name is lower case, with words separated by hyphens. -- [ ] Encourage the use of `tools`, but it's not required. -- [ ] Strongly encourage the use of `model` to specify the model that the chat mode is optimised for. diff --git a/test/global.test.ts b/test/global.test.ts index 4dbc84b18..c71dcad1e 100644 --- a/test/global.test.ts +++ b/test/global.test.ts @@ -3,8 +3,10 @@ * Licensed under the MIT License. See LICENSE.md in the project root for license information. *--------------------------------------------------------------------------------------------*/ +import { registerAppServiceExtensionVariables } from '@microsoft/vscode-azext-azureappservice'; +import { registerAzureUtilsExtensionVariables } from '@microsoft/vscode-azext-azureutils'; import { createTestActionContext, runWithTestActionContext, TestOutputChannel, TestUserInput } from '@microsoft/vscode-azext-dev'; -import { AzExtFsExtra } from '@microsoft/vscode-azext-utils'; +import { AzExtFsExtra, registerUIExtensionVariables } from '@microsoft/vscode-azext-utils'; import * as os from 'os'; import * as path from 'path'; import * as vscode from 'vscode'; @@ -55,6 +57,10 @@ suiteSetup(async function (this: Mocha.Context): Promise { ext.outputChannel = new TestOutputChannel(); + registerAppServiceExtensionVariables(ext); + registerAzureUtilsExtensionVariables(ext); + registerUIExtensionVariables(ext); + registerOnActionStartHandler(context => { // Use `TestUserInput` by default so we get an error if an unexpected call to `context.ui` occurs, rather than timing out context.ui = new TestUserInput(vscode); diff --git a/test/nightly/scenarios/docs/TestScenarios.md b/test/nightly/scenarios/docs/TestScenarios.md new file mode 100644 index 000000000..e69de29bb diff --git a/test/nightly/scenarios/parallelScenarios.ts b/test/nightly/scenarios/parallelScenarios.ts index 59a8ef78e..fb62cf98a 100644 --- a/test/nightly/scenarios/parallelScenarios.ts +++ b/test/nightly/scenarios/parallelScenarios.ts @@ -16,6 +16,7 @@ export interface AzExtFunctionsParallelTestScenario { title: string; scenario?: Promise; runScenario(): Promise; + only?: boolean; } export function generateParallelScenarios(): AzExtFunctionsParallelTestScenario[] { @@ -23,6 +24,7 @@ export function generateParallelScenarios(): AzExtFunctionsParallelTestScenario[ return { title: scenario.label, runScenario: runTestScenario(scenario), + only: scenario.only, }; }); } diff --git a/test/nightly/scenarios/scenarios.test.ts b/test/nightly/scenarios/scenarios.test.ts index a9c763b79..ed0fcae37 100644 --- a/test/nightly/scenarios/scenarios.test.ts +++ b/test/nightly/scenarios/scenarios.test.ts @@ -20,8 +20,13 @@ suite.only('Scenarios', async function (this: Mocha.Suite) { // Todo: This should probably happen elsewhere? await updateGlobalSetting('groupBy', 'resourceType', 'azureResourceGroups'); - for (const s of testScenarios) { - s.scenario = s.runScenario(); + const onlyTestScenario = testScenarios.find(s => s.only); + if (onlyTestScenario) { + onlyTestScenario.scenario = onlyTestScenario.runScenario(); + } else { + for (const s of testScenarios) { + s.scenario = s.runScenario(); + } } }); diff --git a/test/nightly/scenarios/testScenarios/AzExtFunctionsTestScenario.ts b/test/nightly/scenarios/testScenarios/AzExtFunctionsTestScenario.ts index f0a881072..baa08095b 100644 --- a/test/nightly/scenarios/testScenarios/AzExtFunctionsTestScenario.ts +++ b/test/nightly/scenarios/testScenarios/AzExtFunctionsTestScenario.ts @@ -13,7 +13,7 @@ export interface AzExtFunctionsTestScenario { createAndDeployTests: CreateAndDeployTestCase[]; /** - * Indicates this scenario should be executed exclusively. This should only be used during local development. + * Indicates this scenario should be executed exclusively. This should only be used to aid with local development. */ only?: boolean; } @@ -24,7 +24,7 @@ export interface CreateNewProjectTestCase { postTest?: (context: IActionContext, workspaceFolderPath: string, errMsg?: string) => void | Promise; /** - * Indicates this test case should be executed exclusively. This should only be used during local development. + * Indicates this test case should be executed exclusively. This should only be used to aid with local development. */ only?: boolean; } @@ -41,11 +41,12 @@ export interface CreateAndDeployTestCase { inputs: (string | RegExp)[]; preTest?: (context: IActionContext, functionAppId: string, errMsg?: string) => void | Promise; postTest?: (context: IActionContext, functionAppId: string, errMsg?: string) => void | Promise; + asyncWaitTask?: (context: IActionContext) => Promise; }; resourceGroupToDelete?: string; /** - * Indicates this test case should be executed exclusively. This should only be used during local development. + * Indicates this test case should be executed exclusively. This should only be used to aid with local development. */ only?: boolean; } diff --git a/test/nightly/scenarios/testScenarios/durable/azureStorage/DurableAzureStorage.plan.md b/test/nightly/scenarios/testScenarios/durable/azureStorage/DurableAzureStorage.plan.md index 272c47b1f..6973eb3fe 100644 --- a/test/nightly/scenarios/testScenarios/durable/azureStorage/DurableAzureStorage.plan.md +++ b/test/nightly/scenarios/testScenarios/durable/azureStorage/DurableAzureStorage.plan.md @@ -20,29 +20,29 @@ TBD | No. | Workspace Project | Connection Type | Operating System | Plan Type | Comment | |-----|-------------------|-------------------|------------------|------------------------|-------------| -| 1 | 2 | Managed Identity | Linux | Flex Consumption | | -| 2 | 2 | Managed Identity | Linux | Premium | | -| 3 | 2 | Managed Identity | Windows | Premium | | -| 4 | 2 | Managed Identity | Linux | Consumption (Legacy) | | -| 5 | 2 | Managed Identity | Windows | Consumption (Legacy) | | -| 6 | 2 | Managed Identity | Linux | App Service | | -| 7 | 2 | Managed Identity | Windows | App Service | | -| 8 | 2 | Secrets | Linux | Flex Consumption | | -| 9 | 2 | Secrets | Linux | Premium | | -| 10 | 2 | Secrets | Windows | Premium | | -| 11 | 2 | Secrets | Linux | Consumption (Legacy) | | -| 12 | 2 | Secrets | Windows | Consumption (Legacy) | | -| 13 | 2 | Secrets | Linux | App Service | | -| 14 | 2 | Secrets | Windows | App Service | | -| 15 | 4 | Managed Identity | Linux | Flex Consumption | | -| 16 | 4 | Managed Identity | Linux | Premium | | -| 17 | 4 | Secrets | Linux | Flex Consumption | | -| 18 | 4 | Secrets | Linux | Premium | | -| 19 | 5 | Managed Identity | Linux | Flex Consumption | | -| 20 | 5 | Managed Identity | Linux | Premium | | -| 21 | 5 | Secrets | Linux | Flex Consumption | | -| 22 | 5 | Secrets | Linux | Premium | | -| 23 | 6 | Managed Identity | Linux | Flex Consumption | | -| 24 | 6 | Managed Identity | Linux | Premium | | -| 25 | 6 | Secrets | Linux | Flex Consumption | | -| 26 | 6 | Secrets | Linux | Premium | | +| 1 | 1 | Managed Identity | Linux | Flex Consumption | | +| 2 | 1 | Managed Identity | Linux | Premium | | +| 3 | 1 | Managed Identity | Windows | Premium | | +| 4 | 1 | Managed Identity | Linux | Consumption (Legacy) | | +| 5 | 1 | Managed Identity | Windows | Consumption (Legacy) | | +| 6 | 1 | Managed Identity | Linux | App Service | | +| 7 | 1 | Managed Identity | Windows | App Service | | +| 8 | 1 | Secrets | Linux | Flex Consumption | | +| 9 | 1 | Secrets | Linux | Premium | | +| 10 | 1 | Secrets | Windows | Premium | | +| 11 | 1 | Secrets | Linux | Consumption (Legacy) | | +| 12 | 1 | Secrets | Windows | Consumption (Legacy) | | +| 13 | 1 | Secrets | Linux | App Service | | +| 14 | 1 | Secrets | Windows | App Service | | +| 15 | 2 | Managed Identity | Linux | Flex Consumption | | +| 16 | 2 | Managed Identity | Linux | Premium | | +| 17 | 2 | Secrets | Linux | Flex Consumption | | +| 18 | 2 | Secrets | Linux | Premium | | +| 19 | 3 | Managed Identity | Linux | Flex Consumption | | +| 20 | 3 | Managed Identity | Linux | Premium | | +| 21 | 3 | Secrets | Linux | Flex Consumption | | +| 22 | 3 | Secrets | Linux | Premium | | +| 23 | 4 | Managed Identity | Linux | Flex Consumption | | +| 24 | 4 | Managed Identity | Linux | Premium | | +| 25 | 4 | Secrets | Linux | Flex Consumption | | +| 26 | 4 | Secrets | Linux | Premium | | diff --git a/test/nightly/scenarios/testScenarios/durable/azureStorage/azureStorageScenarios.ts b/test/nightly/scenarios/testScenarios/durable/azureStorage/azureStorageScenarios.ts new file mode 100644 index 000000000..9b3bf01bb --- /dev/null +++ b/test/nightly/scenarios/testScenarios/durable/azureStorage/azureStorageScenarios.ts @@ -0,0 +1,13 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { type AzExtFunctionsTestScenario } from '../../AzExtFunctionsTestScenario'; +import { generateTSNodeScenario } from './azureStorageTSNodeScenario'; + +export function generateDurableAzureStorageScenarios(): AzExtFunctionsTestScenario[] { + return [ + generateTSNodeScenario(), + ]; +} diff --git a/test/nightly/scenarios/testScenarios/durable/azureStorage/azureStorageTSNodeScenario.ts b/test/nightly/scenarios/testScenarios/durable/azureStorage/azureStorageTSNodeScenario.ts new file mode 100644 index 000000000..49d515b5d --- /dev/null +++ b/test/nightly/scenarios/testScenarios/durable/azureStorage/azureStorageTSNodeScenario.ts @@ -0,0 +1,65 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { DurableBackend, getRandomAlphanumericString } from "../../../../../../extension.bundle"; +import { ConnectionType, createFunctionAppUtils, CreateMode, OperatingSystem, PlanType } from "../../../../../utils/createFunctionAppUtils"; +import { deployFunctionAppUtils } from "../../../../../utils/deployFunctionAppUtils"; +import { type AzExtFunctionsTestScenario, type CreateAndDeployTestCase } from "../../AzExtFunctionsTestScenario"; + +export function generateTSNodeScenario(): AzExtFunctionsTestScenario { + const folderName: string = 'scenarios-durable-azurestorage-tsnode'; + + return { + only: true, + label: 'durable-azurestorage-tsnode', + folderName, + createNewProjectTest: { + label: 'create-new-project', + inputs: [ + /TypeScript/i, + /v4/i, + /Durable Functions Orchestrator/i, + /Azure Storage/i, + 'durableHello1', + ], + }, + createAndDeployTests: [ + generateCreateAndDeployTest(folderName, CreateMode.Basic, ConnectionType.ManagedIdentity, OperatingSystem.Linux, PlanType.FlexConsumption), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, ConnectionType.ManagedIdentity, OperatingSystem.Linux, PlanType.Premium), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, ConnectionType.ManagedIdentity, OperatingSystem.Windows, PlanType.Premium), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, ConnectionType.ManagedIdentity, OperatingSystem.Linux, PlanType.LegacyConsumption), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, ConnectionType.ManagedIdentity, OperatingSystem.Windows, PlanType.LegacyConsumption), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, ConnectionType.ManagedIdentity, OperatingSystem.Linux, PlanType.AppService), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, ConnectionType.ManagedIdentity, OperatingSystem.Windows, PlanType.AppService), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, ConnectionType.Secrets, OperatingSystem.Linux, PlanType.FlexConsumption), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, ConnectionType.Secrets, OperatingSystem.Linux, PlanType.Premium), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, ConnectionType.Secrets, OperatingSystem.Windows, PlanType.Premium), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, ConnectionType.Secrets, OperatingSystem.Linux, PlanType.LegacyConsumption), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, ConnectionType.Secrets, OperatingSystem.Windows, PlanType.LegacyConsumption), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, ConnectionType.Secrets, OperatingSystem.Linux, PlanType.AppService), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, ConnectionType.Secrets, OperatingSystem.Windows, PlanType.AppService), + ], + }; +} + +function generateCreateAndDeployTest(folderName: string, createMode: CreateMode, connection: ConnectionType, os: OperatingSystem, plan: PlanType): CreateAndDeployTestCase { + const appName: string = getRandomAlphanumericString(); + const description: string = `${createMode}-${connection}-${os}-${plan}`; + + return { + createFunctionApp: { + label: `create-function-app-${description}`, + mode: createMode, + inputs: createMode === CreateMode.Basic ? + createFunctionAppUtils.generateBasicCreateInputs(appName, folderName, connection) : + createFunctionAppUtils.generateAdvancedCreateInputs(appName, folderName, connection, os, plan), + }, + deployFunctionApp: { + label: `deploy-function-app-${description}`, + inputs: deployFunctionAppUtils.generateDurableDeployInputs(appName, DurableBackend.DTS), + }, + resourceGroupToDelete: appName, + }; +} diff --git a/test/nightly/scenarios/testScenarios/durable/dts/DurableDTS.plan.md b/test/nightly/scenarios/testScenarios/durable/dts/DurableDTS.plan.md index a54c60687..5518bd086 100644 --- a/test/nightly/scenarios/testScenarios/durable/dts/DurableDTS.plan.md +++ b/test/nightly/scenarios/testScenarios/durable/dts/DurableDTS.plan.md @@ -1,4 +1,4 @@ -# Durable Task Scheduler - Test Plan +# Durable Task Scheduler (DTS) Test Plan ## I. Create New Project / Create Function diff --git a/test/nightly/scenarios/testScenarios/durable/dts/dtsScenarios.ts b/test/nightly/scenarios/testScenarios/durable/dts/dtsScenarios.ts index 67842b776..c11889e0b 100644 --- a/test/nightly/scenarios/testScenarios/durable/dts/dtsScenarios.ts +++ b/test/nightly/scenarios/testScenarios/durable/dts/dtsScenarios.ts @@ -6,7 +6,7 @@ import { type AzExtFunctionsTestScenario } from '../../AzExtFunctionsTestScenario'; import { generateTSNodeScenario } from './dtsTSNodeScenario'; -export function generateDTSScenarios(): AzExtFunctionsTestScenario[] { +export function generateDurableDTSScenarios(): AzExtFunctionsTestScenario[] { return [ generateTSNodeScenario(), ]; diff --git a/test/nightly/scenarios/testScenarios/durable/dts/dtsTSNodeScenario.ts b/test/nightly/scenarios/testScenarios/durable/dts/dtsTSNodeScenario.ts index 4d0564b02..2acad3cf5 100644 --- a/test/nightly/scenarios/testScenarios/durable/dts/dtsTSNodeScenario.ts +++ b/test/nightly/scenarios/testScenarios/durable/dts/dtsTSNodeScenario.ts @@ -9,10 +9,10 @@ import { deployFunctionAppUtils } from "../../../../../utils/deployFunctionAppUt import { type AzExtFunctionsTestScenario, type CreateAndDeployTestCase } from "../../AzExtFunctionsTestScenario"; export function generateTSNodeScenario(): AzExtFunctionsTestScenario { - const folderName: string = 'scenarios-dts-tsnode'; + const folderName: string = 'scenarios-durable-dts-tsnode'; return { - label: 'dts-tsnode', + label: 'durable-dts-tsnode', folderName, createNewProjectTest: { label: 'create-new-project', @@ -25,12 +25,12 @@ export function generateTSNodeScenario(): AzExtFunctionsTestScenario { ], }, createAndDeployTests: [ - /** 1. */ generateCreateAndDeployTest(folderName, CreateMode.Basic, ConnectionType.ManagedIdentity, OperatingSystem.Linux, PlanType.FlexConsumption), - /** 2. */ generateCreateAndDeployTest(folderName, CreateMode.Advanced, ConnectionType.ManagedIdentity, OperatingSystem.Linux, PlanType.Premium), - /** 3. */ generateCreateAndDeployTest(folderName, CreateMode.Advanced, ConnectionType.ManagedIdentity, OperatingSystem.Windows, PlanType.Premium), - /** 4. */ generateCreateAndDeployTest(folderName, CreateMode.Advanced, ConnectionType.Secrets, OperatingSystem.Linux, PlanType.FlexConsumption), - /** 5. */ generateCreateAndDeployTest(folderName, CreateMode.Advanced, ConnectionType.Secrets, OperatingSystem.Linux, PlanType.Premium), - /** 6. */ generateCreateAndDeployTest(folderName, CreateMode.Advanced, ConnectionType.Secrets, OperatingSystem.Windows, PlanType.Premium), + generateCreateAndDeployTest(folderName, CreateMode.Basic, ConnectionType.ManagedIdentity, OperatingSystem.Linux, PlanType.FlexConsumption), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, ConnectionType.ManagedIdentity, OperatingSystem.Linux, PlanType.Premium), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, ConnectionType.ManagedIdentity, OperatingSystem.Windows, PlanType.Premium), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, ConnectionType.Secrets, OperatingSystem.Linux, PlanType.FlexConsumption), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, ConnectionType.Secrets, OperatingSystem.Linux, PlanType.Premium), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, ConnectionType.Secrets, OperatingSystem.Windows, PlanType.Premium), ], }; } diff --git a/test/nightly/scenarios/testScenarios/testScenarios.ts b/test/nightly/scenarios/testScenarios/testScenarios.ts index ea14ca24b..a6125c74c 100644 --- a/test/nightly/scenarios/testScenarios/testScenarios.ts +++ b/test/nightly/scenarios/testScenarios/testScenarios.ts @@ -4,11 +4,13 @@ *--------------------------------------------------------------------------------------------*/ import { type AzExtFunctionsTestScenario } from "./AzExtFunctionsTestScenario"; -import { generateDTSScenarios } from "./durable/dts/dtsScenarios"; +import { generateDurableAzureStorageScenarios } from "./durable/azureStorage/azureStorageScenarios"; +import { generateDurableDTSScenarios } from "./durable/dts/dtsScenarios"; export function generateTestScenarios(): AzExtFunctionsTestScenario[] { const testScenarios: AzExtFunctionsTestScenario[] = [ - ...generateDTSScenarios(), + ...generateDurableAzureStorageScenarios(), + ...generateDurableDTSScenarios(), ]; return testScenarios; } diff --git a/test/test.code-workspace b/test/test.code-workspace index 1dc010f0e..6ce4988b6 100644 --- a/test/test.code-workspace +++ b/test/test.code-workspace @@ -31,19 +31,23 @@ "path": "../testWorkspace/9" }, { - "name": "scenarios-dts-tsnode", + "name": "scenarios-durable-azurestorage-tsnode", + "path": "../testWorkspace/scenarios/durable-azureStorage/tsnode" + }, + { + "name": "scenarios-durable-dts-tsnode", "path": "../testWorkspace/scenarios/durable-dts/tsnode" }, { - "name": "scenarios-dts-python", + "name": "scenarios-durable-dts-python", "path": "../testWorkspace/scenarios/durable-dts/python" }, { - "name": "scenarios-dts-dotnet-isolated", + "name": "scenarios-durable-dts-dotnet-isolated", "path": "../testWorkspace/scenarios/durable-dts/dotnet-isolated" }, { - "name": "scenarios-dts-dotnet-inproc", + "name": "scenarios-durable-dts-dotnet-inproc", "path": "../testWorkspace/scenarios/durable-dts/dotnet-inproc" } ], diff --git a/test/utils/deployFunctionAppUtils.ts b/test/utils/deployFunctionAppUtils.ts index 1f72029ba..eae75f86e 100644 --- a/test/utils/deployFunctionAppUtils.ts +++ b/test/utils/deployFunctionAppUtils.ts @@ -27,7 +27,9 @@ export namespace deployFunctionAppUtils { return []; case DurableBackend.Storage: default: - return []; + return [ + 'Deploy', + ]; } } } diff --git a/tsconfig.json b/tsconfig.json index 02d9a91c8..4b9d94ad0 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -24,6 +24,7 @@ "exclude": [ "node_modules", ".vscode-test", - "gulpfile.ts" + "gulpfile.ts", + "testWorkspace" ] } From 68fe0655c5f49f8f261de7ad9d8d5708216a9585 Mon Sep 17 00:00:00 2001 From: MicroFish91 <40250218+MicroFish91@users.noreply.github.com> Date: Tue, 30 Dec 2025 15:48:28 -0800 Subject: [PATCH 09/27] Working with scenarios tracker --- test/nightly/createProjectAndDeploy.test.ts | 2 +- test/nightly/functionAppOperations.test.ts | 2 +- test/nightly/global.nightly.test.ts | 10 +- test/nightly/scenarios/ScenariosTracker.ts | 209 ++++++++++++++++++ test/nightly/scenarios/parallelScenarios.ts | 58 +++-- test/nightly/scenarios/scenarios.test.ts | 1 + .../AzExtFunctionsTestScenario.ts | 2 +- .../azureStorageTSNodeScenario.ts | 10 +- .../durable/dts/dtsTSNodeScenario.ts | 2 +- 9 files changed, 269 insertions(+), 27 deletions(-) create mode 100644 test/nightly/scenarios/ScenariosTracker.ts diff --git a/test/nightly/createProjectAndDeploy.test.ts b/test/nightly/createProjectAndDeploy.test.ts index 169677d80..dfddfef67 100644 --- a/test/nightly/createProjectAndDeploy.test.ts +++ b/test/nightly/createProjectAndDeploy.test.ts @@ -90,7 +90,7 @@ async function testCreateProjectAndDeploy(options: ICreateProjectAndDeployOption // TODO: investigate why our SDK calls are throwing errors when app name is over ~12 characters // https://github.com/microsoft/vscode-azurefunctions/issues/4368 const appName: string = 'f' + getRandomAlphanumericString(); - resourceGroupsToDelete.push(appName); + resourceGroupsToDelete.add(appName); await runWithTestActionContext('deploy', async context => { options.deployInputs = options.deployInputs || []; await context.ui.runWithInputs([testWorkspacePath, /create new function app/i, appName, getRotatingLocation(), ...options.deployInputs], async () => { diff --git a/test/nightly/functionAppOperations.test.ts b/test/nightly/functionAppOperations.test.ts index 49681d6c4..903308a56 100644 --- a/test/nightly/functionAppOperations.test.ts +++ b/test/nightly/functionAppOperations.test.ts @@ -33,7 +33,7 @@ suite('Function App Operations', function (this: Mocha.Suite): void { appName = getRandomHexString(); app2Name = getRandomHexString(); rgName = getRandomHexString(); - resourceGroupsToDelete.push(rgName); + resourceGroupsToDelete.add(rgName); saName = getRandomHexString().toLowerCase(); // storage account must have lower case name aiName = getRandomHexString(); location = getRotatingLocation(); diff --git a/test/nightly/global.nightly.test.ts b/test/nightly/global.nightly.test.ts index faef3621f..11ff96076 100644 --- a/test/nightly/global.nightly.test.ts +++ b/test/nightly/global.nightly.test.ts @@ -12,10 +12,12 @@ import { longRunningTestsEnabled } from '../global.test'; import { createSubscriptionContext, subscriptionExperience, type ISubscriptionContext } from '@microsoft/vscode-azext-utils'; import { type AzureSubscription } from '@microsoft/vscode-azureresources-api'; import * as vscode from 'vscode'; +import { ScenariosTracker } from './scenarios/ScenariosTracker'; export let testClient: WebSiteManagementClient; export let subscriptionContext: ISubscriptionContext; -export const resourceGroupsToDelete: string[] = []; +export const resourceGroupsToDelete: Set = new Set(); +export const scenariosTracker = new ScenariosTracker(); // Runs before all nightly tests suiteSetup(async function (this: Mocha.Context): Promise { @@ -42,8 +44,12 @@ suiteTeardown(async function (this: Mocha.Context): Promise { }); async function deleteResourceGroups(): Promise { + const report = scenariosTracker.report(); + console.log(report); + const rgClient: ResourceManagementClient = createAzureClient([await createTestActionContext(), subscriptionContext], ResourceManagementClient); - await Promise.all(resourceGroupsToDelete.map(async resourceGroup => { + + await Promise.allSettled(Array.from(resourceGroupsToDelete).map(async resourceGroup => { if ((await rgClient.resourceGroups.checkExistence(resourceGroup)).body) { console.log(`Started delete of resource group "${resourceGroup}"...`); await rgClient.resourceGroups.beginDeleteAndWait(resourceGroup); diff --git a/test/nightly/scenarios/ScenariosTracker.ts b/test/nightly/scenarios/ScenariosTracker.ts new file mode 100644 index 000000000..c2ac1f699 --- /dev/null +++ b/test/nightly/scenarios/ScenariosTracker.ts @@ -0,0 +1,209 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { nonNullProp, nonNullValue } from "@microsoft/vscode-azext-utils"; + + +export class ScenariosTracker { + private scenarioStatuses: Map = new Map(); + + initScenario(scenarioLabel: string): void { + if (this.scenarioStatuses.has(scenarioLabel)) { + return; + } + this.scenarioStatuses.set(scenarioLabel, { label: scenarioLabel }); + } + + startCreateNewProject(scenarioLabel: string, createNewProjectLabel: string): void { + const scenarioStatus = nonNullValue(this.scenarioStatuses.get(scenarioLabel)); + scenarioStatus.createNewProject = { + label: createNewProjectLabel, + passed: false, + }; + this.scenarioStatuses.set(scenarioLabel, scenarioStatus); + } + + passCreateNewProject(scenarioLabel: string): void { + const scenarioStatus = nonNullValue(this.scenarioStatuses.get(scenarioLabel)); + const createNewProjectStatus = nonNullProp(scenarioStatus, 'createNewProject'); + createNewProjectStatus.passed = true; + this.scenarioStatuses.set(scenarioLabel, scenarioStatus); + } + + failCreateNewProject(scenarioLabel: string, error: string): void { + const scenarioStatus = nonNullValue(this.scenarioStatuses.get(scenarioLabel)); + const createNewProjectStatus = nonNullProp(scenarioStatus, 'createNewProject'); + createNewProjectStatus.passed = false; + createNewProjectStatus.error = error; + this.scenarioStatuses.set(scenarioLabel, scenarioStatus); + } + + initCreateAndDeployTest(scenarioLabel: string): number { + const scenarioStatus = nonNullValue(this.scenarioStatuses.get(scenarioLabel)); + scenarioStatus.createAndDeployTests ??= []; + + const id: number = scenarioStatus.createAndDeployTests.length; + scenarioStatus.createAndDeployTests.push({}); + return id; + } + + startCreateFunctionApp(scenarioLabel: string, createAndDeployTestId: number, createLabel: string): void { + const scenarioStatus = nonNullValue(this.scenarioStatuses.get(scenarioLabel)); + scenarioStatus.createAndDeployTests ??= []; + + const createAndDeployTest = nonNullValue(scenarioStatus.createAndDeployTests[createAndDeployTestId]); + createAndDeployTest.createFunctionApp = { + label: createLabel, + passed: false, + }; + + this.scenarioStatuses.set(scenarioLabel, scenarioStatus); + } + + passCreateFunctionApp(scenarioLabel: string, createAndDeployTestId: number): void { + const scenarioStatus = nonNullValue(this.scenarioStatuses.get(scenarioLabel)); + scenarioStatus.createAndDeployTests ??= []; + + const createAndDeployTest = nonNullValue(scenarioStatus.createAndDeployTests[createAndDeployTestId]); + const createFunctionAppTest = nonNullValue(createAndDeployTest.createFunctionApp); + createFunctionAppTest.passed = true; + + this.scenarioStatuses.set(scenarioLabel, scenarioStatus); + } + + failCreateFunctionApp(scenarioLabel: string, createAndDeployTestId: number, error: string): void { + const scenarioStatus = nonNullValue(this.scenarioStatuses.get(scenarioLabel)); + scenarioStatus.createAndDeployTests ??= []; + + const createAndDeployTest = nonNullValue(scenarioStatus.createAndDeployTests[createAndDeployTestId]); + const createFunctionAppTest = nonNullValue(createAndDeployTest.createFunctionApp); + createFunctionAppTest.passed = false; + createFunctionAppTest.error = error; + + this.scenarioStatuses.set(scenarioLabel, scenarioStatus); + } + + startDeployFunctionApp(scenarioLabel: string, createAndDeployTestId: number, deployLabel: string): void { + const scenarioStatus = nonNullValue(this.scenarioStatuses.get(scenarioLabel)); + scenarioStatus.createAndDeployTests ??= []; + + const createAndDeployTest = nonNullValue(scenarioStatus.createAndDeployTests[createAndDeployTestId]); + createAndDeployTest.deployFunctionApp = { + label: deployLabel, + passed: false, + }; + + this.scenarioStatuses.set(scenarioLabel, scenarioStatus); + } + + passDeployFunctionApp(scenarioLabel: string, createAndDeployTestId: number): void { + const scenarioStatus = nonNullValue(this.scenarioStatuses.get(scenarioLabel)); + scenarioStatus.createAndDeployTests ??= []; + + const createAndDeployTest = nonNullValue(scenarioStatus.createAndDeployTests[createAndDeployTestId]); + const deployFunctionAppTest = nonNullValue(createAndDeployTest.deployFunctionApp); + deployFunctionAppTest.passed = true; + + this.scenarioStatuses.set(scenarioLabel, scenarioStatus); + } + + failDeployFunctionApp(scenarioLabel: string, createAndDeployTestId: number, error: string): void { + const scenarioStatus = nonNullValue(this.scenarioStatuses.get(scenarioLabel)); + scenarioStatus.createAndDeployTests ??= []; + + const createAndDeployTest = nonNullValue(scenarioStatus.createAndDeployTests[createAndDeployTestId]); + const deployFunctionAppTest = nonNullValue(createAndDeployTest.deployFunctionApp); + deployFunctionAppTest.passed = false; + deployFunctionAppTest.error = error; + + this.scenarioStatuses.set(scenarioLabel, scenarioStatus); + } + + report(): string { + if (!this.scenarioStatuses.size) { + return 'No test scenarios recorded.'; + } + + const lines: string[] = []; + const passIcon = '✅'; + const failIcon = '❌'; + const dashIcon = '-'; + const maxErrorLength = 200; + + const getStatusIcon = (passed: boolean | undefined): string => { + if (passed === undefined) { + return dashIcon; + } + return passed ? passIcon : failIcon; + }; + + const truncateError = (error: string | undefined): string => { + if (!error) { + return ''; + } + if (error.length <= maxErrorLength) { + return error; + } + return error.substring(0, maxErrorLength) + '...'; + }; + + lines.push('| # | Scenario | Test | Status | Error |'); + lines.push('|---|----------|------|--------|-------|'); + + let rowNum = 1; + for (const [, scenario] of this.scenarioStatuses) { + const scenarioLabel = scenario.label; + + // Report createNewProject test + if (scenario.createNewProject) { + const status = getStatusIcon(scenario.createNewProject.passed); + const error = truncateError(scenario.createNewProject.error); + const errorCell = error ? ` | ${error} |` : ''; + lines.push(`| ${rowNum++} | ${scenarioLabel} | ${scenario.createNewProject.label} | ${status}${errorCell}`); + } + + // Report createAndDeployTests + if (scenario.createAndDeployTests) { + for (const test of scenario.createAndDeployTests) { + if (test.createFunctionApp) { + const status = getStatusIcon(test.createFunctionApp.passed); + const error = truncateError(test.createFunctionApp.error); + const errorCell = error ? ` | ${error} |` : ''; + lines.push(`| ${rowNum++} | ${scenarioLabel} | ${test.createFunctionApp.label} | ${status}${errorCell}`); + } + if (test.deployFunctionApp) { + const status = getStatusIcon(test.deployFunctionApp.passed); + const error = truncateError(test.deployFunctionApp.error); + const errorCell = error ? ` | ${error} |` : ''; + lines.push(`| ${rowNum++} | ${scenarioLabel} | ${test.deployFunctionApp.label} | ${status}${errorCell}`); + } + } + } + } + + return lines.join('\n'); + } +} + +type ScenarioStatus = { + label: string; + createNewProject?: { + label: string; + passed?: boolean; + error?: string; + }; + createAndDeployTests?: { + createFunctionApp?: { + label: string; + passed?: boolean; + error?: string; + }; + deployFunctionApp?: { + label: string; + passed?: boolean; + error?: string; + }; + }[]; +} diff --git a/test/nightly/scenarios/parallelScenarios.ts b/test/nightly/scenarios/parallelScenarios.ts index fb62cf98a..14c2ef7ef 100644 --- a/test/nightly/scenarios/parallelScenarios.ts +++ b/test/nightly/scenarios/parallelScenarios.ts @@ -4,11 +4,12 @@ *--------------------------------------------------------------------------------------------*/ import { runWithTestActionContext } from '@microsoft/vscode-azext-dev'; -import { AzExtFsExtra } from '@microsoft/vscode-azext-utils'; +import { AzExtFsExtra, parseError } from '@microsoft/vscode-azext-utils'; import * as assert from 'assert'; import { workspace, type Uri, type WorkspaceFolder } from 'vscode'; -import { createFunctionApp, createFunctionAppAdvanced, createNewProjectInternal, deployProductionSlotByFunctionAppId, ext } from '../../../extension.bundle'; +import { createFunctionApp, createFunctionAppAdvanced, createNewProjectInternal, deployProductionSlotByFunctionAppId } from '../../../extension.bundle'; import { CreateMode } from '../../utils/createFunctionAppUtils'; +import { resourceGroupsToDelete, scenariosTracker } from '../global.nightly.test'; import { type AzExtFunctionsTestScenario, type CreateAndDeployTestCase } from './testScenarios/AzExtFunctionsTestScenario'; import { generateTestScenarios } from './testScenarios/testScenarios'; @@ -38,12 +39,19 @@ function runTestScenario(scenario: AzExtFunctionsTestScenario): AzExtFunctionsPa await cleanTestFolder(rootFolder); // 1. Create shared workspace project - ext.outputChannel.appendLog(`[[[ *** ${scenario.label} - ${scenario.createNewProjectTest.label} *** ]]]`); + scenariosTracker.initScenario(scenario.label); + scenariosTracker.startCreateNewProject(scenario.label, scenario.createNewProjectTest.label); await runWithTestActionContext('scenario.createNewProject', async context => { await context.ui.runWithInputs(scenario.createNewProjectTest.inputs, async () => { - await createNewProjectInternal(context, { folderPath: rootFolder.uri.fsPath }); - await scenario.createNewProjectTest.postTest?.(context, rootFolder.uri.fsPath, ''); + try { + await createNewProjectInternal(context, { folderPath: rootFolder.uri.fsPath }); + await scenario.createNewProjectTest.postTest?.(context, rootFolder.uri.fsPath, ''); + scenariosTracker.passCreateNewProject(scenario.label); + } catch (err) { + scenariosTracker.failCreateNewProject(scenario.label, (err as Error).message ?? parseError(err).message); + throw err; + } }); }); @@ -61,30 +69,48 @@ function runTestScenario(scenario: AzExtFunctionsTestScenario): AzExtFunctionsPa } async function startCreateAndDeployTest(scenarioLabel: string, rootFolder: WorkspaceFolder, test: CreateAndDeployTestCase): Promise { + const scenarioTestTrackerId: number = scenariosTracker.initCreateAndDeployTest(scenarioLabel); + + for (const rg of test.resourceGroupsToDelete ?? []) { + resourceGroupsToDelete.add(rg); + } + // 3. Create function app - ext.outputChannel.appendLog(`[[[ *** ${scenarioLabel} - ${test.createFunctionApp.label} *** ]]]`); + scenariosTracker.startCreateFunctionApp(scenarioLabel, scenarioTestTrackerId, test.createFunctionApp.label); let functionAppId: string; await runWithTestActionContext('scenario.createFunctionApp', async context => { await context.ui.runWithInputs(test.createFunctionApp.inputs, async () => { - if (test.createFunctionApp.mode === CreateMode.Basic) { - functionAppId = await createFunctionApp(context); - } else { - functionAppId = await createFunctionAppAdvanced(context); + try { + if (test.createFunctionApp.mode === CreateMode.Basic) { + functionAppId = await createFunctionApp(context); + } else { + functionAppId = await createFunctionAppAdvanced(context); + } + + assert.ok(functionAppId, 'Failed to create function app.'); + await test.createFunctionApp.postTest?.(context, functionAppId, ''); + scenariosTracker.passCreateFunctionApp(scenarioLabel, scenarioTestTrackerId); + } catch (err) { + scenariosTracker.failCreateFunctionApp(scenarioLabel, scenarioTestTrackerId, (err as Error).message ?? parseError(err).message); + throw err; } - - assert.ok(functionAppId, 'Failed to create function app.'); - test.createFunctionApp.postTest?.(context, functionAppId, ''); }); }); // 4. Deploy function app - ext.outputChannel.appendLog(`[[[ *** ${scenarioLabel} - ${test.deployFunctionApp.label} *** ]]]`); + scenariosTracker.startDeployFunctionApp(scenarioLabel, scenarioTestTrackerId, test.deployFunctionApp.label); await runWithTestActionContext('scenario.deploy', async context => { await context.ui.runWithInputs(test.deployFunctionApp.inputs, async () => { - await deployProductionSlotByFunctionAppId(context, functionAppId, rootFolder?.uri); - await test.deployFunctionApp.postTest?.(context, functionAppId, ''); + try { + await deployProductionSlotByFunctionAppId(context, functionAppId, rootFolder?.uri); + await test.deployFunctionApp.postTest?.(context, functionAppId, ''); + scenariosTracker.passDeployFunctionApp(scenarioLabel, scenarioTestTrackerId); + } catch (err) { + scenariosTracker.failDeployFunctionApp(scenarioLabel, scenarioTestTrackerId, (err as Error).message ?? parseError(err).message); + throw err; + } }); }); } diff --git a/test/nightly/scenarios/scenarios.test.ts b/test/nightly/scenarios/scenarios.test.ts index ed0fcae37..3c2c78fb8 100644 --- a/test/nightly/scenarios/scenarios.test.ts +++ b/test/nightly/scenarios/scenarios.test.ts @@ -21,6 +21,7 @@ suite.only('Scenarios', async function (this: Mocha.Suite) { await updateGlobalSetting('groupBy', 'resourceType', 'azureResourceGroups'); const onlyTestScenario = testScenarios.find(s => s.only); + if (onlyTestScenario) { onlyTestScenario.scenario = onlyTestScenario.runScenario(); } else { diff --git a/test/nightly/scenarios/testScenarios/AzExtFunctionsTestScenario.ts b/test/nightly/scenarios/testScenarios/AzExtFunctionsTestScenario.ts index baa08095b..ab9aa8f74 100644 --- a/test/nightly/scenarios/testScenarios/AzExtFunctionsTestScenario.ts +++ b/test/nightly/scenarios/testScenarios/AzExtFunctionsTestScenario.ts @@ -43,7 +43,7 @@ export interface CreateAndDeployTestCase { postTest?: (context: IActionContext, functionAppId: string, errMsg?: string) => void | Promise; asyncWaitTask?: (context: IActionContext) => Promise; }; - resourceGroupToDelete?: string; + resourceGroupsToDelete?: string[]; /** * Indicates this test case should be executed exclusively. This should only be used to aid with local development. diff --git a/test/nightly/scenarios/testScenarios/durable/azureStorage/azureStorageTSNodeScenario.ts b/test/nightly/scenarios/testScenarios/durable/azureStorage/azureStorageTSNodeScenario.ts index 49d515b5d..990680cd3 100644 --- a/test/nightly/scenarios/testScenarios/durable/azureStorage/azureStorageTSNodeScenario.ts +++ b/test/nightly/scenarios/testScenarios/durable/azureStorage/azureStorageTSNodeScenario.ts @@ -18,7 +18,7 @@ export function generateTSNodeScenario(): AzExtFunctionsTestScenario { createNewProjectTest: { label: 'create-new-project', inputs: [ - /TypeScript/i, + /JavaScript/i, /v4/i, /Durable Functions Orchestrator/i, /Azure Storage/i, @@ -50,16 +50,16 @@ function generateCreateAndDeployTest(folderName: string, createMode: CreateMode, return { createFunctionApp: { - label: `create-function-app-${description}`, + label: `create-function-app | ${description}`, mode: createMode, inputs: createMode === CreateMode.Basic ? createFunctionAppUtils.generateBasicCreateInputs(appName, folderName, connection) : createFunctionAppUtils.generateAdvancedCreateInputs(appName, folderName, connection, os, plan), }, deployFunctionApp: { - label: `deploy-function-app-${description}`, - inputs: deployFunctionAppUtils.generateDurableDeployInputs(appName, DurableBackend.DTS), + label: `deploy-function-app | ${description}`, + inputs: deployFunctionAppUtils.generateDurableDeployInputs(appName, DurableBackend.Storage), }, - resourceGroupToDelete: appName, + resourceGroupsToDelete: [appName], }; } diff --git a/test/nightly/scenarios/testScenarios/durable/dts/dtsTSNodeScenario.ts b/test/nightly/scenarios/testScenarios/durable/dts/dtsTSNodeScenario.ts index 2acad3cf5..ce6961a6f 100644 --- a/test/nightly/scenarios/testScenarios/durable/dts/dtsTSNodeScenario.ts +++ b/test/nightly/scenarios/testScenarios/durable/dts/dtsTSNodeScenario.ts @@ -51,6 +51,6 @@ function generateCreateAndDeployTest(folderName: string, createMode: CreateMode, label: `deploy-function-app-${description}`, inputs: deployFunctionAppUtils.generateDurableDeployInputs(appName, DurableBackend.DTS), }, - resourceGroupToDelete: appName, + resourceGroupsToDelete: [appName], }; } From 426f6ab9aa94154f76192fd20211adee4292f0ee Mon Sep 17 00:00:00 2001 From: MicroFish91 <40250218+MicroFish91@users.noreply.github.com> Date: Fri, 2 Jan 2026 14:22:21 -0800 Subject: [PATCH 10/27] Fix some issues, add verify deployment --- test/constants.ts | 12 ++++ .../docs/ScenarioTestCombinations.md | 4 +- test/nightly/scenarios/parallelScenarios.ts | 4 +- .../AzExtFunctionsTestScenario.ts | 6 +- .../azureStorage/DurableAzureStorage.plan.md | 2 +- .../azureStorageJSNodeScenario.ts | 45 +++++++++++++ .../azureStorage/azureStorageScenarios.ts | 6 +- .../azureStorageTSNodeScenario.ts | 65 ------------------- .../durable/azureStorage/pythonScenario.ts | 36 ++++++++++ .../durable/dts/DurableDTS.plan.md | 2 +- .../durable/dts/dtsJSNodeScenario.ts | 37 +++++++++++ .../testScenarios/durable/dts/dtsScenarios.ts | 4 +- .../durable/dts/dtsTSNodeScenario.ts | 56 ---------------- .../durable/generateCreateAndDeployTest.ts | 63 ++++++++++++++++++ .../scenarios/testScenarios/testScenarios.ts | 3 +- test/test.code-workspace | 12 ++-- test/utils/createFunctionAppUtils.ts | 49 +++++++++----- 17 files changed, 251 insertions(+), 155 deletions(-) create mode 100644 test/constants.ts create mode 100644 test/nightly/scenarios/testScenarios/durable/azureStorage/azureStorageJSNodeScenario.ts delete mode 100644 test/nightly/scenarios/testScenarios/durable/azureStorage/azureStorageTSNodeScenario.ts create mode 100644 test/nightly/scenarios/testScenarios/durable/azureStorage/pythonScenario.ts create mode 100644 test/nightly/scenarios/testScenarios/durable/dts/dtsJSNodeScenario.ts delete mode 100644 test/nightly/scenarios/testScenarios/durable/dts/dtsTSNodeScenario.ts create mode 100644 test/nightly/scenarios/testScenarios/durable/generateCreateAndDeployTest.ts diff --git a/test/constants.ts b/test/constants.ts new file mode 100644 index 000000000..52521c518 --- /dev/null +++ b/test/constants.ts @@ -0,0 +1,12 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +// Sometimes this is "orchestrator" or "orchestration" depending on the template feed +export const durableOrchestratorPick: RegExp = /Durable Functions Orch/i; +export const durableOrchestratorName: string = 'durableHello'; + +export const locationDefaultPick: RegExp = /West US 2/i; +export const pythonDefaultPick: RegExp = /Python(\s)?3\.12/i; +export const nodeDefaultPick: RegExp = /Node\.js(\s)?22/i; diff --git a/test/nightly/scenarios/docs/ScenarioTestCombinations.md b/test/nightly/scenarios/docs/ScenarioTestCombinations.md index d497b22b6..3bf552544 100644 --- a/test/nightly/scenarios/docs/ScenarioTestCombinations.md +++ b/test/nightly/scenarios/docs/ScenarioTestCombinations.md @@ -6,8 +6,8 @@ | No. | Language | Runtime | Programming Model | Comment | Selected | |-----|----------|---------|-------------------|------------------------------------|-------------| -| 1 | TS | Node | v3 | Skip unless special requirements | | -| 2 | TS | Node | v4 | | | +| 1 | JS | Node | v3 | Skip unless special requirements | | +| 2 | JS | Node | v4 | | | | 3 | Python | Python | v1 | Skip unless special requirements | | | 4 | Python | Python | v2 | | | | 5 | C# | .NET | isolated | | | diff --git a/test/nightly/scenarios/parallelScenarios.ts b/test/nightly/scenarios/parallelScenarios.ts index 14c2ef7ef..a97e06b73 100644 --- a/test/nightly/scenarios/parallelScenarios.ts +++ b/test/nightly/scenarios/parallelScenarios.ts @@ -89,7 +89,7 @@ async function startCreateAndDeployTest(scenarioLabel: string, rootFolder: Works } assert.ok(functionAppId, 'Failed to create function app.'); - await test.createFunctionApp.postTest?.(context, functionAppId, ''); + await test.createFunctionApp.postTest?.(context, functionAppId); scenariosTracker.passCreateFunctionApp(scenarioLabel, scenarioTestTrackerId); } catch (err) { scenariosTracker.failCreateFunctionApp(scenarioLabel, scenarioTestTrackerId, (err as Error).message ?? parseError(err).message); @@ -105,7 +105,7 @@ async function startCreateAndDeployTest(scenarioLabel: string, rootFolder: Works await context.ui.runWithInputs(test.deployFunctionApp.inputs, async () => { try { await deployProductionSlotByFunctionAppId(context, functionAppId, rootFolder?.uri); - await test.deployFunctionApp.postTest?.(context, functionAppId, ''); + await test.deployFunctionApp.postTest?.(context, functionAppId); scenariosTracker.passDeployFunctionApp(scenarioLabel, scenarioTestTrackerId); } catch (err) { scenariosTracker.failDeployFunctionApp(scenarioLabel, scenarioTestTrackerId, (err as Error).message ?? parseError(err).message); diff --git a/test/nightly/scenarios/testScenarios/AzExtFunctionsTestScenario.ts b/test/nightly/scenarios/testScenarios/AzExtFunctionsTestScenario.ts index ab9aa8f74..ddcb82e54 100644 --- a/test/nightly/scenarios/testScenarios/AzExtFunctionsTestScenario.ts +++ b/test/nightly/scenarios/testScenarios/AzExtFunctionsTestScenario.ts @@ -34,13 +34,13 @@ export interface CreateAndDeployTestCase { label: string; mode: CreateMode; inputs: (string | RegExp)[]; - postTest?: (context: IActionContext, functionAppId: string, errMsg?: string) => void | Promise; + postTest?: (context: IActionContext, functionAppId: string) => void | Promise; }; deployFunctionApp: { label: string; inputs: (string | RegExp)[]; - preTest?: (context: IActionContext, functionAppId: string, errMsg?: string) => void | Promise; - postTest?: (context: IActionContext, functionAppId: string, errMsg?: string) => void | Promise; + preTest?: (context: IActionContext, functionAppId: string) => void | Promise; + postTest?: (context: IActionContext, functionAppId: string) => void | Promise; asyncWaitTask?: (context: IActionContext) => Promise; }; resourceGroupsToDelete?: string[]; diff --git a/test/nightly/scenarios/testScenarios/durable/azureStorage/DurableAzureStorage.plan.md b/test/nightly/scenarios/testScenarios/durable/azureStorage/DurableAzureStorage.plan.md index 6973eb3fe..2d113bc71 100644 --- a/test/nightly/scenarios/testScenarios/durable/azureStorage/DurableAzureStorage.plan.md +++ b/test/nightly/scenarios/testScenarios/durable/azureStorage/DurableAzureStorage.plan.md @@ -6,7 +6,7 @@ | No. | Language | Runtime | Programming Model | Comment | |-----|----------|---------|-------------------|------------------------------------| -| 1 | TS | Node | v4 | | +| 1 | JS | Node | v4 | | | 2 | Python | Python | v2 | | | 3 | C# | .NET | isolated | | | 4 | C# | .NET | in-proc | | diff --git a/test/nightly/scenarios/testScenarios/durable/azureStorage/azureStorageJSNodeScenario.ts b/test/nightly/scenarios/testScenarios/durable/azureStorage/azureStorageJSNodeScenario.ts new file mode 100644 index 000000000..32a4b08ea --- /dev/null +++ b/test/nightly/scenarios/testScenarios/durable/azureStorage/azureStorageJSNodeScenario.ts @@ -0,0 +1,45 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { DurableBackend } from "../../../../../../extension.bundle"; +import { durableOrchestratorName, durableOrchestratorPick } from "../../../../../constants"; +import { ConnectionType, CreateMode, OperatingSystem, PlanType, Runtime } from "../../../../../utils/createFunctionAppUtils"; +import { type AzExtFunctionsTestScenario } from "../../AzExtFunctionsTestScenario"; +import { generateCreateAndDeployTest } from "../generateCreateAndDeployTest"; + +export function generateJSNodeScenario(): AzExtFunctionsTestScenario { + const folderName: string = 'scenarios-durable-azurestorage-jsnode'; + + return { + label: 'durable-azurestorage-jsnode', + folderName, + createNewProjectTest: { + label: 'create-new-project', + inputs: [ + /JavaScript/i, + /v4/i, + durableOrchestratorPick, + /Azure Storage/i, + durableOrchestratorName, + ], + }, + createAndDeployTests: [ + generateCreateAndDeployTest(folderName, CreateMode.Basic, Runtime.Node, ConnectionType.ManagedIdentity, PlanType.FlexConsumption, OperatingSystem.Linux, DurableBackend.Storage), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.ManagedIdentity, PlanType.Premium, OperatingSystem.Linux, DurableBackend.Storage), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.ManagedIdentity, PlanType.Premium, OperatingSystem.Windows, DurableBackend.Storage), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.ManagedIdentity, PlanType.LegacyConsumption, OperatingSystem.Linux, DurableBackend.Storage), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.ManagedIdentity, PlanType.LegacyConsumption, OperatingSystem.Windows, DurableBackend.Storage), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.ManagedIdentity, PlanType.AppService, OperatingSystem.Linux, DurableBackend.Storage), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.ManagedIdentity, PlanType.AppService, OperatingSystem.Windows, DurableBackend.Storage), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.FlexConsumption, OperatingSystem.Linux, DurableBackend.Storage), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.Premium, OperatingSystem.Linux, DurableBackend.Storage), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.Premium, OperatingSystem.Windows, DurableBackend.Storage), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.LegacyConsumption, OperatingSystem.Linux, DurableBackend.Storage), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.LegacyConsumption, OperatingSystem.Windows, DurableBackend.Storage), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.AppService, OperatingSystem.Linux, DurableBackend.Storage), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.AppService, OperatingSystem.Windows, DurableBackend.Storage), + ], + }; +} diff --git a/test/nightly/scenarios/testScenarios/durable/azureStorage/azureStorageScenarios.ts b/test/nightly/scenarios/testScenarios/durable/azureStorage/azureStorageScenarios.ts index 9b3bf01bb..4858cec6d 100644 --- a/test/nightly/scenarios/testScenarios/durable/azureStorage/azureStorageScenarios.ts +++ b/test/nightly/scenarios/testScenarios/durable/azureStorage/azureStorageScenarios.ts @@ -4,10 +4,12 @@ *--------------------------------------------------------------------------------------------*/ import { type AzExtFunctionsTestScenario } from '../../AzExtFunctionsTestScenario'; -import { generateTSNodeScenario } from './azureStorageTSNodeScenario'; +import { generateJSNodeScenario } from './azureStorageJSNodeScenario'; +import { generatePythonScenario } from './pythonScenario'; export function generateDurableAzureStorageScenarios(): AzExtFunctionsTestScenario[] { return [ - generateTSNodeScenario(), + generateJSNodeScenario(), + generatePythonScenario(), ]; } diff --git a/test/nightly/scenarios/testScenarios/durable/azureStorage/azureStorageTSNodeScenario.ts b/test/nightly/scenarios/testScenarios/durable/azureStorage/azureStorageTSNodeScenario.ts deleted file mode 100644 index 990680cd3..000000000 --- a/test/nightly/scenarios/testScenarios/durable/azureStorage/azureStorageTSNodeScenario.ts +++ /dev/null @@ -1,65 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { DurableBackend, getRandomAlphanumericString } from "../../../../../../extension.bundle"; -import { ConnectionType, createFunctionAppUtils, CreateMode, OperatingSystem, PlanType } from "../../../../../utils/createFunctionAppUtils"; -import { deployFunctionAppUtils } from "../../../../../utils/deployFunctionAppUtils"; -import { type AzExtFunctionsTestScenario, type CreateAndDeployTestCase } from "../../AzExtFunctionsTestScenario"; - -export function generateTSNodeScenario(): AzExtFunctionsTestScenario { - const folderName: string = 'scenarios-durable-azurestorage-tsnode'; - - return { - only: true, - label: 'durable-azurestorage-tsnode', - folderName, - createNewProjectTest: { - label: 'create-new-project', - inputs: [ - /JavaScript/i, - /v4/i, - /Durable Functions Orchestrator/i, - /Azure Storage/i, - 'durableHello1', - ], - }, - createAndDeployTests: [ - generateCreateAndDeployTest(folderName, CreateMode.Basic, ConnectionType.ManagedIdentity, OperatingSystem.Linux, PlanType.FlexConsumption), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, ConnectionType.ManagedIdentity, OperatingSystem.Linux, PlanType.Premium), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, ConnectionType.ManagedIdentity, OperatingSystem.Windows, PlanType.Premium), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, ConnectionType.ManagedIdentity, OperatingSystem.Linux, PlanType.LegacyConsumption), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, ConnectionType.ManagedIdentity, OperatingSystem.Windows, PlanType.LegacyConsumption), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, ConnectionType.ManagedIdentity, OperatingSystem.Linux, PlanType.AppService), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, ConnectionType.ManagedIdentity, OperatingSystem.Windows, PlanType.AppService), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, ConnectionType.Secrets, OperatingSystem.Linux, PlanType.FlexConsumption), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, ConnectionType.Secrets, OperatingSystem.Linux, PlanType.Premium), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, ConnectionType.Secrets, OperatingSystem.Windows, PlanType.Premium), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, ConnectionType.Secrets, OperatingSystem.Linux, PlanType.LegacyConsumption), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, ConnectionType.Secrets, OperatingSystem.Windows, PlanType.LegacyConsumption), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, ConnectionType.Secrets, OperatingSystem.Linux, PlanType.AppService), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, ConnectionType.Secrets, OperatingSystem.Windows, PlanType.AppService), - ], - }; -} - -function generateCreateAndDeployTest(folderName: string, createMode: CreateMode, connection: ConnectionType, os: OperatingSystem, plan: PlanType): CreateAndDeployTestCase { - const appName: string = getRandomAlphanumericString(); - const description: string = `${createMode}-${connection}-${os}-${plan}`; - - return { - createFunctionApp: { - label: `create-function-app | ${description}`, - mode: createMode, - inputs: createMode === CreateMode.Basic ? - createFunctionAppUtils.generateBasicCreateInputs(appName, folderName, connection) : - createFunctionAppUtils.generateAdvancedCreateInputs(appName, folderName, connection, os, plan), - }, - deployFunctionApp: { - label: `deploy-function-app | ${description}`, - inputs: deployFunctionAppUtils.generateDurableDeployInputs(appName, DurableBackend.Storage), - }, - resourceGroupsToDelete: [appName], - }; -} diff --git a/test/nightly/scenarios/testScenarios/durable/azureStorage/pythonScenario.ts b/test/nightly/scenarios/testScenarios/durable/azureStorage/pythonScenario.ts new file mode 100644 index 000000000..5ba8bdbf7 --- /dev/null +++ b/test/nightly/scenarios/testScenarios/durable/azureStorage/pythonScenario.ts @@ -0,0 +1,36 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { DurableBackend } from "../../../../../../extension.bundle"; +import { durableOrchestratorName, durableOrchestratorPick, pythonDefaultPick } from "../../../../../constants"; +import { ConnectionType, CreateMode, OperatingSystem, PlanType, Runtime } from "../../../../../utils/createFunctionAppUtils"; +import { type AzExtFunctionsTestScenario } from "../../AzExtFunctionsTestScenario"; +import { generateCreateAndDeployTest } from "../generateCreateAndDeployTest"; + +export function generatePythonScenario(): AzExtFunctionsTestScenario { + const folderName: string = 'scenarios-durable-azurestorage-python'; + + return { + label: 'durable-azurestorage-python', + folderName, + createNewProjectTest: { + label: 'create-new-project', + inputs: [ + /Python/i, + /v2/i, + pythonDefaultPick, + durableOrchestratorPick, + /Azure Storage/i, + durableOrchestratorName, + ], + }, + createAndDeployTests: [ + generateCreateAndDeployTest(folderName, CreateMode.Basic, Runtime.Python, ConnectionType.ManagedIdentity, PlanType.FlexConsumption, OperatingSystem.Linux, DurableBackend.Storage), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Python, ConnectionType.ManagedIdentity, PlanType.Premium, undefined /** os - automatically defaulted */, DurableBackend.Storage), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Python, ConnectionType.Secrets, PlanType.FlexConsumption, OperatingSystem.Linux, DurableBackend.Storage), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Python, ConnectionType.Secrets, PlanType.Premium, undefined /** os - automatically defaulted */, DurableBackend.Storage), + ], + }; +} diff --git a/test/nightly/scenarios/testScenarios/durable/dts/DurableDTS.plan.md b/test/nightly/scenarios/testScenarios/durable/dts/DurableDTS.plan.md index 5518bd086..26d1cc775 100644 --- a/test/nightly/scenarios/testScenarios/durable/dts/DurableDTS.plan.md +++ b/test/nightly/scenarios/testScenarios/durable/dts/DurableDTS.plan.md @@ -6,7 +6,7 @@ | No. | Language | Runtime | Programming Model | Comment | |-----|----------|---------|-------------------|---------| -| 1 | TS | Node | v4 | | +| 1 | JS | Node | v4 | | | 2 | Python | Python | v2 | | | 3 | C# | .NET | isolated | | | 4 | C# | .NET | in-proc | | diff --git a/test/nightly/scenarios/testScenarios/durable/dts/dtsJSNodeScenario.ts b/test/nightly/scenarios/testScenarios/durable/dts/dtsJSNodeScenario.ts new file mode 100644 index 000000000..2f9d3baa9 --- /dev/null +++ b/test/nightly/scenarios/testScenarios/durable/dts/dtsJSNodeScenario.ts @@ -0,0 +1,37 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { DurableBackend } from "../../../../../../extension.bundle"; +import { durableOrchestratorName, durableOrchestratorPick } from "../../../../../constants"; +import { ConnectionType, CreateMode, OperatingSystem, PlanType, Runtime } from "../../../../../utils/createFunctionAppUtils"; +import { type AzExtFunctionsTestScenario } from "../../AzExtFunctionsTestScenario"; +import { generateCreateAndDeployTest } from "../generateCreateAndDeployTest"; + +export function generateJSNodeScenario(): AzExtFunctionsTestScenario { + const folderName: string = 'scenarios-durable-dts-jsnode'; + + return { + label: 'durable-dts-jsnode', + folderName, + createNewProjectTest: { + label: 'create-new-project', + inputs: [ + /JavaScript/i, + /v4/i, + durableOrchestratorPick, + /Durable Task Scheduler/i, + durableOrchestratorName, + ], + }, + createAndDeployTests: [ + generateCreateAndDeployTest(folderName, CreateMode.Basic, Runtime.Node, ConnectionType.ManagedIdentity, PlanType.FlexConsumption, OperatingSystem.Linux, DurableBackend.DTS), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.ManagedIdentity, PlanType.Premium, OperatingSystem.Linux, DurableBackend.DTS), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.ManagedIdentity, PlanType.Premium, OperatingSystem.Windows, DurableBackend.DTS), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.FlexConsumption, OperatingSystem.Linux, DurableBackend.DTS), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.Premium, OperatingSystem.Linux, DurableBackend.DTS), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.Premium, OperatingSystem.Windows, DurableBackend.DTS), + ], + }; +} diff --git a/test/nightly/scenarios/testScenarios/durable/dts/dtsScenarios.ts b/test/nightly/scenarios/testScenarios/durable/dts/dtsScenarios.ts index c11889e0b..151b2485f 100644 --- a/test/nightly/scenarios/testScenarios/durable/dts/dtsScenarios.ts +++ b/test/nightly/scenarios/testScenarios/durable/dts/dtsScenarios.ts @@ -4,10 +4,10 @@ *--------------------------------------------------------------------------------------------*/ import { type AzExtFunctionsTestScenario } from '../../AzExtFunctionsTestScenario'; -import { generateTSNodeScenario } from './dtsTSNodeScenario'; +import { generateJSNodeScenario } from './dtsJSNodeScenario'; export function generateDurableDTSScenarios(): AzExtFunctionsTestScenario[] { return [ - generateTSNodeScenario(), + generateJSNodeScenario(), ]; } diff --git a/test/nightly/scenarios/testScenarios/durable/dts/dtsTSNodeScenario.ts b/test/nightly/scenarios/testScenarios/durable/dts/dtsTSNodeScenario.ts deleted file mode 100644 index ce6961a6f..000000000 --- a/test/nightly/scenarios/testScenarios/durable/dts/dtsTSNodeScenario.ts +++ /dev/null @@ -1,56 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { DurableBackend, getRandomAlphanumericString } from "../../../../../../extension.bundle"; -import { ConnectionType, createFunctionAppUtils, CreateMode, OperatingSystem, PlanType } from "../../../../../utils/createFunctionAppUtils"; -import { deployFunctionAppUtils } from "../../../../../utils/deployFunctionAppUtils"; -import { type AzExtFunctionsTestScenario, type CreateAndDeployTestCase } from "../../AzExtFunctionsTestScenario"; - -export function generateTSNodeScenario(): AzExtFunctionsTestScenario { - const folderName: string = 'scenarios-durable-dts-tsnode'; - - return { - label: 'durable-dts-tsnode', - folderName, - createNewProjectTest: { - label: 'create-new-project', - inputs: [ - /TypeScript/i, - /v4/i, - /Durable Functions Orchestrator/i, - /Durable Task Scheduler/i, - 'durableHello1', - ], - }, - createAndDeployTests: [ - generateCreateAndDeployTest(folderName, CreateMode.Basic, ConnectionType.ManagedIdentity, OperatingSystem.Linux, PlanType.FlexConsumption), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, ConnectionType.ManagedIdentity, OperatingSystem.Linux, PlanType.Premium), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, ConnectionType.ManagedIdentity, OperatingSystem.Windows, PlanType.Premium), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, ConnectionType.Secrets, OperatingSystem.Linux, PlanType.FlexConsumption), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, ConnectionType.Secrets, OperatingSystem.Linux, PlanType.Premium), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, ConnectionType.Secrets, OperatingSystem.Windows, PlanType.Premium), - ], - }; -} - -function generateCreateAndDeployTest(folderName: string, createMode: CreateMode, connection: ConnectionType, os: OperatingSystem, plan: PlanType): CreateAndDeployTestCase { - const appName: string = getRandomAlphanumericString(); - const description: string = `${createMode}-${connection}-${os}-${plan}`; - - return { - createFunctionApp: { - label: `create-function-app-${description}`, - mode: createMode, - inputs: createMode === CreateMode.Basic ? - createFunctionAppUtils.generateBasicCreateInputs(appName, folderName, connection) : - createFunctionAppUtils.generateAdvancedCreateInputs(appName, folderName, connection, os, plan), - }, - deployFunctionApp: { - label: `deploy-function-app-${description}`, - inputs: deployFunctionAppUtils.generateDurableDeployInputs(appName, DurableBackend.DTS), - }, - resourceGroupsToDelete: [appName], - }; -} diff --git a/test/nightly/scenarios/testScenarios/durable/generateCreateAndDeployTest.ts b/test/nightly/scenarios/testScenarios/durable/generateCreateAndDeployTest.ts new file mode 100644 index 000000000..ad463ee5a --- /dev/null +++ b/test/nightly/scenarios/testScenarios/durable/generateCreateAndDeployTest.ts @@ -0,0 +1,63 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { type Site, type WebSiteManagementClient } from "@azure/arm-appservice"; +import { createWebSiteClient } from "@microsoft/vscode-azext-azureappservice"; +import { parseAzureResourceId, type ParsedAzureResourceId } from "@microsoft/vscode-azext-azureutils"; +import { getRandomAlphanumericString, type DurableBackend, type IActionContext } from "../../../../../extension.bundle"; +import { durableOrchestratorName } from "../../../../constants"; +import { createFunctionAppUtils, CreateMode, Runtime, type ConnectionType, type OperatingSystem, type PlanType } from "../../../../utils/createFunctionAppUtils"; +import { deployFunctionAppUtils } from "../../../../utils/deployFunctionAppUtils"; +import { subscriptionContext } from "../../../global.nightly.test"; +import { type CreateAndDeployTestCase } from "../AzExtFunctionsTestScenario"; + +export function generateCreateAndDeployTest(folderName: string, createMode: CreateMode, runtime: Runtime, connection: ConnectionType, plan: PlanType, os?: OperatingSystem, storageType?: DurableBackend): CreateAndDeployTestCase { + const appName: string = getRandomAlphanumericString(); + const osDescription: string = os ? `-${os}` : ''; + const description: string = `${createMode}-${connection}${osDescription}-${plan}`; + + return { + createFunctionApp: { + label: `create-function-app | ${description}`, + mode: createMode, + inputs: createMode === CreateMode.Basic ? + createFunctionAppUtils.generateBasicCreateInputs(appName, folderName, runtime, connection) : + createFunctionAppUtils.generateAdvancedCreateInputs(appName, folderName, runtime, connection, plan, os), + }, + deployFunctionApp: { + label: `deploy-function-app | ${description}`, + inputs: deployFunctionAppUtils.generateDurableDeployInputs(appName, storageType), + postTest: generateVerifyDeployment(runtime), + }, + resourceGroupsToDelete: [appName], + }; +} + +function generateVerifyDeployment(runtime: Runtime) { + return async function verifyDeployment(context: IActionContext, functionAppId: string): Promise { + const client: WebSiteManagementClient = await createWebSiteClient({ ...context, ...subscriptionContext }); + const parsedResource: ParsedAzureResourceId = parseAzureResourceId(functionAppId); + const functionApp: Site = await client.webApps.get(parsedResource.resourceGroup, parsedResource.resourceName); + + let url = `https://${functionApp.defaultHostName}`; + switch (runtime) { + case Runtime.Python: + url += `/api/orchestrators/${durableOrchestratorName}_orchestrator`; + break; + case Runtime.Node: + url += `/api/orchestrators/${durableOrchestratorName}Orchestrator`; + break; + case Runtime.DotNet: + break; + default: + throw new Error('Durable verify deployment not yet supported for this runtime type.'); + } + + const response = await fetch(url); + if (response.status !== 202) { + throw new Error(`Verify Deployment: Orchestrator endpoint responded with ${response.status}.`); + } + }; +} diff --git a/test/nightly/scenarios/testScenarios/testScenarios.ts b/test/nightly/scenarios/testScenarios/testScenarios.ts index a6125c74c..d253fdea2 100644 --- a/test/nightly/scenarios/testScenarios/testScenarios.ts +++ b/test/nightly/scenarios/testScenarios/testScenarios.ts @@ -5,12 +5,11 @@ import { type AzExtFunctionsTestScenario } from "./AzExtFunctionsTestScenario"; import { generateDurableAzureStorageScenarios } from "./durable/azureStorage/azureStorageScenarios"; -import { generateDurableDTSScenarios } from "./durable/dts/dtsScenarios"; export function generateTestScenarios(): AzExtFunctionsTestScenario[] { const testScenarios: AzExtFunctionsTestScenario[] = [ ...generateDurableAzureStorageScenarios(), - ...generateDurableDTSScenarios(), + // ...generateDurableDTSScenarios(), ]; return testScenarios; } diff --git a/test/test.code-workspace b/test/test.code-workspace index 6ce4988b6..da2d7d1bd 100644 --- a/test/test.code-workspace +++ b/test/test.code-workspace @@ -31,12 +31,16 @@ "path": "../testWorkspace/9" }, { - "name": "scenarios-durable-azurestorage-tsnode", - "path": "../testWorkspace/scenarios/durable-azureStorage/tsnode" + "name": "scenarios-durable-azurestorage-jsnode", + "path": "../testWorkspace/scenarios/durable-azurestorage/jsnode" }, { - "name": "scenarios-durable-dts-tsnode", - "path": "../testWorkspace/scenarios/durable-dts/tsnode" + "name": "scenarios-durable-azurestorage-python", + "path": "../testWorkspace/scenarios/durable-azurestorage/python" + }, + { + "name": "scenarios-durable-dts-jsnode", + "path": "../testWorkspace/scenarios/durable-dts/jsnode" }, { "name": "scenarios-durable-dts-python", diff --git a/test/utils/createFunctionAppUtils.ts b/test/utils/createFunctionAppUtils.ts index a44f4f079..ca7637bd9 100644 --- a/test/utils/createFunctionAppUtils.ts +++ b/test/utils/createFunctionAppUtils.ts @@ -3,26 +3,28 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ +import { locationDefaultPick, nodeDefaultPick, pythonDefaultPick } from "../constants"; + export namespace createFunctionAppUtils { - export function generateBasicCreateInputs(appName: string, folderName: string, connection: ConnectionType): (string | RegExp)[] { + export function generateBasicCreateInputs(appName: string, folderName: string, runtime: Runtime, connection: ConnectionType): (string | RegExp)[] { return [ folderName, appName, - /West US 2/i, - /Node\.js 22/i, + locationDefaultPick, + getRuntimePick(runtime), new RegExp(connection, 'i'), ]; } - export function generateAdvancedCreateInputs(appName: string, folderName: string, connection: ConnectionType, os: OperatingSystem, plan: PlanType): (string | RegExp)[] { + export function generateAdvancedCreateInputs(appName: string, folderName: string, runtime: Runtime, connection: ConnectionType, plan: PlanType, os?: OperatingSystem): (string | RegExp)[] { switch (plan) { case PlanType.FlexConsumption: return [ folderName, appName, new RegExp(plan, 'i'), - /West US 2/i, - /Node\.js 22/i, + locationDefaultPick, + getRuntimePick(runtime), '2048', '100', /Create new resource group/i, @@ -38,9 +40,9 @@ export namespace createFunctionAppUtils { folderName, appName, new RegExp(plan, 'i'), - /West US 2/i, - /Node\.js 22/i, - new RegExp(os, 'i'), + locationDefaultPick, + getRuntimePick(runtime), + ...(os ? [new RegExp(os, 'i')] : []), /Create new app service plan/i, appName, /EP1/i, @@ -57,9 +59,9 @@ export namespace createFunctionAppUtils { folderName, appName, new RegExp(plan, 'i'), - /West US 2/i, - /Node\.js 22/i, - new RegExp(os, 'i'), + locationDefaultPick, + getRuntimePick(runtime), + ...(os ? [new RegExp(os, 'i')] : []), /Create new resource group/i, appName, ...getConnectionTypeInputs(connection), @@ -73,9 +75,9 @@ export namespace createFunctionAppUtils { folderName, appName, new RegExp(plan, 'i'), - /West US 2/i, - /Node\.js 22/i, - new RegExp(os, 'i'), + locationDefaultPick, + getRuntimePick(runtime), + ...(os ? [new RegExp(os, 'i')] : []), /Create new app service plan/i, appName, /S1/i, @@ -90,6 +92,17 @@ export namespace createFunctionAppUtils { } } + function getRuntimePick(runtime: Runtime): RegExp | string { + switch (runtime) { + case Runtime.Python: + return pythonDefaultPick; + case Runtime.Node: + return nodeDefaultPick; + default: + throw new Error(`Runtime "${runtime}" not yet supported in "createFunctionAppUtils.generateBasicCreateInputs".`); + } + } + function getConnectionTypeInputs(connection: ConnectionType): (string | RegExp)[] { return connection === ConnectionType.ManagedIdentity ? [new RegExp(connection, 'i'), /Create new user[- ]assigned identity/i] : @@ -118,3 +131,9 @@ export enum CreateMode { Basic = 'Basic', Advanced = 'Advanced', } + +export enum Runtime { + Node = 'Node', + Python = 'Python', + DotNet = '.NET', +} From 6c47b652d142269f22d89ffbd3705e691765b2e2 Mon Sep 17 00:00:00 2001 From: MicroFish91 <40250218+MicroFish91@users.noreply.github.com> Date: Mon, 5 Jan 2026 19:47:01 -0800 Subject: [PATCH 11/27] Dotnet working, added post tests even if deployment failed --- test/constants.ts | 1 + test/nightly/scenarios/parallelScenarios.ts | 30 +++++++++--- .../AzExtFunctionsTestScenario.ts | 2 - .../azureStorage/DurableAzureStorage.plan.md | 26 ++++++---- .../azureStorage/azureStorageScenarios.ts | 2 + .../durable/azureStorage/dotnetIsolated.ts | 48 +++++++++++++++++++ .../durable/azureStorage/pythonScenario.ts | 6 ++- .../durable/generateCreateAndDeployTest.ts | 5 +- test/test.code-workspace | 8 ++++ test/utils/createFunctionAppUtils.ts | 6 ++- 10 files changed, 109 insertions(+), 25 deletions(-) create mode 100644 test/nightly/scenarios/testScenarios/durable/azureStorage/dotnetIsolated.ts diff --git a/test/constants.ts b/test/constants.ts index 52521c518..9ae56c6bf 100644 --- a/test/constants.ts +++ b/test/constants.ts @@ -10,3 +10,4 @@ export const durableOrchestratorName: string = 'durableHello'; export const locationDefaultPick: RegExp = /West US 2/i; export const pythonDefaultPick: RegExp = /Python(\s)?3\.12/i; export const nodeDefaultPick: RegExp = /Node\.js(\s)?22/i; +export const dotnetIsolatedDefaultPick: RegExp = /\.NET(\s)?10(\s)?Isolated/i; diff --git a/test/nightly/scenarios/parallelScenarios.ts b/test/nightly/scenarios/parallelScenarios.ts index a97e06b73..c2d421957 100644 --- a/test/nightly/scenarios/parallelScenarios.ts +++ b/test/nightly/scenarios/parallelScenarios.ts @@ -24,14 +24,14 @@ export function generateParallelScenarios(): AzExtFunctionsParallelTestScenario[ return generateTestScenarios().map(scenario => { return { title: scenario.label, - runScenario: runTestScenario(scenario), + runScenario: generateRunScenario(scenario), only: scenario.only, }; }); } -function runTestScenario(scenario: AzExtFunctionsTestScenario): AzExtFunctionsParallelTestScenario['runScenario'] { - return async () => { +function generateRunScenario(scenario: AzExtFunctionsTestScenario): AzExtFunctionsParallelTestScenario['runScenario'] { + return async function runScenario() { const workspaceFolderUri: Uri = getWorkspaceFolderUri(scenario.folderName); const rootFolder = workspace.getWorkspaceFolder(workspaceFolderUri); assert.ok(rootFolder, `Failed to retrieve root workspace folder for scenario ${scenario.label}.`); @@ -103,13 +103,29 @@ async function startCreateAndDeployTest(scenarioLabel: string, rootFolder: Works await runWithTestActionContext('scenario.deploy', async context => { await context.ui.runWithInputs(test.deployFunctionApp.inputs, async () => { + let error: unknown; try { await deployProductionSlotByFunctionAppId(context, functionAppId, rootFolder?.uri); - await test.deployFunctionApp.postTest?.(context, functionAppId); + } catch (deployError) { + error = deployError; + } + + if (test.deployFunctionApp.postTest) { + try { + await test.deployFunctionApp.postTest(context, functionAppId); + if ((error as Error).message) { + (error as Error).message = '**Function app deployment errored but still passed post test deployment verification. ' + (error as Error).message; + } + } catch (postTestError) { + error ??= postTestError; + } + } + + if (!error) { scenariosTracker.passDeployFunctionApp(scenarioLabel, scenarioTestTrackerId); - } catch (err) { - scenariosTracker.failDeployFunctionApp(scenarioLabel, scenarioTestTrackerId, (err as Error).message ?? parseError(err).message); - throw err; + } else { + scenariosTracker.failDeployFunctionApp(scenarioLabel, scenarioTestTrackerId, (error as Error).message ?? parseError(error).message); + throw error; } }); }); diff --git a/test/nightly/scenarios/testScenarios/AzExtFunctionsTestScenario.ts b/test/nightly/scenarios/testScenarios/AzExtFunctionsTestScenario.ts index ddcb82e54..0c9c377c8 100644 --- a/test/nightly/scenarios/testScenarios/AzExtFunctionsTestScenario.ts +++ b/test/nightly/scenarios/testScenarios/AzExtFunctionsTestScenario.ts @@ -39,9 +39,7 @@ export interface CreateAndDeployTestCase { deployFunctionApp: { label: string; inputs: (string | RegExp)[]; - preTest?: (context: IActionContext, functionAppId: string) => void | Promise; postTest?: (context: IActionContext, functionAppId: string) => void | Promise; - asyncWaitTask?: (context: IActionContext) => Promise; }; resourceGroupsToDelete?: string[]; diff --git a/test/nightly/scenarios/testScenarios/durable/azureStorage/DurableAzureStorage.plan.md b/test/nightly/scenarios/testScenarios/durable/azureStorage/DurableAzureStorage.plan.md index 2d113bc71..0e912cd85 100644 --- a/test/nightly/scenarios/testScenarios/durable/azureStorage/DurableAzureStorage.plan.md +++ b/test/nightly/scenarios/testScenarios/durable/azureStorage/DurableAzureStorage.plan.md @@ -35,14 +35,20 @@ TBD | 13 | 1 | Secrets | Linux | App Service | | | 14 | 1 | Secrets | Windows | App Service | | | 15 | 2 | Managed Identity | Linux | Flex Consumption | | -| 16 | 2 | Managed Identity | Linux | Premium | | +| 16 | 2 | Managed Identity | Windows | Premium | | | 17 | 2 | Secrets | Linux | Flex Consumption | | -| 18 | 2 | Secrets | Linux | Premium | | -| 19 | 3 | Managed Identity | Linux | Flex Consumption | | -| 20 | 3 | Managed Identity | Linux | Premium | | -| 21 | 3 | Secrets | Linux | Flex Consumption | | -| 22 | 3 | Secrets | Linux | Premium | | -| 23 | 4 | Managed Identity | Linux | Flex Consumption | | -| 24 | 4 | Managed Identity | Linux | Premium | | -| 25 | 4 | Secrets | Linux | Flex Consumption | | -| 26 | 4 | Secrets | Linux | Premium | | +| 18 | 2 | Secrets | Windows | Premium | | +| 19 | 2 | Secrets | Linux | Consumption (Legacy) | | +| 20 | 2 | Secrets | Windows | App Service | | +| 21 | 3 | Managed Identity | Linux | Flex Consumption | | +| 22 | 3 | Managed Identity | Windows | Premium | | +| 23 | 3 | Secrets | Linux | Flex Consumption | | +| 24 | 3 | Secrets | Windows | Premium | | +| 25 | 3 | Secrets | Linux | Consumption (Legacy) | | +| 26 | 3 | Secrets | Windows | App Service | | +| 27 | 4 | Managed Identity | Linux | Flex Consumption | | +| 28 | 4 | Managed Identity | Windows | Premium | | +| 29 | 4 | Secrets | Linux | Flex Consumption | | +| 30 | 4 | Secrets | Windows | Premium | | +| 31 | 4 | Secrets | Linux | Consumption (Legacy) | | +| 32 | 4 | Secrets | Windows | App Service | | diff --git a/test/nightly/scenarios/testScenarios/durable/azureStorage/azureStorageScenarios.ts b/test/nightly/scenarios/testScenarios/durable/azureStorage/azureStorageScenarios.ts index 4858cec6d..1d005c968 100644 --- a/test/nightly/scenarios/testScenarios/durable/azureStorage/azureStorageScenarios.ts +++ b/test/nightly/scenarios/testScenarios/durable/azureStorage/azureStorageScenarios.ts @@ -5,11 +5,13 @@ import { type AzExtFunctionsTestScenario } from '../../AzExtFunctionsTestScenario'; import { generateJSNodeScenario } from './azureStorageJSNodeScenario'; +import { generateDotnetIsolatedScenario } from './dotnetIsolated'; import { generatePythonScenario } from './pythonScenario'; export function generateDurableAzureStorageScenarios(): AzExtFunctionsTestScenario[] { return [ generateJSNodeScenario(), generatePythonScenario(), + generateDotnetIsolatedScenario(), ]; } diff --git a/test/nightly/scenarios/testScenarios/durable/azureStorage/dotnetIsolated.ts b/test/nightly/scenarios/testScenarios/durable/azureStorage/dotnetIsolated.ts new file mode 100644 index 000000000..636126559 --- /dev/null +++ b/test/nightly/scenarios/testScenarios/durable/azureStorage/dotnetIsolated.ts @@ -0,0 +1,48 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { DurableBackend } from "../../../../../../extension.bundle"; +import { durableOrchestratorName, durableOrchestratorPick } from "../../../../../constants"; +import { ConnectionType, CreateMode, OperatingSystem, PlanType, Runtime } from "../../../../../utils/createFunctionAppUtils"; +import { type AzExtFunctionsTestScenario } from "../../AzExtFunctionsTestScenario"; +import { generateCreateAndDeployTest } from "../generateCreateAndDeployTest"; + +export function generateDotnetIsolatedScenario(): AzExtFunctionsTestScenario { + const folderName: string = 'scenarios-durable-azurestorage-dotnetisolated'; + + return { + only: true, + label: 'durable-azurestorage-dotnetisolated', + folderName, + createNewProjectTest: { + label: 'create-new-project', + inputs: [ + /C#/i, + // Todo: This needs to also run a match on the description because isolated vs. inproc differentiates there + /\.NET 10/i, + durableOrchestratorPick, + /Azure Storage/i, + durableOrchestratorName, + 'Company.Function', + ], + }, + createAndDeployTests: [ + generateCreateAndDeployTest(folderName, CreateMode.Basic, Runtime.DotNetIsolated, ConnectionType.ManagedIdentity, PlanType.FlexConsumption, OperatingSystem.Linux, DurableBackend.Storage), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.DotNetIsolated, ConnectionType.ManagedIdentity, PlanType.Premium, OperatingSystem.Linux, DurableBackend.Storage), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.DotNetIsolated, ConnectionType.ManagedIdentity, PlanType.Premium, OperatingSystem.Windows, DurableBackend.Storage), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.DotNetIsolated, ConnectionType.ManagedIdentity, PlanType.LegacyConsumption, OperatingSystem.Linux, DurableBackend.Storage), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.DotNetIsolated, ConnectionType.ManagedIdentity, PlanType.LegacyConsumption, OperatingSystem.Windows, DurableBackend.Storage), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.DotNetIsolated, ConnectionType.ManagedIdentity, PlanType.AppService, OperatingSystem.Linux, DurableBackend.Storage), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.DotNetIsolated, ConnectionType.ManagedIdentity, PlanType.AppService, OperatingSystem.Windows, DurableBackend.Storage), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.DotNetIsolated, ConnectionType.Secrets, PlanType.FlexConsumption, OperatingSystem.Linux, DurableBackend.Storage), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.DotNetIsolated, ConnectionType.Secrets, PlanType.Premium, OperatingSystem.Linux, DurableBackend.Storage), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.DotNetIsolated, ConnectionType.Secrets, PlanType.Premium, OperatingSystem.Windows, DurableBackend.Storage), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.DotNetIsolated, ConnectionType.Secrets, PlanType.LegacyConsumption, OperatingSystem.Linux, DurableBackend.Storage), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.DotNetIsolated, ConnectionType.Secrets, PlanType.LegacyConsumption, OperatingSystem.Windows, DurableBackend.Storage), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.DotNetIsolated, ConnectionType.Secrets, PlanType.AppService, OperatingSystem.Linux, DurableBackend.Storage), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.DotNetIsolated, ConnectionType.Secrets, PlanType.AppService, OperatingSystem.Windows, DurableBackend.Storage), + ], + }; +} diff --git a/test/nightly/scenarios/testScenarios/durable/azureStorage/pythonScenario.ts b/test/nightly/scenarios/testScenarios/durable/azureStorage/pythonScenario.ts index 5ba8bdbf7..8842f02c8 100644 --- a/test/nightly/scenarios/testScenarios/durable/azureStorage/pythonScenario.ts +++ b/test/nightly/scenarios/testScenarios/durable/azureStorage/pythonScenario.ts @@ -28,9 +28,11 @@ export function generatePythonScenario(): AzExtFunctionsTestScenario { }, createAndDeployTests: [ generateCreateAndDeployTest(folderName, CreateMode.Basic, Runtime.Python, ConnectionType.ManagedIdentity, PlanType.FlexConsumption, OperatingSystem.Linux, DurableBackend.Storage), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Python, ConnectionType.ManagedIdentity, PlanType.Premium, undefined /** os - automatically defaulted */, DurableBackend.Storage), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Python, ConnectionType.ManagedIdentity, PlanType.Premium, OperatingSystem.Windows, DurableBackend.Storage), generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Python, ConnectionType.Secrets, PlanType.FlexConsumption, OperatingSystem.Linux, DurableBackend.Storage), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Python, ConnectionType.Secrets, PlanType.Premium, undefined /** os - automatically defaulted */, DurableBackend.Storage), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Python, ConnectionType.Secrets, PlanType.Premium, OperatingSystem.Windows, DurableBackend.Storage), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Python, ConnectionType.Secrets, PlanType.LegacyConsumption, OperatingSystem.Linux, DurableBackend.Storage), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Python, ConnectionType.Secrets, PlanType.AppService, OperatingSystem.Windows, DurableBackend.Storage), ], }; } diff --git a/test/nightly/scenarios/testScenarios/durable/generateCreateAndDeployTest.ts b/test/nightly/scenarios/testScenarios/durable/generateCreateAndDeployTest.ts index ad463ee5a..0ff60585f 100644 --- a/test/nightly/scenarios/testScenarios/durable/generateCreateAndDeployTest.ts +++ b/test/nightly/scenarios/testScenarios/durable/generateCreateAndDeployTest.ts @@ -49,10 +49,11 @@ function generateVerifyDeployment(runtime: Runtime) { case Runtime.Node: url += `/api/orchestrators/${durableOrchestratorName}Orchestrator`; break; - case Runtime.DotNet: + case Runtime.DotNetIsolated: + url += `/api/${durableOrchestratorName}_HttpStart`; break; default: - throw new Error('Durable verify deployment not yet supported for this runtime type.'); + throw new Error('Durable verify deployment not yet implemented for this runtime type.'); } const response = await fetch(url); diff --git a/test/test.code-workspace b/test/test.code-workspace index da2d7d1bd..02de5fdb9 100644 --- a/test/test.code-workspace +++ b/test/test.code-workspace @@ -38,6 +38,14 @@ "name": "scenarios-durable-azurestorage-python", "path": "../testWorkspace/scenarios/durable-azurestorage/python" }, + { + "name": "scenarios-durable-azurestorage-dotnetinproc", + "path": "../testWorkspace/scenarios/durable-azurestorage/dotnetinproc" + }, + { + "name": "scenarios-durable-azurestorage-dotnetisolated", + "path": "../testWorkspace/scenarios/durable-azurestorage/dotnetisolated" + }, { "name": "scenarios-durable-dts-jsnode", "path": "../testWorkspace/scenarios/durable-dts/jsnode" diff --git a/test/utils/createFunctionAppUtils.ts b/test/utils/createFunctionAppUtils.ts index ca7637bd9..7bbc28e9b 100644 --- a/test/utils/createFunctionAppUtils.ts +++ b/test/utils/createFunctionAppUtils.ts @@ -3,7 +3,7 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { locationDefaultPick, nodeDefaultPick, pythonDefaultPick } from "../constants"; +import { dotnetIsolatedDefaultPick, locationDefaultPick, nodeDefaultPick, pythonDefaultPick } from "../constants"; export namespace createFunctionAppUtils { export function generateBasicCreateInputs(appName: string, folderName: string, runtime: Runtime, connection: ConnectionType): (string | RegExp)[] { @@ -98,6 +98,8 @@ export namespace createFunctionAppUtils { return pythonDefaultPick; case Runtime.Node: return nodeDefaultPick; + case Runtime.DotNetIsolated: + return dotnetIsolatedDefaultPick; default: throw new Error(`Runtime "${runtime}" not yet supported in "createFunctionAppUtils.generateBasicCreateInputs".`); } @@ -135,5 +137,5 @@ export enum CreateMode { export enum Runtime { Node = 'Node', Python = 'Python', - DotNet = '.NET', + DotNetIsolated = '.NET Isolated', } From 856e7b1805d419a632b32cb78e37d9bf015b4ae6 Mon Sep 17 00:00:00 2001 From: MicroFish91 <40250218+MicroFish91@users.noreply.github.com> Date: Tue, 6 Jan 2026 15:58:00 -0800 Subject: [PATCH 12/27] Enable most of the durable tests azureStorage + DTS --- test/nightly/scenarios/ScenariosTracker.ts | 95 +++++++++---------- test/nightly/scenarios/parallelScenarios.ts | 32 ++++--- .../durable/azureStorage/dotnetIsolated.ts | 1 - .../durable/azureStorage/pythonScenario.ts | 17 ++-- .../scenarios/testScenarios/testScenarios.ts | 3 +- 5 files changed, 80 insertions(+), 68 deletions(-) diff --git a/test/nightly/scenarios/ScenariosTracker.ts b/test/nightly/scenarios/ScenariosTracker.ts index c2ac1f699..1bba1654f 100644 --- a/test/nightly/scenarios/ScenariosTracker.ts +++ b/test/nightly/scenarios/ScenariosTracker.ts @@ -18,24 +18,21 @@ export class ScenariosTracker { startCreateNewProject(scenarioLabel: string, createNewProjectLabel: string): void { const scenarioStatus = nonNullValue(this.scenarioStatuses.get(scenarioLabel)); - scenarioStatus.createNewProject = { - label: createNewProjectLabel, - passed: false, - }; + scenarioStatus.createNewProject = { label: createNewProjectLabel }; this.scenarioStatuses.set(scenarioLabel, scenarioStatus); } passCreateNewProject(scenarioLabel: string): void { const scenarioStatus = nonNullValue(this.scenarioStatuses.get(scenarioLabel)); const createNewProjectStatus = nonNullProp(scenarioStatus, 'createNewProject'); - createNewProjectStatus.passed = true; + createNewProjectStatus.status = 'pass'; this.scenarioStatuses.set(scenarioLabel, scenarioStatus); } failCreateNewProject(scenarioLabel: string, error: string): void { const scenarioStatus = nonNullValue(this.scenarioStatuses.get(scenarioLabel)); const createNewProjectStatus = nonNullProp(scenarioStatus, 'createNewProject'); - createNewProjectStatus.passed = false; + createNewProjectStatus.status = 'fail'; createNewProjectStatus.error = error; this.scenarioStatuses.set(scenarioLabel, scenarioStatus); } @@ -54,10 +51,7 @@ export class ScenariosTracker { scenarioStatus.createAndDeployTests ??= []; const createAndDeployTest = nonNullValue(scenarioStatus.createAndDeployTests[createAndDeployTestId]); - createAndDeployTest.createFunctionApp = { - label: createLabel, - passed: false, - }; + createAndDeployTest.createFunctionApp = { label: createLabel }; this.scenarioStatuses.set(scenarioLabel, scenarioStatus); } @@ -68,7 +62,7 @@ export class ScenariosTracker { const createAndDeployTest = nonNullValue(scenarioStatus.createAndDeployTests[createAndDeployTestId]); const createFunctionAppTest = nonNullValue(createAndDeployTest.createFunctionApp); - createFunctionAppTest.passed = true; + createFunctionAppTest.status = 'pass'; this.scenarioStatuses.set(scenarioLabel, scenarioStatus); } @@ -79,7 +73,7 @@ export class ScenariosTracker { const createAndDeployTest = nonNullValue(scenarioStatus.createAndDeployTests[createAndDeployTestId]); const createFunctionAppTest = nonNullValue(createAndDeployTest.createFunctionApp); - createFunctionAppTest.passed = false; + createFunctionAppTest.status = 'fail'; createFunctionAppTest.error = error; this.scenarioStatuses.set(scenarioLabel, scenarioStatus); @@ -90,10 +84,7 @@ export class ScenariosTracker { scenarioStatus.createAndDeployTests ??= []; const createAndDeployTest = nonNullValue(scenarioStatus.createAndDeployTests[createAndDeployTestId]); - createAndDeployTest.deployFunctionApp = { - label: deployLabel, - passed: false, - }; + createAndDeployTest.deployFunctionApp = { label: deployLabel }; this.scenarioStatuses.set(scenarioLabel, scenarioStatus); } @@ -104,7 +95,19 @@ export class ScenariosTracker { const createAndDeployTest = nonNullValue(scenarioStatus.createAndDeployTests[createAndDeployTestId]); const deployFunctionAppTest = nonNullValue(createAndDeployTest.deployFunctionApp); - deployFunctionAppTest.passed = true; + deployFunctionAppTest.status = 'pass'; + + this.scenarioStatuses.set(scenarioLabel, scenarioStatus); + } + + warnDeployFunctionApp(scenarioLabel: string, createAndDeployTestId: number, error?: string): void { + const scenarioStatus = nonNullValue(this.scenarioStatuses.get(scenarioLabel)); + scenarioStatus.createAndDeployTests ??= []; + + const createAndDeployTest = nonNullValue(scenarioStatus.createAndDeployTests[createAndDeployTestId]); + const deployFunctionAppTest = nonNullValue(createAndDeployTest.deployFunctionApp); + deployFunctionAppTest.status = 'warn'; + deployFunctionAppTest.error = error; this.scenarioStatuses.set(scenarioLabel, scenarioStatus); } @@ -115,7 +118,7 @@ export class ScenariosTracker { const createAndDeployTest = nonNullValue(scenarioStatus.createAndDeployTests[createAndDeployTestId]); const deployFunctionAppTest = nonNullValue(createAndDeployTest.deployFunctionApp); - deployFunctionAppTest.passed = false; + deployFunctionAppTest.status = 'fail'; deployFunctionAppTest.error = error; this.scenarioStatuses.set(scenarioLabel, scenarioStatus); @@ -127,16 +130,16 @@ export class ScenariosTracker { } const lines: string[] = []; - const passIcon = '✅'; - const failIcon = '❌'; - const dashIcon = '-'; + const icons: Record = { + pass: '✅', + warn: '⚠️', + fail: '❌', + undefined: '-', + }; const maxErrorLength = 200; - const getStatusIcon = (passed: boolean | undefined): string => { - if (passed === undefined) { - return dashIcon; - } - return passed ? passIcon : failIcon; + const getStatusIcon = (status: TestStatus | undefined): string => { + return icons[status ?? 'undefined']; }; const truncateError = (error: string | undefined): string => { @@ -158,26 +161,26 @@ export class ScenariosTracker { // Report createNewProject test if (scenario.createNewProject) { - const status = getStatusIcon(scenario.createNewProject.passed); + const statusIcon = getStatusIcon(scenario.createNewProject.status); const error = truncateError(scenario.createNewProject.error); const errorCell = error ? ` | ${error} |` : ''; - lines.push(`| ${rowNum++} | ${scenarioLabel} | ${scenario.createNewProject.label} | ${status}${errorCell}`); + lines.push(`| ${rowNum++} | ${scenarioLabel} | ${scenario.createNewProject.label} | ${statusIcon}${errorCell}`); } // Report createAndDeployTests if (scenario.createAndDeployTests) { for (const test of scenario.createAndDeployTests) { if (test.createFunctionApp) { - const status = getStatusIcon(test.createFunctionApp.passed); + const statusIcon = getStatusIcon(test.createFunctionApp.status); const error = truncateError(test.createFunctionApp.error); const errorCell = error ? ` | ${error} |` : ''; - lines.push(`| ${rowNum++} | ${scenarioLabel} | ${test.createFunctionApp.label} | ${status}${errorCell}`); + lines.push(`| ${rowNum++} | ${scenarioLabel} | ${test.createFunctionApp.label} | ${statusIcon}${errorCell}`); } if (test.deployFunctionApp) { - const status = getStatusIcon(test.deployFunctionApp.passed); + const statusIcon = getStatusIcon(test.deployFunctionApp.status); const error = truncateError(test.deployFunctionApp.error); const errorCell = error ? ` | ${error} |` : ''; - lines.push(`| ${rowNum++} | ${scenarioLabel} | ${test.deployFunctionApp.label} | ${status}${errorCell}`); + lines.push(`| ${rowNum++} | ${scenarioLabel} | ${test.deployFunctionApp.label} | ${statusIcon}${errorCell}`); } } } @@ -187,23 +190,19 @@ export class ScenariosTracker { } } +type TestStatus = 'pass' | 'warn' | 'fail'; + +type TestResult = { + label: string; + status?: TestStatus; + error?: string; +}; + type ScenarioStatus = { label: string; - createNewProject?: { - label: string; - passed?: boolean; - error?: string; - }; + createNewProject?: TestResult; createAndDeployTests?: { - createFunctionApp?: { - label: string; - passed?: boolean; - error?: string; - }; - deployFunctionApp?: { - label: string; - passed?: boolean; - error?: string; - }; + createFunctionApp?: TestResult; + deployFunctionApp?: TestResult; }[]; -} +}; diff --git a/test/nightly/scenarios/parallelScenarios.ts b/test/nightly/scenarios/parallelScenarios.ts index c2d421957..704af903b 100644 --- a/test/nightly/scenarios/parallelScenarios.ts +++ b/test/nightly/scenarios/parallelScenarios.ts @@ -103,30 +103,40 @@ async function startCreateAndDeployTest(scenarioLabel: string, rootFolder: Works await runWithTestActionContext('scenario.deploy', async context => { await context.ui.runWithInputs(test.deployFunctionApp.inputs, async () => { - let error: unknown; + let deployError: unknown; try { - await deployProductionSlotByFunctionAppId(context, functionAppId, rootFolder?.uri); - } catch (deployError) { - error = deployError; + await deployProductionSlotByFunctionAppId(context, functionAppId, rootFolder.uri); + } catch (err) { + deployError = err; } + let postTestError: unknown; if (test.deployFunctionApp.postTest) { try { await test.deployFunctionApp.postTest(context, functionAppId); - if ((error as Error).message) { - (error as Error).message = '**Function app deployment errored but still passed post test deployment verification. ' + (error as Error).message; - } - } catch (postTestError) { - error ??= postTestError; + } catch (err) { + postTestError = err; } } + const error = deployError ?? postTestError; if (!error) { scenariosTracker.passDeployFunctionApp(scenarioLabel, scenarioTestTrackerId); + return; + } + + let errorMessage: string = (error as Error).message ?? parseError(error).message; + + // Deploy failed but verification still passed - warn instead of showing an outright fail + // This is a known issue, TODO: Add issue link + if (deployError && !postTestError) { + errorMessage = '** Deploy-Fail; Verify-Pass ** ' + errorMessage; + scenariosTracker.warnDeployFunctionApp(scenarioLabel, scenarioTestTrackerId, errorMessage); } else { - scenariosTracker.failDeployFunctionApp(scenarioLabel, scenarioTestTrackerId, (error as Error).message ?? parseError(error).message); - throw error; + scenariosTracker.failDeployFunctionApp(scenarioLabel, scenarioTestTrackerId, errorMessage); } + + throw error; }); }); } diff --git a/test/nightly/scenarios/testScenarios/durable/azureStorage/dotnetIsolated.ts b/test/nightly/scenarios/testScenarios/durable/azureStorage/dotnetIsolated.ts index 636126559..d13867b13 100644 --- a/test/nightly/scenarios/testScenarios/durable/azureStorage/dotnetIsolated.ts +++ b/test/nightly/scenarios/testScenarios/durable/azureStorage/dotnetIsolated.ts @@ -13,7 +13,6 @@ export function generateDotnetIsolatedScenario(): AzExtFunctionsTestScenario { const folderName: string = 'scenarios-durable-azurestorage-dotnetisolated'; return { - only: true, label: 'durable-azurestorage-dotnetisolated', folderName, createNewProjectTest: { diff --git a/test/nightly/scenarios/testScenarios/durable/azureStorage/pythonScenario.ts b/test/nightly/scenarios/testScenarios/durable/azureStorage/pythonScenario.ts index 8842f02c8..9b5631623 100644 --- a/test/nightly/scenarios/testScenarios/durable/azureStorage/pythonScenario.ts +++ b/test/nightly/scenarios/testScenarios/durable/azureStorage/pythonScenario.ts @@ -5,7 +5,7 @@ import { DurableBackend } from "../../../../../../extension.bundle"; import { durableOrchestratorName, durableOrchestratorPick, pythonDefaultPick } from "../../../../../constants"; -import { ConnectionType, CreateMode, OperatingSystem, PlanType, Runtime } from "../../../../../utils/createFunctionAppUtils"; +import { ConnectionType, CreateMode, PlanType, Runtime } from "../../../../../utils/createFunctionAppUtils"; import { type AzExtFunctionsTestScenario } from "../../AzExtFunctionsTestScenario"; import { generateCreateAndDeployTest } from "../generateCreateAndDeployTest"; @@ -27,12 +27,15 @@ export function generatePythonScenario(): AzExtFunctionsTestScenario { ], }, createAndDeployTests: [ - generateCreateAndDeployTest(folderName, CreateMode.Basic, Runtime.Python, ConnectionType.ManagedIdentity, PlanType.FlexConsumption, OperatingSystem.Linux, DurableBackend.Storage), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Python, ConnectionType.ManagedIdentity, PlanType.Premium, OperatingSystem.Windows, DurableBackend.Storage), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Python, ConnectionType.Secrets, PlanType.FlexConsumption, OperatingSystem.Linux, DurableBackend.Storage), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Python, ConnectionType.Secrets, PlanType.Premium, OperatingSystem.Windows, DurableBackend.Storage), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Python, ConnectionType.Secrets, PlanType.LegacyConsumption, OperatingSystem.Linux, DurableBackend.Storage), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Python, ConnectionType.Secrets, PlanType.AppService, OperatingSystem.Windows, DurableBackend.Storage), + // OS gets automatically defaulted to Linux + generateCreateAndDeployTest(folderName, CreateMode.Basic, Runtime.Python, ConnectionType.ManagedIdentity, PlanType.FlexConsumption, undefined /** os */, DurableBackend.Storage), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Python, ConnectionType.ManagedIdentity, PlanType.Premium, undefined /** os */, DurableBackend.Storage), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Python, ConnectionType.ManagedIdentity, PlanType.LegacyConsumption, undefined /** os */, DurableBackend.Storage), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Python, ConnectionType.ManagedIdentity, PlanType.AppService, undefined /** os */, DurableBackend.Storage), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Python, ConnectionType.Secrets, PlanType.FlexConsumption, undefined /** os */, DurableBackend.Storage), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Python, ConnectionType.Secrets, PlanType.Premium, undefined /** os */, DurableBackend.Storage), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Python, ConnectionType.Secrets, PlanType.LegacyConsumption, undefined /** os */, DurableBackend.Storage), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Python, ConnectionType.Secrets, PlanType.AppService, undefined /** os */, DurableBackend.Storage), ], }; } diff --git a/test/nightly/scenarios/testScenarios/testScenarios.ts b/test/nightly/scenarios/testScenarios/testScenarios.ts index d253fdea2..a6125c74c 100644 --- a/test/nightly/scenarios/testScenarios/testScenarios.ts +++ b/test/nightly/scenarios/testScenarios/testScenarios.ts @@ -5,11 +5,12 @@ import { type AzExtFunctionsTestScenario } from "./AzExtFunctionsTestScenario"; import { generateDurableAzureStorageScenarios } from "./durable/azureStorage/azureStorageScenarios"; +import { generateDurableDTSScenarios } from "./durable/dts/dtsScenarios"; export function generateTestScenarios(): AzExtFunctionsTestScenario[] { const testScenarios: AzExtFunctionsTestScenario[] = [ ...generateDurableAzureStorageScenarios(), - // ...generateDurableDTSScenarios(), + ...generateDurableDTSScenarios(), ]; return testScenarios; } From 564b423885d61ddff9dfe8c082e115f91018386f Mon Sep 17 00:00:00 2001 From: MicroFish91 <40250218+MicroFish91@users.noreply.github.com> Date: Tue, 6 Jan 2026 18:12:31 -0800 Subject: [PATCH 13/27] Add scenario testing for http triggers js --- test/constants.ts | 8 +++ .../generateCreateAndDeployTest.ts | 61 +++++++++++++++++++ .../httpTrigger/httpTriggerJSNodeScenario.ts | 43 +++++++++++++ .../httpTrigger/httpTriggerScenarios.ts | 13 ++++ .../scenarios/testScenarios/testScenarios.ts | 5 ++ test/test.code-workspace | 8 +-- 6 files changed, 134 insertions(+), 4 deletions(-) create mode 100644 test/nightly/scenarios/testScenarios/httpTrigger/generateCreateAndDeployTest.ts create mode 100644 test/nightly/scenarios/testScenarios/httpTrigger/httpTriggerJSNodeScenario.ts create mode 100644 test/nightly/scenarios/testScenarios/httpTrigger/httpTriggerScenarios.ts diff --git a/test/constants.ts b/test/constants.ts index 9ae56c6bf..9fc317768 100644 --- a/test/constants.ts +++ b/test/constants.ts @@ -7,6 +7,14 @@ export const durableOrchestratorPick: RegExp = /Durable Functions Orch/i; export const durableOrchestratorName: string = 'durableHello'; +export const httpTriggerName: string = 'httpTrigger'; + +// Language picks +export const jsLanguagePick: RegExp = /JavaScript/i; + +// Framework picks +export const jsModelV4Pick: RegExp = /v4/i; + export const locationDefaultPick: RegExp = /West US 2/i; export const pythonDefaultPick: RegExp = /Python(\s)?3\.12/i; export const nodeDefaultPick: RegExp = /Node\.js(\s)?22/i; diff --git a/test/nightly/scenarios/testScenarios/httpTrigger/generateCreateAndDeployTest.ts b/test/nightly/scenarios/testScenarios/httpTrigger/generateCreateAndDeployTest.ts new file mode 100644 index 000000000..504d61d7d --- /dev/null +++ b/test/nightly/scenarios/testScenarios/httpTrigger/generateCreateAndDeployTest.ts @@ -0,0 +1,61 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { type Site, type WebSiteManagementClient } from "@azure/arm-appservice"; +import { createWebSiteClient } from "@microsoft/vscode-azext-azureappservice"; +import { parseAzureResourceId, type ParsedAzureResourceId } from "@microsoft/vscode-azext-azureutils"; +import { getRandomAlphanumericString, type IActionContext } from "../../../../../extension.bundle"; +import { httpTriggerName } from "../../../../constants"; +import { createFunctionAppUtils, CreateMode, Runtime, type ConnectionType, type OperatingSystem, type PlanType } from "../../../../utils/createFunctionAppUtils"; +import { subscriptionContext } from "../../../global.nightly.test"; +import { type CreateAndDeployTestCase } from "../AzExtFunctionsTestScenario"; + +export function generateCreateAndDeployTest(folderName: string, createMode: CreateMode, runtime: Runtime, connection: ConnectionType, plan: PlanType, os?: OperatingSystem): CreateAndDeployTestCase { + const appName: string = getRandomAlphanumericString(); + const osDescription: string = os ? `-${os}` : ''; + const description: string = `${createMode}-${connection}${osDescription}-${plan}`; + + return { + createFunctionApp: { + label: `create-function-app | ${description}`, + mode: createMode, + inputs: createMode === CreateMode.Basic ? + createFunctionAppUtils.generateBasicCreateInputs(appName, folderName, runtime, connection) : + createFunctionAppUtils.generateAdvancedCreateInputs(appName, folderName, runtime, connection, plan, os), + }, + deployFunctionApp: { + label: `deploy-function-app | ${description}`, + inputs: ['Deploy'], + postTest: generateVerifyDeployment(runtime), + }, + resourceGroupsToDelete: [appName], + }; +} + +function generateVerifyDeployment(runtime: Runtime) { + return async function verifyDeployment(context: IActionContext, functionAppId: string): Promise { + const client: WebSiteManagementClient = await createWebSiteClient({ ...context, ...subscriptionContext }); + const parsedResource: ParsedAzureResourceId = parseAzureResourceId(functionAppId); + const functionApp: Site = await client.webApps.get(parsedResource.resourceGroup, parsedResource.resourceName); + + let url = `https://${functionApp.defaultHostName}`; + switch (runtime) { + // case Runtime.Python: + // break; + case Runtime.Node: + url += `/api/${httpTriggerName}`; + break; + // case Runtime.DotNetIsolated: + // break; + default: + throw new Error('Durable verify deployment not yet implemented for this runtime type.'); + } + + const response = await fetch(url); + if (response.status !== 200) { + throw new Error(`Verify Deployment: Http trigger endpoint responded with ${response.status}.`); + } + }; +} diff --git a/test/nightly/scenarios/testScenarios/httpTrigger/httpTriggerJSNodeScenario.ts b/test/nightly/scenarios/testScenarios/httpTrigger/httpTriggerJSNodeScenario.ts new file mode 100644 index 000000000..ba9cb4c2b --- /dev/null +++ b/test/nightly/scenarios/testScenarios/httpTrigger/httpTriggerJSNodeScenario.ts @@ -0,0 +1,43 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { httpTriggerName, jsLanguagePick, jsModelV4Pick } from "../../../../constants"; +import { ConnectionType, CreateMode, OperatingSystem, PlanType, Runtime } from "../../../../utils/createFunctionAppUtils"; +import { type AzExtFunctionsTestScenario } from "../AzExtFunctionsTestScenario"; +import { generateCreateAndDeployTest } from "./generateCreateAndDeployTest"; + +export function generateJSNodeScenario(): AzExtFunctionsTestScenario { + const folderName: string = 'scenarios-http-trigger-jsnode'; + + return { + label: 'http-trigger-jsnode', + folderName, + createNewProjectTest: { + label: 'create-new-project', + inputs: [ + jsLanguagePick, + jsModelV4Pick, + /HTTP\s?Trigger/i, + httpTriggerName, + ], + }, + createAndDeployTests: [ + generateCreateAndDeployTest(folderName, CreateMode.Basic, Runtime.Node, ConnectionType.ManagedIdentity, PlanType.FlexConsumption, OperatingSystem.Linux), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.ManagedIdentity, PlanType.Premium, OperatingSystem.Linux), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.ManagedIdentity, PlanType.Premium, OperatingSystem.Windows), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.ManagedIdentity, PlanType.LegacyConsumption, OperatingSystem.Linux), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.ManagedIdentity, PlanType.LegacyConsumption, OperatingSystem.Windows), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.ManagedIdentity, PlanType.AppService, OperatingSystem.Linux), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.ManagedIdentity, PlanType.AppService, OperatingSystem.Windows), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.FlexConsumption, OperatingSystem.Linux), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.Premium, OperatingSystem.Linux), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.Premium, OperatingSystem.Windows), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.LegacyConsumption, OperatingSystem.Linux), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.LegacyConsumption, OperatingSystem.Windows), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.AppService, OperatingSystem.Linux), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.AppService, OperatingSystem.Windows), + ], + }; +} diff --git a/test/nightly/scenarios/testScenarios/httpTrigger/httpTriggerScenarios.ts b/test/nightly/scenarios/testScenarios/httpTrigger/httpTriggerScenarios.ts new file mode 100644 index 000000000..0f5456c5e --- /dev/null +++ b/test/nightly/scenarios/testScenarios/httpTrigger/httpTriggerScenarios.ts @@ -0,0 +1,13 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { type AzExtFunctionsTestScenario } from "../AzExtFunctionsTestScenario"; +import { generateJSNodeScenario } from "./httpTriggerJSNodeScenario"; + +export function generateHttpTriggerScenarios(): AzExtFunctionsTestScenario[] { + return [ + generateJSNodeScenario(), + ]; +} diff --git a/test/nightly/scenarios/testScenarios/testScenarios.ts b/test/nightly/scenarios/testScenarios/testScenarios.ts index a6125c74c..607fc3c68 100644 --- a/test/nightly/scenarios/testScenarios/testScenarios.ts +++ b/test/nightly/scenarios/testScenarios/testScenarios.ts @@ -6,9 +6,14 @@ import { type AzExtFunctionsTestScenario } from "./AzExtFunctionsTestScenario"; import { generateDurableAzureStorageScenarios } from "./durable/azureStorage/azureStorageScenarios"; import { generateDurableDTSScenarios } from "./durable/dts/dtsScenarios"; +import { generateHttpTriggerScenarios } from "./httpTrigger/httpTriggerScenarios"; export function generateTestScenarios(): AzExtFunctionsTestScenario[] { const testScenarios: AzExtFunctionsTestScenario[] = [ + // Trigger scenarios + ...generateHttpTriggerScenarios(), + + // Durable scenarios ...generateDurableAzureStorageScenarios(), ...generateDurableDTSScenarios(), ]; diff --git a/test/test.code-workspace b/test/test.code-workspace index 02de5fdb9..7f452a321 100644 --- a/test/test.code-workspace +++ b/test/test.code-workspace @@ -30,6 +30,10 @@ { "path": "../testWorkspace/9" }, + { + "name": "scenarios-http-trigger-jsnode", + "path": "../testWorkspace/scenarios/http-trigger/jsnode" + }, { "name": "scenarios-durable-azurestorage-jsnode", "path": "../testWorkspace/scenarios/durable-azurestorage/jsnode" @@ -38,10 +42,6 @@ "name": "scenarios-durable-azurestorage-python", "path": "../testWorkspace/scenarios/durable-azurestorage/python" }, - { - "name": "scenarios-durable-azurestorage-dotnetinproc", - "path": "../testWorkspace/scenarios/durable-azurestorage/dotnetinproc" - }, { "name": "scenarios-durable-azurestorage-dotnetisolated", "path": "../testWorkspace/scenarios/durable-azurestorage/dotnetisolated" From 810d4562df3f3ba98cefe36dcb9443e417e46938 Mon Sep 17 00:00:00 2001 From: MicroFish91 <40250218+MicroFish91@users.noreply.github.com> Date: Fri, 9 Jan 2026 14:21:23 -0800 Subject: [PATCH 14/27] Update test combinations markdown --- .../docs/ScenarioTestCombinations.md | 198 +++++++++--------- test/nightly/scenarios/docs/TestScenarios.md | 0 2 files changed, 99 insertions(+), 99 deletions(-) delete mode 100644 test/nightly/scenarios/docs/TestScenarios.md diff --git a/test/nightly/scenarios/docs/ScenarioTestCombinations.md b/test/nightly/scenarios/docs/ScenarioTestCombinations.md index 3bf552544..2973e8ca3 100644 --- a/test/nightly/scenarios/docs/ScenarioTestCombinations.md +++ b/test/nightly/scenarios/docs/ScenarioTestCombinations.md @@ -2,7 +2,7 @@ ## I. Create New Project / Create Function -### Workspace Project Test Matrix +### Workspace Project - Language / Runtime Test Matrix | No. | Language | Runtime | Programming Model | Comment | Selected | |-----|----------|---------|-------------------|------------------------------------|-------------| @@ -20,101 +20,101 @@ TBD ### Create / Deploy Test Matrix -| No. | Workspace Project | Connection Type | Operating System | Plan Type | Comment | Selected | -|-----|-------------------|-------------------|------------------|------------------------|-------------|-------------| -| 1 | 1 | Managed Identity | Linux | Flex Consumption | Skip | | -| 2 | 1 | Managed Identity | Windows | Flex Consumption | Skip | | -| 3 | 1 | Managed Identity | Linux | Premium | Skip | | -| 4 | 1 | Managed Identity | Windows | Premium | Skip | | -| 5 | 1 | Managed Identity | Linux | Consumption (Legacy) | Skip | | -| 6 | 1 | Managed Identity | Windows | Consumption (Legacy) | Skip | | -| 7 | 1 | Managed Identity | Linux | App Service | Skip | | -| 8 | 1 | Managed Identity | Windows | App Service | Skip | | -| 9 | 1 | Secrets | Linux | Flex Consumption | Skip | | -| 10 | 1 | Secrets | Windows | Flex Consumption | Skip | | -| 11 | 1 | Secrets | Linux | Premium | Skip | | -| 12 | 1 | Secrets | Windows | Premium | Skip | | -| 13 | 1 | Secrets | Linux | Consumption (Legacy) | Skip | | -| 14 | 1 | Secrets | Windows | Consumption (Legacy) | Skip | | -| 15 | 1 | Secrets | Linux | App Service | Skip | | -| 16 | 1 | Secrets | Windows | App Service | Skip | | -| 17 | 2 | Managed Identity | Linux | Flex Consumption | | | -| 18 | 2 | Managed Identity | Windows | Flex Consumption | Not Offered | | -| 19 | 2 | Managed Identity | Linux | Premium | | | -| 20 | 2 | Managed Identity | Windows | Premium | | | -| 21 | 2 | Managed Identity | Linux | Consumption (Legacy) | | | -| 22 | 2 | Managed Identity | Windows | Consumption (Legacy) | | | -| 23 | 2 | Managed Identity | Linux | App Service | | | -| 24 | 2 | Managed Identity | Windows | App Service | | | -| 25 | 2 | Secrets | Linux | Flex Consumption | | | -| 26 | 2 | Secrets | Windows | Flex Consumption | Not Offered | | -| 27 | 2 | Secrets | Linux | Premium | | | -| 28 | 2 | Secrets | Windows | Premium | | | -| 29 | 2 | Secrets | Linux | Consumption (Legacy) | | | -| 30 | 2 | Secrets | Windows | Consumption (Legacy) | | | -| 31 | 2 | Secrets | Linux | App Service | | | -| 32 | 2 | Secrets | Windows | App Service | | | -| 33 | 3 | Managed Identity | Linux | Flex Consumption | Skip | | -| 34 | 3 | Managed Identity | Windows | Flex Consumption | Skip | | -| 35 | 3 | Managed Identity | Linux | Premium | Skip | | -| 36 | 3 | Managed Identity | Windows | Premium | Skip | | -| 37 | 3 | Managed Identity | Linux | Consumption (Legacy) | Skip | | -| 38 | 3 | Managed Identity | Windows | Consumption (Legacy) | Skip | | -| 39 | 3 | Managed Identity | Linux | App Service | Skip | | -| 40 | 3 | Managed Identity | Windows | App Service | Skip | | -| 41 | 3 | Secrets | Linux | Flex Consumption | Skip | | -| 42 | 3 | Secrets | Windows | Flex Consumption | Skip | | -| 43 | 3 | Secrets | Linux | Premium | Skip | | -| 44 | 3 | Secrets | Windows | Premium | Skip | | -| 45 | 3 | Secrets | Linux | Consumption (Legacy) | Skip | | -| 46 | 3 | Secrets | Windows | Consumption (Legacy) | Skip | | -| 47 | 3 | Secrets | Linux | App Service | Skip | | -| 48 | 3 | Secrets | Windows | App Service | Skip | | -| 49 | 4 | Managed Identity | Linux | Flex Consumption | | | -| 50 | 4 | Managed Identity | Windows | Flex Consumption | Not Offered | | -| 51 | 4 | Managed Identity | Linux | Premium | | | -| 52 | 4 | Managed Identity | Windows | Premium | | | -| 53 | 4 | Managed Identity | Linux | Consumption (Legacy) | | | -| 54 | 4 | Managed Identity | Windows | Consumption (Legacy) | | | -| 55 | 4 | Managed Identity | Linux | App Service | | | -| 56 | 4 | Managed Identity | Windows | App Service | | | -| 57 | 4 | Secrets | Linux | Flex Consumption | | | -| 58 | 4 | Secrets | Windows | Flex Consumption | Not Offered | | -| 59 | 4 | Secrets | Linux | Premium | | | -| 60 | 4 | Secrets | Windows | Premium | | | -| 61 | 4 | Secrets | Linux | Consumption (Legacy) | | | -| 62 | 4 | Secrets | Windows | Consumption (Legacy) | | | -| 63 | 4 | Secrets | Linux | App Service | | | -| 64 | 4 | Secrets | Windows | App Service | | | -| 65 | 5 | Managed Identity | Linux | Flex Consumption | | | -| 66 | 5 | Managed Identity | Windows | Flex Consumption | Not Offered | | -| 67 | 5 | Managed Identity | Linux | Premium | | | -| 68 | 5 | Managed Identity | Windows | Premium | | | -| 69 | 5 | Managed Identity | Linux | Consumption (Legacy) | | | -| 70 | 5 | Managed Identity | Windows | Consumption (Legacy) | | | -| 71 | 5 | Managed Identity | Linux | App Service | | | -| 72 | 5 | Managed Identity | Windows | App Service | | | -| 73 | 5 | Secrets | Linux | Flex Consumption | | | -| 74 | 5 | Secrets | Windows | Flex Consumption | Not Offered | | -| 75 | 5 | Secrets | Linux | Premium | | | -| 76 | 5 | Secrets | Windows | Premium | | | -| 77 | 5 | Secrets | Linux | Consumption (Legacy) | | | -| 78 | 5 | Secrets | Windows | Consumption (Legacy) | | | -| 79 | 5 | Secrets | Linux | App Service | | | -| 80 | 5 | Secrets | Windows | App Service | | | -| 81 | 6 | Managed Identity | Linux | Flex Consumption | | | -| 82 | 6 | Managed Identity | Windows | Flex Consumption | Not Offered | | -| 83 | 6 | Managed Identity | Linux | Premium | | | -| 84 | 6 | Managed Identity | Windows | Premium | | | -| 85 | 6 | Managed Identity | Linux | Consumption (Legacy) | | | -| 86 | 6 | Managed Identity | Windows | Consumption (Legacy) | | | -| 87 | 6 | Managed Identity | Linux | App Service | | | -| 88 | 6 | Managed Identity | Windows | App Service | | | -| 89 | 6 | Secrets | Linux | Flex Consumption | | | -| 90 | 6 | Secrets | Windows | Flex Consumption | Not Offered | | -| 91 | 6 | Secrets | Linux | Premium | | | -| 92 | 6 | Secrets | Windows | Premium | | | -| 93 | 6 | Secrets | Linux | Consumption (Legacy) | | | -| 94 | 6 | Secrets | Windows | Consumption (Legacy) | | | -| 95 | 6 | Secrets | Linux | App Service | | | -| 96 | 6 | Secrets | Windows | App Service | | | +| No. | Workspace Project | Storage Connection Type | Operating System | Plan Type | Comment | Selected | +|-----|-------------------|----------------------------|------------------|---------------------|-------------|----------| +| 1 | 1 | Managed Identity | Linux | Flex Consumption | Skip | | +| 2 | 1 | Managed Identity | Windows | Flex Consumption | Skip | | +| 3 | 1 | Managed Identity | Linux | Premium | Skip | | +| 4 | 1 | Managed Identity | Windows | Premium | Skip | | +| 5 | 1 | Managed Identity | Linux | Consumption (Legacy)| Skip | | +| 6 | 1 | Managed Identity | Windows | Consumption (Legacy)| Skip | | +| 7 | 1 | Managed Identity | Linux | App Service | Skip | | +| 8 | 1 | Managed Identity | Windows | App Service | Skip | | +| 9 | 1 | Secrets | Linux | Flex Consumption | Skip | | +| 10 | 1 | Secrets | Windows | Flex Consumption | Skip | | +| 11 | 1 | Secrets | Linux | Premium | Skip | | +| 12 | 1 | Secrets | Windows | Premium | Skip | | +| 13 | 1 | Secrets | Linux | Consumption (Legacy)| Skip | | +| 14 | 1 | Secrets | Windows | Consumption (Legacy)| Skip | | +| 15 | 1 | Secrets | Linux | App Service | Skip | | +| 16 | 1 | Secrets | Windows | App Service | Skip | | +| 17 | 2 | Managed Identity | Linux | Flex Consumption | | | +| 18 | 2 | Managed Identity | Windows | Flex Consumption | Not Offered | | +| 19 | 2 | Managed Identity | Linux | Premium | | | +| 20 | 2 | Managed Identity | Windows | Premium | | | +| 21 | 2 | Managed Identity | Linux | Consumption (Legacy)| | | +| 22 | 2 | Managed Identity | Windows | Consumption (Legacy)| | | +| 23 | 2 | Managed Identity | Linux | App Service | | | +| 24 | 2 | Managed Identity | Windows | App Service | | | +| 25 | 2 | Secrets | Linux | Flex Consumption | | | +| 26 | 2 | Secrets | Windows | Flex Consumption | Not Offered | | +| 27 | 2 | Secrets | Linux | Premium | | | +| 28 | 2 | Secrets | Windows | Premium | | | +| 29 | 2 | Secrets | Linux | Consumption (Legacy)| | | +| 30 | 2 | Secrets | Windows | Consumption (Legacy)| | | +| 31 | 2 | Secrets | Linux | App Service | | | +| 32 | 2 | Secrets | Windows | App Service | | | +| 33 | 3 | Managed Identity | Linux | Flex Consumption | Skip | | +| 34 | 3 | Managed Identity | Windows | Flex Consumption | Skip | | +| 35 | 3 | Managed Identity | Linux | Premium | Skip | | +| 36 | 3 | Managed Identity | Windows | Premium | Skip | | +| 37 | 3 | Managed Identity | Linux | Consumption (Legacy)| Skip | | +| 38 | 3 | Managed Identity | Windows | Consumption (Legacy)| Skip | | +| 39 | 3 | Managed Identity | Linux | App Service | Skip | | +| 40 | 3 | Managed Identity | Windows | App Service | Skip | | +| 41 | 3 | Secrets | Linux | Flex Consumption | Skip | | +| 42 | 3 | Secrets | Windows | Flex Consumption | Skip | | +| 43 | 3 | Secrets | Linux | Premium | Skip | | +| 44 | 3 | Secrets | Windows | Premium | Skip | | +| 45 | 3 | Secrets | Linux | Consumption (Legacy)| Skip | | +| 46 | 3 | Secrets | Windows | Consumption (Legacy)| Skip | | +| 47 | 3 | Secrets | Linux | App Service | Skip | | +| 48 | 3 | Secrets | Windows | App Service | Skip | | +| 49 | 4 | Managed Identity | Linux | Flex Consumption | | | +| 50 | 4 | Managed Identity | Windows | Flex Consumption | Not Offered | | +| 51 | 4 | Managed Identity | Linux | Premium | | | +| 52 | 4 | Managed Identity | Windows | Premium | | | +| 53 | 4 | Managed Identity | Linux | Consumption (Legacy)| | | +| 54 | 4 | Managed Identity | Windows | Consumption (Legacy)| | | +| 55 | 4 | Managed Identity | Linux | App Service | | | +| 56 | 4 | Managed Identity | Windows | App Service | | | +| 57 | 4 | Secrets | Linux | Flex Consumption | | | +| 58 | 4 | Secrets | Windows | Flex Consumption | Not Offered | | +| 59 | 4 | Secrets | Linux | Premium | | | +| 60 | 4 | Secrets | Windows | Premium | | | +| 61 | 4 | Secrets | Linux | Consumption (Legacy)| | | +| 62 | 4 | Secrets | Windows | Consumption (Legacy)| | | +| 63 | 4 | Secrets | Linux | App Service | | | +| 64 | 4 | Secrets | Windows | App Service | | | +| 65 | 5 | Managed Identity | Linux | Flex Consumption | | | +| 66 | 5 | Managed Identity | Windows | Flex Consumption | Not Offered | | +| 67 | 5 | Managed Identity | Linux | Premium | | | +| 68 | 5 | Managed Identity | Windows | Premium | | | +| 69 | 5 | Managed Identity | Linux | Consumption (Legacy)| | | +| 70 | 5 | Managed Identity | Windows | Consumption (Legacy)| | | +| 71 | 5 | Managed Identity | Linux | App Service | | | +| 72 | 5 | Managed Identity | Windows | App Service | | | +| 73 | 5 | Secrets | Linux | Flex Consumption | | | +| 74 | 5 | Secrets | Windows | Flex Consumption | Not Offered | | +| 75 | 5 | Secrets | Linux | Premium | | | +| 76 | 5 | Secrets | Windows | Premium | | | +| 77 | 5 | Secrets | Linux | Consumption (Legacy)| | | +| 78 | 5 | Secrets | Windows | Consumption (Legacy)| | | +| 79 | 5 | Secrets | Linux | App Service | | | +| 80 | 5 | Secrets | Windows | App Service | | | +| 81 | 6 | Managed Identity | Linux | Flex Consumption | | | +| 82 | 6 | Managed Identity | Windows | Flex Consumption | Not Offered | | +| 83 | 6 | Managed Identity | Linux | Premium | | | +| 84 | 6 | Managed Identity | Windows | Premium | | | +| 85 | 6 | Managed Identity | Linux | Consumption (Legacy)| | | +| 86 | 6 | Managed Identity | Windows | Consumption (Legacy)| | | +| 87 | 6 | Managed Identity | Linux | App Service | | | +| 88 | 6 | Managed Identity | Windows | App Service | | | +| 89 | 6 | Secrets | Linux | Flex Consumption | | | +| 90 | 6 | Secrets | Windows | Flex Consumption | Not Offered | | +| 91 | 6 | Secrets | Linux | Premium | | | +| 92 | 6 | Secrets | Windows | Premium | | | +| 93 | 6 | Secrets | Linux | Consumption (Legacy)| | | +| 94 | 6 | Secrets | Windows | Consumption (Legacy)| | | +| 95 | 6 | Secrets | Linux | App Service | | | +| 96 | 6 | Secrets | Windows | App Service | | | diff --git a/test/nightly/scenarios/docs/TestScenarios.md b/test/nightly/scenarios/docs/TestScenarios.md deleted file mode 100644 index e69de29bb..000000000 From 60b703e2117db122cd3cf24bae1a8cd6e87b9659 Mon Sep 17 00:00:00 2001 From: MicroFish91 <40250218+MicroFish91@users.noreply.github.com> Date: Fri, 9 Jan 2026 16:26:20 -0800 Subject: [PATCH 15/27] Update plan.md's --- .../azureStorage/DurableAzureStorage.plan.md | 38 ++++++++++--------- .../httpTrigger/HttpTrigger.plan.md | 33 ++++++++++++++++ 2 files changed, 54 insertions(+), 17 deletions(-) create mode 100644 test/nightly/scenarios/testScenarios/httpTrigger/HttpTrigger.plan.md diff --git a/test/nightly/scenarios/testScenarios/durable/azureStorage/DurableAzureStorage.plan.md b/test/nightly/scenarios/testScenarios/durable/azureStorage/DurableAzureStorage.plan.md index 0e912cd85..1d90066d7 100644 --- a/test/nightly/scenarios/testScenarios/durable/azureStorage/DurableAzureStorage.plan.md +++ b/test/nightly/scenarios/testScenarios/durable/azureStorage/DurableAzureStorage.plan.md @@ -35,20 +35,24 @@ TBD | 13 | 1 | Secrets | Linux | App Service | | | 14 | 1 | Secrets | Windows | App Service | | | 15 | 2 | Managed Identity | Linux | Flex Consumption | | -| 16 | 2 | Managed Identity | Windows | Premium | | -| 17 | 2 | Secrets | Linux | Flex Consumption | | -| 18 | 2 | Secrets | Windows | Premium | | -| 19 | 2 | Secrets | Linux | Consumption (Legacy) | | -| 20 | 2 | Secrets | Windows | App Service | | -| 21 | 3 | Managed Identity | Linux | Flex Consumption | | -| 22 | 3 | Managed Identity | Windows | Premium | | -| 23 | 3 | Secrets | Linux | Flex Consumption | | -| 24 | 3 | Secrets | Windows | Premium | | -| 25 | 3 | Secrets | Linux | Consumption (Legacy) | | -| 26 | 3 | Secrets | Windows | App Service | | -| 27 | 4 | Managed Identity | Linux | Flex Consumption | | -| 28 | 4 | Managed Identity | Windows | Premium | | -| 29 | 4 | Secrets | Linux | Flex Consumption | | -| 30 | 4 | Secrets | Windows | Premium | | -| 31 | 4 | Secrets | Linux | Consumption (Legacy) | | -| 32 | 4 | Secrets | Windows | App Service | | +| 16 | 2 | Managed Identity | Linux | Premium | | +| 17 | 2 | Managed Identity | Linux | Consumption (Legacy) | | +| 18 | 2 | Managed Identity | Linux | App Service | | +| 19 | 2 | Secrets | Linux | Flex Consumption | | +| 20 | 2 | Secrets | Linux | Premium | | +| 21 | 2 | Secrets | Linux | Consumption (Legacy) | | +| 22 | 2 | Secrets | Linux | App Service | | +| 23 | 3 | Managed Identity | Linux | Flex Consumption | | +| 24 | 3 | Managed Identity | Linux | Premium | | +| 25 | 3 | Managed Identity | Windows | Premium | | +| 26 | 3 | Managed Identity | Linux | Consumption (Legacy) | | +| 27 | 3 | Managed Identity | Windows | Consumption (Legacy) | | +| 28 | 3 | Managed Identity | Linux | App Service | | +| 29 | 3 | Managed Identity | Windows | App Service | | +| 30 | 3 | Secrets | Linux | Flex Consumption | | +| 31 | 3 | Secrets | Linux | Premium | | +| 32 | 3 | Secrets | Windows | Premium | | +| 33 | 3 | Secrets | Linux | Consumption (Legacy) | | +| 34 | 3 | Secrets | Windows | Consumption (Legacy) | | +| 35 | 3 | Secrets | Linux | App Service | | +| 36 | 3 | Secrets | Windows | App Service | | diff --git a/test/nightly/scenarios/testScenarios/httpTrigger/HttpTrigger.plan.md b/test/nightly/scenarios/testScenarios/httpTrigger/HttpTrigger.plan.md new file mode 100644 index 000000000..9927377b8 --- /dev/null +++ b/test/nightly/scenarios/testScenarios/httpTrigger/HttpTrigger.plan.md @@ -0,0 +1,33 @@ +# HTTP Trigger Test Plan + +## I. Create New Project / Create Function + +### Workspace Project Test Matrix + +| No. | Language | Runtime | Programming Model | Comment | +|-----|----------|---------|-------------------|---------| +| 1 | JS | Node | v4 | | + +## II. Debug +TBD + +## III. Create Function App / Deployment + +### Create / Deploy Test Matrix + +| No. | Workspace Project | Connection Type | Operating System | Plan Type | Comment | +|-----|-------------------|-------------------|------------------|------------------------|-------------| +| 1 | 1 | Managed Identity | Linux | Flex Consumption | | +| 2 | 1 | Managed Identity | Linux | Premium | | +| 3 | 1 | Managed Identity | Windows | Premium | | +| 4 | 1 | Managed Identity | Linux | Consumption (Legacy) | | +| 5 | 1 | Managed Identity | Windows | Consumption (Legacy) | | +| 6 | 1 | Managed Identity | Linux | App Service | | +| 7 | 1 | Managed Identity | Windows | App Service | | +| 8 | 1 | Secrets | Linux | Flex Consumption | | +| 9 | 1 | Secrets | Linux | Premium | | +| 10 | 1 | Secrets | Windows | Premium | | +| 11 | 1 | Secrets | Linux | Consumption (Legacy) | | +| 12 | 1 | Secrets | Windows | Consumption (Legacy) | | +| 13 | 1 | Secrets | Linux | App Service | | +| 14 | 1 | Secrets | Windows | App Service | | From 15cf768765780de31f9205965a95efc7d7de5c3f Mon Sep 17 00:00:00 2001 From: MicroFish91 <40250218+MicroFish91@users.noreply.github.com> Date: Tue, 20 Jan 2026 15:20:17 -0800 Subject: [PATCH 16/27] Add core vs extended - unfinished --- .vscode/launch.json | 1 + test/constants.ts | 24 ++++++++++++++----- test/nightly/global.nightly.test.ts | 4 +++- test/nightly/scenarios/ScenariosTracker.ts | 1 - test/nightly/scenarios/parallelScenarios.ts | 19 ++++++++++----- test/nightly/scenarios/scenarios.test.ts | 19 ++++++++------- .../AzExtFunctionsTestScenario.ts | 3 ++- .../azureStorageJSNodeScenario.ts | 17 ++++++++----- .../durable/azureStorage/dotnetIsolated.ts | 12 +++++----- .../durable/azureStorage/pythonScenario.ts | 12 +++++----- .../durable/dts/dtsJSNodeScenario.ts | 2 +- .../httpTrigger/httpTriggerJSNodeScenario.ts | 13 ++++++---- test/test.code-workspace | 16 ++++++------- test/utils/createFunctionAppUtils.ts | 8 +++---- 14 files changed, 93 insertions(+), 58 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 2a435fd16..ec29a9607 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -77,6 +77,7 @@ "FUNC_PATH": "func", "AZFUNC_UPDATE_BACKUP_TEMPLATES": "", "AzCode_EnableLongRunningTestsLocal": "true", + "AzCode_OnlyLongRunningTestScenario": "", } }, { diff --git a/test/constants.ts b/test/constants.ts index 9fc317768..21dc68843 100644 --- a/test/constants.ts +++ b/test/constants.ts @@ -3,19 +3,31 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -// Sometimes this is "orchestrator" or "orchestration" depending on the template feed -export const durableOrchestratorPick: RegExp = /Durable Functions Orch/i; -export const durableOrchestratorName: string = 'durableHello'; +// Start section create-new-project +// Project type picks +export const httpTriggerPick: RegExp = /HTTP\s?Trigger/i; +export const durableOrchestratorPick: RegExp = /Durable Functions Orch/i; // Sometimes this is "orchestrator" or "orchestration" depending on the template feed +export const durableAzureStoragePick: RegExp = /Azure Storage/i; +// Default names export const httpTriggerName: string = 'httpTrigger'; +export const durableOrchestratorName: string = 'durableHello'; +export const dotnetNamespaceName: string = 'Company.Function'; // Language picks export const jsLanguagePick: RegExp = /JavaScript/i; +export const pythonLanguagePick: RegExp = /Python/i; +export const cSharpLanguagePick: RegExp = /C#/i; // Framework picks export const jsModelV4Pick: RegExp = /v4/i; +export const pythonModelV2Pick: RegExp = /v2/i; +export const dotnetIsolatedPick: RegExp = /\.NET 10/i; +// Start section create-function-app export const locationDefaultPick: RegExp = /West US 2/i; -export const pythonDefaultPick: RegExp = /Python(\s)?3\.12/i; -export const nodeDefaultPick: RegExp = /Node\.js(\s)?22/i; -export const dotnetIsolatedDefaultPick: RegExp = /\.NET(\s)?10(\s)?Isolated/i; + +// Default runtime picks +export const pythonRuntimePick: RegExp = /Python(\s)?3\.12/i; +export const nodeRuntimePick: RegExp = /Node\.js(\s)?22/i; +export const dotnetIsolatedRuntimePick: RegExp = /\.NET(\s)?10(\s)?Isolated/i; diff --git a/test/nightly/global.nightly.test.ts b/test/nightly/global.nightly.test.ts index 11ff96076..73fb2c05d 100644 --- a/test/nightly/global.nightly.test.ts +++ b/test/nightly/global.nightly.test.ts @@ -6,7 +6,7 @@ import { WebSiteManagementClient } from '@azure/arm-appservice'; import { ResourceManagementClient } from '@azure/arm-resources'; import { createTestActionContext } from '@microsoft/vscode-azext-dev'; -import { AzureAccountTreeItemWithProjects, createAzureClient, ext } from '../../extension.bundle'; +import { AzureAccountTreeItemWithProjects, createAzureClient, ext, updateGlobalSetting } from '../../extension.bundle'; import { longRunningTestsEnabled } from '../global.test'; import { createSubscriptionContext, subscriptionExperience, type ISubscriptionContext } from '@microsoft/vscode-azext-utils'; @@ -25,6 +25,8 @@ suiteSetup(async function (this: Mocha.Context): Promise { this.timeout(2 * 60 * 1000); await vscode.commands.executeCommand('azureResourceGroups.logIn'); + await updateGlobalSetting('groupBy', 'resourceType', 'azureResourceGroups'); + ext.azureAccountTreeItem = new AzureAccountTreeItemWithProjects(); const testContext = await createTestActionContext(); const subscription: AzureSubscription = await subscriptionExperience(testContext, ext.rgApi.appResourceTree); diff --git a/test/nightly/scenarios/ScenariosTracker.ts b/test/nightly/scenarios/ScenariosTracker.ts index 1bba1654f..696e9b096 100644 --- a/test/nightly/scenarios/ScenariosTracker.ts +++ b/test/nightly/scenarios/ScenariosTracker.ts @@ -5,7 +5,6 @@ import { nonNullProp, nonNullValue } from "@microsoft/vscode-azext-utils"; - export class ScenariosTracker { private scenarioStatuses: Map = new Map(); diff --git a/test/nightly/scenarios/parallelScenarios.ts b/test/nightly/scenarios/parallelScenarios.ts index 704af903b..ea1a8ea05 100644 --- a/test/nightly/scenarios/parallelScenarios.ts +++ b/test/nightly/scenarios/parallelScenarios.ts @@ -16,7 +16,7 @@ import { generateTestScenarios } from './testScenarios/testScenarios'; export interface AzExtFunctionsParallelTestScenario { title: string; scenario?: Promise; - runScenario(): Promise; + runScenario(testPlan: TestPlan): Promise; only?: boolean; } @@ -31,7 +31,7 @@ export function generateParallelScenarios(): AzExtFunctionsParallelTestScenario[ } function generateRunScenario(scenario: AzExtFunctionsTestScenario): AzExtFunctionsParallelTestScenario['runScenario'] { - return async function runScenario() { + return async function runScenario(testPlan: TestPlan) { const workspaceFolderUri: Uri = getWorkspaceFolderUri(scenario.folderName); const rootFolder = workspace.getWorkspaceFolder(workspaceFolderUri); assert.ok(rootFolder, `Failed to retrieve root workspace folder for scenario ${scenario.label}.`); @@ -55,13 +55,15 @@ function generateRunScenario(scenario: AzExtFunctionsTestScenario): AzExtFunctio }); }); - // 2. Immediately spin off all the create and deploy tests - const onlyTestCase: CreateAndDeployTestCase | undefined = scenario.createAndDeployTests.find(test => test.only); + // 2. Start all create and deploy tests for the scenario + const createAndDeployTests: CreateAndDeployTestCase[] | undefined = testPlan === 'core' ? scenario.createAndDeployTestsCore : scenario.createAndDeployTestsExtended ?? []; + + const onlyTestCase: CreateAndDeployTestCase | undefined = createAndDeployTests.find(test => test.only); if (onlyTestCase) { await startCreateAndDeployTest(scenario.label, rootFolder, onlyTestCase); } else { - const createAndDeployTests: Promise[] = scenario.createAndDeployTests.map(test => startCreateAndDeployTest(scenario.label, rootFolder, test)); - await Promise.allSettled(createAndDeployTests); + const createAndDeployTasks: Promise[] = createAndDeployTests.map(test => startCreateAndDeployTest(scenario.label, rootFolder, test)); + await Promise.allSettled(createAndDeployTasks); } await cleanTestFolder(rootFolder); @@ -159,3 +161,8 @@ export function getWorkspaceFolderUri(folderName: string): Uri { throw new Error(`Unable to find workspace folder "${folderName}"`); } + +export enum TestPlan { + Core = 'core', + Extended = 'extended', +} diff --git a/test/nightly/scenarios/scenarios.test.ts b/test/nightly/scenarios/scenarios.test.ts index 3c2c78fb8..e37b9878a 100644 --- a/test/nightly/scenarios/scenarios.test.ts +++ b/test/nightly/scenarios/scenarios.test.ts @@ -3,30 +3,33 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { updateGlobalSetting } from '../../../extension.bundle'; import { longRunningTestsEnabled } from '../../global.test'; -import { generateParallelScenarios, type AzExtFunctionsParallelTestScenario } from './parallelScenarios'; +import { generateParallelScenarios, TestPlan, type AzExtFunctionsParallelTestScenario } from './parallelScenarios'; const testScenarios: AzExtFunctionsParallelTestScenario[] = generateParallelScenarios(); suite.only('Scenarios', async function (this: Mocha.Suite) { // Unfortunately, durable task schedulers sometimes take ~30m to provision - this.timeout(60 * 60 * 1000); + this.timeout(45 * 60 * 1000); suiteSetup(async function (this: Mocha.Context) { if (!longRunningTestsEnabled) { this.skip(); } - // Todo: This should probably happen elsewhere? - await updateGlobalSetting('groupBy', 'resourceType', 'azureResourceGroups'); - const onlyTestScenario = testScenarios.find(s => s.only); + const onlyTestScenario = testScenarios.find(s => { + if (process.env.AzCode_OnlyLongRunningTestScenario) { + return s.title === process.env.AzCode_OnlyLongRunningTestScenario; + } + return s.only; + }); + const testPlan: TestPlan = process.env.AzCode_OnlyLongRunningTestScenario ? TestPlan.Extended : TestPlan.Core; if (onlyTestScenario) { - onlyTestScenario.scenario = onlyTestScenario.runScenario(); + onlyTestScenario.scenario = onlyTestScenario.runScenario(testPlan); } else { for (const s of testScenarios) { - s.scenario = s.runScenario(); + s.scenario = s.runScenario(TestPlan.Core); } } }); diff --git a/test/nightly/scenarios/testScenarios/AzExtFunctionsTestScenario.ts b/test/nightly/scenarios/testScenarios/AzExtFunctionsTestScenario.ts index 0c9c377c8..064d4a00c 100644 --- a/test/nightly/scenarios/testScenarios/AzExtFunctionsTestScenario.ts +++ b/test/nightly/scenarios/testScenarios/AzExtFunctionsTestScenario.ts @@ -10,7 +10,8 @@ export interface AzExtFunctionsTestScenario { label: string; folderName: string; createNewProjectTest: CreateNewProjectTestCase; - createAndDeployTests: CreateAndDeployTestCase[]; + createAndDeployTestsCore: CreateAndDeployTestCase[]; + createAndDeployTestsExtended?: CreateAndDeployTestCase[]; /** * Indicates this scenario should be executed exclusively. This should only be used to aid with local development. diff --git a/test/nightly/scenarios/testScenarios/durable/azureStorage/azureStorageJSNodeScenario.ts b/test/nightly/scenarios/testScenarios/durable/azureStorage/azureStorageJSNodeScenario.ts index 32a4b08ea..0e83ecbdd 100644 --- a/test/nightly/scenarios/testScenarios/durable/azureStorage/azureStorageJSNodeScenario.ts +++ b/test/nightly/scenarios/testScenarios/durable/azureStorage/azureStorageJSNodeScenario.ts @@ -4,13 +4,13 @@ *--------------------------------------------------------------------------------------------*/ import { DurableBackend } from "../../../../../../extension.bundle"; -import { durableOrchestratorName, durableOrchestratorPick } from "../../../../../constants"; +import { durableAzureStoragePick, durableOrchestratorName, durableOrchestratorPick, jsLanguagePick, jsModelV4Pick } from "../../../../../constants"; import { ConnectionType, CreateMode, OperatingSystem, PlanType, Runtime } from "../../../../../utils/createFunctionAppUtils"; import { type AzExtFunctionsTestScenario } from "../../AzExtFunctionsTestScenario"; import { generateCreateAndDeployTest } from "../generateCreateAndDeployTest"; export function generateJSNodeScenario(): AzExtFunctionsTestScenario { - const folderName: string = 'scenarios-durable-azurestorage-jsnode'; + const folderName: string = 'scenario-durable-azurestorage-jsnode'; return { label: 'durable-azurestorage-jsnode', @@ -18,14 +18,19 @@ export function generateJSNodeScenario(): AzExtFunctionsTestScenario { createNewProjectTest: { label: 'create-new-project', inputs: [ - /JavaScript/i, - /v4/i, + jsLanguagePick, + jsModelV4Pick, durableOrchestratorPick, - /Azure Storage/i, + durableAzureStoragePick, durableOrchestratorName, ], }, - createAndDeployTests: [ + createAndDeployTestsCore: [ + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.LegacyConsumption, OperatingSystem.Linux), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.AppService, OperatingSystem.Windows), + // generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.Premium, OperatingSystem.Linux), + ], + createAndDeployTestsExtended: [ generateCreateAndDeployTest(folderName, CreateMode.Basic, Runtime.Node, ConnectionType.ManagedIdentity, PlanType.FlexConsumption, OperatingSystem.Linux, DurableBackend.Storage), generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.ManagedIdentity, PlanType.Premium, OperatingSystem.Linux, DurableBackend.Storage), generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.ManagedIdentity, PlanType.Premium, OperatingSystem.Windows, DurableBackend.Storage), diff --git a/test/nightly/scenarios/testScenarios/durable/azureStorage/dotnetIsolated.ts b/test/nightly/scenarios/testScenarios/durable/azureStorage/dotnetIsolated.ts index d13867b13..adbb0ce14 100644 --- a/test/nightly/scenarios/testScenarios/durable/azureStorage/dotnetIsolated.ts +++ b/test/nightly/scenarios/testScenarios/durable/azureStorage/dotnetIsolated.ts @@ -4,13 +4,13 @@ *--------------------------------------------------------------------------------------------*/ import { DurableBackend } from "../../../../../../extension.bundle"; -import { durableOrchestratorName, durableOrchestratorPick } from "../../../../../constants"; +import { cSharpLanguagePick, dotnetIsolatedPick, dotnetNamespaceName, durableAzureStoragePick, durableOrchestratorName, durableOrchestratorPick } from "../../../../../constants"; import { ConnectionType, CreateMode, OperatingSystem, PlanType, Runtime } from "../../../../../utils/createFunctionAppUtils"; import { type AzExtFunctionsTestScenario } from "../../AzExtFunctionsTestScenario"; import { generateCreateAndDeployTest } from "../generateCreateAndDeployTest"; export function generateDotnetIsolatedScenario(): AzExtFunctionsTestScenario { - const folderName: string = 'scenarios-durable-azurestorage-dotnetisolated'; + const folderName: string = 'scenario-durable-azurestorage-dotnetisolated'; return { label: 'durable-azurestorage-dotnetisolated', @@ -18,13 +18,13 @@ export function generateDotnetIsolatedScenario(): AzExtFunctionsTestScenario { createNewProjectTest: { label: 'create-new-project', inputs: [ - /C#/i, + cSharpLanguagePick, // Todo: This needs to also run a match on the description because isolated vs. inproc differentiates there - /\.NET 10/i, + dotnetIsolatedPick, durableOrchestratorPick, - /Azure Storage/i, + durableAzureStoragePick, durableOrchestratorName, - 'Company.Function', + dotnetNamespaceName, ], }, createAndDeployTests: [ diff --git a/test/nightly/scenarios/testScenarios/durable/azureStorage/pythonScenario.ts b/test/nightly/scenarios/testScenarios/durable/azureStorage/pythonScenario.ts index 9b5631623..9d7004df6 100644 --- a/test/nightly/scenarios/testScenarios/durable/azureStorage/pythonScenario.ts +++ b/test/nightly/scenarios/testScenarios/durable/azureStorage/pythonScenario.ts @@ -4,13 +4,13 @@ *--------------------------------------------------------------------------------------------*/ import { DurableBackend } from "../../../../../../extension.bundle"; -import { durableOrchestratorName, durableOrchestratorPick, pythonDefaultPick } from "../../../../../constants"; +import { durableAzureStoragePick, durableOrchestratorName, durableOrchestratorPick, pythonLanguagePick, pythonModelV2Pick, pythonRuntimePick } from "../../../../../constants"; import { ConnectionType, CreateMode, PlanType, Runtime } from "../../../../../utils/createFunctionAppUtils"; import { type AzExtFunctionsTestScenario } from "../../AzExtFunctionsTestScenario"; import { generateCreateAndDeployTest } from "../generateCreateAndDeployTest"; export function generatePythonScenario(): AzExtFunctionsTestScenario { - const folderName: string = 'scenarios-durable-azurestorage-python'; + const folderName: string = 'scenario-durable-azurestorage-python'; return { label: 'durable-azurestorage-python', @@ -18,11 +18,11 @@ export function generatePythonScenario(): AzExtFunctionsTestScenario { createNewProjectTest: { label: 'create-new-project', inputs: [ - /Python/i, - /v2/i, - pythonDefaultPick, + pythonLanguagePick, + pythonModelV2Pick, + pythonRuntimePick, durableOrchestratorPick, - /Azure Storage/i, + durableAzureStoragePick, durableOrchestratorName, ], }, diff --git a/test/nightly/scenarios/testScenarios/durable/dts/dtsJSNodeScenario.ts b/test/nightly/scenarios/testScenarios/durable/dts/dtsJSNodeScenario.ts index 2f9d3baa9..5495940a6 100644 --- a/test/nightly/scenarios/testScenarios/durable/dts/dtsJSNodeScenario.ts +++ b/test/nightly/scenarios/testScenarios/durable/dts/dtsJSNodeScenario.ts @@ -10,7 +10,7 @@ import { type AzExtFunctionsTestScenario } from "../../AzExtFunctionsTestScenari import { generateCreateAndDeployTest } from "../generateCreateAndDeployTest"; export function generateJSNodeScenario(): AzExtFunctionsTestScenario { - const folderName: string = 'scenarios-durable-dts-jsnode'; + const folderName: string = 'scenario-durable-dts-jsnode'; return { label: 'durable-dts-jsnode', diff --git a/test/nightly/scenarios/testScenarios/httpTrigger/httpTriggerJSNodeScenario.ts b/test/nightly/scenarios/testScenarios/httpTrigger/httpTriggerJSNodeScenario.ts index ba9cb4c2b..c1aa4613b 100644 --- a/test/nightly/scenarios/testScenarios/httpTrigger/httpTriggerJSNodeScenario.ts +++ b/test/nightly/scenarios/testScenarios/httpTrigger/httpTriggerJSNodeScenario.ts @@ -3,13 +3,13 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { httpTriggerName, jsLanguagePick, jsModelV4Pick } from "../../../../constants"; +import { httpTriggerName, httpTriggerPick, jsLanguagePick, jsModelV4Pick } from "../../../../constants"; import { ConnectionType, CreateMode, OperatingSystem, PlanType, Runtime } from "../../../../utils/createFunctionAppUtils"; import { type AzExtFunctionsTestScenario } from "../AzExtFunctionsTestScenario"; import { generateCreateAndDeployTest } from "./generateCreateAndDeployTest"; export function generateJSNodeScenario(): AzExtFunctionsTestScenario { - const folderName: string = 'scenarios-http-trigger-jsnode'; + const folderName: string = 'scenario-http-trigger-jsnode'; return { label: 'http-trigger-jsnode', @@ -19,11 +19,16 @@ export function generateJSNodeScenario(): AzExtFunctionsTestScenario { inputs: [ jsLanguagePick, jsModelV4Pick, - /HTTP\s?Trigger/i, + httpTriggerPick, httpTriggerName, ], }, - createAndDeployTests: [ + createAndDeployTestsCore: [ + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.LegacyConsumption, OperatingSystem.Linux), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.AppService, OperatingSystem.Windows), + // generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.Premium, OperatingSystem.Linux), + ], + createAndDeployTestsExtended: [ generateCreateAndDeployTest(folderName, CreateMode.Basic, Runtime.Node, ConnectionType.ManagedIdentity, PlanType.FlexConsumption, OperatingSystem.Linux), generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.ManagedIdentity, PlanType.Premium, OperatingSystem.Linux), generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.ManagedIdentity, PlanType.Premium, OperatingSystem.Windows), diff --git a/test/test.code-workspace b/test/test.code-workspace index 7f452a321..3fc76131f 100644 --- a/test/test.code-workspace +++ b/test/test.code-workspace @@ -31,35 +31,35 @@ "path": "../testWorkspace/9" }, { - "name": "scenarios-http-trigger-jsnode", + "name": "scenario-http-trigger-jsnode", "path": "../testWorkspace/scenarios/http-trigger/jsnode" }, { - "name": "scenarios-durable-azurestorage-jsnode", + "name": "scenario-durable-azurestorage-jsnode", "path": "../testWorkspace/scenarios/durable-azurestorage/jsnode" }, { - "name": "scenarios-durable-azurestorage-python", + "name": "scenario-durable-azurestorage-python", "path": "../testWorkspace/scenarios/durable-azurestorage/python" }, { - "name": "scenarios-durable-azurestorage-dotnetisolated", + "name": "scenario-durable-azurestorage-dotnetisolated", "path": "../testWorkspace/scenarios/durable-azurestorage/dotnetisolated" }, { - "name": "scenarios-durable-dts-jsnode", + "name": "scenario-durable-dts-jsnode", "path": "../testWorkspace/scenarios/durable-dts/jsnode" }, { - "name": "scenarios-durable-dts-python", + "name": "scenario-durable-dts-python", "path": "../testWorkspace/scenarios/durable-dts/python" }, { - "name": "scenarios-durable-dts-dotnet-isolated", + "name": "scenario-durable-dts-dotnet-isolated", "path": "../testWorkspace/scenarios/durable-dts/dotnet-isolated" }, { - "name": "scenarios-durable-dts-dotnet-inproc", + "name": "scenario-durable-dts-dotnet-inproc", "path": "../testWorkspace/scenarios/durable-dts/dotnet-inproc" } ], diff --git a/test/utils/createFunctionAppUtils.ts b/test/utils/createFunctionAppUtils.ts index 7bbc28e9b..eb31a66f2 100644 --- a/test/utils/createFunctionAppUtils.ts +++ b/test/utils/createFunctionAppUtils.ts @@ -3,7 +3,7 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { dotnetIsolatedDefaultPick, locationDefaultPick, nodeDefaultPick, pythonDefaultPick } from "../constants"; +import { dotnetIsolatedRuntimePick, locationDefaultPick, nodeRuntimePick, pythonRuntimePick } from "../constants"; export namespace createFunctionAppUtils { export function generateBasicCreateInputs(appName: string, folderName: string, runtime: Runtime, connection: ConnectionType): (string | RegExp)[] { @@ -95,11 +95,11 @@ export namespace createFunctionAppUtils { function getRuntimePick(runtime: Runtime): RegExp | string { switch (runtime) { case Runtime.Python: - return pythonDefaultPick; + return pythonRuntimePick; case Runtime.Node: - return nodeDefaultPick; + return nodeRuntimePick; case Runtime.DotNetIsolated: - return dotnetIsolatedDefaultPick; + return dotnetIsolatedRuntimePick; default: throw new Error(`Runtime "${runtime}" not yet supported in "createFunctionAppUtils.generateBasicCreateInputs".`); } From 97b9d0ae2205dacd9db45d3f6e34d81353073f45 Mon Sep 17 00:00:00 2001 From: MicroFish91 <40250218+MicroFish91@users.noreply.github.com> Date: Tue, 20 Jan 2026 17:38:57 -0800 Subject: [PATCH 17/27] Remove extra features --- .github/agents/test-scenario-planner.agent.md | 42 ------ .../agents/test-scenario-reviewer.agent.md | 31 ----- test/nightly/scenarios/ScenariosTracker.ts | 9 +- .../docs/ScenarioTestCombinations.md | 120 ------------------ test/nightly/scenarios/parallelScenarios.ts | 33 +++-- test/nightly/scenarios/scenarios.test.ts | 10 +- .../AzExtFunctionsTestScenario.ts | 104 ++++++++++++++- .../azureStorage/DurableAzureStorage.plan.md | 58 --------- .../azureStorage/azureStorageScenarios.ts | 4 - .../durable/azureStorage/dotnetIsolated.ts | 47 ------- .../durable/azureStorage/pythonScenario.ts | 41 ------ .../durable/dts/DurableDTS.plan.md | 52 -------- .../durable/dts/dtsJSNodeScenario.ts | 37 ------ .../testScenarios/durable/dts/dtsScenarios.ts | 13 -- .../durable/generateCreateAndDeployTest.ts | 16 +-- .../httpTrigger/HttpTrigger.plan.md | 33 ----- .../generateCreateAndDeployTest.ts | 61 --------- .../httpTrigger/httpTriggerJSNodeScenario.ts | 48 ------- .../httpTrigger/httpTriggerScenarios.ts | 13 -- .../scenarios/testScenarios/testScenarios.ts | 5 +- test/utils/createFunctionAppUtils.ts | 24 ++-- test/utils/deployFunctionAppUtils.ts | 22 +--- 22 files changed, 160 insertions(+), 663 deletions(-) delete mode 100644 .github/agents/test-scenario-planner.agent.md delete mode 100644 .github/agents/test-scenario-reviewer.agent.md delete mode 100644 test/nightly/scenarios/docs/ScenarioTestCombinations.md delete mode 100644 test/nightly/scenarios/testScenarios/durable/azureStorage/DurableAzureStorage.plan.md delete mode 100644 test/nightly/scenarios/testScenarios/durable/azureStorage/dotnetIsolated.ts delete mode 100644 test/nightly/scenarios/testScenarios/durable/azureStorage/pythonScenario.ts delete mode 100644 test/nightly/scenarios/testScenarios/durable/dts/DurableDTS.plan.md delete mode 100644 test/nightly/scenarios/testScenarios/durable/dts/dtsJSNodeScenario.ts delete mode 100644 test/nightly/scenarios/testScenarios/durable/dts/dtsScenarios.ts delete mode 100644 test/nightly/scenarios/testScenarios/httpTrigger/HttpTrigger.plan.md delete mode 100644 test/nightly/scenarios/testScenarios/httpTrigger/generateCreateAndDeployTest.ts delete mode 100644 test/nightly/scenarios/testScenarios/httpTrigger/httpTriggerJSNodeScenario.ts delete mode 100644 test/nightly/scenarios/testScenarios/httpTrigger/httpTriggerScenarios.ts diff --git a/.github/agents/test-scenario-planner.agent.md b/.github/agents/test-scenario-planner.agent.md deleted file mode 100644 index 0cd7b12e5..000000000 --- a/.github/agents/test-scenario-planner.agent.md +++ /dev/null @@ -1,42 +0,0 @@ ---- -name: AzExtFunc - Test Planner -description: A tool to help Azure Functions extension internal developers plan test scenarios for new features. -model: GPT-4o (copilot) -tools: ['execute/runInTerminal', 'read', 'edit', 'search'] -handoffs: - - label: Review the plan - agent: AzExtFunc - Test Reviewer - prompt: Review and clean up the test plan. - send: true ---- - -## Planning Instructions - -You are in planning mode. Your task is to generate a test plan for a new feature. -Don't make any code edits, just focus on generating a the plan. - -The plan should take the form of a Markdown document, with test matrices for major functions commands. - - -## References - -- All Scenario Test Combinations: This is what should be initially offered to the user unless specified otherwise: -`vscode-azurefunctions/test/nightly/scenarios/docs/ScenarioTestCombinations.md` - -- Plan Example: The final target will eventually be a stripped-down version that looks like the document that follows. This should always be your example reference target when making future changes: -`vscode-azurefunctions/test/nightly/scenarios/testScenarios/durable/dts/DurableDTS.plan.md` - -## Steps - -1. Start by having the user provide the empty directory to place their new plan (do not ask for the name). This directory should be somewhere within the `testScenarios` folder. -1. Next run the `cp` command in the terminal to directly copy the markdown for all scenario test combinations into this directory. If you can't find the file or fail to copy it in any way, please say so to the user. -1. Change the title and file name to reflect the new test plan we are creating. Do not create a separate file, always change the existing file so we don't end up with duplicates. Inspect the file path to inform your file and title naming decisions. -1. Next, ask the user to mark rows that should be tested under the `Selected` column. - - If the user asks you to help with this directly through chat, you can help them do this. -1. Once the user confirms that all changes have been made, handoff to the reviewer agent. - -## Additional Instructions - -1. Keep the information simple for the user. Give them only next step options, do not enumerate all the steps nor should you overload the user with information unless asked. -2. Do not copy over sections that do not exist in the example target provided. -3. Always only give only one next step for the user, keep the info direct and short. diff --git a/.github/agents/test-scenario-reviewer.agent.md b/.github/agents/test-scenario-reviewer.agent.md deleted file mode 100644 index 188ad1e85..000000000 --- a/.github/agents/test-scenario-reviewer.agent.md +++ /dev/null @@ -1,31 +0,0 @@ ---- -name: AzExtFunc - Test Reviewer -description: A tool to help Azure Functions internal developers review their test scenarios before requesting Copilot to implement them. -model: GPT-5 mini (copilot) -tools: ['read', 'edit'] ---- - -## Review Instructions - -You are an agent specialized in reviewing test scenario plans to ensure they are ready to be built. Test plan docs should end in `.plan.md`. -You will ensure the test matrices in the provided file are trimmed down and organized to only the relevant tests. - -## References - -- Example Plan: The final target will eventually be a stripped-down version that looks like the document that follows. This should always be your example reference target when making future changes: -`vscode-azurefunctions/test/nightly/scenarios/testScenarios/durable/dts/DurableDTS.plan.md` - -## Steps - -1. Ask the user for a test plan if one was not provided. -1. Check that the plan title matches the containing directory, use the example plan for formatting examples. -1. Ensure the file name matches the example plan reference as well. The name should end with `.plan.md`. -1. If there are any rows that are not selected, confirm and delete them. -1. If the selected column is still there, remove it. -1. Re-number the rows if needed. -1. Warn if not all listed workspace projects are referenced in the final test matrix. The final list should only show what is used. - -## Additional Instructions - -- Only perform one step at a time, ask where appropriate. -- Keep provided information minimal, don't overload the user with context. diff --git a/test/nightly/scenarios/ScenariosTracker.ts b/test/nightly/scenarios/ScenariosTracker.ts index 696e9b096..db2b1aa50 100644 --- a/test/nightly/scenarios/ScenariosTracker.ts +++ b/test/nightly/scenarios/ScenariosTracker.ts @@ -123,9 +123,10 @@ export class ScenariosTracker { this.scenarioStatuses.set(scenarioLabel, scenarioStatus); } - report(): string { + report(): void { if (!this.scenarioStatuses.size) { - return 'No test scenarios recorded.'; + console.log('No test scenarios recorded.'); + return; } const lines: string[] = []; @@ -138,7 +139,7 @@ export class ScenariosTracker { const maxErrorLength = 200; const getStatusIcon = (status: TestStatus | undefined): string => { - return icons[status ?? 'undefined']; + return icons[String(status)]; }; const truncateError = (error: string | undefined): string => { @@ -185,7 +186,7 @@ export class ScenariosTracker { } } - return lines.join('\n'); + console.log(lines.join('\n')); } } diff --git a/test/nightly/scenarios/docs/ScenarioTestCombinations.md b/test/nightly/scenarios/docs/ScenarioTestCombinations.md deleted file mode 100644 index 2973e8ca3..000000000 --- a/test/nightly/scenarios/docs/ScenarioTestCombinations.md +++ /dev/null @@ -1,120 +0,0 @@ -# Scenario Test Combinations - -## I. Create New Project / Create Function - -### Workspace Project - Language / Runtime Test Matrix - -| No. | Language | Runtime | Programming Model | Comment | Selected | -|-----|----------|---------|-------------------|------------------------------------|-------------| -| 1 | JS | Node | v3 | Skip unless special requirements | | -| 2 | JS | Node | v4 | | | -| 3 | Python | Python | v1 | Skip unless special requirements | | -| 4 | Python | Python | v2 | | | -| 5 | C# | .NET | isolated | | | -| 6 | C# | .NET | in-proc | | | - -## II. Debug -TBD - -## III. Create Function App / Deployment - -### Create / Deploy Test Matrix - -| No. | Workspace Project | Storage Connection Type | Operating System | Plan Type | Comment | Selected | -|-----|-------------------|----------------------------|------------------|---------------------|-------------|----------| -| 1 | 1 | Managed Identity | Linux | Flex Consumption | Skip | | -| 2 | 1 | Managed Identity | Windows | Flex Consumption | Skip | | -| 3 | 1 | Managed Identity | Linux | Premium | Skip | | -| 4 | 1 | Managed Identity | Windows | Premium | Skip | | -| 5 | 1 | Managed Identity | Linux | Consumption (Legacy)| Skip | | -| 6 | 1 | Managed Identity | Windows | Consumption (Legacy)| Skip | | -| 7 | 1 | Managed Identity | Linux | App Service | Skip | | -| 8 | 1 | Managed Identity | Windows | App Service | Skip | | -| 9 | 1 | Secrets | Linux | Flex Consumption | Skip | | -| 10 | 1 | Secrets | Windows | Flex Consumption | Skip | | -| 11 | 1 | Secrets | Linux | Premium | Skip | | -| 12 | 1 | Secrets | Windows | Premium | Skip | | -| 13 | 1 | Secrets | Linux | Consumption (Legacy)| Skip | | -| 14 | 1 | Secrets | Windows | Consumption (Legacy)| Skip | | -| 15 | 1 | Secrets | Linux | App Service | Skip | | -| 16 | 1 | Secrets | Windows | App Service | Skip | | -| 17 | 2 | Managed Identity | Linux | Flex Consumption | | | -| 18 | 2 | Managed Identity | Windows | Flex Consumption | Not Offered | | -| 19 | 2 | Managed Identity | Linux | Premium | | | -| 20 | 2 | Managed Identity | Windows | Premium | | | -| 21 | 2 | Managed Identity | Linux | Consumption (Legacy)| | | -| 22 | 2 | Managed Identity | Windows | Consumption (Legacy)| | | -| 23 | 2 | Managed Identity | Linux | App Service | | | -| 24 | 2 | Managed Identity | Windows | App Service | | | -| 25 | 2 | Secrets | Linux | Flex Consumption | | | -| 26 | 2 | Secrets | Windows | Flex Consumption | Not Offered | | -| 27 | 2 | Secrets | Linux | Premium | | | -| 28 | 2 | Secrets | Windows | Premium | | | -| 29 | 2 | Secrets | Linux | Consumption (Legacy)| | | -| 30 | 2 | Secrets | Windows | Consumption (Legacy)| | | -| 31 | 2 | Secrets | Linux | App Service | | | -| 32 | 2 | Secrets | Windows | App Service | | | -| 33 | 3 | Managed Identity | Linux | Flex Consumption | Skip | | -| 34 | 3 | Managed Identity | Windows | Flex Consumption | Skip | | -| 35 | 3 | Managed Identity | Linux | Premium | Skip | | -| 36 | 3 | Managed Identity | Windows | Premium | Skip | | -| 37 | 3 | Managed Identity | Linux | Consumption (Legacy)| Skip | | -| 38 | 3 | Managed Identity | Windows | Consumption (Legacy)| Skip | | -| 39 | 3 | Managed Identity | Linux | App Service | Skip | | -| 40 | 3 | Managed Identity | Windows | App Service | Skip | | -| 41 | 3 | Secrets | Linux | Flex Consumption | Skip | | -| 42 | 3 | Secrets | Windows | Flex Consumption | Skip | | -| 43 | 3 | Secrets | Linux | Premium | Skip | | -| 44 | 3 | Secrets | Windows | Premium | Skip | | -| 45 | 3 | Secrets | Linux | Consumption (Legacy)| Skip | | -| 46 | 3 | Secrets | Windows | Consumption (Legacy)| Skip | | -| 47 | 3 | Secrets | Linux | App Service | Skip | | -| 48 | 3 | Secrets | Windows | App Service | Skip | | -| 49 | 4 | Managed Identity | Linux | Flex Consumption | | | -| 50 | 4 | Managed Identity | Windows | Flex Consumption | Not Offered | | -| 51 | 4 | Managed Identity | Linux | Premium | | | -| 52 | 4 | Managed Identity | Windows | Premium | | | -| 53 | 4 | Managed Identity | Linux | Consumption (Legacy)| | | -| 54 | 4 | Managed Identity | Windows | Consumption (Legacy)| | | -| 55 | 4 | Managed Identity | Linux | App Service | | | -| 56 | 4 | Managed Identity | Windows | App Service | | | -| 57 | 4 | Secrets | Linux | Flex Consumption | | | -| 58 | 4 | Secrets | Windows | Flex Consumption | Not Offered | | -| 59 | 4 | Secrets | Linux | Premium | | | -| 60 | 4 | Secrets | Windows | Premium | | | -| 61 | 4 | Secrets | Linux | Consumption (Legacy)| | | -| 62 | 4 | Secrets | Windows | Consumption (Legacy)| | | -| 63 | 4 | Secrets | Linux | App Service | | | -| 64 | 4 | Secrets | Windows | App Service | | | -| 65 | 5 | Managed Identity | Linux | Flex Consumption | | | -| 66 | 5 | Managed Identity | Windows | Flex Consumption | Not Offered | | -| 67 | 5 | Managed Identity | Linux | Premium | | | -| 68 | 5 | Managed Identity | Windows | Premium | | | -| 69 | 5 | Managed Identity | Linux | Consumption (Legacy)| | | -| 70 | 5 | Managed Identity | Windows | Consumption (Legacy)| | | -| 71 | 5 | Managed Identity | Linux | App Service | | | -| 72 | 5 | Managed Identity | Windows | App Service | | | -| 73 | 5 | Secrets | Linux | Flex Consumption | | | -| 74 | 5 | Secrets | Windows | Flex Consumption | Not Offered | | -| 75 | 5 | Secrets | Linux | Premium | | | -| 76 | 5 | Secrets | Windows | Premium | | | -| 77 | 5 | Secrets | Linux | Consumption (Legacy)| | | -| 78 | 5 | Secrets | Windows | Consumption (Legacy)| | | -| 79 | 5 | Secrets | Linux | App Service | | | -| 80 | 5 | Secrets | Windows | App Service | | | -| 81 | 6 | Managed Identity | Linux | Flex Consumption | | | -| 82 | 6 | Managed Identity | Windows | Flex Consumption | Not Offered | | -| 83 | 6 | Managed Identity | Linux | Premium | | | -| 84 | 6 | Managed Identity | Windows | Premium | | | -| 85 | 6 | Managed Identity | Linux | Consumption (Legacy)| | | -| 86 | 6 | Managed Identity | Windows | Consumption (Legacy)| | | -| 87 | 6 | Managed Identity | Linux | App Service | | | -| 88 | 6 | Managed Identity | Windows | App Service | | | -| 89 | 6 | Secrets | Linux | Flex Consumption | | | -| 90 | 6 | Secrets | Windows | Flex Consumption | Not Offered | | -| 91 | 6 | Secrets | Linux | Premium | | | -| 92 | 6 | Secrets | Windows | Premium | | | -| 93 | 6 | Secrets | Linux | Consumption (Legacy)| | | -| 94 | 6 | Secrets | Windows | Consumption (Legacy)| | | -| 95 | 6 | Secrets | Linux | App Service | | | -| 96 | 6 | Secrets | Windows | App Service | | | diff --git a/test/nightly/scenarios/parallelScenarios.ts b/test/nightly/scenarios/parallelScenarios.ts index ea1a8ea05..eca7ba53b 100644 --- a/test/nightly/scenarios/parallelScenarios.ts +++ b/test/nightly/scenarios/parallelScenarios.ts @@ -13,10 +13,29 @@ import { resourceGroupsToDelete, scenariosTracker } from '../global.nightly.test import { type AzExtFunctionsTestScenario, type CreateAndDeployTestCase } from './testScenarios/AzExtFunctionsTestScenario'; import { generateTestScenarios } from './testScenarios/testScenarios'; +/** + * A wrapper for {@link AzExtFunctionsTestScenario} which holds a scenario that has been prepared for concurrent test execution. + */ export interface AzExtFunctionsParallelTestScenario { + /** + * A descriptive title for the test scenario that will logged in the final test report. + */ title: string; + + /** + * A promise representing the running scenario. Holds the scenario execution that gets awaited. + */ scenario?: Promise; - runScenario(testPlan: TestPlan): Promise; + + /** + * Starts the concurrent test scenario based on the specified test level. + * @param testLevel - Specifies which level of tests to run: 'core' vs. 'extended' + */ + runScenario(testLevel: TestLevel): Promise; + + /** + * Indicates this scenario should be executed exclusively. This should only be used to aid with local development. + */ only?: boolean; } @@ -31,7 +50,7 @@ export function generateParallelScenarios(): AzExtFunctionsParallelTestScenario[ } function generateRunScenario(scenario: AzExtFunctionsTestScenario): AzExtFunctionsParallelTestScenario['runScenario'] { - return async function runScenario(testPlan: TestPlan) { + return async function runScenario(testLevel: TestLevel) { const workspaceFolderUri: Uri = getWorkspaceFolderUri(scenario.folderName); const rootFolder = workspace.getWorkspaceFolder(workspaceFolderUri); assert.ok(rootFolder, `Failed to retrieve root workspace folder for scenario ${scenario.label}.`); @@ -46,7 +65,6 @@ function generateRunScenario(scenario: AzExtFunctionsTestScenario): AzExtFunctio await context.ui.runWithInputs(scenario.createNewProjectTest.inputs, async () => { try { await createNewProjectInternal(context, { folderPath: rootFolder.uri.fsPath }); - await scenario.createNewProjectTest.postTest?.(context, rootFolder.uri.fsPath, ''); scenariosTracker.passCreateNewProject(scenario.label); } catch (err) { scenariosTracker.failCreateNewProject(scenario.label, (err as Error).message ?? parseError(err).message); @@ -56,7 +74,7 @@ function generateRunScenario(scenario: AzExtFunctionsTestScenario): AzExtFunctio }); // 2. Start all create and deploy tests for the scenario - const createAndDeployTests: CreateAndDeployTestCase[] | undefined = testPlan === 'core' ? scenario.createAndDeployTestsCore : scenario.createAndDeployTestsExtended ?? []; + const createAndDeployTests: CreateAndDeployTestCase[] | undefined = testLevel === 'core' ? scenario.createAndDeployTestsCore : scenario.createAndDeployTestsExtended ?? []; const onlyTestCase: CreateAndDeployTestCase | undefined = createAndDeployTests.find(test => test.only); if (onlyTestCase) { @@ -91,7 +109,6 @@ async function startCreateAndDeployTest(scenarioLabel: string, rootFolder: Works } assert.ok(functionAppId, 'Failed to create function app.'); - await test.createFunctionApp.postTest?.(context, functionAppId); scenariosTracker.passCreateFunctionApp(scenarioLabel, scenarioTestTrackerId); } catch (err) { scenariosTracker.failCreateFunctionApp(scenarioLabel, scenarioTestTrackerId, (err as Error).message ?? parseError(err).message); @@ -130,7 +147,7 @@ async function startCreateAndDeployTest(scenarioLabel: string, rootFolder: Works let errorMessage: string = (error as Error).message ?? parseError(error).message; // Deploy failed but verification still passed - warn instead of showing an outright fail - // This is a known issue, TODO: Add issue link + // There is at least one known issue that this checks for - TODO: Add issue link if (deployError && !postTestError) { errorMessage = '** Deploy-Fail; Verify-Pass ** ' + errorMessage; scenariosTracker.warnDeployFunctionApp(scenarioLabel, scenarioTestTrackerId, errorMessage); @@ -150,7 +167,7 @@ async function cleanTestFolder(testFolder: WorkspaceFolder) { export function getWorkspaceFolderUri(folderName: string): Uri { const workspaceFolders: readonly WorkspaceFolder[] | undefined = workspace.workspaceFolders; if (!workspaceFolders || workspaceFolders.length === 0) { - throw new Error('No workspace is open'); + throw new Error('No workspace folder available.'); } else { for (const workspaceFolder of workspaceFolders) { if (workspaceFolder.name === folderName) { @@ -162,7 +179,7 @@ export function getWorkspaceFolderUri(folderName: string): Uri { throw new Error(`Unable to find workspace folder "${folderName}"`); } -export enum TestPlan { +export enum TestLevel { Core = 'core', Extended = 'extended', } diff --git a/test/nightly/scenarios/scenarios.test.ts b/test/nightly/scenarios/scenarios.test.ts index e37b9878a..f19df0c83 100644 --- a/test/nightly/scenarios/scenarios.test.ts +++ b/test/nightly/scenarios/scenarios.test.ts @@ -4,12 +4,12 @@ *--------------------------------------------------------------------------------------------*/ import { longRunningTestsEnabled } from '../../global.test'; -import { generateParallelScenarios, TestPlan, type AzExtFunctionsParallelTestScenario } from './parallelScenarios'; +import { generateParallelScenarios, TestLevel, type AzExtFunctionsParallelTestScenario } from './parallelScenarios'; const testScenarios: AzExtFunctionsParallelTestScenario[] = generateParallelScenarios(); suite.only('Scenarios', async function (this: Mocha.Suite) { - // Unfortunately, durable task schedulers sometimes take ~30m to provision + // Leave ample time since some tasks are known to take a while (e.g. durable task schedulers sometimes take ~30m to provision) this.timeout(45 * 60 * 1000); suiteSetup(async function (this: Mocha.Context) { @@ -23,13 +23,13 @@ suite.only('Scenarios', async function (this: Mocha.Suite) { } return s.only; }); - const testPlan: TestPlan = process.env.AzCode_OnlyLongRunningTestScenario ? TestPlan.Extended : TestPlan.Core; + const testLevel: TestLevel = process.env.AzCode_OnlyLongRunningTestScenario ? TestLevel.Extended : TestLevel.Core; if (onlyTestScenario) { - onlyTestScenario.scenario = onlyTestScenario.runScenario(testPlan); + onlyTestScenario.scenario = onlyTestScenario.runScenario(testLevel); } else { for (const s of testScenarios) { - s.scenario = s.runScenario(TestPlan.Core); + s.scenario = s.runScenario(TestLevel.Core); } } }); diff --git a/test/nightly/scenarios/testScenarios/AzExtFunctionsTestScenario.ts b/test/nightly/scenarios/testScenarios/AzExtFunctionsTestScenario.ts index 064d4a00c..b01609965 100644 --- a/test/nightly/scenarios/testScenarios/AzExtFunctionsTestScenario.ts +++ b/test/nightly/scenarios/testScenarios/AzExtFunctionsTestScenario.ts @@ -6,11 +6,70 @@ import { type IActionContext } from "../../../../extension.bundle"; import { type CreateMode } from "../../../utils/createFunctionAppUtils"; +/** + * Defines a test scenario for Azure Functions extension testing. + * + * Each scenario follows this execution flow: + * 1. A single workspace project is created first (`createNewProject`) + * 2. Multiple create-and-deploy test cases branch off from the shared project + * 3. Within each test case, function app creation and deployment execute in series + * + * ``` + * Test Scenario + * │ + * ┌────────┴────────┐ + * │ Create New │ + * │ Project │ + * └────────┬────────┘ + * ┌───────┬───────┼───────┬───────┐ (concurrent) + * ▼ ▼ ▼ ▼ ▼ + * ┌──────┐ ┌──────┐ ┌──────┐ ┌──────┐ ┌──────┐ + * │Create│ │Create│ │Create│ │Create│ │Create│ (in series) + * │ Func │ │ Func │ │ Func │ │ Func │ │ Func │ + * │ App │ │ App │ │ App │ │ App │ │ App │ + * └──┬───┘ └──┬───┘ └──┬───┘ └──┬───┘ └──┬───┘ + * ▼ ▼ ▼ ▼ ▼ + * ┌──────┐ ┌──────┐ ┌──────┐ ┌──────┐ ┌──────┐ + * │Deploy│ │Deploy│ │Deploy│ │Deploy│ │Deploy│ + * │ Func │ │ Func │ │ Func │ │ Func │ │ Func │ + * │ App │ │ App │ │ App │ │ App │ │ App │ + * └──────┘ └──────┘ └──────┘ └──────┘ └──────┘ + * ``` + */ export interface AzExtFunctionsTestScenario { + /** + * A descriptive label for the test scenario. + */ label: string; + + /** + * The name of the folder where the test project will be created. + * Should match the name provided under `test.code-workspace` + */ folderName: string; + + /** + * The test case for creating a new workspace project. + * This test will always execute before creating and deploying to a function app. + */ createNewProjectTest: CreateNewProjectTestCase; + + /** + * Core test cases for creating and deploying function apps. + * These tests execute after project creation completes successfully. + * + * Core tests represent high-value smoke/regression tests that run as part of the + * nightly test suite. Keep this set small and focused on critical paths that we expect to pass. + */ createAndDeployTestsCore: CreateAndDeployTestCase[]; + + /** + * Extended test cases for creating and deploying function apps. + * These tests execute after project creation completes successfully. + * + * Extended tests provide broader, more exploratory coverage and are intended to be + * triggered manually for on-demand, deep-dive inspection rather than nightly runs due to their more comprehensive nature. + */ createAndDeployTestsExtended?: CreateAndDeployTestCase[]; /** @@ -20,9 +79,15 @@ export interface AzExtFunctionsTestScenario { } export interface CreateNewProjectTestCase { + /** + * A descriptive label for the test case. + */ label: string; + + /** + * The sequence of inputs to provide during workspace project creation. + */ inputs: (string | RegExp)[]; - postTest?: (context: IActionContext, workspaceFolderPath: string, errMsg?: string) => void | Promise; /** * Indicates this test case should be executed exclusively. This should only be used to aid with local development. @@ -31,17 +96,52 @@ export interface CreateNewProjectTestCase { } export interface CreateAndDeployTestCase { + /** + * Configuration for creating a function app. + * This step executes first before deployment. + */ createFunctionApp: { + /** + * A descriptive label for the create function app test. + */ label: string; + + /** + * The mode used to create the function app (e.g., basic, advanced). + */ mode: CreateMode; + + /** + * The sequence of inputs to provide during function app creation. + */ inputs: (string | RegExp)[]; - postTest?: (context: IActionContext, functionAppId: string) => void | Promise; }; + + /** + * Configuration for deploying a function app. + * This step executes in series after the workspace project and function app have been created. + */ deployFunctionApp: { + /** + * A descriptive label for the deploy function app test. + */ label: string; + + /** + * The sequence of inputs to provide during function app deployment. + */ inputs: (string | RegExp)[]; + + /** + * An optional callback to run after the deployment test completes. + * Note: Highly recommend implementing this as a verification post step as deployment may succeed initially, but fail at runtime. + */ postTest?: (context: IActionContext, functionAppId: string) => void | Promise; }; + + /** + * Resource groups to delete after the test completes for cleanup. + */ resourceGroupsToDelete?: string[]; /** diff --git a/test/nightly/scenarios/testScenarios/durable/azureStorage/DurableAzureStorage.plan.md b/test/nightly/scenarios/testScenarios/durable/azureStorage/DurableAzureStorage.plan.md deleted file mode 100644 index 1d90066d7..000000000 --- a/test/nightly/scenarios/testScenarios/durable/azureStorage/DurableAzureStorage.plan.md +++ /dev/null @@ -1,58 +0,0 @@ -# Durable Azure Storage Test Plan - -## I. Create New Project / Create Function - -### Workspace Project Test Matrix - -| No. | Language | Runtime | Programming Model | Comment | -|-----|----------|---------|-------------------|------------------------------------| -| 1 | JS | Node | v4 | | -| 2 | Python | Python | v2 | | -| 3 | C# | .NET | isolated | | -| 4 | C# | .NET | in-proc | | - -## II. Debug -TBD - -## III. Create Function App / Deployment - -### Create / Deploy Test Matrix - -| No. | Workspace Project | Connection Type | Operating System | Plan Type | Comment | -|-----|-------------------|-------------------|------------------|------------------------|-------------| -| 1 | 1 | Managed Identity | Linux | Flex Consumption | | -| 2 | 1 | Managed Identity | Linux | Premium | | -| 3 | 1 | Managed Identity | Windows | Premium | | -| 4 | 1 | Managed Identity | Linux | Consumption (Legacy) | | -| 5 | 1 | Managed Identity | Windows | Consumption (Legacy) | | -| 6 | 1 | Managed Identity | Linux | App Service | | -| 7 | 1 | Managed Identity | Windows | App Service | | -| 8 | 1 | Secrets | Linux | Flex Consumption | | -| 9 | 1 | Secrets | Linux | Premium | | -| 10 | 1 | Secrets | Windows | Premium | | -| 11 | 1 | Secrets | Linux | Consumption (Legacy) | | -| 12 | 1 | Secrets | Windows | Consumption (Legacy) | | -| 13 | 1 | Secrets | Linux | App Service | | -| 14 | 1 | Secrets | Windows | App Service | | -| 15 | 2 | Managed Identity | Linux | Flex Consumption | | -| 16 | 2 | Managed Identity | Linux | Premium | | -| 17 | 2 | Managed Identity | Linux | Consumption (Legacy) | | -| 18 | 2 | Managed Identity | Linux | App Service | | -| 19 | 2 | Secrets | Linux | Flex Consumption | | -| 20 | 2 | Secrets | Linux | Premium | | -| 21 | 2 | Secrets | Linux | Consumption (Legacy) | | -| 22 | 2 | Secrets | Linux | App Service | | -| 23 | 3 | Managed Identity | Linux | Flex Consumption | | -| 24 | 3 | Managed Identity | Linux | Premium | | -| 25 | 3 | Managed Identity | Windows | Premium | | -| 26 | 3 | Managed Identity | Linux | Consumption (Legacy) | | -| 27 | 3 | Managed Identity | Windows | Consumption (Legacy) | | -| 28 | 3 | Managed Identity | Linux | App Service | | -| 29 | 3 | Managed Identity | Windows | App Service | | -| 30 | 3 | Secrets | Linux | Flex Consumption | | -| 31 | 3 | Secrets | Linux | Premium | | -| 32 | 3 | Secrets | Windows | Premium | | -| 33 | 3 | Secrets | Linux | Consumption (Legacy) | | -| 34 | 3 | Secrets | Windows | Consumption (Legacy) | | -| 35 | 3 | Secrets | Linux | App Service | | -| 36 | 3 | Secrets | Windows | App Service | | diff --git a/test/nightly/scenarios/testScenarios/durable/azureStorage/azureStorageScenarios.ts b/test/nightly/scenarios/testScenarios/durable/azureStorage/azureStorageScenarios.ts index 1d005c968..602c113c7 100644 --- a/test/nightly/scenarios/testScenarios/durable/azureStorage/azureStorageScenarios.ts +++ b/test/nightly/scenarios/testScenarios/durable/azureStorage/azureStorageScenarios.ts @@ -5,13 +5,9 @@ import { type AzExtFunctionsTestScenario } from '../../AzExtFunctionsTestScenario'; import { generateJSNodeScenario } from './azureStorageJSNodeScenario'; -import { generateDotnetIsolatedScenario } from './dotnetIsolated'; -import { generatePythonScenario } from './pythonScenario'; export function generateDurableAzureStorageScenarios(): AzExtFunctionsTestScenario[] { return [ generateJSNodeScenario(), - generatePythonScenario(), - generateDotnetIsolatedScenario(), ]; } diff --git a/test/nightly/scenarios/testScenarios/durable/azureStorage/dotnetIsolated.ts b/test/nightly/scenarios/testScenarios/durable/azureStorage/dotnetIsolated.ts deleted file mode 100644 index adbb0ce14..000000000 --- a/test/nightly/scenarios/testScenarios/durable/azureStorage/dotnetIsolated.ts +++ /dev/null @@ -1,47 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { DurableBackend } from "../../../../../../extension.bundle"; -import { cSharpLanguagePick, dotnetIsolatedPick, dotnetNamespaceName, durableAzureStoragePick, durableOrchestratorName, durableOrchestratorPick } from "../../../../../constants"; -import { ConnectionType, CreateMode, OperatingSystem, PlanType, Runtime } from "../../../../../utils/createFunctionAppUtils"; -import { type AzExtFunctionsTestScenario } from "../../AzExtFunctionsTestScenario"; -import { generateCreateAndDeployTest } from "../generateCreateAndDeployTest"; - -export function generateDotnetIsolatedScenario(): AzExtFunctionsTestScenario { - const folderName: string = 'scenario-durable-azurestorage-dotnetisolated'; - - return { - label: 'durable-azurestorage-dotnetisolated', - folderName, - createNewProjectTest: { - label: 'create-new-project', - inputs: [ - cSharpLanguagePick, - // Todo: This needs to also run a match on the description because isolated vs. inproc differentiates there - dotnetIsolatedPick, - durableOrchestratorPick, - durableAzureStoragePick, - durableOrchestratorName, - dotnetNamespaceName, - ], - }, - createAndDeployTests: [ - generateCreateAndDeployTest(folderName, CreateMode.Basic, Runtime.DotNetIsolated, ConnectionType.ManagedIdentity, PlanType.FlexConsumption, OperatingSystem.Linux, DurableBackend.Storage), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.DotNetIsolated, ConnectionType.ManagedIdentity, PlanType.Premium, OperatingSystem.Linux, DurableBackend.Storage), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.DotNetIsolated, ConnectionType.ManagedIdentity, PlanType.Premium, OperatingSystem.Windows, DurableBackend.Storage), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.DotNetIsolated, ConnectionType.ManagedIdentity, PlanType.LegacyConsumption, OperatingSystem.Linux, DurableBackend.Storage), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.DotNetIsolated, ConnectionType.ManagedIdentity, PlanType.LegacyConsumption, OperatingSystem.Windows, DurableBackend.Storage), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.DotNetIsolated, ConnectionType.ManagedIdentity, PlanType.AppService, OperatingSystem.Linux, DurableBackend.Storage), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.DotNetIsolated, ConnectionType.ManagedIdentity, PlanType.AppService, OperatingSystem.Windows, DurableBackend.Storage), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.DotNetIsolated, ConnectionType.Secrets, PlanType.FlexConsumption, OperatingSystem.Linux, DurableBackend.Storage), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.DotNetIsolated, ConnectionType.Secrets, PlanType.Premium, OperatingSystem.Linux, DurableBackend.Storage), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.DotNetIsolated, ConnectionType.Secrets, PlanType.Premium, OperatingSystem.Windows, DurableBackend.Storage), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.DotNetIsolated, ConnectionType.Secrets, PlanType.LegacyConsumption, OperatingSystem.Linux, DurableBackend.Storage), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.DotNetIsolated, ConnectionType.Secrets, PlanType.LegacyConsumption, OperatingSystem.Windows, DurableBackend.Storage), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.DotNetIsolated, ConnectionType.Secrets, PlanType.AppService, OperatingSystem.Linux, DurableBackend.Storage), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.DotNetIsolated, ConnectionType.Secrets, PlanType.AppService, OperatingSystem.Windows, DurableBackend.Storage), - ], - }; -} diff --git a/test/nightly/scenarios/testScenarios/durable/azureStorage/pythonScenario.ts b/test/nightly/scenarios/testScenarios/durable/azureStorage/pythonScenario.ts deleted file mode 100644 index 9d7004df6..000000000 --- a/test/nightly/scenarios/testScenarios/durable/azureStorage/pythonScenario.ts +++ /dev/null @@ -1,41 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { DurableBackend } from "../../../../../../extension.bundle"; -import { durableAzureStoragePick, durableOrchestratorName, durableOrchestratorPick, pythonLanguagePick, pythonModelV2Pick, pythonRuntimePick } from "../../../../../constants"; -import { ConnectionType, CreateMode, PlanType, Runtime } from "../../../../../utils/createFunctionAppUtils"; -import { type AzExtFunctionsTestScenario } from "../../AzExtFunctionsTestScenario"; -import { generateCreateAndDeployTest } from "../generateCreateAndDeployTest"; - -export function generatePythonScenario(): AzExtFunctionsTestScenario { - const folderName: string = 'scenario-durable-azurestorage-python'; - - return { - label: 'durable-azurestorage-python', - folderName, - createNewProjectTest: { - label: 'create-new-project', - inputs: [ - pythonLanguagePick, - pythonModelV2Pick, - pythonRuntimePick, - durableOrchestratorPick, - durableAzureStoragePick, - durableOrchestratorName, - ], - }, - createAndDeployTests: [ - // OS gets automatically defaulted to Linux - generateCreateAndDeployTest(folderName, CreateMode.Basic, Runtime.Python, ConnectionType.ManagedIdentity, PlanType.FlexConsumption, undefined /** os */, DurableBackend.Storage), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Python, ConnectionType.ManagedIdentity, PlanType.Premium, undefined /** os */, DurableBackend.Storage), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Python, ConnectionType.ManagedIdentity, PlanType.LegacyConsumption, undefined /** os */, DurableBackend.Storage), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Python, ConnectionType.ManagedIdentity, PlanType.AppService, undefined /** os */, DurableBackend.Storage), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Python, ConnectionType.Secrets, PlanType.FlexConsumption, undefined /** os */, DurableBackend.Storage), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Python, ConnectionType.Secrets, PlanType.Premium, undefined /** os */, DurableBackend.Storage), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Python, ConnectionType.Secrets, PlanType.LegacyConsumption, undefined /** os */, DurableBackend.Storage), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Python, ConnectionType.Secrets, PlanType.AppService, undefined /** os */, DurableBackend.Storage), - ], - }; -} diff --git a/test/nightly/scenarios/testScenarios/durable/dts/DurableDTS.plan.md b/test/nightly/scenarios/testScenarios/durable/dts/DurableDTS.plan.md deleted file mode 100644 index 26d1cc775..000000000 --- a/test/nightly/scenarios/testScenarios/durable/dts/DurableDTS.plan.md +++ /dev/null @@ -1,52 +0,0 @@ -# Durable Task Scheduler (DTS) Test Plan - -## I. Create New Project / Create Function - -### Workspace Project Test Matrix: - -| No. | Language | Runtime | Programming Model | Comment | -|-----|----------|---------|-------------------|---------| -| 1 | JS | Node | v4 | | -| 2 | Python | Python | v2 | | -| 3 | C# | .NET | isolated | | -| 4 | C# | .NET | in-proc | | - -## II. Debug -TBD - -## III. Create Function App / Deployment - -### Create / Deploy Test Matrix: - -| Workspace Project | Connection Type | Operating System | Plan Type | Comment | -|-------------------|-------------------|------------------|-------------------|---------| -| 1 | Managed Identity | Linux | Flex Consumption | | -| 1 | Managed Identity | Linux | Premium | | -| 1 | Managed Identity | Windows | Premium | | -| 1 | Secrets | Linux | Flex Consumption | | -| 1 | Secrets | Linux | Premium | | -| 1 | Secrets | Windows | Premium | | -| 2 | Managed Identity | Linux | Flex Consumption | | -| 2 | Managed Identity | Windows | Flex Consumption | | -| 2 | Managed Identity | Linux | Premium | | -| 2 | Managed Identity | Windows | Premium | | -| 2 | Secrets | Linux | Flex Consumption | | -| 2 | Secrets | Windows | Flex Consumption | | -| 2 | Secrets | Linux | Premium | | -| 2 | Secrets | Windows | Premium | | -| 3 | Managed Identity | Linux | Flex Consumption | | -| 3 | Managed Identity | Windows | Flex Consumption | | -| 3 | Managed Identity | Linux | Premium | | -| 3 | Managed Identity | Windows | Premium | | -| 3 | Secrets | Linux | Flex Consumption | | -| 3 | Secrets | Windows | Flex Consumption | | -| 3 | Secrets | Linux | Premium | | -| 3 | Secrets | Windows | Premium | | -| 4 | Managed Identity | Linux | Flex Consumption | | -| 4 | Managed Identity | Windows | Flex Consumption | | -| 4 | Managed Identity | Linux | Premium | | -| 4 | Managed Identity | Windows | Premium | | -| 4 | Secrets | Linux | Flex Consumption | | -| 4 | Secrets | Windows | Flex Consumption | | -| 4 | Secrets | Linux | Premium | | -| 4 | Secrets | Windows | Premium | | diff --git a/test/nightly/scenarios/testScenarios/durable/dts/dtsJSNodeScenario.ts b/test/nightly/scenarios/testScenarios/durable/dts/dtsJSNodeScenario.ts deleted file mode 100644 index 5495940a6..000000000 --- a/test/nightly/scenarios/testScenarios/durable/dts/dtsJSNodeScenario.ts +++ /dev/null @@ -1,37 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { DurableBackend } from "../../../../../../extension.bundle"; -import { durableOrchestratorName, durableOrchestratorPick } from "../../../../../constants"; -import { ConnectionType, CreateMode, OperatingSystem, PlanType, Runtime } from "../../../../../utils/createFunctionAppUtils"; -import { type AzExtFunctionsTestScenario } from "../../AzExtFunctionsTestScenario"; -import { generateCreateAndDeployTest } from "../generateCreateAndDeployTest"; - -export function generateJSNodeScenario(): AzExtFunctionsTestScenario { - const folderName: string = 'scenario-durable-dts-jsnode'; - - return { - label: 'durable-dts-jsnode', - folderName, - createNewProjectTest: { - label: 'create-new-project', - inputs: [ - /JavaScript/i, - /v4/i, - durableOrchestratorPick, - /Durable Task Scheduler/i, - durableOrchestratorName, - ], - }, - createAndDeployTests: [ - generateCreateAndDeployTest(folderName, CreateMode.Basic, Runtime.Node, ConnectionType.ManagedIdentity, PlanType.FlexConsumption, OperatingSystem.Linux, DurableBackend.DTS), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.ManagedIdentity, PlanType.Premium, OperatingSystem.Linux, DurableBackend.DTS), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.ManagedIdentity, PlanType.Premium, OperatingSystem.Windows, DurableBackend.DTS), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.FlexConsumption, OperatingSystem.Linux, DurableBackend.DTS), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.Premium, OperatingSystem.Linux, DurableBackend.DTS), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.Premium, OperatingSystem.Windows, DurableBackend.DTS), - ], - }; -} diff --git a/test/nightly/scenarios/testScenarios/durable/dts/dtsScenarios.ts b/test/nightly/scenarios/testScenarios/durable/dts/dtsScenarios.ts deleted file mode 100644 index 151b2485f..000000000 --- a/test/nightly/scenarios/testScenarios/durable/dts/dtsScenarios.ts +++ /dev/null @@ -1,13 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { type AzExtFunctionsTestScenario } from '../../AzExtFunctionsTestScenario'; -import { generateJSNodeScenario } from './dtsJSNodeScenario'; - -export function generateDurableDTSScenarios(): AzExtFunctionsTestScenario[] { - return [ - generateJSNodeScenario(), - ]; -} diff --git a/test/nightly/scenarios/testScenarios/durable/generateCreateAndDeployTest.ts b/test/nightly/scenarios/testScenarios/durable/generateCreateAndDeployTest.ts index 0ff60585f..6cdc3b08a 100644 --- a/test/nightly/scenarios/testScenarios/durable/generateCreateAndDeployTest.ts +++ b/test/nightly/scenarios/testScenarios/durable/generateCreateAndDeployTest.ts @@ -13,18 +13,18 @@ import { deployFunctionAppUtils } from "../../../../utils/deployFunctionAppUtils import { subscriptionContext } from "../../../global.nightly.test"; import { type CreateAndDeployTestCase } from "../AzExtFunctionsTestScenario"; -export function generateCreateAndDeployTest(folderName: string, createMode: CreateMode, runtime: Runtime, connection: ConnectionType, plan: PlanType, os?: OperatingSystem, storageType?: DurableBackend): CreateAndDeployTestCase { +export function generateCreateAndDeployTest(folderName: string, createMode: CreateMode, runtime: Runtime, storageConnection: ConnectionType, plan: PlanType, os?: OperatingSystem, storageType?: DurableBackend): CreateAndDeployTestCase { const appName: string = getRandomAlphanumericString(); const osDescription: string = os ? `-${os}` : ''; - const description: string = `${createMode}-${connection}${osDescription}-${plan}`; + const description: string = `${createMode}-${storageConnection}${osDescription}-${plan}`; return { createFunctionApp: { label: `create-function-app | ${description}`, mode: createMode, inputs: createMode === CreateMode.Basic ? - createFunctionAppUtils.generateBasicCreateInputs(appName, folderName, runtime, connection) : - createFunctionAppUtils.generateAdvancedCreateInputs(appName, folderName, runtime, connection, plan, os), + createFunctionAppUtils.generateBasicCreateInputs(appName, folderName, runtime, storageConnection) : + createFunctionAppUtils.generateAdvancedCreateInputs(appName, folderName, runtime, storageConnection, plan, os), }, deployFunctionApp: { label: `deploy-function-app | ${description}`, @@ -43,15 +43,11 @@ function generateVerifyDeployment(runtime: Runtime) { let url = `https://${functionApp.defaultHostName}`; switch (runtime) { - case Runtime.Python: - url += `/api/orchestrators/${durableOrchestratorName}_orchestrator`; - break; + // case Runtime.Python: case Runtime.Node: url += `/api/orchestrators/${durableOrchestratorName}Orchestrator`; break; - case Runtime.DotNetIsolated: - url += `/api/${durableOrchestratorName}_HttpStart`; - break; + // case Runtime.DotNetIsolated: default: throw new Error('Durable verify deployment not yet implemented for this runtime type.'); } diff --git a/test/nightly/scenarios/testScenarios/httpTrigger/HttpTrigger.plan.md b/test/nightly/scenarios/testScenarios/httpTrigger/HttpTrigger.plan.md deleted file mode 100644 index 9927377b8..000000000 --- a/test/nightly/scenarios/testScenarios/httpTrigger/HttpTrigger.plan.md +++ /dev/null @@ -1,33 +0,0 @@ -# HTTP Trigger Test Plan - -## I. Create New Project / Create Function - -### Workspace Project Test Matrix - -| No. | Language | Runtime | Programming Model | Comment | -|-----|----------|---------|-------------------|---------| -| 1 | JS | Node | v4 | | - -## II. Debug -TBD - -## III. Create Function App / Deployment - -### Create / Deploy Test Matrix - -| No. | Workspace Project | Connection Type | Operating System | Plan Type | Comment | -|-----|-------------------|-------------------|------------------|------------------------|-------------| -| 1 | 1 | Managed Identity | Linux | Flex Consumption | | -| 2 | 1 | Managed Identity | Linux | Premium | | -| 3 | 1 | Managed Identity | Windows | Premium | | -| 4 | 1 | Managed Identity | Linux | Consumption (Legacy) | | -| 5 | 1 | Managed Identity | Windows | Consumption (Legacy) | | -| 6 | 1 | Managed Identity | Linux | App Service | | -| 7 | 1 | Managed Identity | Windows | App Service | | -| 8 | 1 | Secrets | Linux | Flex Consumption | | -| 9 | 1 | Secrets | Linux | Premium | | -| 10 | 1 | Secrets | Windows | Premium | | -| 11 | 1 | Secrets | Linux | Consumption (Legacy) | | -| 12 | 1 | Secrets | Windows | Consumption (Legacy) | | -| 13 | 1 | Secrets | Linux | App Service | | -| 14 | 1 | Secrets | Windows | App Service | | diff --git a/test/nightly/scenarios/testScenarios/httpTrigger/generateCreateAndDeployTest.ts b/test/nightly/scenarios/testScenarios/httpTrigger/generateCreateAndDeployTest.ts deleted file mode 100644 index 504d61d7d..000000000 --- a/test/nightly/scenarios/testScenarios/httpTrigger/generateCreateAndDeployTest.ts +++ /dev/null @@ -1,61 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { type Site, type WebSiteManagementClient } from "@azure/arm-appservice"; -import { createWebSiteClient } from "@microsoft/vscode-azext-azureappservice"; -import { parseAzureResourceId, type ParsedAzureResourceId } from "@microsoft/vscode-azext-azureutils"; -import { getRandomAlphanumericString, type IActionContext } from "../../../../../extension.bundle"; -import { httpTriggerName } from "../../../../constants"; -import { createFunctionAppUtils, CreateMode, Runtime, type ConnectionType, type OperatingSystem, type PlanType } from "../../../../utils/createFunctionAppUtils"; -import { subscriptionContext } from "../../../global.nightly.test"; -import { type CreateAndDeployTestCase } from "../AzExtFunctionsTestScenario"; - -export function generateCreateAndDeployTest(folderName: string, createMode: CreateMode, runtime: Runtime, connection: ConnectionType, plan: PlanType, os?: OperatingSystem): CreateAndDeployTestCase { - const appName: string = getRandomAlphanumericString(); - const osDescription: string = os ? `-${os}` : ''; - const description: string = `${createMode}-${connection}${osDescription}-${plan}`; - - return { - createFunctionApp: { - label: `create-function-app | ${description}`, - mode: createMode, - inputs: createMode === CreateMode.Basic ? - createFunctionAppUtils.generateBasicCreateInputs(appName, folderName, runtime, connection) : - createFunctionAppUtils.generateAdvancedCreateInputs(appName, folderName, runtime, connection, plan, os), - }, - deployFunctionApp: { - label: `deploy-function-app | ${description}`, - inputs: ['Deploy'], - postTest: generateVerifyDeployment(runtime), - }, - resourceGroupsToDelete: [appName], - }; -} - -function generateVerifyDeployment(runtime: Runtime) { - return async function verifyDeployment(context: IActionContext, functionAppId: string): Promise { - const client: WebSiteManagementClient = await createWebSiteClient({ ...context, ...subscriptionContext }); - const parsedResource: ParsedAzureResourceId = parseAzureResourceId(functionAppId); - const functionApp: Site = await client.webApps.get(parsedResource.resourceGroup, parsedResource.resourceName); - - let url = `https://${functionApp.defaultHostName}`; - switch (runtime) { - // case Runtime.Python: - // break; - case Runtime.Node: - url += `/api/${httpTriggerName}`; - break; - // case Runtime.DotNetIsolated: - // break; - default: - throw new Error('Durable verify deployment not yet implemented for this runtime type.'); - } - - const response = await fetch(url); - if (response.status !== 200) { - throw new Error(`Verify Deployment: Http trigger endpoint responded with ${response.status}.`); - } - }; -} diff --git a/test/nightly/scenarios/testScenarios/httpTrigger/httpTriggerJSNodeScenario.ts b/test/nightly/scenarios/testScenarios/httpTrigger/httpTriggerJSNodeScenario.ts deleted file mode 100644 index c1aa4613b..000000000 --- a/test/nightly/scenarios/testScenarios/httpTrigger/httpTriggerJSNodeScenario.ts +++ /dev/null @@ -1,48 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { httpTriggerName, httpTriggerPick, jsLanguagePick, jsModelV4Pick } from "../../../../constants"; -import { ConnectionType, CreateMode, OperatingSystem, PlanType, Runtime } from "../../../../utils/createFunctionAppUtils"; -import { type AzExtFunctionsTestScenario } from "../AzExtFunctionsTestScenario"; -import { generateCreateAndDeployTest } from "./generateCreateAndDeployTest"; - -export function generateJSNodeScenario(): AzExtFunctionsTestScenario { - const folderName: string = 'scenario-http-trigger-jsnode'; - - return { - label: 'http-trigger-jsnode', - folderName, - createNewProjectTest: { - label: 'create-new-project', - inputs: [ - jsLanguagePick, - jsModelV4Pick, - httpTriggerPick, - httpTriggerName, - ], - }, - createAndDeployTestsCore: [ - generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.LegacyConsumption, OperatingSystem.Linux), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.AppService, OperatingSystem.Windows), - // generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.Premium, OperatingSystem.Linux), - ], - createAndDeployTestsExtended: [ - generateCreateAndDeployTest(folderName, CreateMode.Basic, Runtime.Node, ConnectionType.ManagedIdentity, PlanType.FlexConsumption, OperatingSystem.Linux), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.ManagedIdentity, PlanType.Premium, OperatingSystem.Linux), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.ManagedIdentity, PlanType.Premium, OperatingSystem.Windows), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.ManagedIdentity, PlanType.LegacyConsumption, OperatingSystem.Linux), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.ManagedIdentity, PlanType.LegacyConsumption, OperatingSystem.Windows), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.ManagedIdentity, PlanType.AppService, OperatingSystem.Linux), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.ManagedIdentity, PlanType.AppService, OperatingSystem.Windows), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.FlexConsumption, OperatingSystem.Linux), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.Premium, OperatingSystem.Linux), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.Premium, OperatingSystem.Windows), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.LegacyConsumption, OperatingSystem.Linux), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.LegacyConsumption, OperatingSystem.Windows), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.AppService, OperatingSystem.Linux), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.AppService, OperatingSystem.Windows), - ], - }; -} diff --git a/test/nightly/scenarios/testScenarios/httpTrigger/httpTriggerScenarios.ts b/test/nightly/scenarios/testScenarios/httpTrigger/httpTriggerScenarios.ts deleted file mode 100644 index 0f5456c5e..000000000 --- a/test/nightly/scenarios/testScenarios/httpTrigger/httpTriggerScenarios.ts +++ /dev/null @@ -1,13 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -import { type AzExtFunctionsTestScenario } from "../AzExtFunctionsTestScenario"; -import { generateJSNodeScenario } from "./httpTriggerJSNodeScenario"; - -export function generateHttpTriggerScenarios(): AzExtFunctionsTestScenario[] { - return [ - generateJSNodeScenario(), - ]; -} diff --git a/test/nightly/scenarios/testScenarios/testScenarios.ts b/test/nightly/scenarios/testScenarios/testScenarios.ts index 607fc3c68..c1c5de737 100644 --- a/test/nightly/scenarios/testScenarios/testScenarios.ts +++ b/test/nightly/scenarios/testScenarios/testScenarios.ts @@ -5,17 +5,14 @@ import { type AzExtFunctionsTestScenario } from "./AzExtFunctionsTestScenario"; import { generateDurableAzureStorageScenarios } from "./durable/azureStorage/azureStorageScenarios"; -import { generateDurableDTSScenarios } from "./durable/dts/dtsScenarios"; -import { generateHttpTriggerScenarios } from "./httpTrigger/httpTriggerScenarios"; export function generateTestScenarios(): AzExtFunctionsTestScenario[] { const testScenarios: AzExtFunctionsTestScenario[] = [ // Trigger scenarios - ...generateHttpTriggerScenarios(), + // ...generateHttpTriggerScenarios(), // Durable scenarios ...generateDurableAzureStorageScenarios(), - ...generateDurableDTSScenarios(), ]; return testScenarios; } diff --git a/test/utils/createFunctionAppUtils.ts b/test/utils/createFunctionAppUtils.ts index eb31a66f2..982dbfb66 100644 --- a/test/utils/createFunctionAppUtils.ts +++ b/test/utils/createFunctionAppUtils.ts @@ -3,20 +3,20 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { dotnetIsolatedRuntimePick, locationDefaultPick, nodeRuntimePick, pythonRuntimePick } from "../constants"; +import { locationDefaultPick, nodeRuntimePick } from "../constants"; export namespace createFunctionAppUtils { - export function generateBasicCreateInputs(appName: string, folderName: string, runtime: Runtime, connection: ConnectionType): (string | RegExp)[] { + export function generateBasicCreateInputs(appName: string, folderName: string, runtime: Runtime, storageConnection: ConnectionType): (string | RegExp)[] { return [ folderName, appName, locationDefaultPick, getRuntimePick(runtime), - new RegExp(connection, 'i'), + new RegExp(storageConnection, 'i'), ]; } - export function generateAdvancedCreateInputs(appName: string, folderName: string, runtime: Runtime, connection: ConnectionType, plan: PlanType, os?: OperatingSystem): (string | RegExp)[] { + export function generateAdvancedCreateInputs(appName: string, folderName: string, runtime: Runtime, storageConnection: ConnectionType, plan: PlanType, os?: OperatingSystem): (string | RegExp)[] { switch (plan) { case PlanType.FlexConsumption: return [ @@ -29,7 +29,7 @@ export namespace createFunctionAppUtils { '100', /Create new resource group/i, appName, - ...getConnectionTypeInputs(connection), + ...getConnectionTypeInputs(storageConnection), /Create new storage account/i, appName, /Create new application insights/i, @@ -48,7 +48,7 @@ export namespace createFunctionAppUtils { /EP1/i, /Create new resource group/i, appName, - ...getConnectionTypeInputs(connection), + ...getConnectionTypeInputs(storageConnection), /Create new storage account/i, appName, /Create new application insights/i, @@ -64,7 +64,7 @@ export namespace createFunctionAppUtils { ...(os ? [new RegExp(os, 'i')] : []), /Create new resource group/i, appName, - ...getConnectionTypeInputs(connection), + ...getConnectionTypeInputs(storageConnection), /Create new storage account/i, appName, /Create new application insights/i, @@ -83,7 +83,7 @@ export namespace createFunctionAppUtils { /S1/i, /Create new resource group/i, appName, - ...getConnectionTypeInputs(connection), + ...getConnectionTypeInputs(storageConnection), /Create new storage account/i, appName, /Create new application insights/i, @@ -94,12 +94,11 @@ export namespace createFunctionAppUtils { function getRuntimePick(runtime: Runtime): RegExp | string { switch (runtime) { - case Runtime.Python: - return pythonRuntimePick; + // case Runtime.Python: case Runtime.Node: return nodeRuntimePick; - case Runtime.DotNetIsolated: - return dotnetIsolatedRuntimePick; + // case Runtime.DotNetIsolated: + // case Runtime.DotNetInProc: default: throw new Error(`Runtime "${runtime}" not yet supported in "createFunctionAppUtils.generateBasicCreateInputs".`); } @@ -138,4 +137,5 @@ export enum Runtime { Node = 'Node', Python = 'Python', DotNetIsolated = '.NET Isolated', + DotNetInProc = '.NET In-Proc' } diff --git a/test/utils/deployFunctionAppUtils.ts b/test/utils/deployFunctionAppUtils.ts index eae75f86e..0010b7d3c 100644 --- a/test/utils/deployFunctionAppUtils.ts +++ b/test/utils/deployFunctionAppUtils.ts @@ -6,25 +6,11 @@ import { DurableBackend } from "../../extension.bundle"; export namespace deployFunctionAppUtils { - export function generateDurableDeployInputs(appName: string, storageType?: DurableBackend): (string | RegExp)[] { + export function generateDurableDeployInputs(_appName: string, storageType?: DurableBackend): (string | RegExp)[] { switch (storageType) { - case DurableBackend.DTS: - return [ - // Todo: Expand regexp capability for context.ui.showWarningMessage - 'Connect Durable Task Scheduler', - /Create New Durable Task Scheduler/i, - appName, - /Create New Durable Task Hub/i, - appName, - /Assign New User[- ]Assigned Identity/i, - /Create New User[- ]Assigned Identity/i, - // Todo: Here too - 'Deploy', - ]; - case DurableBackend.Netherite: - return []; - case DurableBackend.SQL: - return []; + // case DurableBackend.DTS: + // case DurableBackend.Netherite: + // case DurableBackend.SQL: case DurableBackend.Storage: default: return [ From 9a3f6f0238162a141e5770a54b05ff60cc0a7467 Mon Sep 17 00:00:00 2001 From: MicroFish91 <40250218+MicroFish91@users.noreply.github.com> Date: Tue, 20 Jan 2026 18:13:58 -0800 Subject: [PATCH 18/27] Misc improvements --- .vscode/launch.json | 2 +- test/constants.ts | 9 ---- test/nightly/global.nightly.test.ts | 5 +-- test/nightly/scenarios/parallelScenarios.ts | 6 +-- test/nightly/scenarios/scenarios.test.ts | 7 ++- .../AzExtFunctionsTestScenario.ts | 27 ++++++------ .../azureStorageJSNodeScenario.ts | 44 ++++++++++--------- .../durable/generateCreateAndDeployTest.ts | 2 +- test/test.code-workspace | 28 ------------ test/utils/createFunctionAppUtils.ts | 6 +++ 10 files changed, 52 insertions(+), 84 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index ec29a9607..d2f94d337 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -77,7 +77,7 @@ "FUNC_PATH": "func", "AZFUNC_UPDATE_BACKUP_TEMPLATES": "", "AzCode_EnableLongRunningTestsLocal": "true", - "AzCode_OnlyLongRunningTestScenario": "", + "AzCode_ScenarioToIsolateAndExpand": "" } }, { diff --git a/test/constants.ts b/test/constants.ts index 21dc68843..ebaa8e59c 100644 --- a/test/constants.ts +++ b/test/constants.ts @@ -5,29 +5,20 @@ // Start section create-new-project // Project type picks -export const httpTriggerPick: RegExp = /HTTP\s?Trigger/i; export const durableOrchestratorPick: RegExp = /Durable Functions Orch/i; // Sometimes this is "orchestrator" or "orchestration" depending on the template feed export const durableAzureStoragePick: RegExp = /Azure Storage/i; // Default names -export const httpTriggerName: string = 'httpTrigger'; export const durableOrchestratorName: string = 'durableHello'; -export const dotnetNamespaceName: string = 'Company.Function'; // Language picks export const jsLanguagePick: RegExp = /JavaScript/i; -export const pythonLanguagePick: RegExp = /Python/i; -export const cSharpLanguagePick: RegExp = /C#/i; // Framework picks export const jsModelV4Pick: RegExp = /v4/i; -export const pythonModelV2Pick: RegExp = /v2/i; -export const dotnetIsolatedPick: RegExp = /\.NET 10/i; // Start section create-function-app export const locationDefaultPick: RegExp = /West US 2/i; // Default runtime picks -export const pythonRuntimePick: RegExp = /Python(\s)?3\.12/i; export const nodeRuntimePick: RegExp = /Node\.js(\s)?22/i; -export const dotnetIsolatedRuntimePick: RegExp = /\.NET(\s)?10(\s)?Isolated/i; diff --git a/test/nightly/global.nightly.test.ts b/test/nightly/global.nightly.test.ts index 73fb2c05d..c1d0dcf6d 100644 --- a/test/nightly/global.nightly.test.ts +++ b/test/nightly/global.nightly.test.ts @@ -39,6 +39,7 @@ suiteSetup(async function (this: Mocha.Context): Promise { suiteTeardown(async function (this: Mocha.Context): Promise { if (longRunningTestsEnabled) { this.timeout(10 * 60 * 1000); + scenariosTracker.report(); await deleteResourceGroups(); ext.azureAccountTreeItem.dispose(); @@ -46,11 +47,7 @@ suiteTeardown(async function (this: Mocha.Context): Promise { }); async function deleteResourceGroups(): Promise { - const report = scenariosTracker.report(); - console.log(report); - const rgClient: ResourceManagementClient = createAzureClient([await createTestActionContext(), subscriptionContext], ResourceManagementClient); - await Promise.allSettled(Array.from(resourceGroupsToDelete).map(async resourceGroup => { if ((await rgClient.resourceGroups.checkExistence(resourceGroup)).body) { console.log(`Started delete of resource group "${resourceGroup}"...`); diff --git a/test/nightly/scenarios/parallelScenarios.ts b/test/nightly/scenarios/parallelScenarios.ts index eca7ba53b..49aac97d6 100644 --- a/test/nightly/scenarios/parallelScenarios.ts +++ b/test/nightly/scenarios/parallelScenarios.ts @@ -74,9 +74,9 @@ function generateRunScenario(scenario: AzExtFunctionsTestScenario): AzExtFunctio }); // 2. Start all create and deploy tests for the scenario - const createAndDeployTests: CreateAndDeployTestCase[] | undefined = testLevel === 'core' ? scenario.createAndDeployTestsCore : scenario.createAndDeployTestsExtended ?? []; - + const createAndDeployTests: CreateAndDeployTestCase[] = testLevel === TestLevel.Core ? scenario.createAndDeployTests.core : scenario.createAndDeployTests.expanded ?? []; const onlyTestCase: CreateAndDeployTestCase | undefined = createAndDeployTests.find(test => test.only); + if (onlyTestCase) { await startCreateAndDeployTest(scenario.label, rootFolder, onlyTestCase); } else { @@ -181,5 +181,5 @@ export function getWorkspaceFolderUri(folderName: string): Uri { export enum TestLevel { Core = 'core', - Extended = 'extended', + Expanded = 'expanded', } diff --git a/test/nightly/scenarios/scenarios.test.ts b/test/nightly/scenarios/scenarios.test.ts index f19df0c83..45fed5e6b 100644 --- a/test/nightly/scenarios/scenarios.test.ts +++ b/test/nightly/scenarios/scenarios.test.ts @@ -9,7 +9,6 @@ import { generateParallelScenarios, TestLevel, type AzExtFunctionsParallelTestSc const testScenarios: AzExtFunctionsParallelTestScenario[] = generateParallelScenarios(); suite.only('Scenarios', async function (this: Mocha.Suite) { - // Leave ample time since some tasks are known to take a while (e.g. durable task schedulers sometimes take ~30m to provision) this.timeout(45 * 60 * 1000); suiteSetup(async function (this: Mocha.Context) { @@ -18,12 +17,12 @@ suite.only('Scenarios', async function (this: Mocha.Suite) { } const onlyTestScenario = testScenarios.find(s => { - if (process.env.AzCode_OnlyLongRunningTestScenario) { - return s.title === process.env.AzCode_OnlyLongRunningTestScenario; + if (process.env.AzCode_ScenarioToIsolateAndExpand) { + return s.title === process.env.AzCode_ScenarioToIsolateAndExpand; } return s.only; }); - const testLevel: TestLevel = process.env.AzCode_OnlyLongRunningTestScenario ? TestLevel.Extended : TestLevel.Core; + const testLevel: TestLevel = process.env.AzCode_ScenarioToIsolateAndExpand ? TestLevel.Expanded : TestLevel.Core; if (onlyTestScenario) { onlyTestScenario.scenario = onlyTestScenario.runScenario(testLevel); diff --git a/test/nightly/scenarios/testScenarios/AzExtFunctionsTestScenario.ts b/test/nightly/scenarios/testScenarios/AzExtFunctionsTestScenario.ts index b01609965..fe7715780 100644 --- a/test/nightly/scenarios/testScenarios/AzExtFunctionsTestScenario.ts +++ b/test/nightly/scenarios/testScenarios/AzExtFunctionsTestScenario.ts @@ -55,22 +55,23 @@ export interface AzExtFunctionsTestScenario { createNewProjectTest: CreateNewProjectTestCase; /** - * Core test cases for creating and deploying function apps. + * Test cases for creating and deploying function apps. * These tests execute after project creation completes successfully. - * - * Core tests represent high-value smoke/regression tests that run as part of the - * nightly test suite. Keep this set small and focused on critical paths that we expect to pass. */ - createAndDeployTestsCore: CreateAndDeployTestCase[]; + createAndDeployTests: { + /** + * Core test cases representing high-value smoke/regression tests that run as part of the + * nightly test suite. Keep this set small and focused on critical paths that we expect to pass. + */ + core: CreateAndDeployTestCase[]; - /** - * Extended test cases for creating and deploying function apps. - * These tests execute after project creation completes successfully. - * - * Extended tests provide broader, more exploratory coverage and are intended to be - * triggered manually for on-demand, deep-dive inspection rather than nightly runs due to their more comprehensive nature. - */ - createAndDeployTestsExtended?: CreateAndDeployTestCase[]; + /** + * Expanded test cases providing broader, more exploratory coverage. Intended to be + * triggered manually for on-demand, deep-dive inspection rather than nightly runs + * due to their more comprehensive nature. + */ + expanded?: CreateAndDeployTestCase[]; + }; /** * Indicates this scenario should be executed exclusively. This should only be used to aid with local development. diff --git a/test/nightly/scenarios/testScenarios/durable/azureStorage/azureStorageJSNodeScenario.ts b/test/nightly/scenarios/testScenarios/durable/azureStorage/azureStorageJSNodeScenario.ts index 0e83ecbdd..911c2896b 100644 --- a/test/nightly/scenarios/testScenarios/durable/azureStorage/azureStorageJSNodeScenario.ts +++ b/test/nightly/scenarios/testScenarios/durable/azureStorage/azureStorageJSNodeScenario.ts @@ -25,26 +25,28 @@ export function generateJSNodeScenario(): AzExtFunctionsTestScenario { durableOrchestratorName, ], }, - createAndDeployTestsCore: [ - generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.LegacyConsumption, OperatingSystem.Linux), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.AppService, OperatingSystem.Windows), - // generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.Premium, OperatingSystem.Linux), - ], - createAndDeployTestsExtended: [ - generateCreateAndDeployTest(folderName, CreateMode.Basic, Runtime.Node, ConnectionType.ManagedIdentity, PlanType.FlexConsumption, OperatingSystem.Linux, DurableBackend.Storage), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.ManagedIdentity, PlanType.Premium, OperatingSystem.Linux, DurableBackend.Storage), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.ManagedIdentity, PlanType.Premium, OperatingSystem.Windows, DurableBackend.Storage), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.ManagedIdentity, PlanType.LegacyConsumption, OperatingSystem.Linux, DurableBackend.Storage), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.ManagedIdentity, PlanType.LegacyConsumption, OperatingSystem.Windows, DurableBackend.Storage), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.ManagedIdentity, PlanType.AppService, OperatingSystem.Linux, DurableBackend.Storage), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.ManagedIdentity, PlanType.AppService, OperatingSystem.Windows, DurableBackend.Storage), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.FlexConsumption, OperatingSystem.Linux, DurableBackend.Storage), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.Premium, OperatingSystem.Linux, DurableBackend.Storage), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.Premium, OperatingSystem.Windows, DurableBackend.Storage), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.LegacyConsumption, OperatingSystem.Linux, DurableBackend.Storage), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.LegacyConsumption, OperatingSystem.Windows, DurableBackend.Storage), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.AppService, OperatingSystem.Linux, DurableBackend.Storage), - generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.AppService, OperatingSystem.Windows, DurableBackend.Storage), - ], + createAndDeployTests: { + core: [ + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.LegacyConsumption, OperatingSystem.Linux), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.AppService, OperatingSystem.Windows), + // generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.Premium, OperatingSystem.Linux), + ], + expanded: [ + generateCreateAndDeployTest(folderName, CreateMode.Basic, Runtime.Node, ConnectionType.ManagedIdentity, PlanType.FlexConsumption, OperatingSystem.Linux, DurableBackend.Storage), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.ManagedIdentity, PlanType.Premium, OperatingSystem.Linux, DurableBackend.Storage), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.ManagedIdentity, PlanType.Premium, OperatingSystem.Windows, DurableBackend.Storage), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.ManagedIdentity, PlanType.LegacyConsumption, OperatingSystem.Linux, DurableBackend.Storage), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.ManagedIdentity, PlanType.LegacyConsumption, OperatingSystem.Windows, DurableBackend.Storage), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.ManagedIdentity, PlanType.AppService, OperatingSystem.Linux, DurableBackend.Storage), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.ManagedIdentity, PlanType.AppService, OperatingSystem.Windows, DurableBackend.Storage), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.FlexConsumption, OperatingSystem.Linux, DurableBackend.Storage), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.Premium, OperatingSystem.Linux, DurableBackend.Storage), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.Premium, OperatingSystem.Windows, DurableBackend.Storage), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.LegacyConsumption, OperatingSystem.Linux, DurableBackend.Storage), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.LegacyConsumption, OperatingSystem.Windows, DurableBackend.Storage), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.AppService, OperatingSystem.Linux, DurableBackend.Storage), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.AppService, OperatingSystem.Windows, DurableBackend.Storage), + ], + } }; } diff --git a/test/nightly/scenarios/testScenarios/durable/generateCreateAndDeployTest.ts b/test/nightly/scenarios/testScenarios/durable/generateCreateAndDeployTest.ts index 6cdc3b08a..59f517966 100644 --- a/test/nightly/scenarios/testScenarios/durable/generateCreateAndDeployTest.ts +++ b/test/nightly/scenarios/testScenarios/durable/generateCreateAndDeployTest.ts @@ -16,7 +16,7 @@ import { type CreateAndDeployTestCase } from "../AzExtFunctionsTestScenario"; export function generateCreateAndDeployTest(folderName: string, createMode: CreateMode, runtime: Runtime, storageConnection: ConnectionType, plan: PlanType, os?: OperatingSystem, storageType?: DurableBackend): CreateAndDeployTestCase { const appName: string = getRandomAlphanumericString(); const osDescription: string = os ? `-${os}` : ''; - const description: string = `${createMode}-${storageConnection}${osDescription}-${plan}`; + const description: string = `${createMode}-${createFunctionAppUtils.getStorageConnectionDescription(storageConnection)}${osDescription}-${plan}`; return { createFunctionApp: { diff --git a/test/test.code-workspace b/test/test.code-workspace index 3fc76131f..886ebd6c2 100644 --- a/test/test.code-workspace +++ b/test/test.code-workspace @@ -30,38 +30,10 @@ { "path": "../testWorkspace/9" }, - { - "name": "scenario-http-trigger-jsnode", - "path": "../testWorkspace/scenarios/http-trigger/jsnode" - }, { "name": "scenario-durable-azurestorage-jsnode", "path": "../testWorkspace/scenarios/durable-azurestorage/jsnode" }, - { - "name": "scenario-durable-azurestorage-python", - "path": "../testWorkspace/scenarios/durable-azurestorage/python" - }, - { - "name": "scenario-durable-azurestorage-dotnetisolated", - "path": "../testWorkspace/scenarios/durable-azurestorage/dotnetisolated" - }, - { - "name": "scenario-durable-dts-jsnode", - "path": "../testWorkspace/scenarios/durable-dts/jsnode" - }, - { - "name": "scenario-durable-dts-python", - "path": "../testWorkspace/scenarios/durable-dts/python" - }, - { - "name": "scenario-durable-dts-dotnet-isolated", - "path": "../testWorkspace/scenarios/durable-dts/dotnet-isolated" - }, - { - "name": "scenario-durable-dts-dotnet-inproc", - "path": "../testWorkspace/scenarios/durable-dts/dotnet-inproc" - } ], "settings": { "debug.internalConsoleOptions": "neverOpen", diff --git a/test/utils/createFunctionAppUtils.ts b/test/utils/createFunctionAppUtils.ts index 982dbfb66..0e555187f 100644 --- a/test/utils/createFunctionAppUtils.ts +++ b/test/utils/createFunctionAppUtils.ts @@ -92,6 +92,12 @@ export namespace createFunctionAppUtils { } } + export function getStorageConnectionDescription(storageConnection: ConnectionType): string { + return storageConnection === ConnectionType.Secrets ? + 'storage:secrets' : + 'storage:mi'; + } + function getRuntimePick(runtime: Runtime): RegExp | string { switch (runtime) { // case Runtime.Python: From 6036466c2e92c0c26141bbc71af0ffd4e41f4746 Mon Sep 17 00:00:00 2001 From: MicroFish91 <40250218+MicroFish91@users.noreply.github.com> Date: Tue, 20 Jan 2026 18:25:08 -0800 Subject: [PATCH 19/27] Clean up stray code --- src/commands/createFunction/createFunction.ts | 2 -- src/commands/createFunctionApp/createFunctionApp.ts | 2 -- 2 files changed, 4 deletions(-) diff --git a/src/commands/createFunction/createFunction.ts b/src/commands/createFunction/createFunction.ts index 86c3144f6..c1df27c43 100644 --- a/src/commands/createFunction/createFunction.ts +++ b/src/commands/createFunction/createFunction.ts @@ -90,6 +90,4 @@ export async function createFunctionInternal(context: IActionContext, options: a }); await wizard.prompt(); await wizard.execute(); - - console.log("test") } diff --git a/src/commands/createFunctionApp/createFunctionApp.ts b/src/commands/createFunctionApp/createFunctionApp.ts index cb03824da..342a6cfd7 100644 --- a/src/commands/createFunctionApp/createFunctionApp.ts +++ b/src/commands/createFunctionApp/createFunctionApp.ts @@ -42,8 +42,6 @@ export async function createFunctionApp(context: IActionContext & Partial Date: Wed, 21 Jan 2026 12:02:25 -0800 Subject: [PATCH 20/27] Add more comments --- test/nightly/scenarios/ScenariosTracker.ts | 3 ++- test/nightly/scenarios/parallelScenarios.ts | 7 +++---- test/nightly/scenarios/scenarios.test.ts | 5 +++++ .../testScenarios/AzExtFunctionsTestScenario.ts | 9 +++++++-- .../durable/azureStorage/azureStorageScenarios.ts | 2 ++ 5 files changed, 19 insertions(+), 7 deletions(-) diff --git a/test/nightly/scenarios/ScenariosTracker.ts b/test/nightly/scenarios/ScenariosTracker.ts index db2b1aa50..775d8ff07 100644 --- a/test/nightly/scenarios/ScenariosTracker.ts +++ b/test/nightly/scenarios/ScenariosTracker.ts @@ -186,7 +186,8 @@ export class ScenariosTracker { } } - console.log(lines.join('\n')); + const reportCard: string = lines.join('\n'); + console.log(reportCard); } } diff --git a/test/nightly/scenarios/parallelScenarios.ts b/test/nightly/scenarios/parallelScenarios.ts index 49aac97d6..312746362 100644 --- a/test/nightly/scenarios/parallelScenarios.ts +++ b/test/nightly/scenarios/parallelScenarios.ts @@ -144,12 +144,11 @@ async function startCreateAndDeployTest(scenarioLabel: string, rootFolder: Works return; } - let errorMessage: string = (error as Error).message ?? parseError(error).message; + const errorMessage: string = (error as Error).message ?? parseError(error).message; - // Deploy failed but verification still passed - warn instead of showing an outright fail - // There is at least one known issue that this checks for - TODO: Add issue link + // Warning marker indicates deployment failed but verification still passed. + // Example of one known issue that this checks for: https://github.com/microsoft/vscode-azurefunctions/issues/4859 if (deployError && !postTestError) { - errorMessage = '** Deploy-Fail; Verify-Pass ** ' + errorMessage; scenariosTracker.warnDeployFunctionApp(scenarioLabel, scenarioTestTrackerId, errorMessage); } else { scenariosTracker.failDeployFunctionApp(scenarioLabel, scenarioTestTrackerId, errorMessage); diff --git a/test/nightly/scenarios/scenarios.test.ts b/test/nightly/scenarios/scenarios.test.ts index 45fed5e6b..8649af798 100644 --- a/test/nightly/scenarios/scenarios.test.ts +++ b/test/nightly/scenarios/scenarios.test.ts @@ -16,6 +16,11 @@ suite.only('Scenarios', async function (this: Mocha.Suite) { this.skip(); } + // For quickly specifying tests to isolate: + // - Set `AzCode_ScenarioToIsolateAndExpand` env var to match the scenario title to isolate and run with the expanded test suite. TODO: Enable setting this for manual runs in the remote pipelines. + // - Set `only: true` on a scenario definition to more quickly isolate during local development/debugging purposes. Defaults to a test level of "core". + // + // When not running isolated tests, test level will always default to "core" to minimize the number of service requests. const onlyTestScenario = testScenarios.find(s => { if (process.env.AzCode_ScenarioToIsolateAndExpand) { return s.title === process.env.AzCode_ScenarioToIsolateAndExpand; diff --git a/test/nightly/scenarios/testScenarios/AzExtFunctionsTestScenario.ts b/test/nightly/scenarios/testScenarios/AzExtFunctionsTestScenario.ts index fe7715780..f957d7da4 100644 --- a/test/nightly/scenarios/testScenarios/AzExtFunctionsTestScenario.ts +++ b/test/nightly/scenarios/testScenarios/AzExtFunctionsTestScenario.ts @@ -61,14 +61,19 @@ export interface AzExtFunctionsTestScenario { createAndDeployTests: { /** * Core test cases representing high-value smoke/regression tests that run as part of the - * nightly test suite. Keep this set small and focused on critical paths that we expect to pass. + * nightly test suite. + * + * Note: Keep this set small (~1-2) and focused on representative samples that we expect to pass. */ core: CreateAndDeployTestCase[]; /** - * Expanded test cases providing broader, more exploratory coverage. Intended to be + * Expanded number of test cases providing much deeper inspection of the scenario deployment health. Intended to be * triggered manually for on-demand, deep-dive inspection rather than nightly runs * due to their more comprehensive nature. + * + * Note: Initially this may be a more exploratory set of tests whose outcome will eventually need to be confirmed by Function App PMs. + * After that determination, we should pare this down to only those tests that we expect to pass. */ expanded?: CreateAndDeployTestCase[]; }; diff --git a/test/nightly/scenarios/testScenarios/durable/azureStorage/azureStorageScenarios.ts b/test/nightly/scenarios/testScenarios/durable/azureStorage/azureStorageScenarios.ts index 602c113c7..cec6ee775 100644 --- a/test/nightly/scenarios/testScenarios/durable/azureStorage/azureStorageScenarios.ts +++ b/test/nightly/scenarios/testScenarios/durable/azureStorage/azureStorageScenarios.ts @@ -9,5 +9,7 @@ import { generateJSNodeScenario } from './azureStorageJSNodeScenario'; export function generateDurableAzureStorageScenarios(): AzExtFunctionsTestScenario[] { return [ generateJSNodeScenario(), + // generatePythonScenario(), + // generateDotNetIsolatedScenario(), ]; } From 002aa8210c7dae6c19d5d7cc47ecd29a203ef3c8 Mon Sep 17 00:00:00 2001 From: MicroFish91 <40250218+MicroFish91@users.noreply.github.com> Date: Wed, 21 Jan 2026 14:13:19 -0800 Subject: [PATCH 21/27] Reorganize constants --- test/constants.ts | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/test/constants.ts b/test/constants.ts index ebaa8e59c..2faa242b2 100644 --- a/test/constants.ts +++ b/test/constants.ts @@ -3,7 +3,7 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -// Start section create-new-project +// #region create-new-project // Project type picks export const durableOrchestratorPick: RegExp = /Durable Functions Orch/i; // Sometimes this is "orchestrator" or "orchestration" depending on the template feed export const durableAzureStoragePick: RegExp = /Azure Storage/i; @@ -17,8 +17,15 @@ export const jsLanguagePick: RegExp = /JavaScript/i; // Framework picks export const jsModelV4Pick: RegExp = /v4/i; -// Start section create-function-app -export const locationDefaultPick: RegExp = /West US 2/i; +// #endregion + +// ---------------------------- +// #region create-function-app // Default runtime picks export const nodeRuntimePick: RegExp = /Node\.js(\s)?22/i; + +// Location picks +export const locationDefaultPick: RegExp = /West US 2/i; + +// #endregion From 949efdec463ac16f244bd8091481088fdd871f57 Mon Sep 17 00:00:00 2001 From: MicroFish91 <40250218+MicroFish91@users.noreply.github.com> Date: Wed, 21 Jan 2026 14:31:29 -0800 Subject: [PATCH 22/27] Update constants --- test/constants.ts | 7 +++++ test/nightly/scenarios/parallelScenarios.ts | 20 ++++++------- test/utils/createFunctionAppUtils.ts | 32 ++++++++++----------- 3 files changed, 33 insertions(+), 26 deletions(-) diff --git a/test/constants.ts b/test/constants.ts index 2faa242b2..a383f0d7e 100644 --- a/test/constants.ts +++ b/test/constants.ts @@ -25,6 +25,13 @@ export const jsModelV4Pick: RegExp = /v4/i; // Default runtime picks export const nodeRuntimePick: RegExp = /Node\.js(\s)?22/i; +// Create resource picks +export const createNewResourceGroupPick: RegExp = /Create new resource group/i; +export const createNewStorageAccountPick: RegExp = /Create new storage account/i; +export const createNewAppInsightsPick: RegExp = /Create new application insights/i; +export const createNewAppServicePlanPick: RegExp = /Create new app service plan/i; +export const createNewUserAssignedIdentityPick: RegExp = /Create new user[- ]assigned identity/i; + // Location picks export const locationDefaultPick: RegExp = /West US 2/i; diff --git a/test/nightly/scenarios/parallelScenarios.ts b/test/nightly/scenarios/parallelScenarios.ts index 312746362..80d6dec0d 100644 --- a/test/nightly/scenarios/parallelScenarios.ts +++ b/test/nightly/scenarios/parallelScenarios.ts @@ -23,7 +23,7 @@ export interface AzExtFunctionsParallelTestScenario { title: string; /** - * A promise representing the running scenario. Holds the scenario execution that gets awaited. + * Holds a reference to the promise representing the running scenario. This is used so that all scenarios can be concurrently awaited. */ scenario?: Promise; @@ -89,14 +89,14 @@ function generateRunScenario(scenario: AzExtFunctionsTestScenario): AzExtFunctio } async function startCreateAndDeployTest(scenarioLabel: string, rootFolder: WorkspaceFolder, test: CreateAndDeployTestCase): Promise { - const scenarioTestTrackerId: number = scenariosTracker.initCreateAndDeployTest(scenarioLabel); + const testId: number = scenariosTracker.initCreateAndDeployTest(scenarioLabel); for (const rg of test.resourceGroupsToDelete ?? []) { resourceGroupsToDelete.add(rg); } // 3. Create function app - scenariosTracker.startCreateFunctionApp(scenarioLabel, scenarioTestTrackerId, test.createFunctionApp.label); + scenariosTracker.startCreateFunctionApp(scenarioLabel, testId, test.createFunctionApp.label); let functionAppId: string; await runWithTestActionContext('scenario.createFunctionApp', async context => { @@ -109,16 +109,16 @@ async function startCreateAndDeployTest(scenarioLabel: string, rootFolder: Works } assert.ok(functionAppId, 'Failed to create function app.'); - scenariosTracker.passCreateFunctionApp(scenarioLabel, scenarioTestTrackerId); + scenariosTracker.passCreateFunctionApp(scenarioLabel, testId); } catch (err) { - scenariosTracker.failCreateFunctionApp(scenarioLabel, scenarioTestTrackerId, (err as Error).message ?? parseError(err).message); + scenariosTracker.failCreateFunctionApp(scenarioLabel, testId, (err as Error).message ?? parseError(err).message); throw err; } }); }); // 4. Deploy function app - scenariosTracker.startDeployFunctionApp(scenarioLabel, scenarioTestTrackerId, test.deployFunctionApp.label); + scenariosTracker.startDeployFunctionApp(scenarioLabel, testId, test.deployFunctionApp.label); await runWithTestActionContext('scenario.deploy', async context => { await context.ui.runWithInputs(test.deployFunctionApp.inputs, async () => { @@ -140,18 +140,18 @@ async function startCreateAndDeployTest(scenarioLabel: string, rootFolder: Works const error = deployError ?? postTestError; if (!error) { - scenariosTracker.passDeployFunctionApp(scenarioLabel, scenarioTestTrackerId); + scenariosTracker.passDeployFunctionApp(scenarioLabel, testId); return; } const errorMessage: string = (error as Error).message ?? parseError(error).message; // Warning marker indicates deployment failed but verification still passed. - // Example of one known issue that this checks for: https://github.com/microsoft/vscode-azurefunctions/issues/4859 + // Example of a known issue where this happens: https://github.com/microsoft/vscode-azurefunctions/issues/4859 if (deployError && !postTestError) { - scenariosTracker.warnDeployFunctionApp(scenarioLabel, scenarioTestTrackerId, errorMessage); + scenariosTracker.warnDeployFunctionApp(scenarioLabel, testId, errorMessage); } else { - scenariosTracker.failDeployFunctionApp(scenarioLabel, scenarioTestTrackerId, errorMessage); + scenariosTracker.failDeployFunctionApp(scenarioLabel, testId, errorMessage); } throw error; diff --git a/test/utils/createFunctionAppUtils.ts b/test/utils/createFunctionAppUtils.ts index 0e555187f..aac8e093a 100644 --- a/test/utils/createFunctionAppUtils.ts +++ b/test/utils/createFunctionAppUtils.ts @@ -3,7 +3,7 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { locationDefaultPick, nodeRuntimePick } from "../constants"; +import { createNewAppInsightsPick, createNewAppServicePlanPick, createNewResourceGroupPick, createNewStorageAccountPick, createNewUserAssignedIdentityPick, locationDefaultPick, nodeRuntimePick } from "../constants"; export namespace createFunctionAppUtils { export function generateBasicCreateInputs(appName: string, folderName: string, runtime: Runtime, storageConnection: ConnectionType): (string | RegExp)[] { @@ -27,12 +27,12 @@ export namespace createFunctionAppUtils { getRuntimePick(runtime), '2048', '100', - /Create new resource group/i, + createNewResourceGroupPick, appName, ...getConnectionTypeInputs(storageConnection), - /Create new storage account/i, + createNewStorageAccountPick, appName, - /Create new application insights/i, + createNewAppInsightsPick, appName, ]; case PlanType.Premium: @@ -43,15 +43,15 @@ export namespace createFunctionAppUtils { locationDefaultPick, getRuntimePick(runtime), ...(os ? [new RegExp(os, 'i')] : []), - /Create new app service plan/i, + createNewAppServicePlanPick, appName, /EP1/i, - /Create new resource group/i, + createNewResourceGroupPick, appName, ...getConnectionTypeInputs(storageConnection), - /Create new storage account/i, + createNewStorageAccountPick, appName, - /Create new application insights/i, + createNewAppInsightsPick, appName, ]; case PlanType.LegacyConsumption: @@ -62,12 +62,12 @@ export namespace createFunctionAppUtils { locationDefaultPick, getRuntimePick(runtime), ...(os ? [new RegExp(os, 'i')] : []), - /Create new resource group/i, + createNewResourceGroupPick, appName, ...getConnectionTypeInputs(storageConnection), - /Create new storage account/i, + createNewStorageAccountPick, appName, - /Create new application insights/i, + createNewAppInsightsPick, appName, ]; case PlanType.AppService: @@ -78,15 +78,15 @@ export namespace createFunctionAppUtils { locationDefaultPick, getRuntimePick(runtime), ...(os ? [new RegExp(os, 'i')] : []), - /Create new app service plan/i, + createNewAppServicePlanPick, appName, /S1/i, - /Create new resource group/i, + createNewResourceGroupPick, appName, ...getConnectionTypeInputs(storageConnection), - /Create new storage account/i, + createNewStorageAccountPick, appName, - /Create new application insights/i, + createNewAppInsightsPick, appName, ]; } @@ -112,7 +112,7 @@ export namespace createFunctionAppUtils { function getConnectionTypeInputs(connection: ConnectionType): (string | RegExp)[] { return connection === ConnectionType.ManagedIdentity ? - [new RegExp(connection, 'i'), /Create new user[- ]assigned identity/i] : + [new RegExp(connection, 'i'), createNewUserAssignedIdentityPick] : [new RegExp(connection, 'i')]; } } From 760423934f55bbc34e6e3ac5f71a02d0ff66348b Mon Sep 17 00:00:00 2001 From: MicroFish91 <40250218+MicroFish91@users.noreply.github.com> Date: Wed, 21 Jan 2026 16:50:42 -0800 Subject: [PATCH 23/27] Rename some properties and environment variable --- .vscode/launch.json | 2 +- test/nightly/scenarios/ScenariosTracker.ts | 105 +++++++----------- test/nightly/scenarios/parallelScenarios.ts | 8 +- test/nightly/scenarios/scenarios.test.ts | 14 +-- .../AzExtFunctionsTestScenario.ts | 12 +- .../azureStorageJSNodeScenario.ts | 5 +- 6 files changed, 62 insertions(+), 84 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index d2f94d337..998b9540c 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -77,7 +77,7 @@ "FUNC_PATH": "func", "AZFUNC_UPDATE_BACKUP_TEMPLATES": "", "AzCode_EnableLongRunningTestsLocal": "true", - "AzCode_ScenarioToIsolateAndExpand": "" + "AzCode_RunScenarioExtended": "durable-azurestorage-jsnode" } }, { diff --git a/test/nightly/scenarios/ScenariosTracker.ts b/test/nightly/scenarios/ScenariosTracker.ts index 775d8ff07..02fe74e42 100644 --- a/test/nightly/scenarios/ScenariosTracker.ts +++ b/test/nightly/scenarios/ScenariosTracker.ts @@ -8,6 +8,23 @@ import { nonNullProp, nonNullValue } from "@microsoft/vscode-azext-utils"; export class ScenariosTracker { private scenarioStatuses: Map = new Map(); + private getScenarioStatus(scenarioLabel: string): ScenarioStatus { + return nonNullValue(this.scenarioStatuses.get(scenarioLabel)); + } + + private getCreateAndDeployTest(scenarioLabel: string, id: number): CreateAndDeployTest { + const scenario = this.getScenarioStatus(scenarioLabel); + scenario.createAndDeployTests ??= []; + return nonNullValue(scenario.createAndDeployTests[id]); + } + + private updateTestStatus(testResult: TestResult, status: TestStatus, error?: string): void { + testResult.status = status; + if (error) { + testResult.error = error; + } + } + initScenario(scenarioLabel: string): void { if (this.scenarioStatuses.has(scenarioLabel)) { return; @@ -16,28 +33,24 @@ export class ScenariosTracker { } startCreateNewProject(scenarioLabel: string, createNewProjectLabel: string): void { - const scenarioStatus = nonNullValue(this.scenarioStatuses.get(scenarioLabel)); + const scenarioStatus = this.getScenarioStatus(scenarioLabel); scenarioStatus.createNewProject = { label: createNewProjectLabel }; - this.scenarioStatuses.set(scenarioLabel, scenarioStatus); } passCreateNewProject(scenarioLabel: string): void { - const scenarioStatus = nonNullValue(this.scenarioStatuses.get(scenarioLabel)); + const scenarioStatus = this.getScenarioStatus(scenarioLabel); const createNewProjectStatus = nonNullProp(scenarioStatus, 'createNewProject'); - createNewProjectStatus.status = 'pass'; - this.scenarioStatuses.set(scenarioLabel, scenarioStatus); + this.updateTestStatus(createNewProjectStatus, 'pass'); } failCreateNewProject(scenarioLabel: string, error: string): void { - const scenarioStatus = nonNullValue(this.scenarioStatuses.get(scenarioLabel)); + const scenarioStatus = this.getScenarioStatus(scenarioLabel); const createNewProjectStatus = nonNullProp(scenarioStatus, 'createNewProject'); - createNewProjectStatus.status = 'fail'; - createNewProjectStatus.error = error; - this.scenarioStatuses.set(scenarioLabel, scenarioStatus); + this.updateTestStatus(createNewProjectStatus, 'fail', error); } initCreateAndDeployTest(scenarioLabel: string): number { - const scenarioStatus = nonNullValue(this.scenarioStatuses.get(scenarioLabel)); + const scenarioStatus = this.getScenarioStatus(scenarioLabel); scenarioStatus.createAndDeployTests ??= []; const id: number = scenarioStatus.createAndDeployTests.length; @@ -46,81 +59,43 @@ export class ScenariosTracker { } startCreateFunctionApp(scenarioLabel: string, createAndDeployTestId: number, createLabel: string): void { - const scenarioStatus = nonNullValue(this.scenarioStatuses.get(scenarioLabel)); - scenarioStatus.createAndDeployTests ??= []; - - const createAndDeployTest = nonNullValue(scenarioStatus.createAndDeployTests[createAndDeployTestId]); + const createAndDeployTest = this.getCreateAndDeployTest(scenarioLabel, createAndDeployTestId); createAndDeployTest.createFunctionApp = { label: createLabel }; - - this.scenarioStatuses.set(scenarioLabel, scenarioStatus); } passCreateFunctionApp(scenarioLabel: string, createAndDeployTestId: number): void { - const scenarioStatus = nonNullValue(this.scenarioStatuses.get(scenarioLabel)); - scenarioStatus.createAndDeployTests ??= []; - - const createAndDeployTest = nonNullValue(scenarioStatus.createAndDeployTests[createAndDeployTestId]); + const createAndDeployTest = this.getCreateAndDeployTest(scenarioLabel, createAndDeployTestId); const createFunctionAppTest = nonNullValue(createAndDeployTest.createFunctionApp); - createFunctionAppTest.status = 'pass'; - - this.scenarioStatuses.set(scenarioLabel, scenarioStatus); + this.updateTestStatus(createFunctionAppTest, 'pass'); } failCreateFunctionApp(scenarioLabel: string, createAndDeployTestId: number, error: string): void { - const scenarioStatus = nonNullValue(this.scenarioStatuses.get(scenarioLabel)); - scenarioStatus.createAndDeployTests ??= []; - - const createAndDeployTest = nonNullValue(scenarioStatus.createAndDeployTests[createAndDeployTestId]); + const createAndDeployTest = this.getCreateAndDeployTest(scenarioLabel, createAndDeployTestId); const createFunctionAppTest = nonNullValue(createAndDeployTest.createFunctionApp); - createFunctionAppTest.status = 'fail'; - createFunctionAppTest.error = error; - - this.scenarioStatuses.set(scenarioLabel, scenarioStatus); + this.updateTestStatus(createFunctionAppTest, 'fail', error); } startDeployFunctionApp(scenarioLabel: string, createAndDeployTestId: number, deployLabel: string): void { - const scenarioStatus = nonNullValue(this.scenarioStatuses.get(scenarioLabel)); - scenarioStatus.createAndDeployTests ??= []; - - const createAndDeployTest = nonNullValue(scenarioStatus.createAndDeployTests[createAndDeployTestId]); + const createAndDeployTest = this.getCreateAndDeployTest(scenarioLabel, createAndDeployTestId); createAndDeployTest.deployFunctionApp = { label: deployLabel }; - - this.scenarioStatuses.set(scenarioLabel, scenarioStatus); } passDeployFunctionApp(scenarioLabel: string, createAndDeployTestId: number): void { - const scenarioStatus = nonNullValue(this.scenarioStatuses.get(scenarioLabel)); - scenarioStatus.createAndDeployTests ??= []; - - const createAndDeployTest = nonNullValue(scenarioStatus.createAndDeployTests[createAndDeployTestId]); + const createAndDeployTest = this.getCreateAndDeployTest(scenarioLabel, createAndDeployTestId); const deployFunctionAppTest = nonNullValue(createAndDeployTest.deployFunctionApp); - deployFunctionAppTest.status = 'pass'; - - this.scenarioStatuses.set(scenarioLabel, scenarioStatus); + this.updateTestStatus(deployFunctionAppTest, 'pass'); } warnDeployFunctionApp(scenarioLabel: string, createAndDeployTestId: number, error?: string): void { - const scenarioStatus = nonNullValue(this.scenarioStatuses.get(scenarioLabel)); - scenarioStatus.createAndDeployTests ??= []; - - const createAndDeployTest = nonNullValue(scenarioStatus.createAndDeployTests[createAndDeployTestId]); + const createAndDeployTest = this.getCreateAndDeployTest(scenarioLabel, createAndDeployTestId); const deployFunctionAppTest = nonNullValue(createAndDeployTest.deployFunctionApp); - deployFunctionAppTest.status = 'warn'; - deployFunctionAppTest.error = error; - - this.scenarioStatuses.set(scenarioLabel, scenarioStatus); + this.updateTestStatus(deployFunctionAppTest, 'warn', error); } failDeployFunctionApp(scenarioLabel: string, createAndDeployTestId: number, error: string): void { - const scenarioStatus = nonNullValue(this.scenarioStatuses.get(scenarioLabel)); - scenarioStatus.createAndDeployTests ??= []; - - const createAndDeployTest = nonNullValue(scenarioStatus.createAndDeployTests[createAndDeployTestId]); + const createAndDeployTest = this.getCreateAndDeployTest(scenarioLabel, createAndDeployTestId); const deployFunctionAppTest = nonNullValue(createAndDeployTest.deployFunctionApp); - deployFunctionAppTest.status = 'fail'; - deployFunctionAppTest.error = error; - - this.scenarioStatuses.set(scenarioLabel, scenarioStatus); + this.updateTestStatus(deployFunctionAppTest, 'fail', error); } report(): void { @@ -202,8 +177,10 @@ type TestResult = { type ScenarioStatus = { label: string; createNewProject?: TestResult; - createAndDeployTests?: { - createFunctionApp?: TestResult; - deployFunctionApp?: TestResult; - }[]; + createAndDeployTests?: CreateAndDeployTest[]; +} + +type CreateAndDeployTest = { + createFunctionApp?: TestResult; + deployFunctionApp?: TestResult; }; diff --git a/test/nightly/scenarios/parallelScenarios.ts b/test/nightly/scenarios/parallelScenarios.ts index 80d6dec0d..592c3f2d8 100644 --- a/test/nightly/scenarios/parallelScenarios.ts +++ b/test/nightly/scenarios/parallelScenarios.ts @@ -29,7 +29,7 @@ export interface AzExtFunctionsParallelTestScenario { /** * Starts the concurrent test scenario based on the specified test level. - * @param testLevel - Specifies which level of tests to run: 'core' vs. 'extended' + * @param testLevel - Specifies which level of tests to run: 'basic' vs. 'extended' */ runScenario(testLevel: TestLevel): Promise; @@ -74,7 +74,7 @@ function generateRunScenario(scenario: AzExtFunctionsTestScenario): AzExtFunctio }); // 2. Start all create and deploy tests for the scenario - const createAndDeployTests: CreateAndDeployTestCase[] = testLevel === TestLevel.Core ? scenario.createAndDeployTests.core : scenario.createAndDeployTests.expanded ?? []; + const createAndDeployTests: CreateAndDeployTestCase[] = testLevel === TestLevel.Basic ? scenario.createAndDeployTests.basic : scenario.createAndDeployTests.extended ?? []; const onlyTestCase: CreateAndDeployTestCase | undefined = createAndDeployTests.find(test => test.only); if (onlyTestCase) { @@ -179,6 +179,6 @@ export function getWorkspaceFolderUri(folderName: string): Uri { } export enum TestLevel { - Core = 'core', - Expanded = 'expanded', + Basic = 'basic', + Extended = 'extended', } diff --git a/test/nightly/scenarios/scenarios.test.ts b/test/nightly/scenarios/scenarios.test.ts index 8649af798..970b3eaeb 100644 --- a/test/nightly/scenarios/scenarios.test.ts +++ b/test/nightly/scenarios/scenarios.test.ts @@ -17,23 +17,23 @@ suite.only('Scenarios', async function (this: Mocha.Suite) { } // For quickly specifying tests to isolate: - // - Set `AzCode_ScenarioToIsolateAndExpand` env var to match the scenario title to isolate and run with the expanded test suite. TODO: Enable setting this for manual runs in the remote pipelines. - // - Set `only: true` on a scenario definition to more quickly isolate during local development/debugging purposes. Defaults to a test level of "core". + // - Set the `AzCode_RunScenarioExtended` env var to match the scenario label if you want to isolate and run with the extended test suite. TODO: Enable setting this for manual runs in the remote pipelines. + // - Set `only: true` on a scenario definition to more quickly isolate during local development. Note this naturally defaults to a test level of "basic". // - // When not running isolated tests, test level will always default to "core" to minimize the number of service requests. + // When not running isolated tests, test level will always default to "basic" to minimize the number of service requests. const onlyTestScenario = testScenarios.find(s => { - if (process.env.AzCode_ScenarioToIsolateAndExpand) { - return s.title === process.env.AzCode_ScenarioToIsolateAndExpand; + if (process.env.AzCode_RunScenarioExtended) { + return s.title === process.env.AzCode_RunScenarioExtended; } return s.only; }); - const testLevel: TestLevel = process.env.AzCode_ScenarioToIsolateAndExpand ? TestLevel.Expanded : TestLevel.Core; + const testLevel: TestLevel = process.env.AzCode_RunScenarioExtended ? TestLevel.Extended : TestLevel.Basic; if (onlyTestScenario) { onlyTestScenario.scenario = onlyTestScenario.runScenario(testLevel); } else { for (const s of testScenarios) { - s.scenario = s.runScenario(TestLevel.Core); + s.scenario = s.runScenario(TestLevel.Basic); } } }); diff --git a/test/nightly/scenarios/testScenarios/AzExtFunctionsTestScenario.ts b/test/nightly/scenarios/testScenarios/AzExtFunctionsTestScenario.ts index f957d7da4..47546e2cb 100644 --- a/test/nightly/scenarios/testScenarios/AzExtFunctionsTestScenario.ts +++ b/test/nightly/scenarios/testScenarios/AzExtFunctionsTestScenario.ts @@ -60,22 +60,22 @@ export interface AzExtFunctionsTestScenario { */ createAndDeployTests: { /** - * Core test cases representing high-value smoke/regression tests that run as part of the + * Basic test cases representing high-value smoke/regression tests that run as part of the * nightly test suite. * * Note: Keep this set small (~1-2) and focused on representative samples that we expect to pass. */ - core: CreateAndDeployTestCase[]; + basic: CreateAndDeployTestCase[]; /** - * Expanded number of test cases providing much deeper inspection of the scenario deployment health. Intended to be + * Extended number of test cases providing much deeper inspection of the scenario deployment health. Intended to be * triggered manually for on-demand, deep-dive inspection rather than nightly runs * due to their more comprehensive nature. * - * Note: Initially this may be a more exploratory set of tests whose outcome will eventually need to be confirmed by Function App PMs. - * After that determination, we should pare this down to only those tests that we expect to pass. + * Note: Initially this may be a more exploratory set of tests whose outcome will eventually need to be more explicitly confirmed by Function App PMs. + * Upon final determination, we should pare this down to only those tests that we expect to pass. */ - expanded?: CreateAndDeployTestCase[]; + extended?: CreateAndDeployTestCase[]; }; /** diff --git a/test/nightly/scenarios/testScenarios/durable/azureStorage/azureStorageJSNodeScenario.ts b/test/nightly/scenarios/testScenarios/durable/azureStorage/azureStorageJSNodeScenario.ts index 911c2896b..ef9cd3226 100644 --- a/test/nightly/scenarios/testScenarios/durable/azureStorage/azureStorageJSNodeScenario.ts +++ b/test/nightly/scenarios/testScenarios/durable/azureStorage/azureStorageJSNodeScenario.ts @@ -26,12 +26,13 @@ export function generateJSNodeScenario(): AzExtFunctionsTestScenario { ], }, createAndDeployTests: { - core: [ + basic: [ generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.LegacyConsumption, OperatingSystem.Linux), generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.AppService, OperatingSystem.Windows), // generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.Premium, OperatingSystem.Linux), ], - expanded: [ + extended: [ + // Todo: Needs discussion & final approval generateCreateAndDeployTest(folderName, CreateMode.Basic, Runtime.Node, ConnectionType.ManagedIdentity, PlanType.FlexConsumption, OperatingSystem.Linux, DurableBackend.Storage), generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.ManagedIdentity, PlanType.Premium, OperatingSystem.Linux, DurableBackend.Storage), generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.ManagedIdentity, PlanType.Premium, OperatingSystem.Windows, DurableBackend.Storage), From b21d39a1dc3e98d257dfd2c13bd7a5551ce2914d Mon Sep 17 00:00:00 2001 From: MicroFish91 <40250218+MicroFish91@users.noreply.github.com> Date: Wed, 21 Jan 2026 17:02:43 -0800 Subject: [PATCH 24/27] Update basic tests --- .../durable/azureStorage/azureStorageJSNodeScenario.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/nightly/scenarios/testScenarios/durable/azureStorage/azureStorageJSNodeScenario.ts b/test/nightly/scenarios/testScenarios/durable/azureStorage/azureStorageJSNodeScenario.ts index ef9cd3226..a46a9fc17 100644 --- a/test/nightly/scenarios/testScenarios/durable/azureStorage/azureStorageJSNodeScenario.ts +++ b/test/nightly/scenarios/testScenarios/durable/azureStorage/azureStorageJSNodeScenario.ts @@ -29,7 +29,7 @@ export function generateJSNodeScenario(): AzExtFunctionsTestScenario { basic: [ generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.LegacyConsumption, OperatingSystem.Linux), generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.AppService, OperatingSystem.Windows), - // generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.Premium, OperatingSystem.Linux), + generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.Premium, OperatingSystem.Linux), ], extended: [ // Todo: Needs discussion & final approval From 6f7a76c116eb3850a74775d1c0956e9839126746 Mon Sep 17 00:00:00 2001 From: MicroFish91 <40250218+MicroFish91@users.noreply.github.com> Date: Wed, 21 Jan 2026 17:10:10 -0800 Subject: [PATCH 25/27] Update a comment --- test/nightly/scenarios/parallelScenarios.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/nightly/scenarios/parallelScenarios.ts b/test/nightly/scenarios/parallelScenarios.ts index 592c3f2d8..0b05807e8 100644 --- a/test/nightly/scenarios/parallelScenarios.ts +++ b/test/nightly/scenarios/parallelScenarios.ts @@ -14,7 +14,7 @@ import { type AzExtFunctionsTestScenario, type CreateAndDeployTestCase } from '. import { generateTestScenarios } from './testScenarios/testScenarios'; /** - * A wrapper for {@link AzExtFunctionsTestScenario} which holds a scenario that has been prepared for concurrent test execution. + * A wrapper for {@link AzExtFunctionsTestScenario}. Prepares a scenario for concurrent test execution. */ export interface AzExtFunctionsParallelTestScenario { /** From 22939af109d31735b69e73efc61ad7d7cf8ee284 Mon Sep 17 00:00:00 2001 From: MicroFish91 <40250218+MicroFish91@users.noreply.github.com> Date: Mon, 26 Jan 2026 16:12:28 -0800 Subject: [PATCH 26/27] Add createNewProject verification post test --- extension.bundle.ts | 2 ++ .../DurableProjectConfigureStep.ts | 4 ++-- test/nightly/scenarios/parallelScenarios.ts | 1 + test/nightly/scenarios/scenarios.test.ts | 8 +++---- .../AzExtFunctionsTestScenario.ts | 6 ++++++ .../azureStorageJSNodeScenario.ts | 21 +++++++++++++++++-- 6 files changed, 34 insertions(+), 8 deletions(-) diff --git a/extension.bundle.ts b/extension.bundle.ts index 8c4c823f2..64d812dd3 100644 --- a/extension.bundle.ts +++ b/extension.bundle.ts @@ -30,9 +30,11 @@ export * from './src/commands/initProjectForVSCode/initProjectForVSCode'; export * from './src/constants'; export * from './src/utils/durableUtils'; // Export activate/deactivate for main.js +export * from './src/commands/createFunction/durableSteps/DurableProjectConfigureStep'; export { activateInternal, deactivateInternal } from './src/extension'; export * from './src/extensionVariables'; export * from './src/funcConfig/function'; +export * from './src/funcConfig/host'; export * from './src/funcCoreTools/hasMinFuncCliVersion'; export * from './src/FuncVersion'; export * from './src/templates/CentralTemplateProvider'; diff --git a/src/commands/createFunction/durableSteps/DurableProjectConfigureStep.ts b/src/commands/createFunction/durableSteps/DurableProjectConfigureStep.ts index a85de9010..750e8a848 100644 --- a/src/commands/createFunction/durableSteps/DurableProjectConfigureStep.ts +++ b/src/commands/createFunction/durableSteps/DurableProjectConfigureStep.ts @@ -69,7 +69,7 @@ export class DurableProjectConfigureStep exten switch (context.newDurableStorageType) { case DurableBackend.Storage: - hostJson.extensions.durableTask = this.getDefaultStorageTaskConfig(); + hostJson.extensions.durableTask = DurableProjectConfigureStep.getDefaultStorageTaskConfig(); // Omit setting azureWebJobsStorage since it should already be initialized during 'createNewProject' break; case DurableBackend.Netherite: @@ -100,7 +100,7 @@ export class DurableProjectConfigureStep exten await AzExtFsExtra.writeJSON(hostJsonPath, hostJson); } - private getDefaultStorageTaskConfig(): IStorageTaskJson { + static getDefaultStorageTaskConfig(): IStorageTaskJson { return { storageProvider: { type: DurableBackend.Storage, diff --git a/test/nightly/scenarios/parallelScenarios.ts b/test/nightly/scenarios/parallelScenarios.ts index 0b05807e8..1300c957a 100644 --- a/test/nightly/scenarios/parallelScenarios.ts +++ b/test/nightly/scenarios/parallelScenarios.ts @@ -65,6 +65,7 @@ function generateRunScenario(scenario: AzExtFunctionsTestScenario): AzExtFunctio await context.ui.runWithInputs(scenario.createNewProjectTest.inputs, async () => { try { await createNewProjectInternal(context, { folderPath: rootFolder.uri.fsPath }); + await scenario.createNewProjectTest.postTest?.(context); scenariosTracker.passCreateNewProject(scenario.label); } catch (err) { scenariosTracker.failCreateNewProject(scenario.label, (err as Error).message ?? parseError(err).message); diff --git a/test/nightly/scenarios/scenarios.test.ts b/test/nightly/scenarios/scenarios.test.ts index 970b3eaeb..ce0f5a427 100644 --- a/test/nightly/scenarios/scenarios.test.ts +++ b/test/nightly/scenarios/scenarios.test.ts @@ -9,7 +9,7 @@ import { generateParallelScenarios, TestLevel, type AzExtFunctionsParallelTestSc const testScenarios: AzExtFunctionsParallelTestScenario[] = generateParallelScenarios(); suite.only('Scenarios', async function (this: Mocha.Suite) { - this.timeout(45 * 60 * 1000); + this.timeout(40 * 60 * 1000); suiteSetup(async function (this: Mocha.Context) { if (!longRunningTestsEnabled) { @@ -17,10 +17,10 @@ suite.only('Scenarios', async function (this: Mocha.Suite) { } // For quickly specifying tests to isolate: - // - Set the `AzCode_RunScenarioExtended` env var to match the scenario label if you want to isolate and run with the extended test suite. TODO: Enable setting this for manual runs in the remote pipelines. - // - Set `only: true` on a scenario definition to more quickly isolate during local development. Note this naturally defaults to a test level of "basic". + // - Set the `AzCode_RunScenarioExtended` env var to match the scenario label you want to isolate. This will automatically run with the extended test suite. TODO: Enable setting this for manual runs in the remote pipelines. + // - Setting `only: true` on a scenario definition also allows more quickly isolating a run during local development. Note this naturally defaults to a test level of "basic". // - // When not running isolated tests, test level will always default to "basic" to minimize the number of service requests. + // When not running isolated tests, test level will always default to "basic" to minimize the number of service requests to avoid running into 429 ratelimit errors. const onlyTestScenario = testScenarios.find(s => { if (process.env.AzCode_RunScenarioExtended) { return s.title === process.env.AzCode_RunScenarioExtended; diff --git a/test/nightly/scenarios/testScenarios/AzExtFunctionsTestScenario.ts b/test/nightly/scenarios/testScenarios/AzExtFunctionsTestScenario.ts index 47546e2cb..12de7a9f7 100644 --- a/test/nightly/scenarios/testScenarios/AzExtFunctionsTestScenario.ts +++ b/test/nightly/scenarios/testScenarios/AzExtFunctionsTestScenario.ts @@ -95,6 +95,12 @@ export interface CreateNewProjectTestCase { */ inputs: (string | RegExp)[]; + /** + * An optional callback to run after the deployment test completes. + * Great for use as a post test verification step. + */ + postTest?: (context: IActionContext) => void | Promise; + /** * Indicates this test case should be executed exclusively. This should only be used to aid with local development. */ diff --git a/test/nightly/scenarios/testScenarios/durable/azureStorage/azureStorageJSNodeScenario.ts b/test/nightly/scenarios/testScenarios/durable/azureStorage/azureStorageJSNodeScenario.ts index a46a9fc17..6ad8c3b47 100644 --- a/test/nightly/scenarios/testScenarios/durable/azureStorage/azureStorageJSNodeScenario.ts +++ b/test/nightly/scenarios/testScenarios/durable/azureStorage/azureStorageJSNodeScenario.ts @@ -3,7 +3,10 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { DurableBackend } from "../../../../../../extension.bundle"; +import { AzExtFsExtra } from "@microsoft/vscode-azext-utils"; +import * as assert from "assert"; +import * as path from "path"; +import { DurableBackend, DurableProjectConfigureStep, hostFileName, type IActionContext, type IHostJsonV2 } from "../../../../../../extension.bundle"; import { durableAzureStoragePick, durableOrchestratorName, durableOrchestratorPick, jsLanguagePick, jsModelV4Pick } from "../../../../../constants"; import { ConnectionType, CreateMode, OperatingSystem, PlanType, Runtime } from "../../../../../utils/createFunctionAppUtils"; import { type AzExtFunctionsTestScenario } from "../../AzExtFunctionsTestScenario"; @@ -24,6 +27,7 @@ export function generateJSNodeScenario(): AzExtFunctionsTestScenario { durableAzureStoragePick, durableOrchestratorName, ], + postTest: verifyCreateNewProject, }, createAndDeployTests: { basic: [ @@ -32,7 +36,8 @@ export function generateJSNodeScenario(): AzExtFunctionsTestScenario { generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.Secrets, PlanType.Premium, OperatingSystem.Linux), ], extended: [ - // Todo: Needs discussion & final approval + // Placeholder... + // Todo: Will need future discussion & final approval generateCreateAndDeployTest(folderName, CreateMode.Basic, Runtime.Node, ConnectionType.ManagedIdentity, PlanType.FlexConsumption, OperatingSystem.Linux, DurableBackend.Storage), generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.ManagedIdentity, PlanType.Premium, OperatingSystem.Linux, DurableBackend.Storage), generateCreateAndDeployTest(folderName, CreateMode.Advanced, Runtime.Node, ConnectionType.ManagedIdentity, PlanType.Premium, OperatingSystem.Windows, DurableBackend.Storage), @@ -51,3 +56,15 @@ export function generateJSNodeScenario(): AzExtFunctionsTestScenario { } }; } + +async function verifyCreateNewProject(context: IActionContext & { projectPath?: string }): Promise { + if (!context.projectPath) { + throw new Error('Internal Error: Test context is missing the requisite project path.'); + } + + const hostJsonPath: string = path.join(context.projectPath, hostFileName); + const hostJson: IHostJsonV2 = await AzExtFsExtra.readJSON(hostJsonPath) as IHostJsonV2; + hostJson.extensions ??= {}; + + assert.deepStrictEqual(hostJson.extensions.durableTask, DurableProjectConfigureStep.getDefaultStorageTaskConfig(), ''); +} From 124c790b825c5c13dea5a953a0dde67b687d9231 Mon Sep 17 00:00:00 2001 From: MicroFish91 <40250218+MicroFish91@users.noreply.github.com> Date: Mon, 26 Jan 2026 18:14:12 -0800 Subject: [PATCH 27/27] Update storage connection description --- .../testScenarios/durable/generateCreateAndDeployTest.ts | 2 +- test/utils/createFunctionAppUtils.ts | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/test/nightly/scenarios/testScenarios/durable/generateCreateAndDeployTest.ts b/test/nightly/scenarios/testScenarios/durable/generateCreateAndDeployTest.ts index 59f517966..6cdc3b08a 100644 --- a/test/nightly/scenarios/testScenarios/durable/generateCreateAndDeployTest.ts +++ b/test/nightly/scenarios/testScenarios/durable/generateCreateAndDeployTest.ts @@ -16,7 +16,7 @@ import { type CreateAndDeployTestCase } from "../AzExtFunctionsTestScenario"; export function generateCreateAndDeployTest(folderName: string, createMode: CreateMode, runtime: Runtime, storageConnection: ConnectionType, plan: PlanType, os?: OperatingSystem, storageType?: DurableBackend): CreateAndDeployTestCase { const appName: string = getRandomAlphanumericString(); const osDescription: string = os ? `-${os}` : ''; - const description: string = `${createMode}-${createFunctionAppUtils.getStorageConnectionDescription(storageConnection)}${osDescription}-${plan}`; + const description: string = `${createMode}-${storageConnection}${osDescription}-${plan}`; return { createFunctionApp: { diff --git a/test/utils/createFunctionAppUtils.ts b/test/utils/createFunctionAppUtils.ts index aac8e093a..a4306f44c 100644 --- a/test/utils/createFunctionAppUtils.ts +++ b/test/utils/createFunctionAppUtils.ts @@ -92,12 +92,6 @@ export namespace createFunctionAppUtils { } } - export function getStorageConnectionDescription(storageConnection: ConnectionType): string { - return storageConnection === ConnectionType.Secrets ? - 'storage:secrets' : - 'storage:mi'; - } - function getRuntimePick(runtime: Runtime): RegExp | string { switch (runtime) { // case Runtime.Python: