Introduce new scenarios testing framework

.vscode/launch.json

@@ -72,11 +72,12 @@
             "env": {
                 "MOCHA_grep": "", // RegExp of tests to run (empty for all)
                 "MOCHA_timeout": "0", // Disable time-outs
-                "DEBUGTELEMETRY": "v",
+                "DEBUGTELEMETRY": "",
                 "NODE_DEBUG": "",
                 "FUNC_PATH": "func",
                 "AZFUNC_UPDATE_BACKUP_TEMPLATES": "",
-                "AzCode_EnableLongRunningTestsLocal": "",
+                "AzCode_EnableLongRunningTestsLocal": "true",
+                "AzCode_RunScenarioExtended": "durable-azurestorage-jsnode"
             }
         },
         {

extension.bundle.ts

@@ -30,9 +30,11 @@ export * from './src/commands/initProjectForVSCode/initProjectForVSCode';
 export * from './src/constants';
 export * from './src/utils/durableUtils';
 // Export activate/deactivate for main.js
+export * from './src/commands/createFunction/durableSteps/DurableProjectConfigureStep';
 export { activateInternal, deactivateInternal } from './src/extension';
 export * from './src/extensionVariables';
 export * from './src/funcConfig/function';
+export * from './src/funcConfig/host';
 export * from './src/funcCoreTools/hasMinFuncCliVersion';
 export * from './src/FuncVersion';
 export * from './src/templates/CentralTemplateProvider';

src/commands/appSettings/localSettings/getLocalSettingsFile.ts

@@ -15,7 +15,7 @@ import { tryGetFunctionProjectRoot } from '../../createNewProject/verifyIsProjec
  * Otherwise, prompt
  */
 export async function getLocalSettingsFile(context: IActionContext, message: string, workspaceFolder?: vscode.WorkspaceFolder): Promise<string> {
-    workspaceFolder ||= await getRootWorkspaceFolder();
+    workspaceFolder ||= await getRootWorkspaceFolder(context);
     if (workspaceFolder) {
         const projectPath: string | undefined = await tryGetFunctionProjectRoot(context, workspaceFolder);
         if (projectPath) {

src/commands/createFunction/createFunction.ts

@@ -54,7 +54,7 @@ export async function createFunctionInternal(context: IActionContext, options: a
     let workspacePath: string | undefined = options.folderPath;
 
     if (workspacePath === undefined) {
-        workspaceFolder = await getRootWorkspaceFolder();
+        workspaceFolder = await getRootWorkspaceFolder(context);
         workspacePath = workspaceFolder?.uri.fsPath;
     } else {
         workspaceFolder = getContainingWorkspace(workspacePath);

src/commands/createFunction/durableSteps/DurableProjectConfigureStep.ts

@@ -69,7 +69,7 @@ export class DurableProjectConfigureStep<T extends IFunctionWizardContext> exten
 
         switch (context.newDurableStorageType) {
             case DurableBackend.Storage:
-                hostJson.extensions.durableTask = this.getDefaultStorageTaskConfig();
+                hostJson.extensions.durableTask = DurableProjectConfigureStep.getDefaultStorageTaskConfig();
                 // Omit setting azureWebJobsStorage since it should already be initialized during 'createNewProject'
                 break;
             case DurableBackend.Netherite:
@@ -100,7 +100,7 @@ export class DurableProjectConfigureStep<T extends IFunctionWizardContext> exten
         await AzExtFsExtra.writeJSON(hostJsonPath, hostJson);
     }
 
-    private getDefaultStorageTaskConfig(): IStorageTaskJson {
+    static getDefaultStorageTaskConfig(): IStorageTaskJson {
         return {
             storageProvider: {
                 type: DurableBackend.Storage,

src/commands/createFunctionApp/containerImage/detectDockerfile.ts

@@ -17,7 +17,7 @@ export async function detectDockerfile(context: ICreateFunctionAppContext): Prom
         return undefined;
     }
 
-    context.workspaceFolder ??= await getRootWorkspaceFolder() as WorkspaceFolder;
+    context.workspaceFolder ??= await getRootWorkspaceFolder(context) as WorkspaceFolder;
     context.rootPath ??= await tryGetFunctionProjectRoot(context, context.workspaceFolder, 'prompt') ?? context.workspaceFolder.uri.fsPath;
 
     const pattern: RelativePattern = new RelativePattern(context.rootPath, `**/${dockerfileGlobPattern}`);

src/commands/createFunctionApp/createFunctionApp.ts

@@ -3,7 +3,7 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
-import { type AzExtParentTreeItem, type IActionContext } from '@microsoft/vscode-azext-utils';
+import { nonNullValueAndProp, type AzExtParentTreeItem, type IActionContext } from '@microsoft/vscode-azext-utils';
 import { ext } from '../../extensionVariables';
 import { localize } from '../../localize';
 import { type SlotTreeItem } from '../../tree/SlotTreeItem';
@@ -38,14 +38,13 @@ export async function createFunctionApp(context: IActionContext & Partial<ICreat
     context.newResourceGroupName = newResourceGroupName;
     if (!isMultiRootWorkspace()) {
         // only set the workspace if we're not doing in a multiroot project
-        context.workspaceFolder = await getRootWorkspaceFolder();
+        context.workspaceFolder = await getRootWorkspaceFolder(context);
     }
 
     const funcAppNode: SlotTreeItem | ContainerTreeItem = await SubscriptionTreeItem.createChild(context as ICreateFunctionAppContext, node as SubscriptionTreeItem);
-
-    return funcAppNode.fullId;
+    return nonNullValueAndProp(funcAppNode.site, 'id');
 }
 
-export async function createFunctionAppAdvanced(context: IActionContext, subscription?: AzExtParentTreeItem | string, nodesOrNewResourceGroupName?: string | (string | AzExtParentTreeItem)[]): Promise<string | undefined> {
+export async function createFunctionAppAdvanced(context: IActionContext, subscription?: AzExtParentTreeItem | string, nodesOrNewResourceGroupName?: string | (string | AzExtParentTreeItem)[]): Promise<string> {
     return await createFunctionApp({ ...context, advancedCreation: true }, subscription, nodesOrNewResourceGroupName);
 }

src/commands/deploy/deploy.ts

@@ -52,8 +52,8 @@ export async function deployProductionSlot(context: IActionContext, target?: vsc
     await deploy(context, target, undefined);
 }
 
-export async function deployProductionSlotByFunctionAppId(context: IActionContext, functionAppId?: string | {}): Promise<void> {
-    await deploy(context, undefined, functionAppId);
+export async function deployProductionSlotByFunctionAppId(context: IActionContext, functionAppId?: string | {}, target?: vscode.Uri | string | SlotTreeItem): Promise<void> {
+    await deploy(context, target, functionAppId);
 }
 
 export async function deploySlot(context: IActionContext, target?: vscode.Uri | string | SlotTreeItem, functionAppId?: string | {}): Promise<void> {

src/utils/workspace.ts

@@ -20,14 +20,14 @@ export function isMultiRootWorkspace(): boolean {
  * Use sparingly. Prefer storing and passing 'projectPaths' instead.
  * Over-reliance on this function may result in excessive prompting when a user employs a multi-root workspace.
  */
-export async function getRootWorkspaceFolder(): Promise<vscode.WorkspaceFolder | undefined> {
+export async function getRootWorkspaceFolder(context: IActionContext): Promise<vscode.WorkspaceFolder | undefined> {
     if (!vscode.workspace.workspaceFolders || vscode.workspace.workspaceFolders.length === 0) {
         return undefined;
     } else if (vscode.workspace.workspaceFolders.length === 1) {
         return vscode.workspace.workspaceFolders[0];
     } else {
         const placeHolder: string = localize('selectRootWorkspace', 'Select the folder containing your function project');
-        const folder = await vscode.window.showWorkspaceFolderPick({ placeHolder });
+        const folder = await context.ui.showWorkspaceFolderPick({ placeHolder });
         if (!folder) {
             throw new UserCancelledError('selectRootWorkspace');
         }

test/constants.ts

@@ -0,0 +1,38 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+// #region create-new-project
+// Project type picks
+export const durableOrchestratorPick: RegExp = /Durable Functions Orch/i; // Sometimes this is "orchestrator" or "orchestration" depending on the template feed
+export const durableAzureStoragePick: RegExp = /Azure Storage/i;
+
+// Default names
+export const durableOrchestratorName: string = 'durableHello';
+
+// Language picks
+export const jsLanguagePick: RegExp = /JavaScript/i;
+
+// Framework picks
+export const jsModelV4Pick: RegExp = /v4/i;
+
+// #endregion
+
+// ----------------------------
+
+// #region create-function-app
+// Default runtime picks
+export const nodeRuntimePick: RegExp = /Node\.js(\s)?22/i;
+
+// Create resource picks
+export const createNewResourceGroupPick: RegExp = /Create new resource group/i;
+export const createNewStorageAccountPick: RegExp = /Create new storage account/i;
+export const createNewAppInsightsPick: RegExp = /Create new application insights/i;
+export const createNewAppServicePlanPick: RegExp = /Create new app service plan/i;
+export const createNewUserAssignedIdentityPick: RegExp = /Create new user[- ]assigned identity/i;
+
+// Location picks
+export const locationDefaultPick: RegExp = /West US 2/i;
+
+// #endregion

test/global.test.ts

@@ -3,9 +3,10 @@
  *  Licensed under the MIT License. See LICENSE.md in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
+import { registerAppServiceExtensionVariables } from '@microsoft/vscode-azext-azureappservice';
+import { registerAzureUtilsExtensionVariables } from '@microsoft/vscode-azext-azureutils';
 import { createTestActionContext, runWithTestActionContext, TestOutputChannel, TestUserInput } from '@microsoft/vscode-azext-dev';
-import { AzExtFsExtra } from '@microsoft/vscode-azext-utils';
-import * as assert from 'assert';
+import { AzExtFsExtra, registerUIExtensionVariables } from '@microsoft/vscode-azext-utils';
 import * as os from 'os';
 import * as path from 'path';
 import * as vscode from 'vscode';
@@ -56,6 +57,10 @@ suiteSetup(async function (this: Mocha.Context): Promise<void> {
 
     ext.outputChannel = new TestOutputChannel();
 
+    registerAppServiceExtensionVariables(ext);
+    registerAzureUtilsExtensionVariables(ext);
+    registerUIExtensionVariables(ext);
+
     registerOnActionStartHandler(context => {
         // Use `TestUserInput` by default so we get an error if an unexpected call to `context.ui` occurs, rather than timing out
         context.ui = new TestUserInput(vscode);
@@ -170,8 +175,8 @@ async function initTestWorkspaceFolders(): Promise<string[]> {
         const folders: string[] = [];
         for (let i = 0; i < workspaceFolders.length; i++) {
             const workspacePath: string = workspaceFolders[i].uri.fsPath;
-            const folderName = path.basename(workspacePath);
-            assert.equal(folderName, String(i), `Unexpected workspace folder name "${folderName}".`);
+            // const folderName = path.basename(workspacePath);
+            // assert.equal(folderName, String(i), `Unexpected workspace folder name "${folderName}".`);
             await AzExtFsExtra.ensureDir(workspacePath);
             await AzExtFsExtra.emptyDir(workspacePath);
             folders.push(workspacePath);

test/nightly/createProjectAndDeploy.test.ts

@@ -90,7 +90,7 @@ async function testCreateProjectAndDeploy(options: ICreateProjectAndDeployOption
     // TODO: investigate why our SDK calls are throwing errors when app name is over ~12 characters
     // https://github.com/microsoft/vscode-azurefunctions/issues/4368
     const appName: string = 'f' + getRandomAlphanumericString();
-    resourceGroupsToDelete.push(appName);
+    resourceGroupsToDelete.add(appName);
     await runWithTestActionContext('deploy', async context => {
         options.deployInputs = options.deployInputs || [];
         await context.ui.runWithInputs([testWorkspacePath, /create new function app/i, appName, getRotatingLocation(), ...options.deployInputs], async () => {

test/nightly/functionAppOperations.test.ts

@@ -33,7 +33,7 @@ suite('Function App Operations', function (this: Mocha.Suite): void {
         appName = getRandomHexString();
         app2Name = getRandomHexString();
         rgName = getRandomHexString();
-        resourceGroupsToDelete.push(rgName);
+        resourceGroupsToDelete.add(rgName);
         saName = getRandomHexString().toLowerCase(); // storage account must have lower case name
         aiName = getRandomHexString();
         location = getRotatingLocation();

test/nightly/global.nightly.test.ts

@@ -6,23 +6,27 @@
 import { WebSiteManagementClient } from '@azure/arm-appservice';
 import { ResourceManagementClient } from '@azure/arm-resources';
 import { createTestActionContext } from '@microsoft/vscode-azext-dev';
-import { AzureAccountTreeItemWithProjects, createAzureClient, ext } from '../../extension.bundle';
+import { AzureAccountTreeItemWithProjects, createAzureClient, ext, updateGlobalSetting } from '../../extension.bundle';
 import { longRunningTestsEnabled } from '../global.test';
 
 import { createSubscriptionContext, subscriptionExperience, type ISubscriptionContext } from '@microsoft/vscode-azext-utils';
 import { type AzureSubscription } from '@microsoft/vscode-azureresources-api';
 import * as vscode from 'vscode';
+import { ScenariosTracker } from './scenarios/ScenariosTracker';
 
 export let testClient: WebSiteManagementClient;
 export let subscriptionContext: ISubscriptionContext;
-export const resourceGroupsToDelete: string[] = [];
+export const resourceGroupsToDelete: Set<string> = new Set();
+export const scenariosTracker = new ScenariosTracker();
 
 // Runs before all nightly tests
 suiteSetup(async function (this: Mocha.Context): Promise<void> {
     if (longRunningTestsEnabled) {
         this.timeout(2 * 60 * 1000);
 
         await vscode.commands.executeCommand('azureResourceGroups.logIn');
+        await updateGlobalSetting('groupBy', 'resourceType', 'azureResourceGroups');
+
         ext.azureAccountTreeItem = new AzureAccountTreeItemWithProjects();
         const testContext = await createTestActionContext();
         const subscription: AzureSubscription = await subscriptionExperience(testContext, ext.rgApi.appResourceTree);
@@ -35,6 +39,7 @@ suiteSetup(async function (this: Mocha.Context): Promise<void> {
 suiteTeardown(async function (this: Mocha.Context): Promise<void> {
     if (longRunningTestsEnabled) {
         this.timeout(10 * 60 * 1000);
+        scenariosTracker.report();
 
         await deleteResourceGroups();
         ext.azureAccountTreeItem.dispose();
@@ -43,7 +48,7 @@ suiteTeardown(async function (this: Mocha.Context): Promise<void> {
 
 async function deleteResourceGroups(): Promise<void> {
     const rgClient: ResourceManagementClient = createAzureClient([await createTestActionContext(), subscriptionContext], ResourceManagementClient);
-    await Promise.all(resourceGroupsToDelete.map(async resourceGroup => {
+    await Promise.allSettled(Array.from(resourceGroupsToDelete).map(async resourceGroup => {
         if ((await rgClient.resourceGroups.checkExistence(resourceGroup)).body) {
             console.log(`Started delete of resource group "${resourceGroup}"...`);
             await rgClient.resourceGroups.beginDeleteAndWait(resourceGroup);