microsoft · shrja-ms · Jun 10, 2026 · Jun 4, 2026 · Jun 9, 2026 · Jun 9, 2026
@@ -0,0 +1,3 @@
+changes:
+  - section: "Bug Fixes"
+    description: "Azure Backup: Replace InvalidOperationException with ArgumentException/KeyNotFoundException for user-input validation errors so the telemetry classifier correctly categorizes them as Customer errors instead of MCP Tool Bugs"
@@ -1,25 +1,28 @@
 ---
 name: azurebackup-telemetry-report
-description: 'Generate weekly telemetry reports for Azure Backup MCP tools. Runs KQL queries against the Kusto telemetry cluster, analyzes error patterns with 3-way classification (Customer/Azure Service/MCP Tool Bug), compares week-over-week metrics, correlates with merged PRs and releases, and produces an Outlook-compatible HTML report. USE WHEN: weekly telemetry report, Azure Backup MCP telemetry, error analysis, telemetry bugs, weekly report, MCP tool success rate, backup telemetry, error classification.'
+description: 'Generate weekly telemetry reports and customer adoption reports for Azure Backup MCP tools. Runs KQL queries against the Kusto telemetry cluster, analyzes error patterns with 3-way classification (Customer/Azure Service/MCP Tool Bug), identifies customers via P360/C360 cross-cluster joins, compares week-over-week metrics, correlates with merged PRs and releases, and produces an Outlook-compatible HTML report. USE WHEN: weekly telemetry report, Azure Backup MCP telemetry, error analysis, telemetry bugs, weekly report, MCP tool success rate, backup telemetry, error classification, customer usage report, who is using Azure Backup MCP, backup adoption, customer report.'
 argument-hint: 'Generate the Azure Backup MCP weekly telemetry report'
 ---
 
-# Azure Backup MCP — Weekly Telemetry Report Generator
+# Azure Backup MCP — Weekly Telemetry & Customer Report Generator
 
 ## Purpose
 
-Generate a comprehensive weekly telemetry report for the Azure Backup MCP toolset by:
+Generate comprehensive telemetry reports for the Azure Backup MCP toolset by:
 1. Querying live production telemetry from Kusto
 2. Classifying errors into Customer / Azure Service / MCP Tool Bug
-3. Correlating with merged PRs and releases
-4. Producing an Outlook-compatible HTML report
+3. Identifying customers via P360/C360 cross-cluster joins (using `P360_CustomerName`)
+4. Correlating with merged PRs and releases
+5. Producing an Outlook-compatible HTML report
 
 ## When to Use
 
 - Weekly cadence (every Monday or Sunday) to produce the status report
 - After a release to assess error rate impact
 - When triaging production bugs from telemetry data
 - When preparing stakeholder updates on Azure Backup MCP health
+- **Customer adoption reports** for any time range (e.g., last 15 days, 30 days)
+- When asked "who is using Azure Backup MCP?" or "generate customer report"
 
 ## Prerequisites
 
@@ -152,6 +155,14 @@ For the Outlook version, apply these rules:
 
 ## Error Classification Decision Tree
 
+> **Customer Report Queries:** For customer identification, tool adoption, client distribution,
+> and version analysis, refer to [`kql-customer-queries.md`](https://github.com/microsoft/mcp/blob/main/tools/Azure.Mcp.Tools.AzureBackup/skills/azurebackup-telemetry-report/references/kql-customer-queries.md)
+> (queries 14–24). These queries use cross-cluster joins to `mabprod1` (P360) and `icmdataro` (C360)
+> for customer name resolution and external/internal classification.
+>
+> **Key rule:** Always use `P360_CustomerName` (not `CustomerName`) from the P360 table.
+> The `CustomerName` field contains generic tenant names (e.g., "Denis" for Prometeia SpA).
+
 > **Important:** MCP bug exception types must be checked **before** StatusCode.
 > The `HandleException` base class maps `ArgumentNullException` (inherits `ArgumentException`)
 > to HTTP 400, which would incorrectly classify MCP bugs as "Customer" errors if StatusCode

@@ -0,0 +1,177 @@
+# Azure Backup MCP — Customer & Adoption KQL Queries
+
+All queries use the `getAzureMcpEvents_ToolCalls` function from the Azure MCP telemetry cluster.
+
+- **Cluster URI:** `https://ddazureclients.kusto.windows.net`
+- **Database:** `AzureDevExp`
+- **Cross-cluster:** `mabprod1.kusto.windows.net` / `AzureBackup` and `icmdataro.centralus` / `Customer`
+
+Replace `{DAYS}` with the desired time range (e.g., `15` for 15 days).
+
+> **Critical:** Always use `P360_CustomerName` instead of `CustomerName` in P360. The `CustomerName`
+> field contains generic tenant names (e.g., "Denis" for all Prometeia SpA subs). `P360_CustomerName`
+> has the actual company name.
+
+---
+
+## 14. Overall Summary
+
+```kql
+getAzureMcpEvents_ToolCalls(ago({DAYS}d), now())
+| extend ToolName = tostring(customDimensions.toolname)
+| where ToolName contains "azurebackup"
+| summarize
+    TotalCalls = count(),
+    SuccessCount = countif(success == true),
+    FailedCount = countif(success == false),
+    DistinctUsers = dcount(tostring(customDimensions.devdeviceid)),
+    CallsWithSubTag = countif(isnotempty(tostring(customDimensions.azsubscriptionguid)))
+```
+
+## 15. Subscription Breakdown
+
+```kql
+getAzureMcpEvents_ToolCalls(ago({DAYS}d), now())
+| extend ToolName = tostring(customDimensions.toolname),
+         SubId = tostring(customDimensions.azsubscriptionguid),
+         DeviceId = tostring(customDimensions.devdeviceid)
+| where ToolName contains "azurebackup"
+| where isnotempty(SubId)
+| summarize Calls = count(), Users = dcount(DeviceId), ToolsUsed = dcount(ToolName)
+    by SubId
+| order by Calls desc
+```
+
+## 16. Customer Name Resolution (P360 — use P360_CustomerName)
+
+```kql
+// Replace <subscription_ids> with dynamic array of sub IDs from query 15
+cluster('mabprod1.kusto.windows.net').database('AzureBackup').P360CustomerSubscriptions
+| where SubscriptionId in (<subscription_ids>)
+| distinct P360_CustomerName, CustomerName, SubscriptionName, SubscriptionId, UsageType, OfferType
+```
+
+## 17. External vs Internal Classification (C360)
+
+```kql
+// External customers only
+cluster('icmdataro.centralus').database('Customer').C360_CustomerSubscriptions
+| where SubscriptionId in (<subscription_ids>)
+| where UsageType != "Internal"
+| where OfferType != "Internal"
+| distinct CustomerName, SubscriptionId, UsageType, OfferType
+```
+
+```kql
+// Internal (Microsoft) customers
+cluster('icmdataro.centralus').database('Customer').C360_CustomerSubscriptions
+| where SubscriptionId in (<subscription_ids>)
+| where UsageType == "Internal" or OfferType == "Internal"
+| distinct CustomerName, SubscriptionId, UsageType, OfferType
+```
+
+## 18. Non-GUID Subscription Name Resolution
+
+If any `azsubscriptionguid` values are not GUIDs (e.g., "ERM-BPER"), resolve via SubscriptionName:
+
+```kql
+cluster('mabprod1.kusto.windows.net').database('AzureBackup').P360CustomerSubscriptions
+| where SubscriptionName == "<sub_name_value>"
+| distinct P360_CustomerName, SubscriptionName, SubscriptionId
+```
+
+## 19. Tool Usage Ranking
+
+```kql
+getAzureMcpEvents_ToolCalls(ago({DAYS}d), now())
+| extend ToolName = tostring(customDimensions.toolname),
+         DeviceId = tostring(customDimensions.devdeviceid)
+| where ToolName contains "azurebackup"
+| summarize Calls = count(), Succeeded = countif(success == true),
+            Failed = countif(success == false), Users = dcount(DeviceId)
+    by ToolName
+| extend SuccessRate = round(100.0 * Succeeded / Calls, 1)
+| order by Calls desc
+```
+
+## 20. Per-Customer Tool Breakdown
+
+```kql
+// Replace <customer_subs> with dynamic array of subscription IDs for a specific customer
+getAzureMcpEvents_ToolCalls(ago({DAYS}d), now())
+| extend ToolName = tostring(customDimensions.toolname),
+         SubId = tostring(customDimensions.azsubscriptionguid)
+| where ToolName contains "azurebackup"
+| where SubId in (<customer_subs>) or SubId == "<sub_name>"
+| summarize Calls = count() by ToolName
+| order by Calls desc
+```
+
+## 21. Client / Editor Distribution
+
+```kql
+getAzureMcpEvents_ToolCalls(ago({DAYS}d), now())
+| extend ToolName = tostring(customDimensions.toolname),
+         ClientName = tostring(customDimensions.clientname)
+| where ToolName contains "azurebackup"
+| summarize Calls = count(), Users = dcount(tostring(customDimensions.devdeviceid))
+    by ClientName
+| order by Calls desc
+```
+
+## 22. MCP Server Version Distribution
+
+```kql
+getAzureMcpEvents_ToolCalls(ago({DAYS}d), now())
+| extend ToolName = tostring(customDimensions.toolname),
+         Version = tostring(customDimensions.version)
+| where ToolName contains "azurebackup"
+| summarize Calls = count(), Users = dcount(tostring(customDimensions.devdeviceid))
+    by Version
+| order by Calls desc
+```
+
+## 23. Daily Trend (for timechart dashboards)
+
+```kql
+getAzureMcpEvents_ToolCalls(ago({DAYS}d), now())
+| extend ToolName = tostring(customDimensions.toolname)
+| where ToolName contains "azurebackup"
+| summarize Total = count(), Errors = countif(success == false),
+    Users = dcount(tostring(customDimensions.devdeviceid))
+    by bin(timestamp, 1d)
+| extend FailureRate = round(100.0 * Errors / Total, 2)
+| order by timestamp asc
+```
+
+## 24. Version by Client Matrix
+
+```kql
+getAzureMcpEvents_ToolCalls(ago({DAYS}d), now())
+| extend ToolName = tostring(customDimensions.toolname),
+         Version = tostring(customDimensions.version),
+         ClientName = tostring(customDimensions.clientname)
+| where ToolName contains "azurebackup"
+| summarize Calls = count(), Users = dcount(tostring(customDimensions.devdeviceid))
+    by Version, ClientName
+| order by Calls desc
+```
+
+---
+
+## Customer Aggregation Rules
+
+When building the final customer table from queries 15–18:
+
+1. **Group by `P360_CustomerName`**, not by subscription ID — one row per customer
+2. **Merge multi-sub customers**: e.g., Prometeia SpA may have 15+ subscriptions
+3. **Merge non-GUID entries**: If a non-GUID value resolves to an existing customer's subscription via query 18, merge it
+4. **Categorize**: External (C360 UsageType=External) vs Internal (UsageType=Internal)
+5. **~74% of calls lack `azsubscriptionguid`** (pre-beta.14 clients) — note this in the report
+
+## Known Data Quirks
+
+- `azsubscriptionguid` only emitted since beta.14 (~May 28, 2026)
+- `P360.CustomerName` is unreliable (e.g., "Denis" = Prometeia SpA) — always use `P360_CustomerName`
+- Non-GUID subscription values occur when clients pass subscription name instead of GUID
+- Namespace-mode proxy tools (e.g., `get_azure_backup_details_azurebackup_vault_get`) have `toolarea=get_azure_backup_details` but `contains 'azurebackup'` catches them correctly
@@ -95,16 +95,12 @@ public override async Task<CommandResponse> ExecuteAsync(CommandContext context,
         RequestFailedException reqEx when reqEx.Status == (int)HttpStatusCode.Forbidden =>
             $"Authorization failed updating the policy. Details: {reqEx.Message}",
         RequestFailedException reqEx => reqEx.Message,
-        InvalidOperationException ioEx when ioEx.Message.Contains("DPP", StringComparison.OrdinalIgnoreCase) =>
-            "Update is only supported for RSV (Recovery Services vault) policies. DPP policies do not support update.",
-        InvalidOperationException ioEx => ioEx.Message,
         _ => base.GetErrorMessage(ex)
     };
 
     protected override HttpStatusCode GetStatusCode(Exception ex) => ex switch
     {
         ArgumentException => HttpStatusCode.BadRequest,
-        InvalidOperationException => HttpStatusCode.BadRequest,
         RequestFailedException reqEx => (HttpStatusCode)reqEx.Status,
         _ => base.GetStatusCode(ex)
     };

@@ -353,7 +353,7 @@ public async Task<OperationResult> UpdatePolicyAsync(
         var resolved = await ResolveVaultTypeAsync(vaultName, resourceGroup, subscription, vaultType, tenant, retryPolicy, cancellationToken);
         if (!VaultTypeResolver.IsRsv(resolved))
         {
-            throw new InvalidOperationException("Update is only supported for RSV (Recovery Services vault) policies. DPP policies do not support update.");
+            throw new ArgumentException("Update is only supported for RSV (Recovery Services vault) policies. DPP policies do not support update.");
         }
 
         return await rsvOps.UpdatePolicyAsync(vaultName, resourceGroup, subscription, policyName, scheduleTime, dailyRetentionDays, tenant, retryPolicy, cancellationToken);

@@ -520,9 +520,13 @@ public async Task<BackupJobInfo> GetJobAsync(
             // The Azure SDK may throw FormatException when parsing the job's duration field
             // (e.g., non-standard ISO 8601 durations from the service). Fall back to listing
             // all jobs and matching by ID to work around this SDK limitation.
+            // Note: ListJobsAsync may return a partial list if it also hits FormatException
+            // during enumeration — so a null result does NOT mean the job is missing; it may
+            // exist beyond the point where the enumerator broke. Re-throw FormatException
+            // (not KeyNotFoundException) to preserve SDK-parse-failure semantics.
             var jobs = await ListJobsAsync(vaultName, resourceGroup, subscription, tenant, retryPolicy, cancellationToken);
             return jobs.FirstOrDefault(j => j.Name == jobId)
-                ?? throw new InvalidOperationException($"Job '{jobId}' not found. The SDK cannot parse this job's duration field.");
+                ?? throw new FormatException($"Job '{jobId}' exists but the Azure SDK cannot parse its duration field (XmlConvert.ToTimeSpan limitation). This is tracked in azure-sdk-for-net#59306.");
         }
     }
 

@@ -663,7 +663,7 @@ public async Task<OperationResult> UpdatePolicyAsync(
         var existingPolicy = await policyCollection.GetAsync(policyName, cancellationToken);
         var policyData = existingPolicy.Value.Data;
         var policyProperties = policyData.Properties as BackupGenericProtectionPolicy
-            ?? throw new InvalidOperationException($"Policy '{policyName}' has an unsupported properties type.");
+            ?? throw new ArgumentException($"Policy '{policyName}' has an unsupported properties type.", nameof(policyName));
 
         DateTimeOffset? newScheduleTime = null;
         if (!string.IsNullOrWhiteSpace(scheduleTime))
@@ -793,18 +793,18 @@ private static void UpdatePolicyScheduleAndRetention(BackupGenericProtectionPoli
                 break;
 
             default:
-                throw new InvalidOperationException($"Unsupported policy type '{policyProperties.GetType().Name}'. Only IaasVM, VmWorkload (SQL/HANA), and FileShare policies are supported for update.");
+                throw new ArgumentException($"Unsupported policy type '{policyProperties.GetType().Name}'. Only IaasVM, VmWorkload (SQL/HANA), and FileShare policies are supported for update.");
         }
 
         if (!scheduleApplied)
         {
-            throw new InvalidOperationException(
+            throw new ArgumentException(
                 $"Schedule update could not be applied. Policy uses '{policyProperties.GetType().Name}' with a schedule type that is not supported for update. Only SimpleSchedulePolicy is supported.");
         }
 
         if (!retentionApplied)
         {
-            throw new InvalidOperationException(
+            throw new ArgumentException(
                 $"Retention update could not be applied. Policy uses '{policyProperties.GetType().Name}' with a retention type that is not supported for update. Only LongTermRetentionPolicy with a daily schedule is supported.");
         }
     }
@@ -1407,7 +1407,7 @@ public async Task<OperationResult> UndeleteProtectedItemAsync(
             }
             else if (exactMatches.Count > 1)
             {
-                throw new InvalidOperationException(
+                throw new ArgumentException(
                     $"Multiple protected items found with datasource ID '{datasourceId}' in vault '{vaultName}'. " +
                     "Provide --container to disambiguate.");
             }
@@ -1417,7 +1417,7 @@ public async Task<OperationResult> UndeleteProtectedItemAsync(
             }
             else if (prefixMatches.Count > 1)
             {
-                throw new InvalidOperationException(
+                throw new ArgumentException(
                     $"Multiple protected items match datasource ID '{datasourceId}' in vault '{vaultName}' " +
                     "(shared storage account prefix). Provide a more specific datasource ID or --container to disambiguate.");
             }
@@ -1447,7 +1447,7 @@ public async Task<OperationResult> UndeleteProtectedItemAsync(
         // the full protection definition (PolicyId, ContainerName, etc.).
         if (matchedItemData.Properties is not BackupGenericProtectedItem existingProperties)
         {
-            throw new InvalidOperationException(
+            throw new ArgumentException(
                 "The matched protected item does not contain properties required to perform undelete.");
         }
 

@@ -735,7 +735,7 @@ public async Task PolicyUpdate_DppVault_ReturnsNotSupportedError()
     {
         var vaultName = $"{Settings.ResourceBaseName}-dpp";
 
-        // DPP vaults do not support policy update — should return an error response with type InvalidOperationException
+        // DPP vaults do not support policy update — should return an error response with type ArgumentException
         var result = await CallToolAsync(
             "azurebackup_policy_update",
             new()
@@ -749,7 +749,7 @@ public async Task PolicyUpdate_DppVault_ReturnsNotSupportedError()
 
         Assert.NotNull(result);
         var errorType = result.Value.AssertProperty("type");
-        Assert.Equal("InvalidOperationException", errorType.GetString());
+        Assert.Equal("ArgumentException", errorType.GetString());
     }
 
     #endregion

@@ -209,7 +209,7 @@ public async Task ExecuteAsync_HandlesDppNotSupportedError()
             Arg.Is("v"), Arg.Is("rg"), Arg.Is("sub"), Arg.Is("p"),
             Arg.Any<string?>(), Arg.Any<string?>(), Arg.Any<string?>(),
             Arg.Any<string?>(), Arg.Any<RetryPolicyOptions?>(), Arg.Any<CancellationToken>())
-            .ThrowsAsync(new InvalidOperationException("Update is only supported for RSV (Recovery Services vault) policies. DPP policies do not support update."));
+            .ThrowsAsync(new ArgumentException("Update is only supported for RSV (Recovery Services vault) policies. DPP policies do not support update."));
 
         // Act
         var response = await ExecuteCommandAsync(
@@ -224,14 +224,14 @@ public async Task ExecuteAsync_HandlesDppNotSupportedError()
     }
 
     [Fact]
-    public async Task ExecuteAsync_HandlesNonDppInvalidOperationException()
+    public async Task ExecuteAsync_HandlesUnsupportedPolicyTypeError()
     {
-        // Arrange — a non-DPP InvalidOperationException should surface its own message
+        // Arrange — an unsupported policy type is an input error, returns ArgumentException
         Service.UpdatePolicyAsync(
             Arg.Is("v"), Arg.Is("rg"), Arg.Is("sub"), Arg.Is("p"),
             Arg.Any<string?>(), Arg.Any<string?>(), Arg.Any<string?>(),
             Arg.Any<string?>(), Arg.Any<RetryPolicyOptions?>(), Arg.Any<CancellationToken>())
-            .ThrowsAsync(new InvalidOperationException("Unsupported policy type 'SomePolicy'."));
+            .ThrowsAsync(new ArgumentException("Unsupported policy type 'SomePolicy'."));
 
         // Act
         var response = await ExecuteCommandAsync(