Telemetry improvements for tracking experimental feature optout (PowerShell#18762)

Author: JamesWTruher

Diff for: src/System.Management.Automation/engine/ExperimentalFeature/ExperimentalFeature.cs

@@ -157,6 +157,20 @@ static ExperimentalFeature()
             EnabledExperimentalFeatureNames = ProcessEnabledFeatures(enabledFeatures);
         }
 
+        /// <summary>
+        /// We need to notify which features were not enabled.
+        /// </summary>
+        private static void SendTelemetryForDeactivatedFeatures(ReadOnlyBag<string> enabledFeatures)
+        {
+            foreach (var feature in EngineExperimentalFeatures)
+            {
+                if (!enabledFeatures.Contains(feature.Name))
+                {
+                    ApplicationInsightsTelemetry.SendTelemetryMetric(TelemetryType.ExperimentalEngineFeatureDeactivation, feature.Name);
+                }
+            }
+        }
+
         /// <summary>
         /// Process the array of enabled feature names retrieved from configuration.
         /// Ignore invalid feature names and unavailable engine feature names, and
@@ -198,7 +212,9 @@ private static ReadOnlyBag<string> ProcessEnabledFeatures(string[] enabledFeatur
                 }
             }
 
-            return new ReadOnlyBag<string>(new HashSet<string>(list, StringComparer.OrdinalIgnoreCase));
+            ReadOnlyBag<string> features = new(new HashSet<string>(list, StringComparer.OrdinalIgnoreCase));
+            SendTelemetryForDeactivatedFeatures(features);
+            return features;
         }
 
         /// <summary>

Diff for: src/System.Management.Automation/engine/NativeCommandProcessor.cs

@@ -18,6 +18,7 @@
 using System.Text;
 using System.Threading;
 using System.Xml;
+using Microsoft.PowerShell.Telemetry;
 using Dbg = System.Management.Automation.Diagnostics;
 
 namespace System.Management.Automation
@@ -872,8 +873,36 @@ internal override void Complete()
 
                     this.commandRuntime.PipelineProcessor.ExecutionFailed = true;
 
-                    if (!ExperimentalFeature.IsEnabled(ExperimentalFeature.PSNativeCommandErrorActionPreferenceFeatureName)
-                        || !Command.Context.GetBooleanPreference(SpecialVariables.PSNativeCommandUseErrorActionPreferenceVarPath, defaultPref: false, out _))
+                    // Feature is not enabled, so return
+                    if (!ExperimentalFeature.IsEnabled(ExperimentalFeature.PSNativeCommandErrorActionPreferenceFeatureName))
+                    {
+                        return;
+                    }
+
+                    // We send telemetry information only if the feature is enabled.
+                    // This shouldn't be done once, because it's a run-time check we should send telemetry every time.
+                    // Report on the following conditions:
+                    // - The variable is not present
+                    // - The value is not set (variable is null)
+                    // - The value is set to true or false
+                    bool useDefaultSetting;
+                    bool nativeErrorActionPreferenceSetting = Command.Context.GetBooleanPreference(
+                        SpecialVariables.PSNativeCommandUseErrorActionPreferenceVarPath,
+                        defaultPref: false,
+                        out useDefaultSetting);
+
+                    // The variable is unset
+                    if (useDefaultSetting)
+                    {
+                        ApplicationInsightsTelemetry.SendExperimentalUseData(ExperimentalFeature.PSNativeCommandErrorActionPreferenceFeatureName, "unset");
+                        return;
+                    }
+
+                    // Send the value that was set.
+                    ApplicationInsightsTelemetry.SendExperimentalUseData(ExperimentalFeature.PSNativeCommandErrorActionPreferenceFeatureName, nativeErrorActionPreferenceSetting.ToString());
+
+                    // if it was explicitly set to false, return
+                    if (!nativeErrorActionPreferenceSetting)
                     {
                         return;
                     }

Diff for: src/System.Management.Automation/utils/Telemetry.cs

@@ -38,12 +38,31 @@ internal enum TelemetryType
         /// </summary>
         WinCompatModuleLoad,
 
+        /// <summary>
+        /// Send telemetry for experimental module feature deactivation.
+        /// All experimental engine features will be have telemetry.
+        /// </summary>
+        ExperimentalEngineFeatureDeactivation,
+
         /// <summary>
         /// Send telemetry for experimental module feature activation.
         /// All experimental engine features will be have telemetry.
         /// </summary>
         ExperimentalEngineFeatureActivation,
 
+        /// <summary>
+        /// Send telemetry for an experimental feature when use.
+        /// </summary>
+        ExperimentalFeatureUse,
+
+        /// <summary>
+        /// Send telemetry for experimental module feature deactivation.
+        /// Experimental module features will send telemetry based on the module it is in.
+        /// If we send telemetry for the module, we will also do so for any experimental feature
+        /// in that module.
+        /// </summary>
+        ExperimentalModuleFeatureDeactivation,
+
         /// <summary>
         /// Send telemetry for experimental module feature activation.
         /// Experimental module features will send telemetry based on the module it is in.
@@ -126,7 +145,7 @@ public static class ApplicationInsightsTelemetry
         private static readonly HashSet<string> s_knownModules;
 
         /// <summary>Gets a value indicating whether telemetry can be sent.</summary>
-        public static bool CanSendTelemetry { get; private set; }
+        public static bool CanSendTelemetry { get; private set; } = false;
 
         /// <summary>
         /// Initializes static members of the <see cref="ApplicationInsightsTelemetry"/> class.
@@ -703,9 +722,12 @@ internal static void SendTelemetryMetric(TelemetryType metricId, string data)
                     case TelemetryType.PowerShellCreate:
                     case TelemetryType.RemoteSessionOpen:
                     case TelemetryType.ExperimentalEngineFeatureActivation:
+                    case TelemetryType.ExperimentalEngineFeatureDeactivation:
+                    case TelemetryType.ExperimentalFeatureUse:
                         s_telemetryClient.GetMetric(metricName, "uuid", "SessionId", "Detail").TrackValue(metricValue: 1.0, s_uniqueUserIdentifier, s_sessionId, data);
                         break;
                     case TelemetryType.ExperimentalModuleFeatureActivation:
+                    case TelemetryType.ExperimentalModuleFeatureDeactivation:
                         string experimentalFeatureName = GetExperimentalFeatureName(data);
                         s_telemetryClient.GetMetric(metricName, "uuid", "SessionId", "Detail").TrackValue(metricValue: 1.0, s_uniqueUserIdentifier, s_sessionId, experimentalFeatureName);
                         break;
@@ -718,6 +740,21 @@ internal static void SendTelemetryMetric(TelemetryType metricId, string data)
             }
         }
 
+        /// <summary>
+        /// Send additional information about an experimental feature as it is used.
+        /// </summary>
+        /// <param name="featureName">The name of the experimental feature.</param>
+        /// <param name="detail">The details about the experimental feature use.</param>
+        internal static void SendExperimentalUseData(string featureName, string detail)
+        {
+            if (!CanSendTelemetry)
+            {
+                return;
+            }
+
+            ApplicationInsightsTelemetry.SendTelemetryMetric(TelemetryType.ExperimentalFeatureUse, string.Join(":", featureName, detail));
+        }
+
         // Get the experimental feature name. If we can report it, we'll return the name of the feature, otherwise, we'll return "anonymous"
         private static string GetExperimentalFeatureName(string featureNameToValidate)
         {
@@ -779,6 +816,7 @@ internal static void SendPSCoreStartupTelemetry(string mode, double parametersUs
             properties.Add("UUID", s_uniqueUserIdentifier);
             properties.Add("GitCommitID", PSVersionInfo.GitCommitId);
             properties.Add("OSDescription", RuntimeInformation.OSDescription);
+            properties.Add("RuntimeIdentifier", RuntimeInformation.RuntimeIdentifier);
             properties.Add("OSChannel", string.IsNullOrEmpty(channel) ? "unknown" : channel);
             properties.Add("StartMode", string.IsNullOrEmpty(mode) ? "unknown" : mode);