{AKS} az aks create/update: add --enable/--disable-control-plane-metrics

src/aks-preview/HISTORY.rst

@@ -12,6 +12,7 @@ To release a new version, please select a new version number (usually plus 1 to
 Pending
 +++++++
 * `az aks upgrade`: Fix `--node-image-only` to skip Machines mode agent pools, which do not support node image version upgrade.
+* `az aks create / update`: Add `--enable-control-plane-metrics` and `az aks update`: `--disable-control-plane-metrics` flags to opt clusters into Azure Monitor managed Prometheus control plane metrics (kube-apiserver, etcd, etc.) via the first-class API property `azureMonitorProfile.metrics.controlPlane.enabled` (replaces the AFEC-gated preview).
 
 21.0.0b4
 +++++++

src/aks-preview/azext_aks_preview/_help.py

@@ -606,6 +606,9 @@
         - name: --enable-azure-monitor-metrics
           type: bool
           short-summary: Enable Azure Monitor Metrics Profile
+        - name: --enable-control-plane-metrics --enable-cp-metrics
+          type: bool
+          short-summary: Enable collection of Azure Monitor managed Prometheus control plane metrics for managed cluster components (kube-apiserver, etcd, etc). Requires Azure Monitor metrics to be enabled (already enabled or via --enable-azure-monitor-metrics). See aka.ms/aks/controlplane-metrics.
         - name: --azure-monitor-workspace-resource-id
           type: string
           short-summary: Resource ID of the Azure Monitor Workspace
@@ -1332,6 +1335,9 @@
         - name: --enable-azure-monitor-metrics
           type: bool
           short-summary: Enable Azure Monitor Metrics Profile
+        - name: --enable-control-plane-metrics --enable-cp-metrics
+          type: bool
+          short-summary: Enable collection of Azure Monitor managed Prometheus control plane metrics for managed cluster components (kube-apiserver, etcd, etc). Requires Azure Monitor metrics to be enabled (already enabled or via --enable-azure-monitor-metrics). See aka.ms/aks/controlplane-metrics.
         - name: --azure-monitor-workspace-resource-id
           type: string
           short-summary: Resource ID of the Azure Monitor Workspace
@@ -1353,6 +1359,9 @@
         - name: --disable-azure-monitor-metrics
           type: bool
           short-summary: Disable Azure Monitor Metrics Profile. This will delete all DCRA's associated with the cluster, any linked DCRs with the data stream = prometheus-stream and the recording rule groups created by the addon for this AKS cluster.
+        - name: --disable-control-plane-metrics --disable-cp-metrics
+          type: bool
+          short-summary: Disable collection of Azure Monitor managed Prometheus control plane metrics. Leaves Azure Monitor metrics enabled. See aka.ms/aks/controlplane-metrics.
         - name: --enable-azure-monitor-app-monitoring
           type: bool
           short-summary: Enable Azure Monitor Application Monitoring

src/aks-preview/azext_aks_preview/_params.py

@@ -1129,6 +1129,16 @@ def load_arguments(self, _):
         c.argument("ksm_metric_annotations_allow_list")
         c.argument("grafana_resource_id", validator=validate_grafanaresourceid)
         c.argument("enable_windows_recording_rules", action="store_true")
+        c.argument(
+            "enable_control_plane_metrics",
+            options_list=["--enable-control-plane-metrics", "--enable-cp-metrics"],
+            action="store_true",
+            help=(
+                "Enable collection of Azure Monitor managed Prometheus control plane metrics for managed "
+                "cluster components (kube-apiserver, etcd, etc). Requires Azure Monitor metrics to be enabled "
+                "(already enabled or via --enable-azure-monitor-metrics). See aka.ms/aks/controlplane-metrics."
+            ),
+        )
         c.argument("enable_azure_monitor_app_monitoring",
                    is_preview=True,
                    action="store_true"
@@ -1625,6 +1635,26 @@ def load_arguments(self, _):
                 hide=True,
             ),
         )
+        c.argument("disable_azure_monitor_metrics", action="store_true")
+        c.argument(
+            "enable_control_plane_metrics",
+            options_list=["--enable-control-plane-metrics", "--enable-cp-metrics"],
+            action="store_true",
+            help=(
+                "Enable collection of Azure Monitor managed Prometheus control plane metrics for managed "
+                "cluster components (kube-apiserver, etcd, etc). Requires Azure Monitor metrics to be enabled "
+                "(already enabled or via --enable-azure-monitor-metrics). See aka.ms/aks/controlplane-metrics."
+            ),
+        )
+        c.argument(
+            "disable_control_plane_metrics",
+            options_list=["--disable-control-plane-metrics", "--disable-cp-metrics"],
+            action="store_true",
+            help=(
+                "Disable collection of Azure Monitor managed Prometheus control plane metrics. "
+                "Sets azureMonitorProfile.metrics.controlPlane.enabled=false on the cluster."
+            ),
+        )
         c.argument("enable_azure_monitor_app_monitoring",
                    action="store_true",
                    is_preview=True

src/aks-preview/azext_aks_preview/azuremonitormetrics/azuremonitorprofile.py

@@ -18,13 +18,52 @@
     check_azuremonitormetrics_profile,
     rp_registrations
 )
-from azure.cli.core.azclierror import InvalidArgumentValueError
+from azure.cli.command_modules.acs._client_factory import get_container_service_client
+from azure.cli.core.azclierror import InvalidArgumentValueError, UnknownError
 from knack.util import CLIError
 from knack.log import get_logger
 
 logger = get_logger(__name__)
 
 
+# pylint: disable=line-too-long
+def _addon_put_with_control_plane(cmd, cluster_subscription, cluster_resource_group_name, cluster_name):
+    """Sibling of core ``addon_put`` that ALSO flips ``metrics.controlPlane.enabled=True``.
+
+    Used by the greenfield ``aks create --enable-control-plane-metrics`` path. The
+    initial cluster PUT intentionally leaves ``control_plane`` unset so the RP does
+    not schedule the control-plane-metrics collection (CCP) pod before the DCRA is
+    created in postprocessing. Once the DCRA exists, we issue this PUT so the CCP
+    pod is scheduled with its DCRA already in place (race-free).
+    """
+    client = get_container_service_client(cmd.cli_ctx, cluster_subscription).managed_clusters
+    try:
+        mc = client.get(cluster_resource_group_name, cluster_name)
+    except CLIError as e:
+        raise UnknownError(e)
+    # Enable metrics if present and not already enabled (mirrors core addon_put).
+    if hasattr(mc, "azure_monitor_profile") and mc.azure_monitor_profile:
+        if hasattr(mc.azure_monitor_profile, "metrics") and mc.azure_monitor_profile.metrics:
+            if getattr(mc.azure_monitor_profile.metrics, "enabled", None) is False:
+                mc.azure_monitor_profile.metrics.enabled = True
+            # Flip control plane now that DCRA exists.
+            try:
+                from azure.mgmt.containerservice.models import (
+                    ManagedClusterAzureMonitorProfileMetricsControlPlane,
+                )
+                mc.azure_monitor_profile.metrics.control_plane = (
+                    ManagedClusterAzureMonitorProfileMetricsControlPlane(enabled=True)
+                )
+            except ImportError:
+                # Fallback for SDK versions that don't expose the model directly:
+                # set a dict that the generated client will serialize as the property.
+                mc.azure_monitor_profile.metrics.control_plane = {"enabled": True}
+    try:
+        client.begin_create_or_update(cluster_resource_group_name, cluster_name, mc)
+    except Exception as e:
+        raise UnknownError(e)
+
+
 # pylint: disable=line-too-long
 def link_azure_monitor_profile_artifacts(
         cmd,
@@ -49,7 +88,15 @@ def link_azure_monitor_profile_artifacts(
     create_rules(cmd, cluster_subscription, cluster_resource_group_name, cluster_name, azure_monitor_workspace_resource_id, azure_monitor_workspace_location, raw_parameters)
     # if aks cluster create flow -> do a PUT on the AKS cluster to enable the addon
     if create_flow:
-        addon_put(cmd, cluster_subscription, cluster_resource_group_name, cluster_name)
+        # If --enable-control-plane-metrics was specified on create, flip
+        # metrics.controlPlane.enabled HERE (after DCRA creation) instead of on
+        # the initial cluster PUT. This avoids the CCP pod being scheduled before
+        # its DCRA exists (which would cause CrashLoopBackOff until reconciliation).
+        enable_cp = bool(raw_parameters and raw_parameters.get("enable_control_plane_metrics"))
+        if enable_cp:
+            _addon_put_with_control_plane(cmd, cluster_subscription, cluster_resource_group_name, cluster_name)
+        else:
+            addon_put(cmd, cluster_subscription, cluster_resource_group_name, cluster_name)
 
 
 # pylint: disable=line-too-long

src/aks-preview/azext_aks_preview/custom.py

@@ -1136,6 +1136,7 @@ def aks_create(
     ksm_metric_annotations_allow_list=None,
     grafana_resource_id=None,
     enable_windows_recording_rules=False,
+    enable_control_plane_metrics=False,
     # azure monitor profile - app monitoring
     enable_azure_monitor_app_monitoring=False,
     # opentelemetry parameters
@@ -1363,6 +1364,8 @@ def aks_update(
     enable_windows_recording_rules=False,
     disable_azuremonitormetrics=False,
     disable_azure_monitor_metrics=False,
+    enable_control_plane_metrics=False,
+    disable_control_plane_metrics=False,
     # azure monitor profile - app monitoring
     enable_azure_monitor_app_monitoring=False,
     disable_azure_monitor_app_monitoring=False,

src/aks-preview/azext_aks_preview/managed_cluster_decorator.py

@@ -2651,6 +2651,91 @@ def get_disable_azure_monitor_metrics(self) -> bool:
         """
         return self._get_disable_azure_monitor_metrics(enable_validation=True)
 
+    def _get_enable_control_plane_metrics(self, enable_validation: bool = False) -> bool:
+        """Internal function to obtain the value of enable_control_plane_metrics.
+        This function supports the option of enable_validation. When enabled, if both
+        enable_control_plane_metrics and disable_control_plane_metrics are specified, raise a
+        MutuallyExclusiveArgumentError. Additionally, --enable-control-plane-metrics requires Azure
+        Monitor metrics to either already be enabled on the cluster or to be enabled in the same
+        command via --enable-azure-monitor-metrics.
+
+        :return: bool
+        """
+        # Read the original value passed by the command.
+        enable_control_plane_metrics = self.raw_param.get("enable_control_plane_metrics")
+        # In create mode, try to read the property value corresponding to the parameter from the `mc` object.
+        if self.decorator_mode == DecoratorMode.CREATE:
+            if (
+                self.mc and
+                self.mc.azure_monitor_profile and
+                self.mc.azure_monitor_profile.metrics and
+                self.mc.azure_monitor_profile.metrics.control_plane
+            ):
+                enable_control_plane_metrics = self.mc.azure_monitor_profile.metrics.control_plane.enabled
+        # This parameter does not need dynamic completion.
+        if enable_validation:
+            if enable_control_plane_metrics and self._get_disable_control_plane_metrics(False):
+                raise MutuallyExclusiveArgumentError(
+                    "Cannot specify --enable-control-plane-metrics and --disable-control-plane-metrics "
+                    "at the same time."
+                )
+            if enable_control_plane_metrics:
+                # Reject combining enable-control-plane-metrics with disable-azure-monitor-metrics
+                # in the same command — the resulting payload would be inconsistent.
+                if self._get_disable_azure_monitor_metrics(False):
+                    raise MutuallyExclusiveArgumentError(
+                        "Cannot specify --enable-control-plane-metrics together with "
+                        "--disable-azure-monitor-metrics."
+                    )
+                # Must have Azure Monitor metrics enabled (either already or in this command).
+                already_enabled = (
+                    self.mc and
+                    self.mc.azure_monitor_profile and
+                    self.mc.azure_monitor_profile.metrics and
+                    self.mc.azure_monitor_profile.metrics.enabled
+                )
+                enabling_now = self._get_enable_azure_monitor_metrics(False)
+                if not already_enabled and not enabling_now:
+                    raise RequiredArgumentMissingError(
+                        "--enable-control-plane-metrics requires Azure Monitor metrics to be enabled. "
+                        "Specify --enable-azure-monitor-metrics or run on a cluster that already has "
+                        "Azure Monitor metrics enabled."
+                    )
+        return enable_control_plane_metrics
+
+    def get_enable_control_plane_metrics(self) -> bool:
+        """Obtain the value of enable_control_plane_metrics.
+        This function will verify the parameter by default. If both enable_control_plane_metrics and
+        disable_control_plane_metrics are specified, raise a MutuallyExclusiveArgumentError.
+        :return: bool
+        """
+        return self._get_enable_control_plane_metrics(enable_validation=True)
+
+    def _get_disable_control_plane_metrics(self, enable_validation: bool = False) -> bool:
+        """Internal function to obtain the value of disable_control_plane_metrics.
+        This function supports the option of enable_validation. When enabled, if both
+        enable_control_plane_metrics and disable_control_plane_metrics are specified, raise a
+        MutuallyExclusiveArgumentError.
+        :return: bool
+        """
+        # Read the original value passed by the command.
+        disable_control_plane_metrics = self.raw_param.get("disable_control_plane_metrics")
+        if enable_validation:
+            if disable_control_plane_metrics and self._get_enable_control_plane_metrics(False):
+                raise MutuallyExclusiveArgumentError(
+                    "Cannot specify --enable-control-plane-metrics and --disable-control-plane-metrics "
+                    "at the same time."
+                )
+        return disable_control_plane_metrics
+
+    def get_disable_control_plane_metrics(self) -> bool:
+        """Obtain the value of disable_control_plane_metrics.
+        This function will verify the parameter by default. If both enable_control_plane_metrics and
+        disable_control_plane_metrics are specified, raise a MutuallyExclusiveArgumentError.
+        :return: bool
+        """
+        return self._get_disable_control_plane_metrics(enable_validation=True)
+
     def _get_enable_azure_monitor_app_monitoring(self, enable_validation=True) -> bool:
         """Internal function to obtain the value of enable_azure_monitor_app_monitoring.
         This function supports the option of enable_validation. When enabled, if both
@@ -4693,6 +4778,18 @@ def _setup_azure_monitor_metrics(self, mc: ManagedCluster) -> None:
                 metric_annotations_allow_list=str(ksm_metric_annotations_allow_list)
             ))
         mc.azure_monitor_profile.metrics.kube_state_metrics = kube_state_metrics
+
+        # NOTE: control_plane.enabled is intentionally NOT set here on the create flow.
+        # If we set it on this initial PUT, the RP would schedule the control-plane-metrics
+        # collection pod (CCP) before the DCRA (Data Collection Rule Association) has been
+        # created in postprocessing. The CCP would then crash-loop with "DCRA not found"
+        # until the next reconciliation.
+        # Instead, we defer the flip to the addon_put step inside
+        # link_azure_monitor_profile_artifacts (postprocessing_after_mc_created), which
+        # runs *after* DCRA creation. The validator still runs here to surface flag
+        # combination errors early.
+        self.context.get_enable_control_plane_metrics()
+
         self.context.set_intermediate("azuremonitormetrics_addon_enabled", True, overwrite_exists=True)
 
     def _setup_azure_monitor_app_monitoring(self, mc: ManagedCluster) -> None:
@@ -4812,6 +4909,11 @@ def set_up_azure_monitor_profile(self, mc: ManagedCluster) -> ManagedCluster:
         """
         self._ensure_mc(mc)
 
+        # Trigger control-plane-metrics validation even if the parent metrics flag was
+        # not specified, so users get a clear error instead of silent ignore when they
+        # pass --enable-control-plane-metrics on its own.
+        self.context.get_enable_control_plane_metrics()
+
         if self.context.get_enable_azure_monitor_metrics():
             self._setup_azure_monitor_metrics(mc)
 
@@ -7006,6 +7108,30 @@ def update_azure_monitor_profile(self, mc: ManagedCluster) -> ManagedCluster:
         if self.context.get_disable_azure_monitor_metrics():
             self._disable_azure_monitor_metrics(mc)
 
+        # Handle enable / disable of control plane metrics independently of the parent metrics flag,
+        # so users can toggle control plane metrics on a cluster that already has metrics enabled.
+        if self.context.get_enable_control_plane_metrics():
+            if mc.azure_monitor_profile is None:
+                mc.azure_monitor_profile = self.models.ManagedClusterAzureMonitorProfile()  # pylint: disable=no-member
+            if mc.azure_monitor_profile.metrics is None:
+                # Should not normally happen — validation requires metrics to be enabled — but guard
+                # against partially-populated profiles to avoid AttributeError.
+                mc.azure_monitor_profile.metrics = (
+                    self.models.ManagedClusterAzureMonitorProfileMetrics(enabled=True)  # pylint: disable=no-member
+                )
+            mc.azure_monitor_profile.metrics.control_plane = (
+                self.models.ManagedClusterAzureMonitorProfileMetricsControlPlane(enabled=True)  # pylint: disable=no-member
+            )
+
+        if self.context.get_disable_control_plane_metrics():
+            if (
+                mc.azure_monitor_profile and
+                mc.azure_monitor_profile.metrics
+            ):
+                mc.azure_monitor_profile.metrics.control_plane = (
+                    self.models.ManagedClusterAzureMonitorProfileMetricsControlPlane(enabled=False)  # pylint: disable=no-member
+                )
+
         if self.context.get_enable_azure_monitor_app_monitoring():
             if mc.azure_monitor_profile is None:
                 mc.azure_monitor_profile = self.models.ManagedClusterAzureMonitorProfile()