Support attached Fabric Spark compute in azure-ai-ml

sdk/ml/azure-ai-ml/CHANGELOG.md

@@ -3,6 +3,7 @@
 ## 1.36.0 (unreleased)
 
 ### Features Added
+- Added Fabric Spark compute attachment support, including typed attach, detach, get, and list results.
 - Added Spark runtime 3.5 support for model monitoring serverless Spark compute while retaining runtime 3.4 support.
 
 ### Bugs Fixed

sdk/ml/azure-ai-ml/api.md

@@ -5432,6 +5432,41 @@ namespace azure.ai.ml.entities
             ): ...
 
 
+    class azure.ai.ml.entities.FabricSparkCompute(Compute):
+        property base_path: str    # Read-only
+        property created_on: Optional[str]    # Read-only
+        property creation_context: Optional[SystemData]    # Read-only
+        property id: Optional[str]    # Read-only
+        property provisioning_errors: Optional[str]    # Read-only
+        property provisioning_state: Optional[str]    # Read-only
+        property type: Optional[str]    # Read-only
+
+        def __init__(
+                self,
+                *,
+                description: Optional[str] = ...,
+                environment_id: str,
+                fabric_workspace_id: str,
+                identity: Optional[IdentityConfiguration] = ...,
+                lakehouse_id: str,
+                name: str,
+                pool_id: str,
+                pool_name: Optional[str] = ...,
+                tags: Optional[Dict[str, str]] = ...,
+                **kwargs: Any
+            ) -> None: ...
+
+        def __repr__(self) -> str: ...
+
+        def __str__(self) -> str: ...
+
+        def dump(
+                self,
+                dest: Union[str, PathLike, IO[AnyStr]],
+                **kwargs: Any
+            ) -> None: ...
+
+
     class azure.ai.ml.entities.Feature(RestTranslatableMixin):
 
         def __init__(

sdk/ml/azure-ai-ml/api.metadata.yml

@@ -1,3 +1,4 @@
-apiMdSha256: c39d5bf8320fae3388fc50ddcd9e5a7cbca070b46ffced33935ae06133a3364e
+apiMdSha256: 4eceeb67e3f43723bec05fb770198bc746d73dd373881d98619979d4fe64393c
+packageVersion: 1.36.0
 parserVersion: 0.3.31
 pythonVersion: 3.12.13

sdk/ml/azure-ai-ml/azure/ai/ml/_restclient/arm_ml_service/models/__init__.py

@@ -188,6 +188,8 @@
     EstimatedVMPrice,
     EstimatedVMPrices,
     ExternalFQDNResponse,
+    FabricSpark,
+    FabricSparkProperties,
     FQDNEndpoint,
     FQDNEndpointDetail,
     FQDNEndpoints,
@@ -866,6 +868,8 @@
     "EstimatedVMPrice",
     "EstimatedVMPrices",
     "ExternalFQDNResponse",
+    "FabricSpark",
+    "FabricSparkProperties",
     "FQDNEndpoint",
     "FQDNEndpointDetail",
     "FQDNEndpoints",

sdk/ml/azure-ai-ml/azure/ai/ml/_restclient/arm_ml_service/models/_enums.py

@@ -425,6 +425,8 @@ class ComputeType(str, Enum, metaclass=CaseInsensitiveEnumMeta):
     """DATA_LAKE_ANALYTICS."""
     SYNAPSE_SPARK = "SynapseSpark"
     """SYNAPSE_SPARK."""
+    FABRIC_SPARK = "FabricSpark"
+    """FABRIC_SPARK."""
 
 
 class ComputeWeekDay(str, Enum, metaclass=CaseInsensitiveEnumMeta):

sdk/ml/azure-ai-ml/azure/ai/ml/_restclient/arm_ml_service/models/_models.py

@@ -751,6 +751,64 @@ def __init__(self, *args: Any, **kwargs: Any) -> None:
         super().__init__(*args, **kwargs)
 
 
+class FabricSparkProperties(_Model):
+    """An existing Fabric workspace, Lakehouse, published Environment, and custom Spark pool."""
+
+    fabric_workspace_id: str = rest_field(
+        name="fabricWorkspaceId", visibility=["read", "create", "update", "delete", "query"]
+    )
+    lakehouse_id: str = rest_field(name="lakehouseId", visibility=["read", "create", "update", "delete", "query"])
+    environment_id: str = rest_field(name="environmentId", visibility=["read", "create", "update", "delete", "query"])
+    pool_id: str = rest_field(name="poolId", visibility=["read", "create", "update", "delete", "query"])
+    pool_name: Optional[str] = rest_field(name="poolName", visibility=["read"])
+
+    @overload
+    def __init__(
+        self,
+        *,
+        fabric_workspace_id: str,
+        lakehouse_id: str,
+        environment_id: str,
+        pool_id: str,
+        pool_name: Optional[str] = None,
+    ) -> None: ...
+
+    @overload
+    def __init__(self, mapping: Mapping[str, Any]) -> None: ...
+
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        super().__init__(*args, **kwargs)
+
+
+class FabricSpark(Compute, discriminator="FabricSpark"):
+    """An attached Fabric custom Spark pool."""
+
+    properties: Optional["_models.FabricSparkProperties"] = rest_field(
+        visibility=["read", "create", "update", "delete", "query"]
+    )
+    compute_type: Literal[ComputeType.FABRIC_SPARK] = rest_discriminator(
+        name="computeType", visibility=["read", "create", "update", "delete", "query"]
+    )  # type: ignore
+
+    @overload
+    def __init__(
+        self,
+        *,
+        compute_location: Optional[str] = None,
+        description: Optional[str] = None,
+        resource_id: Optional[str] = None,
+        disable_local_auth: Optional[bool] = None,
+        properties: Optional["_models.FabricSparkProperties"] = None,
+    ) -> None: ...
+
+    @overload
+    def __init__(self, mapping: Mapping[str, Any]) -> None: ...
+
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
+        super().__init__(*args, **kwargs)
+        self.compute_type = ComputeType.FABRIC_SPARK  # type: ignore
+
+
 class AKS(Compute, discriminator="AKS"):
     """A Machine Learning compute based on AKS.
 

sdk/ml/azure-ai-ml/azure/ai/ml/_restclient/arm_ml_service/tsp-location.yaml

@@ -1,4 +1,4 @@
-directory: specification/machinelearningservices/MachineLearningServices.Management
-commit: bce80108d7ce39dccc1e9c2df1b1bd99d018bf7f
+directory: specification/machinelearningservices/resource-manager/Microsoft.MachineLearningServices/MachineLearningServices
+commit: 99b98e0bdb68f26290317a5b1ad5e7f6a31cc7e6
 repo: Azure/azure-rest-api-specs
 additionalDirectories: []

sdk/ml/azure-ai-ml/azure/ai/ml/_schema/compute/fabricspark_compute.py

@@ -0,0 +1,23 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+from marshmallow import fields
+
+from azure.ai.ml.constants._compute import ComputeType
+
+from ..core.fields import NestedField, StringTransformedEnum
+from .compute import ComputeSchema, IdentitySchema
+
+
+class FabricSparkComputeSchema(ComputeSchema):
+    type = StringTransformedEnum(allowed_values=[ComputeType.FABRICSPARK], required=True)
+    fabric_workspace_id = fields.Str(required=True)
+    lakehouse_id = fields.Str(required=True)
+    environment_id = fields.Str(required=True)
+    pool_id = fields.Str(required=True)
+    pool_name = fields.Str(dump_only=True)
+    is_attached_compute = fields.Bool(dump_only=True)
+    modified_on = fields.Str(dump_only=True)
+    disable_local_auth = fields.Bool(dump_only=True)
+    identity = NestedField(IdentitySchema)

sdk/ml/azure-ai-ml/azure/ai/ml/constants/_compute.py

@@ -9,8 +9,8 @@ class ComputeType:
     ComputeType can be used to specify the compute type for a compute target. It can also be used to
     specify the compute type for a compute target that is being created.
 
-    Valid values are "managed", "amlcompute", "computeinstance", "virtualmachine", "kubernetes", "DataFactory"
-        , "synapsespark".
+    Valid values are "managed", "amlcompute", "computeinstance", "virtualmachine", "kubernetes", "DataFactory",
+        "synapsespark", and "fabricspark".
     """
 
     MANAGED = "managed"
@@ -27,6 +27,8 @@ class ComputeType:
     """Data Factory is a compute resource that is managed by Azure Machine Learning."""
     SYNAPSESPARK = "synapsespark"
     """Synapse Spark is a compute resource that is managed by Azure Machine Learning."""
+    FABRICSPARK = "fabricspark"
+    """Fabric Spark is an attached custom pool in a Fabric workspace."""
 
 
 class ComputeTier:

sdk/ml/azure-ai-ml/azure/ai/ml/entities/__init__.py

@@ -64,6 +64,7 @@
 from ._compute.compute_instance import AssignedUserConfiguration, ComputeInstance, ComputeInstanceSshSettings
 from ._compute.kubernetes_compute import KubernetesCompute
 from ._compute.synapsespark_compute import AutoPauseSettings, AutoScaleSettings, SynapseSparkCompute
+from ._compute.fabricspark_compute import FabricSparkCompute
 from ._compute.unsupported_compute import UnsupportedCompute
 from ._compute.virtual_machine_compute import VirtualMachineCompute, VirtualMachineSshSettings
 from ._credentials import (
@@ -399,6 +400,7 @@
     "RegistryRegionDetails",
     "Registry",
     "SynapseSparkCompute",
+    "FabricSparkCompute",
     "AutoScaleSettings",
     "AutoPauseSettings",
     "WorkspaceModelReference",

sdk/ml/azure-ai-ml/azure/ai/ml/entities/_compute/compute.py

@@ -26,7 +26,7 @@ class Compute(Resource, RestTranslatableMixin):
     This class should not be instantiated directly. Instead, use one of its subclasses.
 
     :param type: The compute type. Accepted values are "amlcompute", "computeinstance",
-        "virtualmachine", "kubernetes", and "synapsespark".
+        "virtualmachine", "kubernetes", "synapsespark", and "fabricspark".
     :type type: str
     :param name: Name of the compute resource.
     :type name: str
@@ -106,6 +106,7 @@ def _from_rest_object(cls, obj: ComputeResource) -> "Compute":
         from azure.ai.ml.entities import (
             AmlCompute,
             ComputeInstance,
+            FabricSparkCompute,
             KubernetesCompute,
             SynapseSparkCompute,
             UnsupportedCompute,
@@ -118,11 +119,21 @@ def _from_rest_object(cls, obj: ComputeResource) -> "Compute":
             ComputeType.VIRTUALMACHINE.lower(): VirtualMachineCompute,
             ComputeType.KUBERNETES.lower(): KubernetesCompute,
             ComputeType.SYNAPSESPARK.lower(): SynapseSparkCompute,
+            ComputeType.FABRICSPARK.lower(): FabricSparkCompute,
         }
         compute_type = obj.properties.compute_type.lower() if obj.properties.compute_type else None
 
         class_type = cast(
-            Optional[Union[AmlCompute, ComputeInstance, VirtualMachineCompute, KubernetesCompute, SynapseSparkCompute]],
+            Optional[
+                Union[
+                    AmlCompute,
+                    ComputeInstance,
+                    VirtualMachineCompute,
+                    KubernetesCompute,
+                    SynapseSparkCompute,
+                    FabricSparkCompute,
+                ]
+            ],
             mapping.get(compute_type, None),  # type: ignore
         )
         if class_type:
@@ -174,6 +185,7 @@ def _load(
         from azure.ai.ml.entities import (
             AmlCompute,
             ComputeInstance,
+            FabricSparkCompute,
             KubernetesCompute,
             SynapseSparkCompute,
             VirtualMachineCompute,
@@ -197,6 +209,8 @@ def _load(
             if compute_type.lower() == ComputeType.SYNAPSESPARK:
                 _synapse_spark_load_from_dict: Compute = SynapseSparkCompute._load_from_dict(data, context, **kwargs)
                 return _synapse_spark_load_from_dict
+            if compute_type.lower() == ComputeType.FABRICSPARK:
+                return FabricSparkCompute._load_from_dict(data, context, **kwargs)
         msg = f"Unknown compute type: {compute_type}"
         raise ValidationException(
             message=msg,