From 291568bf0e3b2aa5ef12de075d8263f537132593 Mon Sep 17 00:00:00 2001 From: Kevin Hannon Date: Fri, 7 Mar 2025 13:45:03 -0500 Subject: [PATCH 01/42] add kueue api for api-review --- .../generated_openapi/zz_generated.openapi.go | 641 ++++++++++++++++++ openapi/openapi.json | 363 ++++++++++ operator/v1alpha1/register.go | 2 + operator/v1alpha1/types_kueue.go | 299 ++++++++ operator/v1alpha1/zz_generated.deepcopy.go | 350 ++++++++++ 5 files changed, 1655 insertions(+) create mode 100644 operator/v1alpha1/types_kueue.go diff --git a/openapi/generated_openapi/zz_generated.openapi.go b/openapi/generated_openapi/zz_generated.openapi.go index 72470ff0a95..b1234c35e0d 100644 --- a/openapi/generated_openapi/zz_generated.openapi.go +++ b/openapi/generated_openapi/zz_generated.openapi.go @@ -1110,12 +1110,20 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "github.com/openshift/api/operator/v1alpha1.EtcdBackupList": schema_openshift_api_operator_v1alpha1_EtcdBackupList(ref), "github.com/openshift/api/operator/v1alpha1.EtcdBackupSpec": schema_openshift_api_operator_v1alpha1_EtcdBackupSpec(ref), "github.com/openshift/api/operator/v1alpha1.EtcdBackupStatus": schema_openshift_api_operator_v1alpha1_EtcdBackupStatus(ref), + "github.com/openshift/api/operator/v1alpha1.FairSharing": schema_openshift_api_operator_v1alpha1_FairSharing(ref), "github.com/openshift/api/operator/v1alpha1.GenerationHistory": schema_openshift_api_operator_v1alpha1_GenerationHistory(ref), "github.com/openshift/api/operator/v1alpha1.GenericOperatorConfig": schema_openshift_api_operator_v1alpha1_GenericOperatorConfig(ref), "github.com/openshift/api/operator/v1alpha1.ImageContentSourcePolicy": schema_openshift_api_operator_v1alpha1_ImageContentSourcePolicy(ref), "github.com/openshift/api/operator/v1alpha1.ImageContentSourcePolicyList": schema_openshift_api_operator_v1alpha1_ImageContentSourcePolicyList(ref), "github.com/openshift/api/operator/v1alpha1.ImageContentSourcePolicySpec": schema_openshift_api_operator_v1alpha1_ImageContentSourcePolicySpec(ref), + "github.com/openshift/api/operator/v1alpha1.Integrations": schema_openshift_api_operator_v1alpha1_Integrations(ref), + "github.com/openshift/api/operator/v1alpha1.Kueue": schema_openshift_api_operator_v1alpha1_Kueue(ref), + "github.com/openshift/api/operator/v1alpha1.KueueConfiguration": schema_openshift_api_operator_v1alpha1_KueueConfiguration(ref), + "github.com/openshift/api/operator/v1alpha1.KueueList": schema_openshift_api_operator_v1alpha1_KueueList(ref), + "github.com/openshift/api/operator/v1alpha1.KueueOperandSpec": schema_openshift_api_operator_v1alpha1_KueueOperandSpec(ref), + "github.com/openshift/api/operator/v1alpha1.KueueStatus": schema_openshift_api_operator_v1alpha1_KueueStatus(ref), "github.com/openshift/api/operator/v1alpha1.LoggingConfig": schema_openshift_api_operator_v1alpha1_LoggingConfig(ref), + "github.com/openshift/api/operator/v1alpha1.MultiKueue": schema_openshift_api_operator_v1alpha1_MultiKueue(ref), "github.com/openshift/api/operator/v1alpha1.NodeStatus": schema_openshift_api_operator_v1alpha1_NodeStatus(ref), "github.com/openshift/api/operator/v1alpha1.OLM": schema_openshift_api_operator_v1alpha1_OLM(ref), "github.com/openshift/api/operator/v1alpha1.OLMList": schema_openshift_api_operator_v1alpha1_OLMList(ref), @@ -1125,8 +1133,12 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "github.com/openshift/api/operator/v1alpha1.OperatorSpec": schema_openshift_api_operator_v1alpha1_OperatorSpec(ref), "github.com/openshift/api/operator/v1alpha1.OperatorStatus": schema_openshift_api_operator_v1alpha1_OperatorStatus(ref), "github.com/openshift/api/operator/v1alpha1.RepositoryDigestMirrors": schema_openshift_api_operator_v1alpha1_RepositoryDigestMirrors(ref), + "github.com/openshift/api/operator/v1alpha1.RequeuingStrategy": schema_openshift_api_operator_v1alpha1_RequeuingStrategy(ref), + "github.com/openshift/api/operator/v1alpha1.ResourceTransformation": schema_openshift_api_operator_v1alpha1_ResourceTransformation(ref), + "github.com/openshift/api/operator/v1alpha1.Resources": schema_openshift_api_operator_v1alpha1_Resources(ref), "github.com/openshift/api/operator/v1alpha1.StaticPodOperatorStatus": schema_openshift_api_operator_v1alpha1_StaticPodOperatorStatus(ref), "github.com/openshift/api/operator/v1alpha1.VersionAvailability": schema_openshift_api_operator_v1alpha1_VersionAvailability(ref), + "github.com/openshift/api/operator/v1alpha1.WaitForPodsReady": schema_openshift_api_operator_v1alpha1_WaitForPodsReady(ref), "github.com/openshift/api/operatorcontrolplane/v1alpha1.LogEntry": schema_openshift_api_operatorcontrolplane_v1alpha1_LogEntry(ref), "github.com/openshift/api/operatorcontrolplane/v1alpha1.OutageEntry": schema_openshift_api_operatorcontrolplane_v1alpha1_OutageEntry(ref), "github.com/openshift/api/operatorcontrolplane/v1alpha1.PodNetworkConnectivityCheck": schema_openshift_api_operatorcontrolplane_v1alpha1_PodNetworkConnectivityCheck(ref), @@ -56839,6 +56851,42 @@ func schema_openshift_api_operator_v1alpha1_EtcdBackupStatus(ref common.Referenc } } +func schema_openshift_api_operator_v1alpha1_FairSharing(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "enable": { + SchemaProps: spec.SchemaProps{ + Description: "enable indicates whether to enable fair sharing for all cohorts. Defaults to false.", + Default: false, + Type: []string{"boolean"}, + Format: "", + }, + }, + "preemptionStrategies": { + SchemaProps: spec.SchemaProps{ + Description: "preemptionStrategies indicates which constraints should a preemption satisfy. The preemption algorithm will only use the next strategy in the list if the incoming workload (preemptor) doesn't fit after using the previous strategies. Possible values are: - LessThanOrEqualToFinalShare: Only preempt a workload if the share of the preemptor CQ\n with the preemptor workload is less than or equal to the share of the preemptee CQ\n without the workload to be preempted.\n This strategy might favor preemption of smaller workloads in the preemptee CQ,\n regardless of priority or start time, in an effort to keep the share of the CQ\n as high as possible.\n- LessThanInitialShare: Only preempt a workload if the share of the preemptor CQ\n with the incoming workload is strictly less than the share of the preemptee CQ.\n This strategy doesn't depend on the share usage of the workload being preempted.\n As a result, the strategy chooses to preempt workloads with the lowest priority and\n newest start time first.\nThe default strategy is [\"LessThanOrEqualToFinalShare\", \"LessThanInitialShare\"].", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + }, + Required: []string{"enable"}, + }, + }, + } +} + func schema_openshift_api_operator_v1alpha1_GenerationHistory(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ @@ -57073,6 +57121,392 @@ func schema_openshift_api_operator_v1alpha1_ImageContentSourcePolicySpec(ref com } } +func schema_openshift_api_operator_v1alpha1_Integrations(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "frameworks": { + SchemaProps: spec.SchemaProps{ + Description: "List of framework names to be enabled. Possible options:\n - \"batch/job\"\n - \"kubeflow.org/mpijob\"\n - \"ray.io/rayjob\"\n - \"ray.io/raycluster\"\n - \"jobset.x-k8s.io/jobset\"\n - \"kubeflow.org/paddlejob\"\n - \"kubeflow.org/pytorchjob\"\n - \"kubeflow.org/tfjob\"\n - \"kubeflow.org/xgboostjob\"\n - \"workload.codeflare.dev/appwrapper\"\n - \"pod\"\n - \"deployment\" (requires enabling pod integration)\n - \"statefulset\" (requires enabling pod integration)\n - \"leaderworkerset.x-k8s.io/leaderworkerset\" (requires enabling pod integration)", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + "externalFrameworks": { + SchemaProps: spec.SchemaProps{ + Description: "List of GroupVersionKinds that are managed for Kueue by external controllers; the expected format is `Kind.version.group.com`.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + "labelKeysToCopy": { + SchemaProps: spec.SchemaProps{ + Description: "labelKeysToCopy is a list of label keys that should be copied from the job into the workload object. It is not required for the job to have all the labels from this list. If a job does not have some label with the given key from this list, the constructed workload object will be created without this label. In the case of creating a workload from a composable job (pod group), if multiple objects have labels with some key from the list, the values of these labels must match or otherwise the workload creation would fail. The labels are copied only during the workload creation and are not updated even if the labels of the underlying job are changed.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + }, + }, + }, + } +} + +func schema_openshift_api_operator_v1alpha1_Kueue(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "Kueue is the Schema for the kueue API Compatibility level 4: No compatibility is provided, the API can change at any point for any reason. These capabilities should not be used by applications needing long term support.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Type: []string{"string"}, + Format: "", + }, + }, + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + Type: []string{"string"}, + Format: "", + }, + }, + "metadata": { + SchemaProps: spec.SchemaProps{ + Description: "metadata for kueue", + Default: map[string]interface{}{}, + Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"), + }, + }, + "spec": { + SchemaProps: spec.SchemaProps{ + Description: "spec holds user settable values for configuration", + Default: map[string]interface{}{}, + Ref: ref("github.com/openshift/api/operator/v1alpha1.KueueOperandSpec"), + }, + }, + "status": { + SchemaProps: spec.SchemaProps{ + Description: "status holds observed values from the cluster. They may not be overridden.", + Default: map[string]interface{}{}, + Ref: ref("github.com/openshift/api/operator/v1alpha1.KueueStatus"), + }, + }, + }, + Required: []string{"spec"}, + }, + }, + Dependencies: []string{ + "github.com/openshift/api/operator/v1alpha1.KueueOperandSpec", "github.com/openshift/api/operator/v1alpha1.KueueStatus", "k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"}, + } +} + +func schema_openshift_api_operator_v1alpha1_KueueConfiguration(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "waitForPodsReady": { + SchemaProps: spec.SchemaProps{ + Description: "waitForPodsReady configures gang admission", + Default: map[string]interface{}{}, + Ref: ref("github.com/openshift/api/operator/v1alpha1.WaitForPodsReady"), + }, + }, + "integrations": { + SchemaProps: spec.SchemaProps{ + Description: "integrations are the types of integrations Kueue will manager", + Default: map[string]interface{}{}, + Ref: ref("github.com/openshift/api/operator/v1alpha1.Integrations"), + }, + }, + "featureGates": { + SchemaProps: spec.SchemaProps{ + Description: "featureGates are advanced features for Kueue", + Type: []string{"object"}, + AdditionalProperties: &spec.SchemaOrBool{ + Allows: true, + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: false, + Type: []string{"boolean"}, + Format: "", + }, + }, + }, + }, + }, + "resources": { + SchemaProps: spec.SchemaProps{ + Description: "resources provides additional configuration options for handling the resources. Supports https://github.com/kubernetes-sigs/kueue/blob/release-0.10/keps/2937-resource-transformer/README.md", + Default: map[string]interface{}{}, + Ref: ref("github.com/openshift/api/operator/v1alpha1.Resources"), + }, + }, + "manageJobsWithoutQueueName": { + SchemaProps: spec.SchemaProps{ + Description: "ManageJobsWithoutQueueName controls whether or not Kueue reconciles jobs that don't set the annotation kueue.x-k8s.io/queue-name. Allowed values are NoQueueName and QueueName Default will be QueueName", + Type: []string{"string"}, + Format: "", + }, + }, + "managedJobsNamespaceSelector": { + SchemaProps: spec.SchemaProps{ + Description: "ManagedJobsNamespaceSelector can be used to omit some namespaces from ManagedJobsWithoutQueueName", + Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.LabelSelector"), + }, + }, + "fairSharing": { + SchemaProps: spec.SchemaProps{ + Description: "FairSharing controls the fair sharing semantics across the cluster.", + Default: map[string]interface{}{}, + Ref: ref("github.com/openshift/api/operator/v1alpha1.FairSharing"), + }, + }, + "disableMetrics": { + SchemaProps: spec.SchemaProps{ + Description: "Disable Metrics Microshift does not enable metrics by default Default will assume metrics are enabled.", + Type: []string{"boolean"}, + Format: "", + }, + }, + }, + Required: []string{"integrations"}, + }, + }, + Dependencies: []string{ + "github.com/openshift/api/operator/v1alpha1.FairSharing", "github.com/openshift/api/operator/v1alpha1.Integrations", "github.com/openshift/api/operator/v1alpha1.Resources", "github.com/openshift/api/operator/v1alpha1.WaitForPodsReady", "k8s.io/apimachinery/pkg/apis/meta/v1.LabelSelector"}, + } +} + +func schema_openshift_api_operator_v1alpha1_KueueList(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "KueueList contains a list of Kueue\n\nCompatibility level 4: No compatibility is provided, the API can change at any point for any reason. These capabilities should not be used by applications needing long term support.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "kind": { + SchemaProps: spec.SchemaProps{ + Description: "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + Type: []string{"string"}, + Format: "", + }, + }, + "apiVersion": { + SchemaProps: spec.SchemaProps{ + Description: "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + Type: []string{"string"}, + Format: "", + }, + }, + "metadata": { + SchemaProps: spec.SchemaProps{ + Description: "metadata for the list", + Default: map[string]interface{}{}, + Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.ListMeta"), + }, + }, + "items": { + SchemaProps: spec.SchemaProps{ + Description: "items is a slice of kueue", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("github.com/openshift/api/operator/v1alpha1.Kueue"), + }, + }, + }, + }, + }, + }, + Required: []string{"items"}, + }, + }, + Dependencies: []string{ + "github.com/openshift/api/operator/v1alpha1.Kueue", "k8s.io/apimachinery/pkg/apis/meta/v1.ListMeta"}, + } +} + +func schema_openshift_api_operator_v1alpha1_KueueOperandSpec(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "managementState": { + SchemaProps: spec.SchemaProps{ + Description: "managementState indicates whether and how the operator should manage the component", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "logLevel": { + SchemaProps: spec.SchemaProps{ + Description: "logLevel is an intent based logging for an overall component. It does not give fine grained control, but it is a simple way to manage coarse grained logging choices that operators have to interpret for their operands.\n\nValid values are: \"Normal\", \"Debug\", \"Trace\", \"TraceAll\". Defaults to \"Normal\".", + Type: []string{"string"}, + Format: "", + }, + }, + "operatorLogLevel": { + SchemaProps: spec.SchemaProps{ + Description: "operatorLogLevel is an intent based logging for the operator itself. It does not give fine grained control, but it is a simple way to manage coarse grained logging choices that operators have to interpret for themselves.\n\nValid values are: \"Normal\", \"Debug\", \"Trace\", \"TraceAll\". Defaults to \"Normal\".", + Type: []string{"string"}, + Format: "", + }, + }, + "unsupportedConfigOverrides": { + SchemaProps: spec.SchemaProps{ + Description: "unsupportedConfigOverrides overrides the final configuration that was computed by the operator. Red Hat does not support the use of this field. Misuse of this field could lead to unexpected behavior or conflict with other configuration options. Seek guidance from the Red Hat support before using this field. Use of this property blocks cluster upgrades, it must be removed before upgrading your cluster.", + Ref: ref("k8s.io/apimachinery/pkg/runtime.RawExtension"), + }, + }, + "observedConfig": { + SchemaProps: spec.SchemaProps{ + Description: "observedConfig holds a sparse config that controller has observed from the cluster state. It exists in spec because it is an input to the level for the operator", + Ref: ref("k8s.io/apimachinery/pkg/runtime.RawExtension"), + }, + }, + "config": { + SchemaProps: spec.SchemaProps{ + Description: "config that is persisted to a config map", + Default: map[string]interface{}{}, + Ref: ref("github.com/openshift/api/operator/v1alpha1.KueueConfiguration"), + }, + }, + }, + Required: []string{"managementState", "config"}, + }, + }, + Dependencies: []string{ + "github.com/openshift/api/operator/v1alpha1.KueueConfiguration", "k8s.io/apimachinery/pkg/runtime.RawExtension"}, + } +} + +func schema_openshift_api_operator_v1alpha1_KueueStatus(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "KueueStatus defines the observed state of Kueue", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "observedGeneration": { + SchemaProps: spec.SchemaProps{ + Description: "observedGeneration is the last generation change you've dealt with", + Type: []string{"integer"}, + Format: "int64", + }, + }, + "conditions": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-map-keys": []interface{}{ + "type", + }, + "x-kubernetes-list-type": "map", + }, + }, + SchemaProps: spec.SchemaProps{ + Description: "conditions is a list of conditions and their status", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("github.com/openshift/api/operator/v1.OperatorCondition"), + }, + }, + }, + }, + }, + "version": { + SchemaProps: spec.SchemaProps{ + Description: "version is the level this availability applies to", + Type: []string{"string"}, + Format: "", + }, + }, + "readyReplicas": { + SchemaProps: spec.SchemaProps{ + Description: "readyReplicas indicates how many replicas are ready and at the desired state", + Default: 0, + Type: []string{"integer"}, + Format: "int32", + }, + }, + "latestAvailableRevision": { + SchemaProps: spec.SchemaProps{ + Description: "latestAvailableRevision is the deploymentID of the most recent deployment", + Type: []string{"integer"}, + Format: "int32", + }, + }, + "generations": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-map-keys": []interface{}{ + "group", + "resource", + "namespace", + "name", + }, + "x-kubernetes-list-type": "map", + }, + }, + SchemaProps: spec.SchemaProps{ + Description: "generations are used to determine when an item needs to be reconciled or has changed in a way that needs a reaction.", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("github.com/openshift/api/operator/v1.GenerationStatus"), + }, + }, + }, + }, + }, + }, + Required: []string{"readyReplicas"}, + }, + }, + Dependencies: []string{ + "github.com/openshift/api/operator/v1.GenerationStatus", "github.com/openshift/api/operator/v1.OperatorCondition"}, + } +} + func schema_openshift_api_operator_v1alpha1_LoggingConfig(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ @@ -57103,6 +57537,39 @@ func schema_openshift_api_operator_v1alpha1_LoggingConfig(ref common.ReferenceCa } } +func schema_openshift_api_operator_v1alpha1_MultiKueue(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "gcInterval": { + SchemaProps: spec.SchemaProps{ + Description: "GCInterval defines the time interval between two consecutive garbage collection runs. Defaults to 1min. If 0, the garbage collection is disabled.", + Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.Duration"), + }, + }, + "origin": { + SchemaProps: spec.SchemaProps{ + Description: "Origin defines a label value used to track the creator of workloads in the worker clusters. This is used by multikueue in components like its garbage collector to identify remote objects that ware created by this multikueue manager cluster and delete them if their local counterpart no longer exists.", + Type: []string{"string"}, + Format: "", + }, + }, + "workerLostTimeout": { + SchemaProps: spec.SchemaProps{ + Description: "WorkerLostTimeout defines the time a local workload's multikueue admission check state is kept Ready if the connection with its reserving worker cluster is lost.\n\nDefaults to 15 minutes.", + Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.Duration"), + }, + }, + }, + }, + }, + Dependencies: []string{ + "k8s.io/apimachinery/pkg/apis/meta/v1.Duration"}, + } +} + func schema_openshift_api_operator_v1alpha1_NodeStatus(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ @@ -57607,6 +58074,133 @@ func schema_openshift_api_operator_v1alpha1_RepositoryDigestMirrors(ref common.R } } +func schema_openshift_api_operator_v1alpha1_RequeuingStrategy(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "timestamp": { + SchemaProps: spec.SchemaProps{ + Description: "Timestamp defines the timestamp used for re-queuing a Workload that was evicted due to Pod readiness. The possible values are:\n\n- `Eviction` (default) indicates from Workload `Evicted` condition with `PodsReadyTimeout` reason. - `Creation` indicates from Workload .metadata.creationTimestamp.", + Type: []string{"string"}, + Format: "", + }, + }, + "backoffLimitCount": { + SchemaProps: spec.SchemaProps{ + Description: "BackoffLimitCount defines the maximum number of re-queuing retries. Once the number is reached, the workload is deactivated (`.spec.activate`=`false`). When it is null, the workloads will repeatedly and endless re-queueing.\n\nEvery backoff duration is about \"b*2^(n-1)+Rand\" where: - \"b\" represents the base set by \"BackoffBaseSeconds\" parameter, - \"n\" represents the \"workloadStatus.requeueState.count\", - \"Rand\" represents the random jitter. During this time, the workload is taken as an inadmissible and other workloads will have a chance to be admitted. By default, the consecutive requeue delays are around: (60s, 120s, 240s, ...).\n\nDefaults to null.", + Type: []string{"integer"}, + Format: "int32", + }, + }, + "backoffBaseSeconds": { + SchemaProps: spec.SchemaProps{ + Description: "BackoffBaseSeconds defines the base for the exponential backoff for re-queuing an evicted workload.\n\nDefaults to 60.", + Type: []string{"integer"}, + Format: "int32", + }, + }, + "backoffMaxSeconds": { + SchemaProps: spec.SchemaProps{ + Description: "BackoffMaxSeconds defines the maximum backoff time to re-queue an evicted workload.\n\nDefaults to 3600.", + Type: []string{"integer"}, + Format: "int32", + }, + }, + }, + }, + }, + } +} + +func schema_openshift_api_operator_v1alpha1_ResourceTransformation(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "input": { + SchemaProps: spec.SchemaProps{ + Description: "Input is the name of the input resource.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "strategy": { + SchemaProps: spec.SchemaProps{ + Description: "Strategy specifies if the input resource should be replaced or retained. Defaults to Retain", + Type: []string{"string"}, + Format: "", + }, + }, + "outputs": { + SchemaProps: spec.SchemaProps{ + Description: "Outputs specifies the output resources and quantities per unit of input resource. An empty Outputs combined with a `Replace` Strategy causes the Input resource to be ignored by Kueue.", + Type: []string{"object"}, + AdditionalProperties: &spec.SchemaOrBool{ + Allows: true, + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Ref: ref("k8s.io/apimachinery/pkg/api/resource.Quantity"), + }, + }, + }, + }, + }, + }, + Required: []string{"input"}, + }, + }, + Dependencies: []string{ + "k8s.io/apimachinery/pkg/api/resource.Quantity"}, + } +} + +func schema_openshift_api_operator_v1alpha1_Resources(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "excludeResourcePrefixes": { + SchemaProps: spec.SchemaProps{ + Description: "ExcludedResourcePrefixes defines which resources should be ignored by Kueue", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + "transformations": { + SchemaProps: spec.SchemaProps{ + Description: "Transformations defines how to transform PodSpec resources into Workload resource requests. This is intended to be a map with Input as the key (enforced by validation code)", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: map[string]interface{}{}, + Ref: ref("github.com/openshift/api/operator/v1alpha1.ResourceTransformation"), + }, + }, + }, + }, + }, + }, + }, + }, + Dependencies: []string{ + "github.com/openshift/api/operator/v1alpha1.ResourceTransformation"}, + } +} + func schema_openshift_api_operator_v1alpha1_StaticPodOperatorStatus(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ @@ -57761,6 +58355,53 @@ func schema_openshift_api_operator_v1alpha1_VersionAvailability(ref common.Refer } } +func schema_openshift_api_operator_v1alpha1_WaitForPodsReady(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "WaitForPodsReady defines configuration for the Wait For Pods Ready feature, which is used to ensure that all Pods are ready within the specified time.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "enable": { + SchemaProps: spec.SchemaProps{ + Description: "Enable indicates whether to enable wait for pods ready feature. Defaults to false.", + Type: []string{"boolean"}, + Format: "", + }, + }, + "timeout": { + SchemaProps: spec.SchemaProps{ + Description: "Timeout defines the time for an admitted workload to reach the PodsReady=true condition. When the timeout is exceeded, the workload evicted and requeued in the same cluster queue. Defaults to 5min.", + Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.Duration"), + }, + }, + "blockAdmission": { + SchemaProps: spec.SchemaProps{ + Description: "BlockAdmission when true, cluster queue will block admissions for all subsequent jobs until the jobs reach the PodsReady=true condition. This setting is only honored when `Enable` is set to true.", + Type: []string{"boolean"}, + Format: "", + }, + }, + "requeuingStrategy": { + SchemaProps: spec.SchemaProps{ + Description: "RequeuingStrategy defines the strategy for requeuing a Workload.", + Ref: ref("github.com/openshift/api/operator/v1alpha1.RequeuingStrategy"), + }, + }, + "recoveryTimeout": { + SchemaProps: spec.SchemaProps{ + Description: "RecoveryTimeout defines an opt-in timeout, measured since the last transition to the PodsReady=false condition after a Workload is Admitted and running. Such a transition may happen when a Pod failed and the replacement Pod is awaited to be scheduled. After exceeding the timeout the corresponding job gets suspended again and requeued after the backoff delay. The timeout is enforced only if waitForPodsReady.enable=true. If not set, there is no timeout.", + Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.Duration"), + }, + }, + }, + }, + }, + Dependencies: []string{ + "github.com/openshift/api/operator/v1alpha1.RequeuingStrategy", "k8s.io/apimachinery/pkg/apis/meta/v1.Duration"}, + } +} + func schema_openshift_api_operatorcontrolplane_v1alpha1_LogEntry(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ diff --git a/openapi/openapi.json b/openapi/openapi.json index 8e81748664b..81ef39b3b9c 100644 --- a/openapi/openapi.json +++ b/openapi/openapi.json @@ -33206,6 +33206,27 @@ } } }, + "com.github.openshift.api.operator.v1alpha1.FairSharing": { + "type": "object", + "required": [ + "enable" + ], + "properties": { + "enable": { + "description": "enable indicates whether to enable fair sharing for all cohorts. Defaults to false.", + "type": "boolean", + "default": false + }, + "preemptionStrategies": { + "description": "preemptionStrategies indicates which constraints should a preemption satisfy. The preemption algorithm will only use the next strategy in the list if the incoming workload (preemptor) doesn't fit after using the previous strategies. Possible values are: - LessThanOrEqualToFinalShare: Only preempt a workload if the share of the preemptor CQ\n with the preemptor workload is less than or equal to the share of the preemptee CQ\n without the workload to be preempted.\n This strategy might favor preemption of smaller workloads in the preemptee CQ,\n regardless of priority or start time, in an effort to keep the share of the CQ\n as high as possible.\n- LessThanInitialShare: Only preempt a workload if the share of the preemptor CQ\n with the incoming workload is strictly less than the share of the preemptee CQ.\n This strategy doesn't depend on the share usage of the workload being preempted.\n As a result, the strategy chooses to preempt workloads with the lowest priority and\n newest start time first.\nThe default strategy is [\"LessThanOrEqualToFinalShare\", \"LessThanInitialShare\"].", + "type": "array", + "items": { + "type": "string", + "default": "" + } + } + } + }, "com.github.openshift.api.operator.v1alpha1.GenerationHistory": { "description": "GenerationHistory keeps track of the generation for a given resource so that decisions about forced updated can be made. DEPRECATED: Use fields in v1.GenerationStatus instead", "type": "object", @@ -33350,6 +33371,236 @@ } } }, + "com.github.openshift.api.operator.v1alpha1.Integrations": { + "type": "object", + "properties": { + "externalFrameworks": { + "description": "List of GroupVersionKinds that are managed for Kueue by external controllers; the expected format is `Kind.version.group.com`.", + "type": "array", + "items": { + "type": "string", + "default": "" + } + }, + "frameworks": { + "description": "List of framework names to be enabled. Possible options:\n - \"batch/job\"\n - \"kubeflow.org/mpijob\"\n - \"ray.io/rayjob\"\n - \"ray.io/raycluster\"\n - \"jobset.x-k8s.io/jobset\"\n - \"kubeflow.org/paddlejob\"\n - \"kubeflow.org/pytorchjob\"\n - \"kubeflow.org/tfjob\"\n - \"kubeflow.org/xgboostjob\"\n - \"workload.codeflare.dev/appwrapper\"\n - \"pod\"\n - \"deployment\" (requires enabling pod integration)\n - \"statefulset\" (requires enabling pod integration)\n - \"leaderworkerset.x-k8s.io/leaderworkerset\" (requires enabling pod integration)", + "type": "array", + "items": { + "type": "string", + "default": "" + } + }, + "labelKeysToCopy": { + "description": "labelKeysToCopy is a list of label keys that should be copied from the job into the workload object. It is not required for the job to have all the labels from this list. If a job does not have some label with the given key from this list, the constructed workload object will be created without this label. In the case of creating a workload from a composable job (pod group), if multiple objects have labels with some key from the list, the values of these labels must match or otherwise the workload creation would fail. The labels are copied only during the workload creation and are not updated even if the labels of the underlying job are changed.", + "type": "array", + "items": { + "type": "string", + "default": "" + } + } + } + }, + "com.github.openshift.api.operator.v1alpha1.Kueue": { + "description": "Kueue is the Schema for the kueue API Compatibility level 4: No compatibility is provided, the API can change at any point for any reason. These capabilities should not be used by applications needing long term support.", + "type": "object", + "required": [ + "spec" + ], + "properties": { + "apiVersion": { + "description": "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + "type": "string" + }, + "kind": { + "description": "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + "type": "string" + }, + "metadata": { + "description": "metadata for kueue", + "default": {}, + "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta" + }, + "spec": { + "description": "spec holds user settable values for configuration", + "default": {}, + "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.KueueOperandSpec" + }, + "status": { + "description": "status holds observed values from the cluster. They may not be overridden.", + "default": {}, + "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.KueueStatus" + } + } + }, + "com.github.openshift.api.operator.v1alpha1.KueueConfiguration": { + "type": "object", + "required": [ + "integrations" + ], + "properties": { + "disableMetrics": { + "description": "Disable Metrics Microshift does not enable metrics by default Default will assume metrics are enabled.", + "type": "boolean" + }, + "fairSharing": { + "description": "FairSharing controls the fair sharing semantics across the cluster.", + "default": {}, + "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.FairSharing" + }, + "featureGates": { + "description": "featureGates are advanced features for Kueue", + "type": "object", + "additionalProperties": { + "type": "boolean", + "default": false + } + }, + "integrations": { + "description": "integrations are the types of integrations Kueue will manager", + "default": {}, + "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.Integrations" + }, + "manageJobsWithoutQueueName": { + "description": "ManageJobsWithoutQueueName controls whether or not Kueue reconciles jobs that don't set the annotation kueue.x-k8s.io/queue-name. Allowed values are NoQueueName and QueueName Default will be QueueName", + "type": "string" + }, + "managedJobsNamespaceSelector": { + "description": "ManagedJobsNamespaceSelector can be used to omit some namespaces from ManagedJobsWithoutQueueName", + "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.LabelSelector" + }, + "resources": { + "description": "resources provides additional configuration options for handling the resources. Supports https://github.com/kubernetes-sigs/kueue/blob/release-0.10/keps/2937-resource-transformer/README.md", + "default": {}, + "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.Resources" + }, + "waitForPodsReady": { + "description": "waitForPodsReady configures gang admission", + "default": {}, + "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.WaitForPodsReady" + } + } + }, + "com.github.openshift.api.operator.v1alpha1.KueueList": { + "description": "KueueList contains a list of Kueue\n\nCompatibility level 4: No compatibility is provided, the API can change at any point for any reason. These capabilities should not be used by applications needing long term support.", + "type": "object", + "required": [ + "items" + ], + "properties": { + "apiVersion": { + "description": "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources", + "type": "string" + }, + "items": { + "description": "items is a slice of kueue", + "type": "array", + "items": { + "default": {}, + "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.Kueue" + } + }, + "kind": { + "description": "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds", + "type": "string" + }, + "metadata": { + "description": "metadata for the list", + "default": {}, + "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ListMeta" + } + } + }, + "com.github.openshift.api.operator.v1alpha1.KueueOperandSpec": { + "type": "object", + "required": [ + "managementState", + "config" + ], + "properties": { + "config": { + "description": "config that is persisted to a config map", + "default": {}, + "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.KueueConfiguration" + }, + "logLevel": { + "description": "logLevel is an intent based logging for an overall component. It does not give fine grained control, but it is a simple way to manage coarse grained logging choices that operators have to interpret for their operands.\n\nValid values are: \"Normal\", \"Debug\", \"Trace\", \"TraceAll\". Defaults to \"Normal\".", + "type": "string" + }, + "managementState": { + "description": "managementState indicates whether and how the operator should manage the component", + "type": "string", + "default": "" + }, + "observedConfig": { + "description": "observedConfig holds a sparse config that controller has observed from the cluster state. It exists in spec because it is an input to the level for the operator", + "$ref": "#/definitions/io.k8s.apimachinery.pkg.runtime.RawExtension" + }, + "operatorLogLevel": { + "description": "operatorLogLevel is an intent based logging for the operator itself. It does not give fine grained control, but it is a simple way to manage coarse grained logging choices that operators have to interpret for themselves.\n\nValid values are: \"Normal\", \"Debug\", \"Trace\", \"TraceAll\". Defaults to \"Normal\".", + "type": "string" + }, + "unsupportedConfigOverrides": { + "description": "unsupportedConfigOverrides overrides the final configuration that was computed by the operator. Red Hat does not support the use of this field. Misuse of this field could lead to unexpected behavior or conflict with other configuration options. Seek guidance from the Red Hat support before using this field. Use of this property blocks cluster upgrades, it must be removed before upgrading your cluster.", + "$ref": "#/definitions/io.k8s.apimachinery.pkg.runtime.RawExtension" + } + } + }, + "com.github.openshift.api.operator.v1alpha1.KueueStatus": { + "description": "KueueStatus defines the observed state of Kueue", + "type": "object", + "required": [ + "readyReplicas" + ], + "properties": { + "conditions": { + "description": "conditions is a list of conditions and their status", + "type": "array", + "items": { + "default": {}, + "$ref": "#/definitions/com.github.openshift.api.operator.v1.OperatorCondition" + }, + "x-kubernetes-list-map-keys": [ + "type" + ], + "x-kubernetes-list-type": "map" + }, + "generations": { + "description": "generations are used to determine when an item needs to be reconciled or has changed in a way that needs a reaction.", + "type": "array", + "items": { + "default": {}, + "$ref": "#/definitions/com.github.openshift.api.operator.v1.GenerationStatus" + }, + "x-kubernetes-list-map-keys": [ + "group", + "resource", + "namespace", + "name" + ], + "x-kubernetes-list-type": "map" + }, + "latestAvailableRevision": { + "description": "latestAvailableRevision is the deploymentID of the most recent deployment", + "type": "integer", + "format": "int32" + }, + "observedGeneration": { + "description": "observedGeneration is the last generation change you've dealt with", + "type": "integer", + "format": "int64" + }, + "readyReplicas": { + "description": "readyReplicas indicates how many replicas are ready and at the desired state", + "type": "integer", + "format": "int32", + "default": 0 + }, + "version": { + "description": "version is the level this availability applies to", + "type": "string" + } + } + }, "com.github.openshift.api.operator.v1alpha1.LoggingConfig": { "description": "LoggingConfig holds information about configuring logging DEPRECATED: Use v1.LogLevel instead", "type": "object", @@ -33371,6 +33622,23 @@ } } }, + "com.github.openshift.api.operator.v1alpha1.MultiKueue": { + "type": "object", + "properties": { + "gcInterval": { + "description": "GCInterval defines the time interval between two consecutive garbage collection runs. Defaults to 1min. If 0, the garbage collection is disabled.", + "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.Duration" + }, + "origin": { + "description": "Origin defines a label value used to track the creator of workloads in the worker clusters. This is used by multikueue in components like its garbage collector to identify remote objects that ware created by this multikueue manager cluster and delete them if their local counterpart no longer exists.", + "type": "string" + }, + "workerLostTimeout": { + "description": "WorkerLostTimeout defines the time a local workload's multikueue admission check state is kept Ready if the connection with its reserving worker cluster is lost.\n\nDefaults to 15 minutes.", + "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.Duration" + } + } + }, "com.github.openshift.api.operator.v1alpha1.NodeStatus": { "description": "NodeStatus provides information about the current state of a particular node managed by this operator. Deprecated: Use v1.NodeStatus instead", "type": "object", @@ -33684,6 +33952,75 @@ } } }, + "com.github.openshift.api.operator.v1alpha1.RequeuingStrategy": { + "type": "object", + "properties": { + "backoffBaseSeconds": { + "description": "BackoffBaseSeconds defines the base for the exponential backoff for re-queuing an evicted workload.\n\nDefaults to 60.", + "type": "integer", + "format": "int32" + }, + "backoffLimitCount": { + "description": "BackoffLimitCount defines the maximum number of re-queuing retries. Once the number is reached, the workload is deactivated (`.spec.activate`=`false`). When it is null, the workloads will repeatedly and endless re-queueing.\n\nEvery backoff duration is about \"b*2^(n-1)+Rand\" where: - \"b\" represents the base set by \"BackoffBaseSeconds\" parameter, - \"n\" represents the \"workloadStatus.requeueState.count\", - \"Rand\" represents the random jitter. During this time, the workload is taken as an inadmissible and other workloads will have a chance to be admitted. By default, the consecutive requeue delays are around: (60s, 120s, 240s, ...).\n\nDefaults to null.", + "type": "integer", + "format": "int32" + }, + "backoffMaxSeconds": { + "description": "BackoffMaxSeconds defines the maximum backoff time to re-queue an evicted workload.\n\nDefaults to 3600.", + "type": "integer", + "format": "int32" + }, + "timestamp": { + "description": "Timestamp defines the timestamp used for re-queuing a Workload that was evicted due to Pod readiness. The possible values are:\n\n- `Eviction` (default) indicates from Workload `Evicted` condition with `PodsReadyTimeout` reason. - `Creation` indicates from Workload .metadata.creationTimestamp.", + "type": "string" + } + } + }, + "com.github.openshift.api.operator.v1alpha1.ResourceTransformation": { + "type": "object", + "required": [ + "input" + ], + "properties": { + "input": { + "description": "Input is the name of the input resource.", + "type": "string", + "default": "" + }, + "outputs": { + "description": "Outputs specifies the output resources and quantities per unit of input resource. An empty Outputs combined with a `Replace` Strategy causes the Input resource to be ignored by Kueue.", + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/io.k8s.apimachinery.pkg.api.resource.Quantity" + } + }, + "strategy": { + "description": "Strategy specifies if the input resource should be replaced or retained. Defaults to Retain", + "type": "string" + } + } + }, + "com.github.openshift.api.operator.v1alpha1.Resources": { + "type": "object", + "properties": { + "excludeResourcePrefixes": { + "description": "ExcludedResourcePrefixes defines which resources should be ignored by Kueue", + "type": "array", + "items": { + "type": "string", + "default": "" + } + }, + "transformations": { + "description": "Transformations defines how to transform PodSpec resources into Workload resource requests. This is intended to be a map with Input as the key (enforced by validation code)", + "type": "array", + "items": { + "default": {}, + "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.ResourceTransformation" + } + } + } + }, "com.github.openshift.api.operator.v1alpha1.StaticPodOperatorStatus": { "description": "StaticPodOperatorStatus is status for controllers that manage static pods. There are different needs because individual node status must be tracked. DEPRECATED: Use v1.StaticPodOperatorStatus instead", "type": "object", @@ -33783,6 +34120,32 @@ } } }, + "com.github.openshift.api.operator.v1alpha1.WaitForPodsReady": { + "description": "WaitForPodsReady defines configuration for the Wait For Pods Ready feature, which is used to ensure that all Pods are ready within the specified time.", + "type": "object", + "properties": { + "blockAdmission": { + "description": "BlockAdmission when true, cluster queue will block admissions for all subsequent jobs until the jobs reach the PodsReady=true condition. This setting is only honored when `Enable` is set to true.", + "type": "boolean" + }, + "enable": { + "description": "Enable indicates whether to enable wait for pods ready feature. Defaults to false.", + "type": "boolean" + }, + "recoveryTimeout": { + "description": "RecoveryTimeout defines an opt-in timeout, measured since the last transition to the PodsReady=false condition after a Workload is Admitted and running. Such a transition may happen when a Pod failed and the replacement Pod is awaited to be scheduled. After exceeding the timeout the corresponding job gets suspended again and requeued after the backoff delay. The timeout is enforced only if waitForPodsReady.enable=true. If not set, there is no timeout.", + "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.Duration" + }, + "requeuingStrategy": { + "description": "RequeuingStrategy defines the strategy for requeuing a Workload.", + "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.RequeuingStrategy" + }, + "timeout": { + "description": "Timeout defines the time for an admitted workload to reach the PodsReady=true condition. When the timeout is exceeded, the workload evicted and requeued in the same cluster queue. Defaults to 5min.", + "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.Duration" + } + } + }, "com.github.openshift.api.operatorcontrolplane.v1alpha1.LogEntry": { "description": "LogEntry records events", "type": "object", diff --git a/operator/v1alpha1/register.go b/operator/v1alpha1/register.go index 3e9b09ccede..689786f2dbb 100644 --- a/operator/v1alpha1/register.go +++ b/operator/v1alpha1/register.go @@ -41,6 +41,8 @@ func addKnownTypes(scheme *runtime.Scheme) error { &EtcdBackupList{}, &ClusterVersionOperator{}, &ClusterVersionOperatorList{}, + &Kueue{}, + &KueueList{}, ) return nil diff --git a/operator/v1alpha1/types_kueue.go b/operator/v1alpha1/types_kueue.go new file mode 100644 index 00000000000..07602ec5215 --- /dev/null +++ b/operator/v1alpha1/types_kueue.go @@ -0,0 +1,299 @@ +package v1alpha1 + +import ( + operatorv1 "github.com/openshift/api/operator/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// Kueue is the Schema for the kueue API +// Compatibility level 4: No compatibility is provided, the API can change at any point for any reason. These capabilities should not be used by applications needing long term support. +// +openshift:compatibility-gen:level=4 +// +k8s:openapi-gen=true +// +genclient +// +kubebuilder:storageversion +// +kubebuilder:subresource:status +type Kueue struct { + metav1.TypeMeta `json:",inline"` + // metadata for kueue + // +optional + metav1.ObjectMeta `json:"metadata,omitempty"` + + // spec holds user settable values for configuration + // +required + Spec KueueOperandSpec `json:"spec"` + // status holds observed values from the cluster. They may not be overridden. + // +optional + Status KueueStatus `json:"status,omitempty"` +} + +type KueueOperandSpec struct { + operatorv1.OperatorSpec `json:",inline"` + // config that is persisted to a config map + // +required + Config KueueConfiguration `json:"config"` +} + +type ManageJobsWithoutQueueNameOption string + +const ( + // NoQueueName means that all jobs will be gated by Kueue + NoQueueName ManageJobsWithoutQueueNameOption = "NoQueueName" + // QueueName means that the jobs require a queue label. + QueueName ManageJobsWithoutQueueNameOption = "QueueName" +) + +type KueueConfiguration struct { + // waitForPodsReady configures gang admission + // +optional + WaitForPodsReady WaitForPodsReady `json:"waitForPodsReady,omitempty"` + // integrations are the types of integrations Kueue will manager + // +required + Integrations Integrations `json:"integrations"` + // featureGates are advanced features for Kueue + // +optional + FeatureGates map[string]bool `json:"featureGates,omitempty"` + // resources provides additional configuration options for handling the resources. + // Supports https://github.com/kubernetes-sigs/kueue/blob/release-0.10/keps/2937-resource-transformer/README.md + // +optional + Resources Resources `json:"resources,omitempty"` + // ManageJobsWithoutQueueName controls whether or not Kueue reconciles + // jobs that don't set the annotation kueue.x-k8s.io/queue-name. + // Allowed values are NoQueueName and QueueName + // Default will be QueueName + // +optional + ManageJobsWithoutQueueName *ManageJobsWithoutQueueNameOption `json:"manageJobsWithoutQueueName,omitempty"` + // ManagedJobsNamespaceSelector can be used to omit some namespaces from ManagedJobsWithoutQueueName + // +optional + ManagedJobsNamespaceSelector *metav1.LabelSelector `json:"managedJobsNamespaceSelector,omitempty"` + // FairSharing controls the fair sharing semantics across the cluster. + FairSharing FairSharing `json:"fairSharing,omitempty"` + // Disable Metrics + // Microshift does not enable metrics by default + // Default will assume metrics are enabled. + // +optional + DisableMetrics *bool `json:"disableMetrics,omitempty"` +} + +// KueueStatus defines the observed state of Kueue +type KueueStatus struct { + operatorv1.OperatorStatus `json:",inline"` +} + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +// KueueList contains a list of Kueue +// +// Compatibility level 4: No compatibility is provided, the API can change at any point for any reason. These capabilities should not be used by applications needing long term support. +// +openshift:compatibility-gen:level=4 +type KueueList struct { + metav1.TypeMeta `json:",inline"` + // metadata for the list + // +optional + metav1.ListMeta `json:"metadata,omitempty"` + // items is a slice of kueue + // +required + Items []Kueue `json:"items"` +} + +// These structs come directly from Kueue. + +type Resources struct { + // ExcludedResourcePrefixes defines which resources should be ignored by Kueue + ExcludeResourcePrefixes []string `json:"excludeResourcePrefixes,omitempty"` + + // Transformations defines how to transform PodSpec resources into Workload resource requests. + // This is intended to be a map with Input as the key (enforced by validation code) + Transformations []ResourceTransformation `json:"transformations,omitempty"` +} + +type ResourceTransformationStrategy string + +const Retain ResourceTransformationStrategy = "Retain" +const Replace ResourceTransformationStrategy = "Replace" + +type ResourceTransformation struct { + // Input is the name of the input resource. + Input corev1.ResourceName `json:"input"` + + // Strategy specifies if the input resource should be replaced or retained. + // Defaults to Retain + Strategy *ResourceTransformationStrategy `json:"strategy,omitempty"` + + // Outputs specifies the output resources and quantities per unit of input resource. + // An empty Outputs combined with a `Replace` Strategy causes the Input resource to be ignored by Kueue. + Outputs corev1.ResourceList `json:"outputs,omitempty"` +} + +type PreemptionStrategy string + +const ( + LessThanOrEqualToFinalShare PreemptionStrategy = "LessThanOrEqualToFinalShare" + LessThanInitialShare PreemptionStrategy = "LessThanInitialShare" +) + +type FairSharing struct { + // enable indicates whether to enable fair sharing for all cohorts. + // Defaults to false. + Enable bool `json:"enable"` + + // preemptionStrategies indicates which constraints should a preemption satisfy. + // The preemption algorithm will only use the next strategy in the list if the + // incoming workload (preemptor) doesn't fit after using the previous strategies. + // Possible values are: + // - LessThanOrEqualToFinalShare: Only preempt a workload if the share of the preemptor CQ + // with the preemptor workload is less than or equal to the share of the preemptee CQ + // without the workload to be preempted. + // This strategy might favor preemption of smaller workloads in the preemptee CQ, + // regardless of priority or start time, in an effort to keep the share of the CQ + // as high as possible. + // - LessThanInitialShare: Only preempt a workload if the share of the preemptor CQ + // with the incoming workload is strictly less than the share of the preemptee CQ. + // This strategy doesn't depend on the share usage of the workload being preempted. + // As a result, the strategy chooses to preempt workloads with the lowest priority and + // newest start time first. + // The default strategy is ["LessThanOrEqualToFinalShare", "LessThanInitialShare"]. + PreemptionStrategies []PreemptionStrategy `json:"preemptionStrategies,omitempty"` +} + +// WaitForPodsReady defines configuration for the Wait For Pods Ready feature, +// which is used to ensure that all Pods are ready within the specified time. +type WaitForPodsReady struct { + // Enable indicates whether to enable wait for pods ready feature. + // Defaults to false. + Enable bool `json:"enable,omitempty"` + + // Timeout defines the time for an admitted workload to reach the + // PodsReady=true condition. When the timeout is exceeded, the workload + // evicted and requeued in the same cluster queue. + // Defaults to 5min. + // +optional + Timeout *metav1.Duration `json:"timeout,omitempty"` + + // BlockAdmission when true, cluster queue will block admissions for all + // subsequent jobs until the jobs reach the PodsReady=true condition. + // This setting is only honored when `Enable` is set to true. + BlockAdmission *bool `json:"blockAdmission,omitempty"` + + // RequeuingStrategy defines the strategy for requeuing a Workload. + // +optional + RequeuingStrategy *RequeuingStrategy `json:"requeuingStrategy,omitempty"` + + // RecoveryTimeout defines an opt-in timeout, measured since the + // last transition to the PodsReady=false condition after a Workload is Admitted and running. + // Such a transition may happen when a Pod failed and the replacement Pod + // is awaited to be scheduled. + // After exceeding the timeout the corresponding job gets suspended again + // and requeued after the backoff delay. The timeout is enforced only if waitForPodsReady.enable=true. + // If not set, there is no timeout. + // +optional + RecoveryTimeout *metav1.Duration `json:"recoveryTimeout,omitempty"` +} + +type MultiKueue struct { + // GCInterval defines the time interval between two consecutive garbage collection runs. + // Defaults to 1min. If 0, the garbage collection is disabled. + // +optional + GCInterval *metav1.Duration `json:"gcInterval"` + + // Origin defines a label value used to track the creator of workloads in the worker + // clusters. + // This is used by multikueue in components like its garbage collector to identify + // remote objects that ware created by this multikueue manager cluster and delete + // them if their local counterpart no longer exists. + // +optional + Origin *string `json:"origin,omitempty"` + + // WorkerLostTimeout defines the time a local workload's multikueue admission check state is kept Ready + // if the connection with its reserving worker cluster is lost. + // + // Defaults to 15 minutes. + // +optional + WorkerLostTimeout *metav1.Duration `json:"workerLostTimeout,omitempty"` +} + +type RequeuingStrategy struct { + // Timestamp defines the timestamp used for re-queuing a Workload + // that was evicted due to Pod readiness. The possible values are: + // + // - `Eviction` (default) indicates from Workload `Evicted` condition with `PodsReadyTimeout` reason. + // - `Creation` indicates from Workload .metadata.creationTimestamp. + // + // +optional + Timestamp *RequeuingTimestamp `json:"timestamp,omitempty"` + + // BackoffLimitCount defines the maximum number of re-queuing retries. + // Once the number is reached, the workload is deactivated (`.spec.activate`=`false`). + // When it is null, the workloads will repeatedly and endless re-queueing. + // + // Every backoff duration is about "b*2^(n-1)+Rand" where: + // - "b" represents the base set by "BackoffBaseSeconds" parameter, + // - "n" represents the "workloadStatus.requeueState.count", + // - "Rand" represents the random jitter. + // During this time, the workload is taken as an inadmissible and + // other workloads will have a chance to be admitted. + // By default, the consecutive requeue delays are around: (60s, 120s, 240s, ...). + // + // Defaults to null. + // +optional + BackoffLimitCount *int32 `json:"backoffLimitCount,omitempty"` + + // BackoffBaseSeconds defines the base for the exponential backoff for + // re-queuing an evicted workload. + // + // Defaults to 60. + // +optional + BackoffBaseSeconds *int32 `json:"backoffBaseSeconds,omitempty"` + + // BackoffMaxSeconds defines the maximum backoff time to re-queue an evicted workload. + // + // Defaults to 3600. + // +optional + BackoffMaxSeconds *int32 `json:"backoffMaxSeconds,omitempty"` +} + +type RequeuingTimestamp string + +const ( + // CreationTimestamp timestamp (from Workload .metadata.creationTimestamp). + CreationTimestamp RequeuingTimestamp = "Creation" + + // EvictionTimestamp timestamp (from Workload .status.conditions). + EvictionTimestamp RequeuingTimestamp = "Eviction" +) + +type Integrations struct { + // List of framework names to be enabled. + // Possible options: + // - "batch/job" + // - "kubeflow.org/mpijob" + // - "ray.io/rayjob" + // - "ray.io/raycluster" + // - "jobset.x-k8s.io/jobset" + // - "kubeflow.org/paddlejob" + // - "kubeflow.org/pytorchjob" + // - "kubeflow.org/tfjob" + // - "kubeflow.org/xgboostjob" + // - "workload.codeflare.dev/appwrapper" + // - "pod" + // - "deployment" (requires enabling pod integration) + // - "statefulset" (requires enabling pod integration) + // - "leaderworkerset.x-k8s.io/leaderworkerset" (requires enabling pod integration) + Frameworks []string `json:"frameworks,omitempty"` + // List of GroupVersionKinds that are managed for Kueue by external controllers; + // the expected format is `Kind.version.group.com`. + ExternalFrameworks []string `json:"externalFrameworks,omitempty"` + + // labelKeysToCopy is a list of label keys that should be copied from the job into the + // workload object. It is not required for the job to have all the labels from this + // list. If a job does not have some label with the given key from this list, the + // constructed workload object will be created without this label. In the case + // of creating a workload from a composable job (pod group), if multiple objects + // have labels with some key from the list, the values of these labels must + // match or otherwise the workload creation would fail. The labels are copied only + // during the workload creation and are not updated even if the labels of the + // underlying job are changed. + LabelKeysToCopy []string `json:"labelKeysToCopy,omitempty"` +} diff --git a/operator/v1alpha1/zz_generated.deepcopy.go b/operator/v1alpha1/zz_generated.deepcopy.go index f8daa0576b1..16b227da938 100644 --- a/operator/v1alpha1/zz_generated.deepcopy.go +++ b/operator/v1alpha1/zz_generated.deepcopy.go @@ -6,6 +6,7 @@ package v1alpha1 import ( + corev1 "k8s.io/api/core/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -256,6 +257,27 @@ func (in *EtcdBackupStatus) DeepCopy() *EtcdBackupStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FairSharing) DeepCopyInto(out *FairSharing) { + *out = *in + if in.PreemptionStrategies != nil { + in, out := &in.PreemptionStrategies, &out.PreemptionStrategies + *out = make([]PreemptionStrategy, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FairSharing. +func (in *FairSharing) DeepCopy() *FairSharing { + if in == nil { + return nil + } + out := new(FairSharing) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GenerationHistory) DeepCopyInto(out *GenerationHistory) { *out = *in @@ -384,6 +406,175 @@ func (in *ImageContentSourcePolicySpec) DeepCopy() *ImageContentSourcePolicySpec return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Integrations) DeepCopyInto(out *Integrations) { + *out = *in + if in.Frameworks != nil { + in, out := &in.Frameworks, &out.Frameworks + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.ExternalFrameworks != nil { + in, out := &in.ExternalFrameworks, &out.ExternalFrameworks + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.LabelKeysToCopy != nil { + in, out := &in.LabelKeysToCopy, &out.LabelKeysToCopy + *out = make([]string, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Integrations. +func (in *Integrations) DeepCopy() *Integrations { + if in == nil { + return nil + } + out := new(Integrations) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Kueue) DeepCopyInto(out *Kueue) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + in.Status.DeepCopyInto(&out.Status) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Kueue. +func (in *Kueue) DeepCopy() *Kueue { + if in == nil { + return nil + } + out := new(Kueue) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *Kueue) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KueueConfiguration) DeepCopyInto(out *KueueConfiguration) { + *out = *in + in.WaitForPodsReady.DeepCopyInto(&out.WaitForPodsReady) + in.Integrations.DeepCopyInto(&out.Integrations) + if in.FeatureGates != nil { + in, out := &in.FeatureGates, &out.FeatureGates + *out = make(map[string]bool, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + in.Resources.DeepCopyInto(&out.Resources) + if in.ManageJobsWithoutQueueName != nil { + in, out := &in.ManageJobsWithoutQueueName, &out.ManageJobsWithoutQueueName + *out = new(ManageJobsWithoutQueueNameOption) + **out = **in + } + if in.ManagedJobsNamespaceSelector != nil { + in, out := &in.ManagedJobsNamespaceSelector, &out.ManagedJobsNamespaceSelector + *out = new(v1.LabelSelector) + (*in).DeepCopyInto(*out) + } + in.FairSharing.DeepCopyInto(&out.FairSharing) + if in.DisableMetrics != nil { + in, out := &in.DisableMetrics, &out.DisableMetrics + *out = new(bool) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KueueConfiguration. +func (in *KueueConfiguration) DeepCopy() *KueueConfiguration { + if in == nil { + return nil + } + out := new(KueueConfiguration) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KueueList) DeepCopyInto(out *KueueList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]Kueue, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KueueList. +func (in *KueueList) DeepCopy() *KueueList { + if in == nil { + return nil + } + out := new(KueueList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *KueueList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KueueOperandSpec) DeepCopyInto(out *KueueOperandSpec) { + *out = *in + in.OperatorSpec.DeepCopyInto(&out.OperatorSpec) + in.Config.DeepCopyInto(&out.Config) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KueueOperandSpec. +func (in *KueueOperandSpec) DeepCopy() *KueueOperandSpec { + if in == nil { + return nil + } + out := new(KueueOperandSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KueueStatus) DeepCopyInto(out *KueueStatus) { + *out = *in + in.OperatorStatus.DeepCopyInto(&out.OperatorStatus) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KueueStatus. +func (in *KueueStatus) DeepCopy() *KueueStatus { + if in == nil { + return nil + } + out := new(KueueStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LoggingConfig) DeepCopyInto(out *LoggingConfig) { *out = *in @@ -400,6 +591,37 @@ func (in *LoggingConfig) DeepCopy() *LoggingConfig { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MultiKueue) DeepCopyInto(out *MultiKueue) { + *out = *in + if in.GCInterval != nil { + in, out := &in.GCInterval, &out.GCInterval + *out = new(v1.Duration) + **out = **in + } + if in.Origin != nil { + in, out := &in.Origin, &out.Origin + *out = new(string) + **out = **in + } + if in.WorkerLostTimeout != nil { + in, out := &in.WorkerLostTimeout, &out.WorkerLostTimeout + *out = new(v1.Duration) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MultiKueue. +func (in *MultiKueue) DeepCopy() *MultiKueue { + if in == nil { + return nil + } + out := new(MultiKueue) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NodeStatus) DeepCopyInto(out *NodeStatus) { *out = *in @@ -604,6 +826,98 @@ func (in *RepositoryDigestMirrors) DeepCopy() *RepositoryDigestMirrors { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RequeuingStrategy) DeepCopyInto(out *RequeuingStrategy) { + *out = *in + if in.Timestamp != nil { + in, out := &in.Timestamp, &out.Timestamp + *out = new(RequeuingTimestamp) + **out = **in + } + if in.BackoffLimitCount != nil { + in, out := &in.BackoffLimitCount, &out.BackoffLimitCount + *out = new(int32) + **out = **in + } + if in.BackoffBaseSeconds != nil { + in, out := &in.BackoffBaseSeconds, &out.BackoffBaseSeconds + *out = new(int32) + **out = **in + } + if in.BackoffMaxSeconds != nil { + in, out := &in.BackoffMaxSeconds, &out.BackoffMaxSeconds + *out = new(int32) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RequeuingStrategy. +func (in *RequeuingStrategy) DeepCopy() *RequeuingStrategy { + if in == nil { + return nil + } + out := new(RequeuingStrategy) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ResourceTransformation) DeepCopyInto(out *ResourceTransformation) { + *out = *in + if in.Strategy != nil { + in, out := &in.Strategy, &out.Strategy + *out = new(ResourceTransformationStrategy) + **out = **in + } + if in.Outputs != nil { + in, out := &in.Outputs, &out.Outputs + *out = make(corev1.ResourceList, len(*in)) + for key, val := range *in { + (*out)[key] = val.DeepCopy() + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResourceTransformation. +func (in *ResourceTransformation) DeepCopy() *ResourceTransformation { + if in == nil { + return nil + } + out := new(ResourceTransformation) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Resources) DeepCopyInto(out *Resources) { + *out = *in + if in.ExcludeResourcePrefixes != nil { + in, out := &in.ExcludeResourcePrefixes, &out.ExcludeResourcePrefixes + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.Transformations != nil { + in, out := &in.Transformations, &out.Transformations + *out = make([]ResourceTransformation, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Resources. +func (in *Resources) DeepCopy() *Resources { + if in == nil { + return nil + } + out := new(Resources) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *StaticPodOperatorStatus) DeepCopyInto(out *StaticPodOperatorStatus) { *out = *in @@ -653,3 +967,39 @@ func (in *VersionAvailability) DeepCopy() *VersionAvailability { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *WaitForPodsReady) DeepCopyInto(out *WaitForPodsReady) { + *out = *in + if in.Timeout != nil { + in, out := &in.Timeout, &out.Timeout + *out = new(v1.Duration) + **out = **in + } + if in.BlockAdmission != nil { + in, out := &in.BlockAdmission, &out.BlockAdmission + *out = new(bool) + **out = **in + } + if in.RequeuingStrategy != nil { + in, out := &in.RequeuingStrategy, &out.RequeuingStrategy + *out = new(RequeuingStrategy) + (*in).DeepCopyInto(*out) + } + if in.RecoveryTimeout != nil { + in, out := &in.RecoveryTimeout, &out.RecoveryTimeout + *out = new(v1.Duration) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WaitForPodsReady. +func (in *WaitForPodsReady) DeepCopy() *WaitForPodsReady { + if in == nil { + return nil + } + out := new(WaitForPodsReady) + in.DeepCopyInto(out) + return out +} From 1ae6862fc288dde3fb794d8b78e9122df7ac2a11 Mon Sep 17 00:00:00 2001 From: Kevin Hannon Date: Fri, 7 Mar 2025 13:56:23 -0500 Subject: [PATCH 02/42] kueue api review --- .../zz_generated.swagger_doc_generated.go | 124 ++++++++++++++++++ 1 file changed, 124 insertions(+) diff --git a/operator/v1alpha1/zz_generated.swagger_doc_generated.go b/operator/v1alpha1/zz_generated.swagger_doc_generated.go index 9060bf99815..2d223e1051b 100644 --- a/operator/v1alpha1/zz_generated.swagger_doc_generated.go +++ b/operator/v1alpha1/zz_generated.swagger_doc_generated.go @@ -257,6 +257,130 @@ func (RepositoryDigestMirrors) SwaggerDoc() map[string]string { return map_RepositoryDigestMirrors } +var map_FairSharing = map[string]string{ + "enable": "enable indicates whether to enable fair sharing for all cohorts. Defaults to false.", + "preemptionStrategies": "preemptionStrategies indicates which constraints should a preemption satisfy. The preemption algorithm will only use the next strategy in the list if the incoming workload (preemptor) doesn't fit after using the previous strategies. Possible values are: - LessThanOrEqualToFinalShare: Only preempt a workload if the share of the preemptor CQ\n with the preemptor workload is less than or equal to the share of the preemptee CQ\n without the workload to be preempted.\n This strategy might favor preemption of smaller workloads in the preemptee CQ,\n regardless of priority or start time, in an effort to keep the share of the CQ\n as high as possible.\n- LessThanInitialShare: Only preempt a workload if the share of the preemptor CQ\n with the incoming workload is strictly less than the share of the preemptee CQ.\n This strategy doesn't depend on the share usage of the workload being preempted.\n As a result, the strategy chooses to preempt workloads with the lowest priority and\n newest start time first.\nThe default strategy is [\"LessThanOrEqualToFinalShare\", \"LessThanInitialShare\"].", +} + +func (FairSharing) SwaggerDoc() map[string]string { + return map_FairSharing +} + +var map_Integrations = map[string]string{ + "frameworks": "List of framework names to be enabled. Possible options:\n - \"batch/job\"\n - \"kubeflow.org/mpijob\"\n - \"ray.io/rayjob\"\n - \"ray.io/raycluster\"\n - \"jobset.x-k8s.io/jobset\"\n - \"kubeflow.org/paddlejob\"\n - \"kubeflow.org/pytorchjob\"\n - \"kubeflow.org/tfjob\"\n - \"kubeflow.org/xgboostjob\"\n - \"workload.codeflare.dev/appwrapper\"\n - \"pod\"\n - \"deployment\" (requires enabling pod integration)\n - \"statefulset\" (requires enabling pod integration)\n - \"leaderworkerset.x-k8s.io/leaderworkerset\" (requires enabling pod integration)", + "externalFrameworks": "List of GroupVersionKinds that are managed for Kueue by external controllers; the expected format is `Kind.version.group.com`.", + "labelKeysToCopy": "labelKeysToCopy is a list of label keys that should be copied from the job into the workload object. It is not required for the job to have all the labels from this list. If a job does not have some label with the given key from this list, the constructed workload object will be created without this label. In the case of creating a workload from a composable job (pod group), if multiple objects have labels with some key from the list, the values of these labels must match or otherwise the workload creation would fail. The labels are copied only during the workload creation and are not updated even if the labels of the underlying job are changed.", +} + +func (Integrations) SwaggerDoc() map[string]string { + return map_Integrations +} + +var map_Kueue = map[string]string{ + "": "Kueue is the Schema for the kueue API Compatibility level 4: No compatibility is provided, the API can change at any point for any reason. These capabilities should not be used by applications needing long term support.", + "metadata": "metadata for kueue", + "spec": "spec holds user settable values for configuration", + "status": "status holds observed values from the cluster. They may not be overridden.", +} + +func (Kueue) SwaggerDoc() map[string]string { + return map_Kueue +} + +var map_KueueConfiguration = map[string]string{ + "waitForPodsReady": "waitForPodsReady configures gang admission", + "integrations": "integrations are the types of integrations Kueue will manager", + "featureGates": "featureGates are advanced features for Kueue", + "resources": "resources provides additional configuration options for handling the resources. Supports https://github.com/kubernetes-sigs/kueue/blob/release-0.10/keps/2937-resource-transformer/README.md", + "manageJobsWithoutQueueName": "ManageJobsWithoutQueueName controls whether or not Kueue reconciles jobs that don't set the annotation kueue.x-k8s.io/queue-name. Allowed values are NoQueueName and QueueName Default will be QueueName", + "managedJobsNamespaceSelector": "ManagedJobsNamespaceSelector can be used to omit some namespaces from ManagedJobsWithoutQueueName", + "fairSharing": "FairSharing controls the fair sharing semantics across the cluster.", + "disableMetrics": "Disable Metrics Microshift does not enable metrics by default Default will assume metrics are enabled.", +} + +func (KueueConfiguration) SwaggerDoc() map[string]string { + return map_KueueConfiguration +} + +var map_KueueList = map[string]string{ + "": "KueueList contains a list of Kueue\n\nCompatibility level 4: No compatibility is provided, the API can change at any point for any reason. These capabilities should not be used by applications needing long term support.", + "metadata": "metadata for the list", + "items": "items is a slice of kueue", +} + +func (KueueList) SwaggerDoc() map[string]string { + return map_KueueList +} + +var map_KueueOperandSpec = map[string]string{ + "config": "config that is persisted to a config map", +} + +func (KueueOperandSpec) SwaggerDoc() map[string]string { + return map_KueueOperandSpec +} + +var map_KueueStatus = map[string]string{ + "": "KueueStatus defines the observed state of Kueue", +} + +func (KueueStatus) SwaggerDoc() map[string]string { + return map_KueueStatus +} + +var map_MultiKueue = map[string]string{ + "gcInterval": "GCInterval defines the time interval between two consecutive garbage collection runs. Defaults to 1min. If 0, the garbage collection is disabled.", + "origin": "Origin defines a label value used to track the creator of workloads in the worker clusters. This is used by multikueue in components like its garbage collector to identify remote objects that ware created by this multikueue manager cluster and delete them if their local counterpart no longer exists.", + "workerLostTimeout": "WorkerLostTimeout defines the time a local workload's multikueue admission check state is kept Ready if the connection with its reserving worker cluster is lost.\n\nDefaults to 15 minutes.", +} + +func (MultiKueue) SwaggerDoc() map[string]string { + return map_MultiKueue +} + +var map_RequeuingStrategy = map[string]string{ + "timestamp": "Timestamp defines the timestamp used for re-queuing a Workload that was evicted due to Pod readiness. The possible values are:\n\n- `Eviction` (default) indicates from Workload `Evicted` condition with `PodsReadyTimeout` reason. - `Creation` indicates from Workload .metadata.creationTimestamp.", + "backoffLimitCount": "BackoffLimitCount defines the maximum number of re-queuing retries. Once the number is reached, the workload is deactivated (`.spec.activate`=`false`). When it is null, the workloads will repeatedly and endless re-queueing.\n\nEvery backoff duration is about \"b*2^(n-1)+Rand\" where: - \"b\" represents the base set by \"BackoffBaseSeconds\" parameter, - \"n\" represents the \"workloadStatus.requeueState.count\", - \"Rand\" represents the random jitter. During this time, the workload is taken as an inadmissible and other workloads will have a chance to be admitted. By default, the consecutive requeue delays are around: (60s, 120s, 240s, ...).\n\nDefaults to null.", + "backoffBaseSeconds": "BackoffBaseSeconds defines the base for the exponential backoff for re-queuing an evicted workload.\n\nDefaults to 60.", + "backoffMaxSeconds": "BackoffMaxSeconds defines the maximum backoff time to re-queue an evicted workload.\n\nDefaults to 3600.", +} + +func (RequeuingStrategy) SwaggerDoc() map[string]string { + return map_RequeuingStrategy +} + +var map_ResourceTransformation = map[string]string{ + "input": "Input is the name of the input resource.", + "strategy": "Strategy specifies if the input resource should be replaced or retained. Defaults to Retain", + "outputs": "Outputs specifies the output resources and quantities per unit of input resource. An empty Outputs combined with a `Replace` Strategy causes the Input resource to be ignored by Kueue.", +} + +func (ResourceTransformation) SwaggerDoc() map[string]string { + return map_ResourceTransformation +} + +var map_Resources = map[string]string{ + "excludeResourcePrefixes": "ExcludedResourcePrefixes defines which resources should be ignored by Kueue", + "transformations": "Transformations defines how to transform PodSpec resources into Workload resource requests. This is intended to be a map with Input as the key (enforced by validation code)", +} + +func (Resources) SwaggerDoc() map[string]string { + return map_Resources +} + +var map_WaitForPodsReady = map[string]string{ + "": "WaitForPodsReady defines configuration for the Wait For Pods Ready feature, which is used to ensure that all Pods are ready within the specified time.", + "enable": "Enable indicates whether to enable wait for pods ready feature. Defaults to false.", + "timeout": "Timeout defines the time for an admitted workload to reach the PodsReady=true condition. When the timeout is exceeded, the workload evicted and requeued in the same cluster queue. Defaults to 5min.", + "blockAdmission": "BlockAdmission when true, cluster queue will block admissions for all subsequent jobs until the jobs reach the PodsReady=true condition. This setting is only honored when `Enable` is set to true.", + "requeuingStrategy": "RequeuingStrategy defines the strategy for requeuing a Workload.", + "recoveryTimeout": "RecoveryTimeout defines an opt-in timeout, measured since the last transition to the PodsReady=false condition after a Workload is Admitted and running. Such a transition may happen when a Pod failed and the replacement Pod is awaited to be scheduled. After exceeding the timeout the corresponding job gets suspended again and requeued after the backoff delay. The timeout is enforced only if waitForPodsReady.enable=true. If not set, there is no timeout.", +} + +func (WaitForPodsReady) SwaggerDoc() map[string]string { + return map_WaitForPodsReady +} + var map_OLM = map[string]string{ "": "OLM provides information to configure an operator to manage the OLM controllers\n\nCompatibility level 4: No compatibility is provided, the API can change at any point for any reason. These capabilities should not be used by applications needing long term support.", "metadata": "metadata is the standard object's metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata", From 7d9e95ff6ccfaf05683bf2b08f68dc8c4ad8e17d Mon Sep 17 00:00:00 2001 From: Kevin Hannon Date: Fri, 7 Mar 2025 15:44:44 -0500 Subject: [PATCH 03/42] update api with linter --- .../generated_openapi/zz_generated.openapi.go | 92 +++++-------------- openapi/openapi.json | 77 ++++++---------- operator/v1alpha1/types_kueue.go | 92 +++++++++---------- operator/v1alpha1/zz_generated.deepcopy.go | 41 +-------- .../zz_generated.swagger_doc_generated.go | 45 ++++----- 5 files changed, 120 insertions(+), 227 deletions(-) diff --git a/openapi/generated_openapi/zz_generated.openapi.go b/openapi/generated_openapi/zz_generated.openapi.go index b1234c35e0d..6c7621346f5 100644 --- a/openapi/generated_openapi/zz_generated.openapi.go +++ b/openapi/generated_openapi/zz_generated.openapi.go @@ -1123,7 +1123,6 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "github.com/openshift/api/operator/v1alpha1.KueueOperandSpec": schema_openshift_api_operator_v1alpha1_KueueOperandSpec(ref), "github.com/openshift/api/operator/v1alpha1.KueueStatus": schema_openshift_api_operator_v1alpha1_KueueStatus(ref), "github.com/openshift/api/operator/v1alpha1.LoggingConfig": schema_openshift_api_operator_v1alpha1_LoggingConfig(ref), - "github.com/openshift/api/operator/v1alpha1.MultiKueue": schema_openshift_api_operator_v1alpha1_MultiKueue(ref), "github.com/openshift/api/operator/v1alpha1.NodeStatus": schema_openshift_api_operator_v1alpha1_NodeStatus(ref), "github.com/openshift/api/operator/v1alpha1.OLM": schema_openshift_api_operator_v1alpha1_OLM(ref), "github.com/openshift/api/operator/v1alpha1.OLMList": schema_openshift_api_operator_v1alpha1_OLMList(ref), @@ -56860,8 +56859,8 @@ func schema_openshift_api_operator_v1alpha1_FairSharing(ref common.ReferenceCall "enable": { SchemaProps: spec.SchemaProps{ Description: "enable indicates whether to enable fair sharing for all cohorts. Defaults to false.", - Default: false, - Type: []string{"boolean"}, + Default: "", + Type: []string{"string"}, Format: "", }, }, @@ -56881,7 +56880,6 @@ func schema_openshift_api_operator_v1alpha1_FairSharing(ref common.ReferenceCall }, }, }, - Required: []string{"enable"}, }, }, } @@ -57129,7 +57127,7 @@ func schema_openshift_api_operator_v1alpha1_Integrations(ref common.ReferenceCal Properties: map[string]spec.Schema{ "frameworks": { SchemaProps: spec.SchemaProps{ - Description: "List of framework names to be enabled. Possible options:\n - \"batch/job\"\n - \"kubeflow.org/mpijob\"\n - \"ray.io/rayjob\"\n - \"ray.io/raycluster\"\n - \"jobset.x-k8s.io/jobset\"\n - \"kubeflow.org/paddlejob\"\n - \"kubeflow.org/pytorchjob\"\n - \"kubeflow.org/tfjob\"\n - \"kubeflow.org/xgboostjob\"\n - \"workload.codeflare.dev/appwrapper\"\n - \"pod\"\n - \"deployment\" (requires enabling pod integration)\n - \"statefulset\" (requires enabling pod integration)\n - \"leaderworkerset.x-k8s.io/leaderworkerset\" (requires enabling pod integration)", + Description: "frameworks List of framework names to be enabled. Possible options:\n - \"batch/job\"\n - \"kubeflow.org/mpijob\"\n - \"ray.io/rayjob\"\n - \"ray.io/raycluster\"\n - \"jobset.x-k8s.io/jobset\"\n - \"kubeflow.org/paddlejob\"\n - \"kubeflow.org/pytorchjob\"\n - \"kubeflow.org/tfjob\"\n - \"kubeflow.org/xgboostjob\"\n - \"workload.codeflare.dev/appwrapper\"\n - \"pod\"\n - \"deployment\" (requires enabling pod integration)\n - \"statefulset\" (requires enabling pod integration)\n - \"leaderworkerset.x-k8s.io/leaderworkerset\" (requires enabling pod integration)", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -57144,7 +57142,7 @@ func schema_openshift_api_operator_v1alpha1_Integrations(ref common.ReferenceCal }, "externalFrameworks": { SchemaProps: spec.SchemaProps{ - Description: "List of GroupVersionKinds that are managed for Kueue by external controllers; the expected format is `Kind.version.group.com`.", + Description: "externalFrameworks List of GroupVersionKinds that are managed for Kueue by external controllers; the expected format is `Kind.version.group.com`.", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -57257,8 +57255,8 @@ func schema_openshift_api_operator_v1alpha1_KueueConfiguration(ref common.Refere Allows: true, Schema: &spec.Schema{ SchemaProps: spec.SchemaProps{ - Default: false, - Type: []string{"boolean"}, + Default: "", + Type: []string{"string"}, Format: "", }, }, @@ -57274,33 +57272,33 @@ func schema_openshift_api_operator_v1alpha1_KueueConfiguration(ref common.Refere }, "manageJobsWithoutQueueName": { SchemaProps: spec.SchemaProps{ - Description: "ManageJobsWithoutQueueName controls whether or not Kueue reconciles jobs that don't set the annotation kueue.x-k8s.io/queue-name. Allowed values are NoQueueName and QueueName Default will be QueueName", + Description: "manageJobsWithoutQueueName controls whether or not Kueue reconciles jobs that don't set the annotation kueue.x-k8s.io/queue-name. Allowed values are NoQueueName and QueueName Default will be QueueName", Type: []string{"string"}, Format: "", }, }, "managedJobsNamespaceSelector": { SchemaProps: spec.SchemaProps{ - Description: "ManagedJobsNamespaceSelector can be used to omit some namespaces from ManagedJobsWithoutQueueName", + Description: "managedJobsNamespaceSelector can be used to omit some namespaces from ManagedJobsWithoutQueueName", Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.LabelSelector"), }, }, "fairSharing": { SchemaProps: spec.SchemaProps{ - Description: "FairSharing controls the fair sharing semantics across the cluster.", + Description: "fairSharing controls the fair sharing semantics across the cluster.", Default: map[string]interface{}{}, Ref: ref("github.com/openshift/api/operator/v1alpha1.FairSharing"), }, }, - "disableMetrics": { + "metrics": { SchemaProps: spec.SchemaProps{ - Description: "Disable Metrics Microshift does not enable metrics by default Default will assume metrics are enabled.", - Type: []string{"boolean"}, + Description: "Metrics Microshift does not enable metrics by default Default will assume metrics are enabled.", + Type: []string{"string"}, Format: "", }, }, }, - Required: []string{"integrations"}, + Required: []string{"integrations", "metrics"}, }, }, Dependencies: []string{ @@ -57537,39 +57535,6 @@ func schema_openshift_api_operator_v1alpha1_LoggingConfig(ref common.ReferenceCa } } -func schema_openshift_api_operator_v1alpha1_MultiKueue(ref common.ReferenceCallback) common.OpenAPIDefinition { - return common.OpenAPIDefinition{ - Schema: spec.Schema{ - SchemaProps: spec.SchemaProps{ - Type: []string{"object"}, - Properties: map[string]spec.Schema{ - "gcInterval": { - SchemaProps: spec.SchemaProps{ - Description: "GCInterval defines the time interval between two consecutive garbage collection runs. Defaults to 1min. If 0, the garbage collection is disabled.", - Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.Duration"), - }, - }, - "origin": { - SchemaProps: spec.SchemaProps{ - Description: "Origin defines a label value used to track the creator of workloads in the worker clusters. This is used by multikueue in components like its garbage collector to identify remote objects that ware created by this multikueue manager cluster and delete them if their local counterpart no longer exists.", - Type: []string{"string"}, - Format: "", - }, - }, - "workerLostTimeout": { - SchemaProps: spec.SchemaProps{ - Description: "WorkerLostTimeout defines the time a local workload's multikueue admission check state is kept Ready if the connection with its reserving worker cluster is lost.\n\nDefaults to 15 minutes.", - Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.Duration"), - }, - }, - }, - }, - }, - Dependencies: []string{ - "k8s.io/apimachinery/pkg/apis/meta/v1.Duration"}, - } -} - func schema_openshift_api_operator_v1alpha1_NodeStatus(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ @@ -58082,14 +58047,14 @@ func schema_openshift_api_operator_v1alpha1_RequeuingStrategy(ref common.Referen Properties: map[string]spec.Schema{ "timestamp": { SchemaProps: spec.SchemaProps{ - Description: "Timestamp defines the timestamp used for re-queuing a Workload that was evicted due to Pod readiness. The possible values are:\n\n- `Eviction` (default) indicates from Workload `Evicted` condition with `PodsReadyTimeout` reason. - `Creation` indicates from Workload .metadata.creationTimestamp.", + Description: "timestamp defines the timestamp used for re-queuing a Workload that was evicted due to Pod readiness. The possible values are:\n\n- `Eviction` (default) indicates from Workload `Evicted` condition with `PodsReadyTimeout` reason. - `Creation` indicates from Workload .metadata.creationTimestamp.", Type: []string{"string"}, Format: "", }, }, "backoffLimitCount": { SchemaProps: spec.SchemaProps{ - Description: "BackoffLimitCount defines the maximum number of re-queuing retries. Once the number is reached, the workload is deactivated (`.spec.activate`=`false`). When it is null, the workloads will repeatedly and endless re-queueing.\n\nEvery backoff duration is about \"b*2^(n-1)+Rand\" where: - \"b\" represents the base set by \"BackoffBaseSeconds\" parameter, - \"n\" represents the \"workloadStatus.requeueState.count\", - \"Rand\" represents the random jitter. During this time, the workload is taken as an inadmissible and other workloads will have a chance to be admitted. By default, the consecutive requeue delays are around: (60s, 120s, 240s, ...).\n\nDefaults to null.", + Description: "backoffLimitCount defines the maximum number of re-queuing retries. Once the number is reached, the workload is deactivated (`.spec.activate`=`false`). When it is null, the workloads will repeatedly and endless re-queueing.\n\nEvery backoff duration is about \"b*2^(n-1)+Rand\" where: - \"b\" represents the base set by \"BackoffBaseSeconds\" parameter, - \"n\" represents the \"workloadStatus.requeueState.count\", - \"Rand\" represents the random jitter. During this time, the workload is taken as an inadmissible and other workloads will have a chance to be admitted. By default, the consecutive requeue delays are around: (60s, 120s, 240s, ...).\n\nDefaults to null.", Type: []string{"integer"}, Format: "int32", }, @@ -58122,7 +58087,7 @@ func schema_openshift_api_operator_v1alpha1_ResourceTransformation(ref common.Re Properties: map[string]spec.Schema{ "input": { SchemaProps: spec.SchemaProps{ - Description: "Input is the name of the input resource.", + Description: "input is the name of the input resource.", Default: "", Type: []string{"string"}, Format: "", @@ -58130,14 +58095,14 @@ func schema_openshift_api_operator_v1alpha1_ResourceTransformation(ref common.Re }, "strategy": { SchemaProps: spec.SchemaProps{ - Description: "Strategy specifies if the input resource should be replaced or retained. Defaults to Retain", + Description: "strategy specifies if the input resource should be replaced or retained. Defaults to Retain", Type: []string{"string"}, Format: "", }, }, "outputs": { SchemaProps: spec.SchemaProps{ - Description: "Outputs specifies the output resources and quantities per unit of input resource. An empty Outputs combined with a `Replace` Strategy causes the Input resource to be ignored by Kueue.", + Description: "outputs specifies the output resources and quantities per unit of input resource. An empty Outputs combined with a `Replace` Strategy causes the Input resource to be ignored by Kueue.", Type: []string{"object"}, AdditionalProperties: &spec.SchemaOrBool{ Allows: true, @@ -58166,7 +58131,7 @@ func schema_openshift_api_operator_v1alpha1_Resources(ref common.ReferenceCallba Properties: map[string]spec.Schema{ "excludeResourcePrefixes": { SchemaProps: spec.SchemaProps{ - Description: "ExcludedResourcePrefixes defines which resources should be ignored by Kueue", + Description: "excludedResourcePrefixes defines which resources should be ignored by Kueue", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -58181,7 +58146,7 @@ func schema_openshift_api_operator_v1alpha1_Resources(ref common.ReferenceCallba }, "transformations": { SchemaProps: spec.SchemaProps{ - Description: "Transformations defines how to transform PodSpec resources into Workload resource requests. This is intended to be a map with Input as the key (enforced by validation code)", + Description: "transformations defines how to transform PodSpec resources into Workload resource requests. This is intended to be a map with Input as the key (enforced by validation code)", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -58362,35 +58327,28 @@ func schema_openshift_api_operator_v1alpha1_WaitForPodsReady(ref common.Referenc Description: "WaitForPodsReady defines configuration for the Wait For Pods Ready feature, which is used to ensure that all Pods are ready within the specified time.", Type: []string{"object"}, Properties: map[string]spec.Schema{ - "enable": { - SchemaProps: spec.SchemaProps{ - Description: "Enable indicates whether to enable wait for pods ready feature. Defaults to false.", - Type: []string{"boolean"}, - Format: "", - }, - }, "timeout": { SchemaProps: spec.SchemaProps{ - Description: "Timeout defines the time for an admitted workload to reach the PodsReady=true condition. When the timeout is exceeded, the workload evicted and requeued in the same cluster queue. Defaults to 5min.", + Description: "timeout defines the time for an admitted workload to reach the PodsReady=true condition. When the timeout is exceeded, the workload evicted and requeued in the same cluster queue. Defaults to 5min.", Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.Duration"), }, }, "blockAdmission": { SchemaProps: spec.SchemaProps{ - Description: "BlockAdmission when true, cluster queue will block admissions for all subsequent jobs until the jobs reach the PodsReady=true condition. This setting is only honored when `Enable` is set to true.", - Type: []string{"boolean"}, + Description: "blockAdmission when true, cluster queue will block admissions for all subsequent jobs until the jobs reach the PodsReady=true condition. This setting is only honored when `Enable` is set to true.", + Type: []string{"string"}, Format: "", }, }, "requeuingStrategy": { SchemaProps: spec.SchemaProps{ - Description: "RequeuingStrategy defines the strategy for requeuing a Workload.", + Description: "requeuingStrategy defines the strategy for requeuing a Workload.", Ref: ref("github.com/openshift/api/operator/v1alpha1.RequeuingStrategy"), }, }, "recoveryTimeout": { SchemaProps: spec.SchemaProps{ - Description: "RecoveryTimeout defines an opt-in timeout, measured since the last transition to the PodsReady=false condition after a Workload is Admitted and running. Such a transition may happen when a Pod failed and the replacement Pod is awaited to be scheduled. After exceeding the timeout the corresponding job gets suspended again and requeued after the backoff delay. The timeout is enforced only if waitForPodsReady.enable=true. If not set, there is no timeout.", + Description: "recoveryTimeout defines an opt-in timeout, measured since the last transition to the PodsReady=false condition after a Workload is Admitted and running. Such a transition may happen when a Pod failed and the replacement Pod is awaited to be scheduled. After exceeding the timeout the corresponding job gets suspended again and requeued after the backoff delay. The timeout is enforced only if waitForPodsReady.enable=true. If not set, there is no timeout.", Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.Duration"), }, }, diff --git a/openapi/openapi.json b/openapi/openapi.json index 81ef39b3b9c..b7c51ce00d2 100644 --- a/openapi/openapi.json +++ b/openapi/openapi.json @@ -33208,14 +33208,11 @@ }, "com.github.openshift.api.operator.v1alpha1.FairSharing": { "type": "object", - "required": [ - "enable" - ], "properties": { "enable": { "description": "enable indicates whether to enable fair sharing for all cohorts. Defaults to false.", - "type": "boolean", - "default": false + "type": "string", + "default": "" }, "preemptionStrategies": { "description": "preemptionStrategies indicates which constraints should a preemption satisfy. The preemption algorithm will only use the next strategy in the list if the incoming workload (preemptor) doesn't fit after using the previous strategies. Possible values are: - LessThanOrEqualToFinalShare: Only preempt a workload if the share of the preemptor CQ\n with the preemptor workload is less than or equal to the share of the preemptee CQ\n without the workload to be preempted.\n This strategy might favor preemption of smaller workloads in the preemptee CQ,\n regardless of priority or start time, in an effort to keep the share of the CQ\n as high as possible.\n- LessThanInitialShare: Only preempt a workload if the share of the preemptor CQ\n with the incoming workload is strictly less than the share of the preemptee CQ.\n This strategy doesn't depend on the share usage of the workload being preempted.\n As a result, the strategy chooses to preempt workloads with the lowest priority and\n newest start time first.\nThe default strategy is [\"LessThanOrEqualToFinalShare\", \"LessThanInitialShare\"].", @@ -33375,7 +33372,7 @@ "type": "object", "properties": { "externalFrameworks": { - "description": "List of GroupVersionKinds that are managed for Kueue by external controllers; the expected format is `Kind.version.group.com`.", + "description": "externalFrameworks List of GroupVersionKinds that are managed for Kueue by external controllers; the expected format is `Kind.version.group.com`.", "type": "array", "items": { "type": "string", @@ -33383,7 +33380,7 @@ } }, "frameworks": { - "description": "List of framework names to be enabled. Possible options:\n - \"batch/job\"\n - \"kubeflow.org/mpijob\"\n - \"ray.io/rayjob\"\n - \"ray.io/raycluster\"\n - \"jobset.x-k8s.io/jobset\"\n - \"kubeflow.org/paddlejob\"\n - \"kubeflow.org/pytorchjob\"\n - \"kubeflow.org/tfjob\"\n - \"kubeflow.org/xgboostjob\"\n - \"workload.codeflare.dev/appwrapper\"\n - \"pod\"\n - \"deployment\" (requires enabling pod integration)\n - \"statefulset\" (requires enabling pod integration)\n - \"leaderworkerset.x-k8s.io/leaderworkerset\" (requires enabling pod integration)", + "description": "frameworks List of framework names to be enabled. Possible options:\n - \"batch/job\"\n - \"kubeflow.org/mpijob\"\n - \"ray.io/rayjob\"\n - \"ray.io/raycluster\"\n - \"jobset.x-k8s.io/jobset\"\n - \"kubeflow.org/paddlejob\"\n - \"kubeflow.org/pytorchjob\"\n - \"kubeflow.org/tfjob\"\n - \"kubeflow.org/xgboostjob\"\n - \"workload.codeflare.dev/appwrapper\"\n - \"pod\"\n - \"deployment\" (requires enabling pod integration)\n - \"statefulset\" (requires enabling pod integration)\n - \"leaderworkerset.x-k8s.io/leaderworkerset\" (requires enabling pod integration)", "type": "array", "items": { "type": "string", @@ -33435,15 +33432,12 @@ "com.github.openshift.api.operator.v1alpha1.KueueConfiguration": { "type": "object", "required": [ - "integrations" + "integrations", + "metrics" ], "properties": { - "disableMetrics": { - "description": "Disable Metrics Microshift does not enable metrics by default Default will assume metrics are enabled.", - "type": "boolean" - }, "fairSharing": { - "description": "FairSharing controls the fair sharing semantics across the cluster.", + "description": "fairSharing controls the fair sharing semantics across the cluster.", "default": {}, "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.FairSharing" }, @@ -33451,8 +33445,8 @@ "description": "featureGates are advanced features for Kueue", "type": "object", "additionalProperties": { - "type": "boolean", - "default": false + "type": "string", + "default": "" } }, "integrations": { @@ -33461,13 +33455,17 @@ "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.Integrations" }, "manageJobsWithoutQueueName": { - "description": "ManageJobsWithoutQueueName controls whether or not Kueue reconciles jobs that don't set the annotation kueue.x-k8s.io/queue-name. Allowed values are NoQueueName and QueueName Default will be QueueName", + "description": "manageJobsWithoutQueueName controls whether or not Kueue reconciles jobs that don't set the annotation kueue.x-k8s.io/queue-name. Allowed values are NoQueueName and QueueName Default will be QueueName", "type": "string" }, "managedJobsNamespaceSelector": { - "description": "ManagedJobsNamespaceSelector can be used to omit some namespaces from ManagedJobsWithoutQueueName", + "description": "managedJobsNamespaceSelector can be used to omit some namespaces from ManagedJobsWithoutQueueName", "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.LabelSelector" }, + "metrics": { + "description": "Metrics Microshift does not enable metrics by default Default will assume metrics are enabled.", + "type": "string" + }, "resources": { "description": "resources provides additional configuration options for handling the resources. Supports https://github.com/kubernetes-sigs/kueue/blob/release-0.10/keps/2937-resource-transformer/README.md", "default": {}, @@ -33622,23 +33620,6 @@ } } }, - "com.github.openshift.api.operator.v1alpha1.MultiKueue": { - "type": "object", - "properties": { - "gcInterval": { - "description": "GCInterval defines the time interval between two consecutive garbage collection runs. Defaults to 1min. If 0, the garbage collection is disabled.", - "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.Duration" - }, - "origin": { - "description": "Origin defines a label value used to track the creator of workloads in the worker clusters. This is used by multikueue in components like its garbage collector to identify remote objects that ware created by this multikueue manager cluster and delete them if their local counterpart no longer exists.", - "type": "string" - }, - "workerLostTimeout": { - "description": "WorkerLostTimeout defines the time a local workload's multikueue admission check state is kept Ready if the connection with its reserving worker cluster is lost.\n\nDefaults to 15 minutes.", - "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.Duration" - } - } - }, "com.github.openshift.api.operator.v1alpha1.NodeStatus": { "description": "NodeStatus provides information about the current state of a particular node managed by this operator. Deprecated: Use v1.NodeStatus instead", "type": "object", @@ -33961,7 +33942,7 @@ "format": "int32" }, "backoffLimitCount": { - "description": "BackoffLimitCount defines the maximum number of re-queuing retries. Once the number is reached, the workload is deactivated (`.spec.activate`=`false`). When it is null, the workloads will repeatedly and endless re-queueing.\n\nEvery backoff duration is about \"b*2^(n-1)+Rand\" where: - \"b\" represents the base set by \"BackoffBaseSeconds\" parameter, - \"n\" represents the \"workloadStatus.requeueState.count\", - \"Rand\" represents the random jitter. During this time, the workload is taken as an inadmissible and other workloads will have a chance to be admitted. By default, the consecutive requeue delays are around: (60s, 120s, 240s, ...).\n\nDefaults to null.", + "description": "backoffLimitCount defines the maximum number of re-queuing retries. Once the number is reached, the workload is deactivated (`.spec.activate`=`false`). When it is null, the workloads will repeatedly and endless re-queueing.\n\nEvery backoff duration is about \"b*2^(n-1)+Rand\" where: - \"b\" represents the base set by \"BackoffBaseSeconds\" parameter, - \"n\" represents the \"workloadStatus.requeueState.count\", - \"Rand\" represents the random jitter. During this time, the workload is taken as an inadmissible and other workloads will have a chance to be admitted. By default, the consecutive requeue delays are around: (60s, 120s, 240s, ...).\n\nDefaults to null.", "type": "integer", "format": "int32" }, @@ -33971,7 +33952,7 @@ "format": "int32" }, "timestamp": { - "description": "Timestamp defines the timestamp used for re-queuing a Workload that was evicted due to Pod readiness. The possible values are:\n\n- `Eviction` (default) indicates from Workload `Evicted` condition with `PodsReadyTimeout` reason. - `Creation` indicates from Workload .metadata.creationTimestamp.", + "description": "timestamp defines the timestamp used for re-queuing a Workload that was evicted due to Pod readiness. The possible values are:\n\n- `Eviction` (default) indicates from Workload `Evicted` condition with `PodsReadyTimeout` reason. - `Creation` indicates from Workload .metadata.creationTimestamp.", "type": "string" } } @@ -33983,19 +33964,19 @@ ], "properties": { "input": { - "description": "Input is the name of the input resource.", + "description": "input is the name of the input resource.", "type": "string", "default": "" }, "outputs": { - "description": "Outputs specifies the output resources and quantities per unit of input resource. An empty Outputs combined with a `Replace` Strategy causes the Input resource to be ignored by Kueue.", + "description": "outputs specifies the output resources and quantities per unit of input resource. An empty Outputs combined with a `Replace` Strategy causes the Input resource to be ignored by Kueue.", "type": "object", "additionalProperties": { "$ref": "#/definitions/io.k8s.apimachinery.pkg.api.resource.Quantity" } }, "strategy": { - "description": "Strategy specifies if the input resource should be replaced or retained. Defaults to Retain", + "description": "strategy specifies if the input resource should be replaced or retained. Defaults to Retain", "type": "string" } } @@ -34004,7 +33985,7 @@ "type": "object", "properties": { "excludeResourcePrefixes": { - "description": "ExcludedResourcePrefixes defines which resources should be ignored by Kueue", + "description": "excludedResourcePrefixes defines which resources should be ignored by Kueue", "type": "array", "items": { "type": "string", @@ -34012,7 +33993,7 @@ } }, "transformations": { - "description": "Transformations defines how to transform PodSpec resources into Workload resource requests. This is intended to be a map with Input as the key (enforced by validation code)", + "description": "transformations defines how to transform PodSpec resources into Workload resource requests. This is intended to be a map with Input as the key (enforced by validation code)", "type": "array", "items": { "default": {}, @@ -34125,23 +34106,19 @@ "type": "object", "properties": { "blockAdmission": { - "description": "BlockAdmission when true, cluster queue will block admissions for all subsequent jobs until the jobs reach the PodsReady=true condition. This setting is only honored when `Enable` is set to true.", - "type": "boolean" - }, - "enable": { - "description": "Enable indicates whether to enable wait for pods ready feature. Defaults to false.", - "type": "boolean" + "description": "blockAdmission when true, cluster queue will block admissions for all subsequent jobs until the jobs reach the PodsReady=true condition. This setting is only honored when `Enable` is set to true.", + "type": "string" }, "recoveryTimeout": { - "description": "RecoveryTimeout defines an opt-in timeout, measured since the last transition to the PodsReady=false condition after a Workload is Admitted and running. Such a transition may happen when a Pod failed and the replacement Pod is awaited to be scheduled. After exceeding the timeout the corresponding job gets suspended again and requeued after the backoff delay. The timeout is enforced only if waitForPodsReady.enable=true. If not set, there is no timeout.", + "description": "recoveryTimeout defines an opt-in timeout, measured since the last transition to the PodsReady=false condition after a Workload is Admitted and running. Such a transition may happen when a Pod failed and the replacement Pod is awaited to be scheduled. After exceeding the timeout the corresponding job gets suspended again and requeued after the backoff delay. The timeout is enforced only if waitForPodsReady.enable=true. If not set, there is no timeout.", "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.Duration" }, "requeuingStrategy": { - "description": "RequeuingStrategy defines the strategy for requeuing a Workload.", + "description": "requeuingStrategy defines the strategy for requeuing a Workload.", "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.RequeuingStrategy" }, "timeout": { - "description": "Timeout defines the time for an admitted workload to reach the PodsReady=true condition. When the timeout is exceeded, the workload evicted and requeued in the same cluster queue. Defaults to 5min.", + "description": "timeout defines the time for an admitted workload to reach the PodsReady=true condition. When the timeout is exceeded, the workload evicted and requeued in the same cluster queue. Defaults to 5min.", "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.Duration" } } diff --git a/operator/v1alpha1/types_kueue.go b/operator/v1alpha1/types_kueue.go index 07602ec5215..c79e004af6e 100644 --- a/operator/v1alpha1/types_kueue.go +++ b/operator/v1alpha1/types_kueue.go @@ -13,6 +13,7 @@ import ( // +openshift:compatibility-gen:level=4 // +k8s:openapi-gen=true // +genclient +// +genclient:nonNamespaced // +kubebuilder:storageversion // +kubebuilder:subresource:status type Kueue struct { @@ -45,6 +46,13 @@ const ( QueueName ManageJobsWithoutQueueNameOption = "QueueName" ) +type EnabledOrDisabled string + +const ( + Enabled EnabledOrDisabled = "Enabled" + Disabled EnabledOrDisabled = "Disabled" +) + type KueueConfiguration struct { // waitForPodsReady configures gang admission // +optional @@ -54,27 +62,30 @@ type KueueConfiguration struct { Integrations Integrations `json:"integrations"` // featureGates are advanced features for Kueue // +optional - FeatureGates map[string]bool `json:"featureGates,omitempty"` + FeatureGates map[string]EnabledOrDisabled `json:"featureGates,omitempty"` // resources provides additional configuration options for handling the resources. // Supports https://github.com/kubernetes-sigs/kueue/blob/release-0.10/keps/2937-resource-transformer/README.md // +optional Resources Resources `json:"resources,omitempty"` - // ManageJobsWithoutQueueName controls whether or not Kueue reconciles + // manageJobsWithoutQueueName controls whether or not Kueue reconciles // jobs that don't set the annotation kueue.x-k8s.io/queue-name. // Allowed values are NoQueueName and QueueName // Default will be QueueName + // +kubebuilder:default=QueueName // +optional ManageJobsWithoutQueueName *ManageJobsWithoutQueueNameOption `json:"manageJobsWithoutQueueName,omitempty"` - // ManagedJobsNamespaceSelector can be used to omit some namespaces from ManagedJobsWithoutQueueName + // managedJobsNamespaceSelector can be used to omit some namespaces from ManagedJobsWithoutQueueName + // Only valid if ManagedJobsWithoutQueueName is NoQueueName // +optional ManagedJobsNamespaceSelector *metav1.LabelSelector `json:"managedJobsNamespaceSelector,omitempty"` - // FairSharing controls the fair sharing semantics across the cluster. + // fairSharing controls the fair sharing semantics across the cluster. FairSharing FairSharing `json:"fairSharing,omitempty"` - // Disable Metrics + // ,etrics // Microshift does not enable metrics by default // Default will assume metrics are enabled. - // +optional - DisableMetrics *bool `json:"disableMetrics,omitempty"` + // +kubebuilder:default=Enabled + // +required + Metrics *EnabledOrDisabled `json:"metrics,omitempty"` } // KueueStatus defines the observed state of Kueue @@ -101,11 +112,13 @@ type KueueList struct { // These structs come directly from Kueue. type Resources struct { - // ExcludedResourcePrefixes defines which resources should be ignored by Kueue + // excludedResourcePrefixes defines which resources should be ignored by Kueue + // +optional ExcludeResourcePrefixes []string `json:"excludeResourcePrefixes,omitempty"` - // Transformations defines how to transform PodSpec resources into Workload resource requests. + // transformations defines how to transform PodSpec resources into Workload resource requests. // This is intended to be a map with Input as the key (enforced by validation code) + // +optional Transformations []ResourceTransformation `json:"transformations,omitempty"` } @@ -115,15 +128,20 @@ const Retain ResourceTransformationStrategy = "Retain" const Replace ResourceTransformationStrategy = "Replace" type ResourceTransformation struct { - // Input is the name of the input resource. + // input is the name of the input resource. + // +required Input corev1.ResourceName `json:"input"` - // Strategy specifies if the input resource should be replaced or retained. + // strategy specifies if the input resource should be replaced or retained. // Defaults to Retain + // +kubebuilder:default=Retain + // +kubebuilder:validation:MaxLength=6 + // +optional Strategy *ResourceTransformationStrategy `json:"strategy,omitempty"` - // Outputs specifies the output resources and quantities per unit of input resource. + // outputs specifies the output resources and quantities per unit of input resource. // An empty Outputs combined with a `Replace` Strategy causes the Input resource to be ignored by Kueue. + // +optional Outputs corev1.ResourceList `json:"outputs,omitempty"` } @@ -137,7 +155,9 @@ const ( type FairSharing struct { // enable indicates whether to enable fair sharing for all cohorts. // Defaults to false. - Enable bool `json:"enable"` + // +kubebuilder:default=Disabled + // +optional + Enable EnabledOrDisabled `json:"enable"` // preemptionStrategies indicates which constraints should a preemption satisfy. // The preemption algorithm will only use the next strategy in the list if the @@ -161,27 +181,24 @@ type FairSharing struct { // WaitForPodsReady defines configuration for the Wait For Pods Ready feature, // which is used to ensure that all Pods are ready within the specified time. type WaitForPodsReady struct { - // Enable indicates whether to enable wait for pods ready feature. - // Defaults to false. - Enable bool `json:"enable,omitempty"` - - // Timeout defines the time for an admitted workload to reach the + // timeout defines the time for an admitted workload to reach the // PodsReady=true condition. When the timeout is exceeded, the workload // evicted and requeued in the same cluster queue. // Defaults to 5min. // +optional Timeout *metav1.Duration `json:"timeout,omitempty"` - // BlockAdmission when true, cluster queue will block admissions for all + // blockAdmission when true, cluster queue will block admissions for all // subsequent jobs until the jobs reach the PodsReady=true condition. // This setting is only honored when `Enable` is set to true. - BlockAdmission *bool `json:"blockAdmission,omitempty"` + // +optional + BlockAdmission *EnabledOrDisabled `json:"blockAdmission,omitempty"` - // RequeuingStrategy defines the strategy for requeuing a Workload. + // requeuingStrategy defines the strategy for requeuing a Workload. // +optional RequeuingStrategy *RequeuingStrategy `json:"requeuingStrategy,omitempty"` - // RecoveryTimeout defines an opt-in timeout, measured since the + // recoveryTimeout defines an opt-in timeout, measured since the // last transition to the PodsReady=false condition after a Workload is Admitted and running. // Such a transition may happen when a Pod failed and the replacement Pod // is awaited to be scheduled. @@ -192,30 +209,8 @@ type WaitForPodsReady struct { RecoveryTimeout *metav1.Duration `json:"recoveryTimeout,omitempty"` } -type MultiKueue struct { - // GCInterval defines the time interval between two consecutive garbage collection runs. - // Defaults to 1min. If 0, the garbage collection is disabled. - // +optional - GCInterval *metav1.Duration `json:"gcInterval"` - - // Origin defines a label value used to track the creator of workloads in the worker - // clusters. - // This is used by multikueue in components like its garbage collector to identify - // remote objects that ware created by this multikueue manager cluster and delete - // them if their local counterpart no longer exists. - // +optional - Origin *string `json:"origin,omitempty"` - - // WorkerLostTimeout defines the time a local workload's multikueue admission check state is kept Ready - // if the connection with its reserving worker cluster is lost. - // - // Defaults to 15 minutes. - // +optional - WorkerLostTimeout *metav1.Duration `json:"workerLostTimeout,omitempty"` -} - type RequeuingStrategy struct { - // Timestamp defines the timestamp used for re-queuing a Workload + // timestamp defines the timestamp used for re-queuing a Workload // that was evicted due to Pod readiness. The possible values are: // // - `Eviction` (default) indicates from Workload `Evicted` condition with `PodsReadyTimeout` reason. @@ -224,7 +219,7 @@ type RequeuingStrategy struct { // +optional Timestamp *RequeuingTimestamp `json:"timestamp,omitempty"` - // BackoffLimitCount defines the maximum number of re-queuing retries. + // backoffLimitCount defines the maximum number of re-queuing retries. // Once the number is reached, the workload is deactivated (`.spec.activate`=`false`). // When it is null, the workloads will repeatedly and endless re-queueing. // @@ -244,12 +239,14 @@ type RequeuingStrategy struct { // re-queuing an evicted workload. // // Defaults to 60. + // +kubebuilder:default=60 // +optional BackoffBaseSeconds *int32 `json:"backoffBaseSeconds,omitempty"` // BackoffMaxSeconds defines the maximum backoff time to re-queue an evicted workload. // // Defaults to 3600. + // +kubebuilder:default=3600 // +optional BackoffMaxSeconds *int32 `json:"backoffMaxSeconds,omitempty"` } @@ -265,6 +262,7 @@ const ( ) type Integrations struct { + // frameworks // List of framework names to be enabled. // Possible options: // - "batch/job" @@ -281,7 +279,9 @@ type Integrations struct { // - "deployment" (requires enabling pod integration) // - "statefulset" (requires enabling pod integration) // - "leaderworkerset.x-k8s.io/leaderworkerset" (requires enabling pod integration) + // +kubebuilder:validation:items:MaxLength=14 Frameworks []string `json:"frameworks,omitempty"` + // externalFrameworks // List of GroupVersionKinds that are managed for Kueue by external controllers; // the expected format is `Kind.version.group.com`. ExternalFrameworks []string `json:"externalFrameworks,omitempty"` diff --git a/operator/v1alpha1/zz_generated.deepcopy.go b/operator/v1alpha1/zz_generated.deepcopy.go index 16b227da938..ea5bdf23e2d 100644 --- a/operator/v1alpha1/zz_generated.deepcopy.go +++ b/operator/v1alpha1/zz_generated.deepcopy.go @@ -472,7 +472,7 @@ func (in *KueueConfiguration) DeepCopyInto(out *KueueConfiguration) { in.Integrations.DeepCopyInto(&out.Integrations) if in.FeatureGates != nil { in, out := &in.FeatureGates, &out.FeatureGates - *out = make(map[string]bool, len(*in)) + *out = make(map[string]EnabledOrDisabled, len(*in)) for key, val := range *in { (*out)[key] = val } @@ -489,9 +489,9 @@ func (in *KueueConfiguration) DeepCopyInto(out *KueueConfiguration) { (*in).DeepCopyInto(*out) } in.FairSharing.DeepCopyInto(&out.FairSharing) - if in.DisableMetrics != nil { - in, out := &in.DisableMetrics, &out.DisableMetrics - *out = new(bool) + if in.Metrics != nil { + in, out := &in.Metrics, &out.Metrics + *out = new(EnabledOrDisabled) **out = **in } return @@ -591,37 +591,6 @@ func (in *LoggingConfig) DeepCopy() *LoggingConfig { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *MultiKueue) DeepCopyInto(out *MultiKueue) { - *out = *in - if in.GCInterval != nil { - in, out := &in.GCInterval, &out.GCInterval - *out = new(v1.Duration) - **out = **in - } - if in.Origin != nil { - in, out := &in.Origin, &out.Origin - *out = new(string) - **out = **in - } - if in.WorkerLostTimeout != nil { - in, out := &in.WorkerLostTimeout, &out.WorkerLostTimeout - *out = new(v1.Duration) - **out = **in - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MultiKueue. -func (in *MultiKueue) DeepCopy() *MultiKueue { - if in == nil { - return nil - } - out := new(MultiKueue) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NodeStatus) DeepCopyInto(out *NodeStatus) { *out = *in @@ -978,7 +947,7 @@ func (in *WaitForPodsReady) DeepCopyInto(out *WaitForPodsReady) { } if in.BlockAdmission != nil { in, out := &in.BlockAdmission, &out.BlockAdmission - *out = new(bool) + *out = new(EnabledOrDisabled) **out = **in } if in.RequeuingStrategy != nil { diff --git a/operator/v1alpha1/zz_generated.swagger_doc_generated.go b/operator/v1alpha1/zz_generated.swagger_doc_generated.go index 2d223e1051b..93ddf349fa3 100644 --- a/operator/v1alpha1/zz_generated.swagger_doc_generated.go +++ b/operator/v1alpha1/zz_generated.swagger_doc_generated.go @@ -267,8 +267,8 @@ func (FairSharing) SwaggerDoc() map[string]string { } var map_Integrations = map[string]string{ - "frameworks": "List of framework names to be enabled. Possible options:\n - \"batch/job\"\n - \"kubeflow.org/mpijob\"\n - \"ray.io/rayjob\"\n - \"ray.io/raycluster\"\n - \"jobset.x-k8s.io/jobset\"\n - \"kubeflow.org/paddlejob\"\n - \"kubeflow.org/pytorchjob\"\n - \"kubeflow.org/tfjob\"\n - \"kubeflow.org/xgboostjob\"\n - \"workload.codeflare.dev/appwrapper\"\n - \"pod\"\n - \"deployment\" (requires enabling pod integration)\n - \"statefulset\" (requires enabling pod integration)\n - \"leaderworkerset.x-k8s.io/leaderworkerset\" (requires enabling pod integration)", - "externalFrameworks": "List of GroupVersionKinds that are managed for Kueue by external controllers; the expected format is `Kind.version.group.com`.", + "frameworks": "frameworks List of framework names to be enabled. Possible options:\n - \"batch/job\"\n - \"kubeflow.org/mpijob\"\n - \"ray.io/rayjob\"\n - \"ray.io/raycluster\"\n - \"jobset.x-k8s.io/jobset\"\n - \"kubeflow.org/paddlejob\"\n - \"kubeflow.org/pytorchjob\"\n - \"kubeflow.org/tfjob\"\n - \"kubeflow.org/xgboostjob\"\n - \"workload.codeflare.dev/appwrapper\"\n - \"pod\"\n - \"deployment\" (requires enabling pod integration)\n - \"statefulset\" (requires enabling pod integration)\n - \"leaderworkerset.x-k8s.io/leaderworkerset\" (requires enabling pod integration)", + "externalFrameworks": "externalFrameworks List of GroupVersionKinds that are managed for Kueue by external controllers; the expected format is `Kind.version.group.com`.", "labelKeysToCopy": "labelKeysToCopy is a list of label keys that should be copied from the job into the workload object. It is not required for the job to have all the labels from this list. If a job does not have some label with the given key from this list, the constructed workload object will be created without this label. In the case of creating a workload from a composable job (pod group), if multiple objects have labels with some key from the list, the values of these labels must match or otherwise the workload creation would fail. The labels are copied only during the workload creation and are not updated even if the labels of the underlying job are changed.", } @@ -292,10 +292,10 @@ var map_KueueConfiguration = map[string]string{ "integrations": "integrations are the types of integrations Kueue will manager", "featureGates": "featureGates are advanced features for Kueue", "resources": "resources provides additional configuration options for handling the resources. Supports https://github.com/kubernetes-sigs/kueue/blob/release-0.10/keps/2937-resource-transformer/README.md", - "manageJobsWithoutQueueName": "ManageJobsWithoutQueueName controls whether or not Kueue reconciles jobs that don't set the annotation kueue.x-k8s.io/queue-name. Allowed values are NoQueueName and QueueName Default will be QueueName", - "managedJobsNamespaceSelector": "ManagedJobsNamespaceSelector can be used to omit some namespaces from ManagedJobsWithoutQueueName", - "fairSharing": "FairSharing controls the fair sharing semantics across the cluster.", - "disableMetrics": "Disable Metrics Microshift does not enable metrics by default Default will assume metrics are enabled.", + "manageJobsWithoutQueueName": "manageJobsWithoutQueueName controls whether or not Kueue reconciles jobs that don't set the annotation kueue.x-k8s.io/queue-name. Allowed values are NoQueueName and QueueName Default will be QueueName", + "managedJobsNamespaceSelector": "managedJobsNamespaceSelector can be used to omit some namespaces from ManagedJobsWithoutQueueName Only valid if ManagedJobsWithoutQueueName is NoQueueName", + "fairSharing": "fairSharing controls the fair sharing semantics across the cluster.", + "metrics": ",etrics Microshift does not enable metrics by default Default will assume metrics are enabled.", } func (KueueConfiguration) SwaggerDoc() map[string]string { @@ -328,19 +328,9 @@ func (KueueStatus) SwaggerDoc() map[string]string { return map_KueueStatus } -var map_MultiKueue = map[string]string{ - "gcInterval": "GCInterval defines the time interval between two consecutive garbage collection runs. Defaults to 1min. If 0, the garbage collection is disabled.", - "origin": "Origin defines a label value used to track the creator of workloads in the worker clusters. This is used by multikueue in components like its garbage collector to identify remote objects that ware created by this multikueue manager cluster and delete them if their local counterpart no longer exists.", - "workerLostTimeout": "WorkerLostTimeout defines the time a local workload's multikueue admission check state is kept Ready if the connection with its reserving worker cluster is lost.\n\nDefaults to 15 minutes.", -} - -func (MultiKueue) SwaggerDoc() map[string]string { - return map_MultiKueue -} - var map_RequeuingStrategy = map[string]string{ - "timestamp": "Timestamp defines the timestamp used for re-queuing a Workload that was evicted due to Pod readiness. The possible values are:\n\n- `Eviction` (default) indicates from Workload `Evicted` condition with `PodsReadyTimeout` reason. - `Creation` indicates from Workload .metadata.creationTimestamp.", - "backoffLimitCount": "BackoffLimitCount defines the maximum number of re-queuing retries. Once the number is reached, the workload is deactivated (`.spec.activate`=`false`). When it is null, the workloads will repeatedly and endless re-queueing.\n\nEvery backoff duration is about \"b*2^(n-1)+Rand\" where: - \"b\" represents the base set by \"BackoffBaseSeconds\" parameter, - \"n\" represents the \"workloadStatus.requeueState.count\", - \"Rand\" represents the random jitter. During this time, the workload is taken as an inadmissible and other workloads will have a chance to be admitted. By default, the consecutive requeue delays are around: (60s, 120s, 240s, ...).\n\nDefaults to null.", + "timestamp": "timestamp defines the timestamp used for re-queuing a Workload that was evicted due to Pod readiness. The possible values are:\n\n- `Eviction` (default) indicates from Workload `Evicted` condition with `PodsReadyTimeout` reason. - `Creation` indicates from Workload .metadata.creationTimestamp.", + "backoffLimitCount": "backoffLimitCount defines the maximum number of re-queuing retries. Once the number is reached, the workload is deactivated (`.spec.activate`=`false`). When it is null, the workloads will repeatedly and endless re-queueing.\n\nEvery backoff duration is about \"b*2^(n-1)+Rand\" where: - \"b\" represents the base set by \"BackoffBaseSeconds\" parameter, - \"n\" represents the \"workloadStatus.requeueState.count\", - \"Rand\" represents the random jitter. During this time, the workload is taken as an inadmissible and other workloads will have a chance to be admitted. By default, the consecutive requeue delays are around: (60s, 120s, 240s, ...).\n\nDefaults to null.", "backoffBaseSeconds": "BackoffBaseSeconds defines the base for the exponential backoff for re-queuing an evicted workload.\n\nDefaults to 60.", "backoffMaxSeconds": "BackoffMaxSeconds defines the maximum backoff time to re-queue an evicted workload.\n\nDefaults to 3600.", } @@ -350,9 +340,9 @@ func (RequeuingStrategy) SwaggerDoc() map[string]string { } var map_ResourceTransformation = map[string]string{ - "input": "Input is the name of the input resource.", - "strategy": "Strategy specifies if the input resource should be replaced or retained. Defaults to Retain", - "outputs": "Outputs specifies the output resources and quantities per unit of input resource. An empty Outputs combined with a `Replace` Strategy causes the Input resource to be ignored by Kueue.", + "input": "input is the name of the input resource.", + "strategy": "strategy specifies if the input resource should be replaced or retained. Defaults to Retain", + "outputs": "outputs specifies the output resources and quantities per unit of input resource. An empty Outputs combined with a `Replace` Strategy causes the Input resource to be ignored by Kueue.", } func (ResourceTransformation) SwaggerDoc() map[string]string { @@ -360,8 +350,8 @@ func (ResourceTransformation) SwaggerDoc() map[string]string { } var map_Resources = map[string]string{ - "excludeResourcePrefixes": "ExcludedResourcePrefixes defines which resources should be ignored by Kueue", - "transformations": "Transformations defines how to transform PodSpec resources into Workload resource requests. This is intended to be a map with Input as the key (enforced by validation code)", + "excludeResourcePrefixes": "excludedResourcePrefixes defines which resources should be ignored by Kueue", + "transformations": "transformations defines how to transform PodSpec resources into Workload resource requests. This is intended to be a map with Input as the key (enforced by validation code)", } func (Resources) SwaggerDoc() map[string]string { @@ -370,11 +360,10 @@ func (Resources) SwaggerDoc() map[string]string { var map_WaitForPodsReady = map[string]string{ "": "WaitForPodsReady defines configuration for the Wait For Pods Ready feature, which is used to ensure that all Pods are ready within the specified time.", - "enable": "Enable indicates whether to enable wait for pods ready feature. Defaults to false.", - "timeout": "Timeout defines the time for an admitted workload to reach the PodsReady=true condition. When the timeout is exceeded, the workload evicted and requeued in the same cluster queue. Defaults to 5min.", - "blockAdmission": "BlockAdmission when true, cluster queue will block admissions for all subsequent jobs until the jobs reach the PodsReady=true condition. This setting is only honored when `Enable` is set to true.", - "requeuingStrategy": "RequeuingStrategy defines the strategy for requeuing a Workload.", - "recoveryTimeout": "RecoveryTimeout defines an opt-in timeout, measured since the last transition to the PodsReady=false condition after a Workload is Admitted and running. Such a transition may happen when a Pod failed and the replacement Pod is awaited to be scheduled. After exceeding the timeout the corresponding job gets suspended again and requeued after the backoff delay. The timeout is enforced only if waitForPodsReady.enable=true. If not set, there is no timeout.", + "timeout": "timeout defines the time for an admitted workload to reach the PodsReady=true condition. When the timeout is exceeded, the workload evicted and requeued in the same cluster queue. Defaults to 5min.", + "blockAdmission": "blockAdmission when true, cluster queue will block admissions for all subsequent jobs until the jobs reach the PodsReady=true condition. This setting is only honored when `Enable` is set to true.", + "requeuingStrategy": "requeuingStrategy defines the strategy for requeuing a Workload.", + "recoveryTimeout": "recoveryTimeout defines an opt-in timeout, measured since the last transition to the PodsReady=false condition after a Workload is Admitted and running. Such a transition may happen when a Pod failed and the replacement Pod is awaited to be scheduled. After exceeding the timeout the corresponding job gets suspended again and requeued after the backoff delay. The timeout is enforced only if waitForPodsReady.enable=true. If not set, there is no timeout.", } func (WaitForPodsReady) SwaggerDoc() map[string]string { From 00a45be0f88807d7dcfbf3ea133697fb8c3fa237 Mon Sep 17 00:00:00 2001 From: Kevin Hannon Date: Fri, 7 Mar 2025 15:59:39 -0500 Subject: [PATCH 04/42] api updates --- operator/v1alpha1/types_kueue.go | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/operator/v1alpha1/types_kueue.go b/operator/v1alpha1/types_kueue.go index c79e004af6e..eb4c5f17250 100644 --- a/operator/v1alpha1/types_kueue.go +++ b/operator/v1alpha1/types_kueue.go @@ -37,6 +37,7 @@ type KueueOperandSpec struct { Config KueueConfiguration `json:"config"` } +// +kubebuilder:validation:Enum="NoQueueName;QueueName" type ManageJobsWithoutQueueNameOption string const ( @@ -46,6 +47,7 @@ const ( QueueName ManageJobsWithoutQueueNameOption = "QueueName" ) +// +kubebuilder:validation:Enum="Enabled;Disabled" type EnabledOrDisabled string const ( @@ -79,12 +81,14 @@ type KueueConfiguration struct { // +optional ManagedJobsNamespaceSelector *metav1.LabelSelector `json:"managedJobsNamespaceSelector,omitempty"` // fairSharing controls the fair sharing semantics across the cluster. + // +optional FairSharing FairSharing `json:"fairSharing,omitempty"` - // ,etrics + // metrics allows one to change if metrics + // are enabled or disabled. // Microshift does not enable metrics by default // Default will assume metrics are enabled. // +kubebuilder:default=Enabled - // +required + // +optional Metrics *EnabledOrDisabled `json:"metrics,omitempty"` } @@ -105,14 +109,15 @@ type KueueList struct { // +optional metav1.ListMeta `json:"metadata,omitempty"` // items is a slice of kueue + // this is a cluster scoped resource and there can only be 1 kueue + // +kubebuilder:validation:MaxItems=1 // +required Items []Kueue `json:"items"` } // These structs come directly from Kueue. - type Resources struct { - // excludedResourcePrefixes defines which resources should be ignored by Kueue + // excludeResourcePrefixes defines which resources should be ignored by Kueue // +optional ExcludeResourcePrefixes []string `json:"excludeResourcePrefixes,omitempty"` @@ -122,6 +127,7 @@ type Resources struct { Transformations []ResourceTransformation `json:"transformations,omitempty"` } +// +kubebuilder:validation:Enum=Retain;Replace type ResourceTransformationStrategy string const Retain ResourceTransformationStrategy = "Retain" @@ -133,9 +139,7 @@ type ResourceTransformation struct { Input corev1.ResourceName `json:"input"` // strategy specifies if the input resource should be replaced or retained. - // Defaults to Retain // +kubebuilder:default=Retain - // +kubebuilder:validation:MaxLength=6 // +optional Strategy *ResourceTransformationStrategy `json:"strategy,omitempty"` @@ -145,6 +149,7 @@ type ResourceTransformation struct { Outputs corev1.ResourceList `json:"outputs,omitempty"` } +// +kubebuilder:validation:Enum=LessThanOrEqualToFinalShare;LessThanInitialShare type PreemptionStrategy string const ( @@ -216,6 +221,8 @@ type RequeuingStrategy struct { // - `Eviction` (default) indicates from Workload `Evicted` condition with `PodsReadyTimeout` reason. // - `Creation` indicates from Workload .metadata.creationTimestamp. // + // +kubebuilder:default=Eviction + // +kubebuilder:validation:MaxLength=8 // +optional Timestamp *RequeuingTimestamp `json:"timestamp,omitempty"` @@ -235,7 +242,7 @@ type RequeuingStrategy struct { // +optional BackoffLimitCount *int32 `json:"backoffLimitCount,omitempty"` - // BackoffBaseSeconds defines the base for the exponential backoff for + // backoffBaseSeconds defines the base for the exponential backoff for // re-queuing an evicted workload. // // Defaults to 60. @@ -243,7 +250,7 @@ type RequeuingStrategy struct { // +optional BackoffBaseSeconds *int32 `json:"backoffBaseSeconds,omitempty"` - // BackoffMaxSeconds defines the maximum backoff time to re-queue an evicted workload. + // backoffMaxSeconds defines the maximum backoff time to re-queue an evicted workload. // // Defaults to 3600. // +kubebuilder:default=3600 @@ -262,8 +269,7 @@ const ( ) type Integrations struct { - // frameworks - // List of framework names to be enabled. + // frameworks are a list of names to be enabled. // Possible options: // - "batch/job" // - "kubeflow.org/mpijob" @@ -279,10 +285,10 @@ type Integrations struct { // - "deployment" (requires enabling pod integration) // - "statefulset" (requires enabling pod integration) // - "leaderworkerset.x-k8s.io/leaderworkerset" (requires enabling pod integration) - // +kubebuilder:validation:items:MaxLength=14 + // +kubebuilder:validation:MaxItems=14 Frameworks []string `json:"frameworks,omitempty"` - // externalFrameworks - // List of GroupVersionKinds that are managed for Kueue by external controllers; + // externalFrameworks are a list of GroupVersionKinds + // that are managed for Kueue by external controllers; // the expected format is `Kind.version.group.com`. ExternalFrameworks []string `json:"externalFrameworks,omitempty"` From 03bd30c8122c53ac794deb6d2f9f2096498cc97b Mon Sep 17 00:00:00 2001 From: Kevin Hannon Date: Mon, 10 Mar 2025 16:25:03 -0400 Subject: [PATCH 05/42] make linter pass for kueue --- operator/v1alpha1/types_kueue.go | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/operator/v1alpha1/types_kueue.go b/operator/v1alpha1/types_kueue.go index eb4c5f17250..a0a11dd0fed 100644 --- a/operator/v1alpha1/types_kueue.go +++ b/operator/v1alpha1/types_kueue.go @@ -8,14 +8,18 @@ import ( // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object -// Kueue is the Schema for the kueue API +// Kueue is the CRD to represent the kueue operator +// This CRD defines the configuration that the Kueue // Compatibility level 4: No compatibility is provided, the API can change at any point for any reason. These capabilities should not be used by applications needing long term support. // +openshift:compatibility-gen:level=4 +// +kubebuilder:object:root=true +// +kubebuilder:resource:path=kueue,scope=Cluster // +k8s:openapi-gen=true // +genclient // +genclient:nonNamespaced // +kubebuilder:storageversion // +kubebuilder:subresource:status +// +kubebuilder:validation:XValidation:rule="self.metadata.name == 'cluster'",message="olm is a singleton, .metadata.name must be 'cluster'" type Kueue struct { metav1.TypeMeta `json:",inline"` // metadata for kueue @@ -119,11 +123,15 @@ type KueueList struct { type Resources struct { // excludeResourcePrefixes defines which resources should be ignored by Kueue // +optional + // +kubebuilder:validation:items:MaxLength=64 + // +kubebuilder:validation:MaxItems=64 ExcludeResourcePrefixes []string `json:"excludeResourcePrefixes,omitempty"` // transformations defines how to transform PodSpec resources into Workload resource requests. // This is intended to be a map with Input as the key (enforced by validation code) // +optional + // +kubebuilder:validation:items:MaxLength=64 + // +kubebuilder:validation:MaxItems=64 Transformations []ResourceTransformation `json:"transformations,omitempty"` } @@ -180,6 +188,8 @@ type FairSharing struct { // As a result, the strategy chooses to preempt workloads with the lowest priority and // newest start time first. // The default strategy is ["LessThanOrEqualToFinalShare", "LessThanInitialShare"]. + // +optional + // +kubebuilder:validation:MaxItems=2 PreemptionStrategies []PreemptionStrategy `json:"preemptionStrategies,omitempty"` } @@ -286,10 +296,16 @@ type Integrations struct { // - "statefulset" (requires enabling pod integration) // - "leaderworkerset.x-k8s.io/leaderworkerset" (requires enabling pod integration) // +kubebuilder:validation:MaxItems=14 - Frameworks []string `json:"frameworks,omitempty"` + // +kubebuilder:validation:items:MaxLength=64 + // +required + Frameworks []string `json:"frameworks"` // externalFrameworks are a list of GroupVersionKinds // that are managed for Kueue by external controllers; // the expected format is `Kind.version.group.com`. + // As far as + // +optional + // +kubebuilder:validation:items:MaxLength=64 + // +kubebuilder:validation:MaxItems=4 ExternalFrameworks []string `json:"externalFrameworks,omitempty"` // labelKeysToCopy is a list of label keys that should be copied from the job into the @@ -301,5 +317,8 @@ type Integrations struct { // match or otherwise the workload creation would fail. The labels are copied only // during the workload creation and are not updated even if the labels of the // underlying job are changed. + // +kubebuilder:validation:items:MaxLength=64 + // +kubebuilder:validation:MaxItems=64 + // +optional LabelKeysToCopy []string `json:"labelKeysToCopy,omitempty"` } From 6bd4e6f4102127c309eb263975fe8698ba327e6a Mon Sep 17 00:00:00 2001 From: Kevin Hannon Date: Tue, 11 Mar 2025 14:15:58 -0400 Subject: [PATCH 06/42] update apis --- .../generated_openapi/zz_generated.openapi.go | 26 +++++++++--------- openapi/openapi.json | 27 ++++++++++--------- ..._generated.featuregated-crd-manifests.yaml | 21 +++++++++++++++ .../AAA_ungated.yaml | 20 ++++++++++++++ .../zz_generated.swagger_doc_generated.go | 19 ++++++------- 5 files changed, 80 insertions(+), 33 deletions(-) create mode 100644 operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml diff --git a/openapi/generated_openapi/zz_generated.openapi.go b/openapi/generated_openapi/zz_generated.openapi.go index 6c7621346f5..db6de6953e0 100644 --- a/openapi/generated_openapi/zz_generated.openapi.go +++ b/openapi/generated_openapi/zz_generated.openapi.go @@ -57127,7 +57127,7 @@ func schema_openshift_api_operator_v1alpha1_Integrations(ref common.ReferenceCal Properties: map[string]spec.Schema{ "frameworks": { SchemaProps: spec.SchemaProps{ - Description: "frameworks List of framework names to be enabled. Possible options:\n - \"batch/job\"\n - \"kubeflow.org/mpijob\"\n - \"ray.io/rayjob\"\n - \"ray.io/raycluster\"\n - \"jobset.x-k8s.io/jobset\"\n - \"kubeflow.org/paddlejob\"\n - \"kubeflow.org/pytorchjob\"\n - \"kubeflow.org/tfjob\"\n - \"kubeflow.org/xgboostjob\"\n - \"workload.codeflare.dev/appwrapper\"\n - \"pod\"\n - \"deployment\" (requires enabling pod integration)\n - \"statefulset\" (requires enabling pod integration)\n - \"leaderworkerset.x-k8s.io/leaderworkerset\" (requires enabling pod integration)", + Description: "frameworks are a list of names to be enabled. Possible options:\n - \"batch/job\"\n - \"kubeflow.org/mpijob\"\n - \"ray.io/rayjob\"\n - \"ray.io/raycluster\"\n - \"jobset.x-k8s.io/jobset\"\n - \"kubeflow.org/paddlejob\"\n - \"kubeflow.org/pytorchjob\"\n - \"kubeflow.org/tfjob\"\n - \"kubeflow.org/xgboostjob\"\n - \"workload.codeflare.dev/appwrapper\"\n - \"pod\"\n - \"deployment\" (requires enabling pod integration)\n - \"statefulset\" (requires enabling pod integration)\n - \"leaderworkerset.x-k8s.io/leaderworkerset\" (requires enabling pod integration)", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -57142,7 +57142,7 @@ func schema_openshift_api_operator_v1alpha1_Integrations(ref common.ReferenceCal }, "externalFrameworks": { SchemaProps: spec.SchemaProps{ - Description: "externalFrameworks List of GroupVersionKinds that are managed for Kueue by external controllers; the expected format is `Kind.version.group.com`.", + Description: "externalFrameworks are a list of GroupVersionKinds that are managed for Kueue by external controllers; the expected format is `Kind.version.group.com`. As far as", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -57171,6 +57171,7 @@ func schema_openshift_api_operator_v1alpha1_Integrations(ref common.ReferenceCal }, }, }, + Required: []string{"frameworks"}, }, }, } @@ -57180,7 +57181,7 @@ func schema_openshift_api_operator_v1alpha1_Kueue(ref common.ReferenceCallback) return common.OpenAPIDefinition{ Schema: spec.Schema{ SchemaProps: spec.SchemaProps{ - Description: "Kueue is the Schema for the kueue API Compatibility level 4: No compatibility is provided, the API can change at any point for any reason. These capabilities should not be used by applications needing long term support.", + Description: "Kueue is the CRD to represent the kueue operator This CRD defines the configuration that the Kueue Compatibility level 4: No compatibility is provided, the API can change at any point for any reason. These capabilities should not be used by applications needing long term support.", Type: []string{"object"}, Properties: map[string]spec.Schema{ "kind": { @@ -57279,7 +57280,7 @@ func schema_openshift_api_operator_v1alpha1_KueueConfiguration(ref common.Refere }, "managedJobsNamespaceSelector": { SchemaProps: spec.SchemaProps{ - Description: "managedJobsNamespaceSelector can be used to omit some namespaces from ManagedJobsWithoutQueueName", + Description: "managedJobsNamespaceSelector can be used to omit some namespaces from ManagedJobsWithoutQueueName Only valid if ManagedJobsWithoutQueueName is NoQueueName", Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.LabelSelector"), }, }, @@ -57292,13 +57293,13 @@ func schema_openshift_api_operator_v1alpha1_KueueConfiguration(ref common.Refere }, "metrics": { SchemaProps: spec.SchemaProps{ - Description: "Metrics Microshift does not enable metrics by default Default will assume metrics are enabled.", + Description: "metrics allows one to change if metrics are enabled or disabled. Microshift does not enable metrics by default Default will assume metrics are enabled.", Type: []string{"string"}, Format: "", }, }, }, - Required: []string{"integrations", "metrics"}, + Required: []string{"integrations"}, }, }, Dependencies: []string{ @@ -57336,7 +57337,7 @@ func schema_openshift_api_operator_v1alpha1_KueueList(ref common.ReferenceCallba }, "items": { SchemaProps: spec.SchemaProps{ - Description: "items is a slice of kueue", + Description: "items is a slice of kueue this is a cluster scoped resource and there can only be 1 kueue", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -58061,14 +58062,14 @@ func schema_openshift_api_operator_v1alpha1_RequeuingStrategy(ref common.Referen }, "backoffBaseSeconds": { SchemaProps: spec.SchemaProps{ - Description: "BackoffBaseSeconds defines the base for the exponential backoff for re-queuing an evicted workload.\n\nDefaults to 60.", + Description: "backoffBaseSeconds defines the base for the exponential backoff for re-queuing an evicted workload.\n\nDefaults to 60.", Type: []string{"integer"}, Format: "int32", }, }, "backoffMaxSeconds": { SchemaProps: spec.SchemaProps{ - Description: "BackoffMaxSeconds defines the maximum backoff time to re-queue an evicted workload.\n\nDefaults to 3600.", + Description: "backoffMaxSeconds defines the maximum backoff time to re-queue an evicted workload.\n\nDefaults to 3600.", Type: []string{"integer"}, Format: "int32", }, @@ -58095,7 +58096,7 @@ func schema_openshift_api_operator_v1alpha1_ResourceTransformation(ref common.Re }, "strategy": { SchemaProps: spec.SchemaProps{ - Description: "strategy specifies if the input resource should be replaced or retained. Defaults to Retain", + Description: "strategy specifies if the input resource should be replaced or retained.", Type: []string{"string"}, Format: "", }, @@ -58127,11 +58128,12 @@ func schema_openshift_api_operator_v1alpha1_Resources(ref common.ReferenceCallba return common.OpenAPIDefinition{ Schema: spec.Schema{ SchemaProps: spec.SchemaProps{ - Type: []string{"object"}, + Description: "These structs come directly from Kueue.", + Type: []string{"object"}, Properties: map[string]spec.Schema{ "excludeResourcePrefixes": { SchemaProps: spec.SchemaProps{ - Description: "excludedResourcePrefixes defines which resources should be ignored by Kueue", + Description: "excludeResourcePrefixes defines which resources should be ignored by Kueue", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ diff --git a/openapi/openapi.json b/openapi/openapi.json index b7c51ce00d2..0e57eace064 100644 --- a/openapi/openapi.json +++ b/openapi/openapi.json @@ -33370,9 +33370,12 @@ }, "com.github.openshift.api.operator.v1alpha1.Integrations": { "type": "object", + "required": [ + "frameworks" + ], "properties": { "externalFrameworks": { - "description": "externalFrameworks List of GroupVersionKinds that are managed for Kueue by external controllers; the expected format is `Kind.version.group.com`.", + "description": "externalFrameworks are a list of GroupVersionKinds that are managed for Kueue by external controllers; the expected format is `Kind.version.group.com`. As far as", "type": "array", "items": { "type": "string", @@ -33380,7 +33383,7 @@ } }, "frameworks": { - "description": "frameworks List of framework names to be enabled. Possible options:\n - \"batch/job\"\n - \"kubeflow.org/mpijob\"\n - \"ray.io/rayjob\"\n - \"ray.io/raycluster\"\n - \"jobset.x-k8s.io/jobset\"\n - \"kubeflow.org/paddlejob\"\n - \"kubeflow.org/pytorchjob\"\n - \"kubeflow.org/tfjob\"\n - \"kubeflow.org/xgboostjob\"\n - \"workload.codeflare.dev/appwrapper\"\n - \"pod\"\n - \"deployment\" (requires enabling pod integration)\n - \"statefulset\" (requires enabling pod integration)\n - \"leaderworkerset.x-k8s.io/leaderworkerset\" (requires enabling pod integration)", + "description": "frameworks are a list of names to be enabled. Possible options:\n - \"batch/job\"\n - \"kubeflow.org/mpijob\"\n - \"ray.io/rayjob\"\n - \"ray.io/raycluster\"\n - \"jobset.x-k8s.io/jobset\"\n - \"kubeflow.org/paddlejob\"\n - \"kubeflow.org/pytorchjob\"\n - \"kubeflow.org/tfjob\"\n - \"kubeflow.org/xgboostjob\"\n - \"workload.codeflare.dev/appwrapper\"\n - \"pod\"\n - \"deployment\" (requires enabling pod integration)\n - \"statefulset\" (requires enabling pod integration)\n - \"leaderworkerset.x-k8s.io/leaderworkerset\" (requires enabling pod integration)", "type": "array", "items": { "type": "string", @@ -33398,7 +33401,7 @@ } }, "com.github.openshift.api.operator.v1alpha1.Kueue": { - "description": "Kueue is the Schema for the kueue API Compatibility level 4: No compatibility is provided, the API can change at any point for any reason. These capabilities should not be used by applications needing long term support.", + "description": "Kueue is the CRD to represent the kueue operator This CRD defines the configuration that the Kueue Compatibility level 4: No compatibility is provided, the API can change at any point for any reason. These capabilities should not be used by applications needing long term support.", "type": "object", "required": [ "spec" @@ -33432,8 +33435,7 @@ "com.github.openshift.api.operator.v1alpha1.KueueConfiguration": { "type": "object", "required": [ - "integrations", - "metrics" + "integrations" ], "properties": { "fairSharing": { @@ -33459,11 +33461,11 @@ "type": "string" }, "managedJobsNamespaceSelector": { - "description": "managedJobsNamespaceSelector can be used to omit some namespaces from ManagedJobsWithoutQueueName", + "description": "managedJobsNamespaceSelector can be used to omit some namespaces from ManagedJobsWithoutQueueName Only valid if ManagedJobsWithoutQueueName is NoQueueName", "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.LabelSelector" }, "metrics": { - "description": "Metrics Microshift does not enable metrics by default Default will assume metrics are enabled.", + "description": "metrics allows one to change if metrics are enabled or disabled. Microshift does not enable metrics by default Default will assume metrics are enabled.", "type": "string" }, "resources": { @@ -33490,7 +33492,7 @@ "type": "string" }, "items": { - "description": "items is a slice of kueue", + "description": "items is a slice of kueue this is a cluster scoped resource and there can only be 1 kueue", "type": "array", "items": { "default": {}, @@ -33937,7 +33939,7 @@ "type": "object", "properties": { "backoffBaseSeconds": { - "description": "BackoffBaseSeconds defines the base for the exponential backoff for re-queuing an evicted workload.\n\nDefaults to 60.", + "description": "backoffBaseSeconds defines the base for the exponential backoff for re-queuing an evicted workload.\n\nDefaults to 60.", "type": "integer", "format": "int32" }, @@ -33947,7 +33949,7 @@ "format": "int32" }, "backoffMaxSeconds": { - "description": "BackoffMaxSeconds defines the maximum backoff time to re-queue an evicted workload.\n\nDefaults to 3600.", + "description": "backoffMaxSeconds defines the maximum backoff time to re-queue an evicted workload.\n\nDefaults to 3600.", "type": "integer", "format": "int32" }, @@ -33976,16 +33978,17 @@ } }, "strategy": { - "description": "strategy specifies if the input resource should be replaced or retained. Defaults to Retain", + "description": "strategy specifies if the input resource should be replaced or retained.", "type": "string" } } }, "com.github.openshift.api.operator.v1alpha1.Resources": { + "description": "These structs come directly from Kueue.", "type": "object", "properties": { "excludeResourcePrefixes": { - "description": "excludedResourcePrefixes defines which resources should be ignored by Kueue", + "description": "excludeResourcePrefixes defines which resources should be ignored by Kueue", "type": "array", "items": { "type": "string", diff --git a/operator/v1alpha1/zz_generated.featuregated-crd-manifests.yaml b/operator/v1alpha1/zz_generated.featuregated-crd-manifests.yaml index 0d595be801b..d9ff920ea50 100644 --- a/operator/v1alpha1/zz_generated.featuregated-crd-manifests.yaml +++ b/operator/v1alpha1/zz_generated.featuregated-crd-manifests.yaml @@ -66,6 +66,27 @@ imagecontentsourcepolicies.operator.openshift.io: TopLevelFeatureGates: [] Version: v1alpha1 +kueue.operator.openshift.io: + Annotations: {} + ApprovedPRNumber: "" + CRDName: kueue.operator.openshift.io + Capability: "" + Category: "" + FeatureGates: [] + FilenameOperatorName: "" + FilenameOperatorOrdering: "" + FilenameRunLevel: "" + GroupName: operator.openshift.io + HasStatus: true + KindName: Kueue + Labels: {} + PluralName: kueue + PrinterColumns: [] + Scope: Cluster + ShortNames: null + TopLevelFeatureGates: [] + Version: v1alpha1 + olms.operator.openshift.io: Annotations: include.release.openshift.io/ibm-cloud-managed: "false" diff --git a/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml b/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml new file mode 100644 index 00000000000..69425f4f5e5 --- /dev/null +++ b/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml @@ -0,0 +1,20 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + feature-gate.release.openshift.io/: "true" + name: kueue.operator.openshift.io +spec: + group: operator.openshift.io + names: + kind: Kueue + listKind: KueueList + plural: kueue + singular: kueue + scope: Cluster + versions: + - name: v1alpha1 + served: true + storage: true + subresources: + status: {} diff --git a/operator/v1alpha1/zz_generated.swagger_doc_generated.go b/operator/v1alpha1/zz_generated.swagger_doc_generated.go index 93ddf349fa3..03cbfbf4ecb 100644 --- a/operator/v1alpha1/zz_generated.swagger_doc_generated.go +++ b/operator/v1alpha1/zz_generated.swagger_doc_generated.go @@ -267,8 +267,8 @@ func (FairSharing) SwaggerDoc() map[string]string { } var map_Integrations = map[string]string{ - "frameworks": "frameworks List of framework names to be enabled. Possible options:\n - \"batch/job\"\n - \"kubeflow.org/mpijob\"\n - \"ray.io/rayjob\"\n - \"ray.io/raycluster\"\n - \"jobset.x-k8s.io/jobset\"\n - \"kubeflow.org/paddlejob\"\n - \"kubeflow.org/pytorchjob\"\n - \"kubeflow.org/tfjob\"\n - \"kubeflow.org/xgboostjob\"\n - \"workload.codeflare.dev/appwrapper\"\n - \"pod\"\n - \"deployment\" (requires enabling pod integration)\n - \"statefulset\" (requires enabling pod integration)\n - \"leaderworkerset.x-k8s.io/leaderworkerset\" (requires enabling pod integration)", - "externalFrameworks": "externalFrameworks List of GroupVersionKinds that are managed for Kueue by external controllers; the expected format is `Kind.version.group.com`.", + "frameworks": "frameworks are a list of names to be enabled. Possible options:\n - \"batch/job\"\n - \"kubeflow.org/mpijob\"\n - \"ray.io/rayjob\"\n - \"ray.io/raycluster\"\n - \"jobset.x-k8s.io/jobset\"\n - \"kubeflow.org/paddlejob\"\n - \"kubeflow.org/pytorchjob\"\n - \"kubeflow.org/tfjob\"\n - \"kubeflow.org/xgboostjob\"\n - \"workload.codeflare.dev/appwrapper\"\n - \"pod\"\n - \"deployment\" (requires enabling pod integration)\n - \"statefulset\" (requires enabling pod integration)\n - \"leaderworkerset.x-k8s.io/leaderworkerset\" (requires enabling pod integration)", + "externalFrameworks": "externalFrameworks are a list of GroupVersionKinds that are managed for Kueue by external controllers; the expected format is `Kind.version.group.com`. As far as", "labelKeysToCopy": "labelKeysToCopy is a list of label keys that should be copied from the job into the workload object. It is not required for the job to have all the labels from this list. If a job does not have some label with the given key from this list, the constructed workload object will be created without this label. In the case of creating a workload from a composable job (pod group), if multiple objects have labels with some key from the list, the values of these labels must match or otherwise the workload creation would fail. The labels are copied only during the workload creation and are not updated even if the labels of the underlying job are changed.", } @@ -277,7 +277,7 @@ func (Integrations) SwaggerDoc() map[string]string { } var map_Kueue = map[string]string{ - "": "Kueue is the Schema for the kueue API Compatibility level 4: No compatibility is provided, the API can change at any point for any reason. These capabilities should not be used by applications needing long term support.", + "": "Kueue is the CRD to represent the kueue operator This CRD defines the configuration that the Kueue Compatibility level 4: No compatibility is provided, the API can change at any point for any reason. These capabilities should not be used by applications needing long term support.", "metadata": "metadata for kueue", "spec": "spec holds user settable values for configuration", "status": "status holds observed values from the cluster. They may not be overridden.", @@ -295,7 +295,7 @@ var map_KueueConfiguration = map[string]string{ "manageJobsWithoutQueueName": "manageJobsWithoutQueueName controls whether or not Kueue reconciles jobs that don't set the annotation kueue.x-k8s.io/queue-name. Allowed values are NoQueueName and QueueName Default will be QueueName", "managedJobsNamespaceSelector": "managedJobsNamespaceSelector can be used to omit some namespaces from ManagedJobsWithoutQueueName Only valid if ManagedJobsWithoutQueueName is NoQueueName", "fairSharing": "fairSharing controls the fair sharing semantics across the cluster.", - "metrics": ",etrics Microshift does not enable metrics by default Default will assume metrics are enabled.", + "metrics": "metrics allows one to change if metrics are enabled or disabled. Microshift does not enable metrics by default Default will assume metrics are enabled.", } func (KueueConfiguration) SwaggerDoc() map[string]string { @@ -305,7 +305,7 @@ func (KueueConfiguration) SwaggerDoc() map[string]string { var map_KueueList = map[string]string{ "": "KueueList contains a list of Kueue\n\nCompatibility level 4: No compatibility is provided, the API can change at any point for any reason. These capabilities should not be used by applications needing long term support.", "metadata": "metadata for the list", - "items": "items is a slice of kueue", + "items": "items is a slice of kueue this is a cluster scoped resource and there can only be 1 kueue", } func (KueueList) SwaggerDoc() map[string]string { @@ -331,8 +331,8 @@ func (KueueStatus) SwaggerDoc() map[string]string { var map_RequeuingStrategy = map[string]string{ "timestamp": "timestamp defines the timestamp used for re-queuing a Workload that was evicted due to Pod readiness. The possible values are:\n\n- `Eviction` (default) indicates from Workload `Evicted` condition with `PodsReadyTimeout` reason. - `Creation` indicates from Workload .metadata.creationTimestamp.", "backoffLimitCount": "backoffLimitCount defines the maximum number of re-queuing retries. Once the number is reached, the workload is deactivated (`.spec.activate`=`false`). When it is null, the workloads will repeatedly and endless re-queueing.\n\nEvery backoff duration is about \"b*2^(n-1)+Rand\" where: - \"b\" represents the base set by \"BackoffBaseSeconds\" parameter, - \"n\" represents the \"workloadStatus.requeueState.count\", - \"Rand\" represents the random jitter. During this time, the workload is taken as an inadmissible and other workloads will have a chance to be admitted. By default, the consecutive requeue delays are around: (60s, 120s, 240s, ...).\n\nDefaults to null.", - "backoffBaseSeconds": "BackoffBaseSeconds defines the base for the exponential backoff for re-queuing an evicted workload.\n\nDefaults to 60.", - "backoffMaxSeconds": "BackoffMaxSeconds defines the maximum backoff time to re-queue an evicted workload.\n\nDefaults to 3600.", + "backoffBaseSeconds": "backoffBaseSeconds defines the base for the exponential backoff for re-queuing an evicted workload.\n\nDefaults to 60.", + "backoffMaxSeconds": "backoffMaxSeconds defines the maximum backoff time to re-queue an evicted workload.\n\nDefaults to 3600.", } func (RequeuingStrategy) SwaggerDoc() map[string]string { @@ -341,7 +341,7 @@ func (RequeuingStrategy) SwaggerDoc() map[string]string { var map_ResourceTransformation = map[string]string{ "input": "input is the name of the input resource.", - "strategy": "strategy specifies if the input resource should be replaced or retained. Defaults to Retain", + "strategy": "strategy specifies if the input resource should be replaced or retained.", "outputs": "outputs specifies the output resources and quantities per unit of input resource. An empty Outputs combined with a `Replace` Strategy causes the Input resource to be ignored by Kueue.", } @@ -350,7 +350,8 @@ func (ResourceTransformation) SwaggerDoc() map[string]string { } var map_Resources = map[string]string{ - "excludeResourcePrefixes": "excludedResourcePrefixes defines which resources should be ignored by Kueue", + "": "These structs come directly from Kueue.", + "excludeResourcePrefixes": "excludeResourcePrefixes defines which resources should be ignored by Kueue", "transformations": "transformations defines how to transform PodSpec resources into Workload resource requests. This is intended to be a map with Input as the key (enforced by validation code)", } From cab346d46593157c467db9a7bee67070a97097f3 Mon Sep 17 00:00:00 2001 From: Kevin Hannon Date: Wed, 12 Mar 2025 16:55:17 -0400 Subject: [PATCH 07/42] resolving api changes --- .../generated_openapi/zz_generated.openapi.go | 24 ++++----- openapi/openapi.json | 12 ++--- operator/v1alpha1/types_kueue.go | 54 ++++++++++++------- operator/v1alpha1/zz_generated.deepcopy.go | 2 +- .../zz_generated.swagger_doc_generated.go | 12 ++--- 5 files changed, 59 insertions(+), 45 deletions(-) diff --git a/openapi/generated_openapi/zz_generated.openapi.go b/openapi/generated_openapi/zz_generated.openapi.go index db6de6953e0..2f0b424b32d 100644 --- a/openapi/generated_openapi/zz_generated.openapi.go +++ b/openapi/generated_openapi/zz_generated.openapi.go @@ -56858,7 +56858,7 @@ func schema_openshift_api_operator_v1alpha1_FairSharing(ref common.ReferenceCall Properties: map[string]spec.Schema{ "enable": { SchemaProps: spec.SchemaProps{ - Description: "enable indicates whether to enable fair sharing for all cohorts. Defaults to false.", + Description: "enable indicates whether to enable fair sharing for all cohorts.", Default: "", Type: []string{"string"}, Format: "", @@ -57241,16 +57241,9 @@ func schema_openshift_api_operator_v1alpha1_KueueConfiguration(ref common.Refere Ref: ref("github.com/openshift/api/operator/v1alpha1.WaitForPodsReady"), }, }, - "integrations": { - SchemaProps: spec.SchemaProps{ - Description: "integrations are the types of integrations Kueue will manager", - Default: map[string]interface{}{}, - Ref: ref("github.com/openshift/api/operator/v1alpha1.Integrations"), - }, - }, "featureGates": { SchemaProps: spec.SchemaProps{ - Description: "featureGates are advanced features for Kueue", + Description: "featureGates are advanced features for Kueue if ManagementState is Unmanaged Otherwise we will fail at validation time.", Type: []string{"object"}, AdditionalProperties: &spec.SchemaOrBool{ Allows: true, @@ -57264,6 +57257,13 @@ func schema_openshift_api_operator_v1alpha1_KueueConfiguration(ref common.Refere }, }, }, + "integrations": { + SchemaProps: spec.SchemaProps{ + Description: "integrations are the types of integrations Kueue will manage", + Default: map[string]interface{}{}, + Ref: ref("github.com/openshift/api/operator/v1alpha1.Integrations"), + }, + }, "resources": { SchemaProps: spec.SchemaProps{ Description: "resources provides additional configuration options for handling the resources. Supports https://github.com/kubernetes-sigs/kueue/blob/release-0.10/keps/2937-resource-transformer/README.md", @@ -57273,14 +57273,14 @@ func schema_openshift_api_operator_v1alpha1_KueueConfiguration(ref common.Refere }, "manageJobsWithoutQueueName": { SchemaProps: spec.SchemaProps{ - Description: "manageJobsWithoutQueueName controls whether or not Kueue reconciles jobs that don't set the annotation kueue.x-k8s.io/queue-name. Allowed values are NoQueueName and QueueName Default will be QueueName", + Description: "manageJobsWithoutQueueName controls whether or not Kueue reconciles jobs that don't set the annotation kueue.x-k8s.io/queue-name.", Type: []string{"string"}, Format: "", }, }, "managedJobsNamespaceSelector": { SchemaProps: spec.SchemaProps{ - Description: "managedJobsNamespaceSelector can be used to omit some namespaces from ManagedJobsWithoutQueueName Only valid if ManagedJobsWithoutQueueName is NoQueueName", + Description: "managedJobsNamespaceSelector can be used to omit some namespaces from ManagedJobsWithoutQueueName Only valid if ManagedJobsWithoutQueueName is QueueNameOptional", Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.LabelSelector"), }, }, @@ -57400,7 +57400,7 @@ func schema_openshift_api_operator_v1alpha1_KueueOperandSpec(ref common.Referenc }, "config": { SchemaProps: spec.SchemaProps{ - Description: "config that is persisted to a config map", + Description: "config is the desired configuration for the kueue operator.", Default: map[string]interface{}{}, Ref: ref("github.com/openshift/api/operator/v1alpha1.KueueConfiguration"), }, diff --git a/openapi/openapi.json b/openapi/openapi.json index 0e57eace064..5421bf61a87 100644 --- a/openapi/openapi.json +++ b/openapi/openapi.json @@ -33210,7 +33210,7 @@ "type": "object", "properties": { "enable": { - "description": "enable indicates whether to enable fair sharing for all cohorts. Defaults to false.", + "description": "enable indicates whether to enable fair sharing for all cohorts.", "type": "string", "default": "" }, @@ -33444,7 +33444,7 @@ "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.FairSharing" }, "featureGates": { - "description": "featureGates are advanced features for Kueue", + "description": "featureGates are advanced features for Kueue if ManagementState is Unmanaged Otherwise we will fail at validation time.", "type": "object", "additionalProperties": { "type": "string", @@ -33452,16 +33452,16 @@ } }, "integrations": { - "description": "integrations are the types of integrations Kueue will manager", + "description": "integrations are the types of integrations Kueue will manage", "default": {}, "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.Integrations" }, "manageJobsWithoutQueueName": { - "description": "manageJobsWithoutQueueName controls whether or not Kueue reconciles jobs that don't set the annotation kueue.x-k8s.io/queue-name. Allowed values are NoQueueName and QueueName Default will be QueueName", + "description": "manageJobsWithoutQueueName controls whether or not Kueue reconciles jobs that don't set the annotation kueue.x-k8s.io/queue-name.", "type": "string" }, "managedJobsNamespaceSelector": { - "description": "managedJobsNamespaceSelector can be used to omit some namespaces from ManagedJobsWithoutQueueName Only valid if ManagedJobsWithoutQueueName is NoQueueName", + "description": "managedJobsNamespaceSelector can be used to omit some namespaces from ManagedJobsWithoutQueueName Only valid if ManagedJobsWithoutQueueName is QueueNameOptional", "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.LabelSelector" }, "metrics": { @@ -33518,7 +33518,7 @@ ], "properties": { "config": { - "description": "config that is persisted to a config map", + "description": "config is the desired configuration for the kueue operator.", "default": {}, "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.KueueConfiguration" }, diff --git a/operator/v1alpha1/types_kueue.go b/operator/v1alpha1/types_kueue.go index a0a11dd0fed..7b2e15c521e 100644 --- a/operator/v1alpha1/types_kueue.go +++ b/operator/v1alpha1/types_kueue.go @@ -36,22 +36,29 @@ type Kueue struct { type KueueOperandSpec struct { operatorv1.OperatorSpec `json:",inline"` - // config that is persisted to a config map + // config is the desired configuration + // for the kueue operator. // +required Config KueueConfiguration `json:"config"` } -// +kubebuilder:validation:Enum="NoQueueName;QueueName" +// +kubebuilder:validation:Enum=QueueNameOptional;QueueNameRequired type ManageJobsWithoutQueueNameOption string +// ManageJobsWithoutQueueName allows to control what kind of +// jobs kueue will manage. +// Kueue jobs usually require kueue.x-k8s.io/queue-name on each job +// to be opt-in for Kueue. const ( - // NoQueueName means that all jobs will be gated by Kueue - NoQueueName ManageJobsWithoutQueueNameOption = "NoQueueName" - // QueueName means that the jobs require a queue label. - QueueName ManageJobsWithoutQueueNameOption = "QueueName" + // QueueNameOptional means kueue will assume all workloads + // are to be gated by Kueue. + // This must be used with ManagedJobsNamespaceSelector. + QueueNameOptional ManageJobsWithoutQueueNameOption = "QueueNameOptional" + // QueueNameRequired means that the jobs require a queue label. + QueueNameRequired ManageJobsWithoutQueueNameOption = "QueueNameRequired" ) -// +kubebuilder:validation:Enum="Enabled;Disabled" +// +kubebuilder:validation:Enum=Enabled;Disabled type EnabledOrDisabled string const ( @@ -59,29 +66,39 @@ const ( Disabled EnabledOrDisabled = "Disabled" ) +// Feature gates is unresolved +// Option 1: +// Drop the feature gate entirely +// Feature gate modification is not supported. +// Option 2: +// Allow for modifications of feature gates if operator is an Unmanaged state +// We could validate that user set operator in "Unmanaged" state as part of operator v1 API +// When this is set, specific experimental fields will be enabled. + type KueueConfiguration struct { // waitForPodsReady configures gang admission // +optional WaitForPodsReady WaitForPodsReady `json:"waitForPodsReady,omitempty"` - // integrations are the types of integrations Kueue will manager - // +required - Integrations Integrations `json:"integrations"` // featureGates are advanced features for Kueue + // TODO: we could only allow features + // if ManagementState is Unmanaged + // Otherwise we will fail at validation time. // +optional FeatureGates map[string]EnabledOrDisabled `json:"featureGates,omitempty"` + // integrations are the types of integrations Kueue will manage + // +required + Integrations Integrations `json:"integrations"` // resources provides additional configuration options for handling the resources. // Supports https://github.com/kubernetes-sigs/kueue/blob/release-0.10/keps/2937-resource-transformer/README.md // +optional Resources Resources `json:"resources,omitempty"` // manageJobsWithoutQueueName controls whether or not Kueue reconciles // jobs that don't set the annotation kueue.x-k8s.io/queue-name. - // Allowed values are NoQueueName and QueueName - // Default will be QueueName - // +kubebuilder:default=QueueName + // +kubebuilder:default=QueueNameRequired // +optional ManageJobsWithoutQueueName *ManageJobsWithoutQueueNameOption `json:"manageJobsWithoutQueueName,omitempty"` // managedJobsNamespaceSelector can be used to omit some namespaces from ManagedJobsWithoutQueueName - // Only valid if ManagedJobsWithoutQueueName is NoQueueName + // Only valid if ManagedJobsWithoutQueueName is QueueNameOptional // +optional ManagedJobsNamespaceSelector *metav1.LabelSelector `json:"managedJobsNamespaceSelector,omitempty"` // fairSharing controls the fair sharing semantics across the cluster. @@ -91,7 +108,6 @@ type KueueConfiguration struct { // are enabled or disabled. // Microshift does not enable metrics by default // Default will assume metrics are enabled. - // +kubebuilder:default=Enabled // +optional Metrics *EnabledOrDisabled `json:"metrics,omitempty"` } @@ -124,18 +140,18 @@ type Resources struct { // excludeResourcePrefixes defines which resources should be ignored by Kueue // +optional // +kubebuilder:validation:items:MaxLength=64 - // +kubebuilder:validation:MaxItems=64 + // +kubebuilder:validation:MaxItems=16 ExcludeResourcePrefixes []string `json:"excludeResourcePrefixes,omitempty"` // transformations defines how to transform PodSpec resources into Workload resource requests. // This is intended to be a map with Input as the key (enforced by validation code) // +optional // +kubebuilder:validation:items:MaxLength=64 - // +kubebuilder:validation:MaxItems=64 + // +kubebuilder:validation:MaxItems=16 Transformations []ResourceTransformation `json:"transformations,omitempty"` } -// +kubebuilder:validation:Enum=Retain;Replace +// +kubebuilder:validation:Enum="Retain";"Replace" type ResourceTransformationStrategy string const Retain ResourceTransformationStrategy = "Retain" @@ -167,7 +183,6 @@ const ( type FairSharing struct { // enable indicates whether to enable fair sharing for all cohorts. - // Defaults to false. // +kubebuilder:default=Disabled // +optional Enable EnabledOrDisabled `json:"enable"` @@ -231,7 +246,6 @@ type RequeuingStrategy struct { // - `Eviction` (default) indicates from Workload `Evicted` condition with `PodsReadyTimeout` reason. // - `Creation` indicates from Workload .metadata.creationTimestamp. // - // +kubebuilder:default=Eviction // +kubebuilder:validation:MaxLength=8 // +optional Timestamp *RequeuingTimestamp `json:"timestamp,omitempty"` diff --git a/operator/v1alpha1/zz_generated.deepcopy.go b/operator/v1alpha1/zz_generated.deepcopy.go index ea5bdf23e2d..4ee68793b67 100644 --- a/operator/v1alpha1/zz_generated.deepcopy.go +++ b/operator/v1alpha1/zz_generated.deepcopy.go @@ -469,7 +469,6 @@ func (in *Kueue) DeepCopyObject() runtime.Object { func (in *KueueConfiguration) DeepCopyInto(out *KueueConfiguration) { *out = *in in.WaitForPodsReady.DeepCopyInto(&out.WaitForPodsReady) - in.Integrations.DeepCopyInto(&out.Integrations) if in.FeatureGates != nil { in, out := &in.FeatureGates, &out.FeatureGates *out = make(map[string]EnabledOrDisabled, len(*in)) @@ -477,6 +476,7 @@ func (in *KueueConfiguration) DeepCopyInto(out *KueueConfiguration) { (*out)[key] = val } } + in.Integrations.DeepCopyInto(&out.Integrations) in.Resources.DeepCopyInto(&out.Resources) if in.ManageJobsWithoutQueueName != nil { in, out := &in.ManageJobsWithoutQueueName, &out.ManageJobsWithoutQueueName diff --git a/operator/v1alpha1/zz_generated.swagger_doc_generated.go b/operator/v1alpha1/zz_generated.swagger_doc_generated.go index 03cbfbf4ecb..3b7a1ba7740 100644 --- a/operator/v1alpha1/zz_generated.swagger_doc_generated.go +++ b/operator/v1alpha1/zz_generated.swagger_doc_generated.go @@ -258,7 +258,7 @@ func (RepositoryDigestMirrors) SwaggerDoc() map[string]string { } var map_FairSharing = map[string]string{ - "enable": "enable indicates whether to enable fair sharing for all cohorts. Defaults to false.", + "enable": "enable indicates whether to enable fair sharing for all cohorts.", "preemptionStrategies": "preemptionStrategies indicates which constraints should a preemption satisfy. The preemption algorithm will only use the next strategy in the list if the incoming workload (preemptor) doesn't fit after using the previous strategies. Possible values are: - LessThanOrEqualToFinalShare: Only preempt a workload if the share of the preemptor CQ\n with the preemptor workload is less than or equal to the share of the preemptee CQ\n without the workload to be preempted.\n This strategy might favor preemption of smaller workloads in the preemptee CQ,\n regardless of priority or start time, in an effort to keep the share of the CQ\n as high as possible.\n- LessThanInitialShare: Only preempt a workload if the share of the preemptor CQ\n with the incoming workload is strictly less than the share of the preemptee CQ.\n This strategy doesn't depend on the share usage of the workload being preempted.\n As a result, the strategy chooses to preempt workloads with the lowest priority and\n newest start time first.\nThe default strategy is [\"LessThanOrEqualToFinalShare\", \"LessThanInitialShare\"].", } @@ -289,11 +289,11 @@ func (Kueue) SwaggerDoc() map[string]string { var map_KueueConfiguration = map[string]string{ "waitForPodsReady": "waitForPodsReady configures gang admission", - "integrations": "integrations are the types of integrations Kueue will manager", - "featureGates": "featureGates are advanced features for Kueue", + "featureGates": "featureGates are advanced features for Kueue if ManagementState is Unmanaged Otherwise we will fail at validation time.", + "integrations": "integrations are the types of integrations Kueue will manage", "resources": "resources provides additional configuration options for handling the resources. Supports https://github.com/kubernetes-sigs/kueue/blob/release-0.10/keps/2937-resource-transformer/README.md", - "manageJobsWithoutQueueName": "manageJobsWithoutQueueName controls whether or not Kueue reconciles jobs that don't set the annotation kueue.x-k8s.io/queue-name. Allowed values are NoQueueName and QueueName Default will be QueueName", - "managedJobsNamespaceSelector": "managedJobsNamespaceSelector can be used to omit some namespaces from ManagedJobsWithoutQueueName Only valid if ManagedJobsWithoutQueueName is NoQueueName", + "manageJobsWithoutQueueName": "manageJobsWithoutQueueName controls whether or not Kueue reconciles jobs that don't set the annotation kueue.x-k8s.io/queue-name.", + "managedJobsNamespaceSelector": "managedJobsNamespaceSelector can be used to omit some namespaces from ManagedJobsWithoutQueueName Only valid if ManagedJobsWithoutQueueName is QueueNameOptional", "fairSharing": "fairSharing controls the fair sharing semantics across the cluster.", "metrics": "metrics allows one to change if metrics are enabled or disabled. Microshift does not enable metrics by default Default will assume metrics are enabled.", } @@ -313,7 +313,7 @@ func (KueueList) SwaggerDoc() map[string]string { } var map_KueueOperandSpec = map[string]string{ - "config": "config that is persisted to a config map", + "config": "config is the desired configuration for the kueue operator.", } func (KueueOperandSpec) SwaggerDoc() map[string]string { From 5c036c03283b2af588aa7ec1a44985926345fbf8 Mon Sep 17 00:00:00 2001 From: Kevin Hannon Date: Wed, 12 Mar 2025 21:05:31 -0400 Subject: [PATCH 08/42] option 3 --- operator/v1alpha1/types_kueue.go | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/operator/v1alpha1/types_kueue.go b/operator/v1alpha1/types_kueue.go index 7b2e15c521e..fe85a7486ae 100644 --- a/operator/v1alpha1/types_kueue.go +++ b/operator/v1alpha1/types_kueue.go @@ -68,13 +68,20 @@ const ( // Feature gates is unresolved // Option 1: -// Drop the feature gate entirely -// Feature gate modification is not supported. +// +// Drop the feature gate entirely +// Feature gate modification is not supported. +// // Option 2: -// Allow for modifications of feature gates if operator is an Unmanaged state -// We could validate that user set operator in "Unmanaged" state as part of operator v1 API -// When this is set, specific experimental fields will be enabled. - +// +// Allow for modifications of feature gates if operator is an Unmanaged state +// We could validate that user set operator in "Unmanaged" state as part of operator v1 API +// When this is set, specific experimental fields will be enabled. +// +// Option 3: +// +// Allow for a users to specifiy a configmap that matches kueue configuration +// Operator will fetch the configmap and assume its valid. type KueueConfiguration struct { // waitForPodsReady configures gang admission // +optional From 5d4942c5ee05825b90c47d6de2fa4e0601b8de4d Mon Sep 17 00:00:00 2001 From: Kevin Hannon Date: Thu, 13 Mar 2025 19:39:37 -0400 Subject: [PATCH 09/42] address comments --- .../generated_openapi/zz_generated.openapi.go | 114 +--------------- openapi/openapi.json | 64 +-------- operator/v1alpha1/types_kueue.go | 122 ++---------------- operator/v1alpha1/zz_generated.deepcopy.go | 80 ------------ .../zz_generated.swagger_doc_generated.go | 29 +---- 5 files changed, 20 insertions(+), 389 deletions(-) diff --git a/openapi/generated_openapi/zz_generated.openapi.go b/openapi/generated_openapi/zz_generated.openapi.go index 2f0b424b32d..6c2ac02f752 100644 --- a/openapi/generated_openapi/zz_generated.openapi.go +++ b/openapi/generated_openapi/zz_generated.openapi.go @@ -1132,12 +1132,10 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "github.com/openshift/api/operator/v1alpha1.OperatorSpec": schema_openshift_api_operator_v1alpha1_OperatorSpec(ref), "github.com/openshift/api/operator/v1alpha1.OperatorStatus": schema_openshift_api_operator_v1alpha1_OperatorStatus(ref), "github.com/openshift/api/operator/v1alpha1.RepositoryDigestMirrors": schema_openshift_api_operator_v1alpha1_RepositoryDigestMirrors(ref), - "github.com/openshift/api/operator/v1alpha1.RequeuingStrategy": schema_openshift_api_operator_v1alpha1_RequeuingStrategy(ref), "github.com/openshift/api/operator/v1alpha1.ResourceTransformation": schema_openshift_api_operator_v1alpha1_ResourceTransformation(ref), "github.com/openshift/api/operator/v1alpha1.Resources": schema_openshift_api_operator_v1alpha1_Resources(ref), "github.com/openshift/api/operator/v1alpha1.StaticPodOperatorStatus": schema_openshift_api_operator_v1alpha1_StaticPodOperatorStatus(ref), "github.com/openshift/api/operator/v1alpha1.VersionAvailability": schema_openshift_api_operator_v1alpha1_VersionAvailability(ref), - "github.com/openshift/api/operator/v1alpha1.WaitForPodsReady": schema_openshift_api_operator_v1alpha1_WaitForPodsReady(ref), "github.com/openshift/api/operatorcontrolplane/v1alpha1.LogEntry": schema_openshift_api_operatorcontrolplane_v1alpha1_LogEntry(ref), "github.com/openshift/api/operatorcontrolplane/v1alpha1.OutageEntry": schema_openshift_api_operatorcontrolplane_v1alpha1_OutageEntry(ref), "github.com/openshift/api/operatorcontrolplane/v1alpha1.PodNetworkConnectivityCheck": schema_openshift_api_operatorcontrolplane_v1alpha1_PodNetworkConnectivityCheck(ref), @@ -57207,7 +57205,7 @@ func schema_openshift_api_operator_v1alpha1_Kueue(ref common.ReferenceCallback) }, "spec": { SchemaProps: spec.SchemaProps{ - Description: "spec holds user settable values for configuration", + Description: "spec holds user settable values for configuration kueue configuration must not be changed once the object exists to change the configuration, one can delete the object and create a new object.", Default: map[string]interface{}{}, Ref: ref("github.com/openshift/api/operator/v1alpha1.KueueOperandSpec"), }, @@ -57232,31 +57230,9 @@ func schema_openshift_api_operator_v1alpha1_KueueConfiguration(ref common.Refere return common.OpenAPIDefinition{ Schema: spec.Schema{ SchemaProps: spec.SchemaProps{ - Type: []string{"object"}, + Description: "Feature gates is unresolved Option 1:\n\n\tDrop the feature gate entirely\n\tFeature gate modification is not supported.\n\nOption 2:\n\n\tAllow for modifications of feature gates if operator is an Unmanaged state\n\tWe could validate that user set operator in \"Unmanaged\" state as part of operator v1 API\n\tWhen this is set, specific experimental fields will be enabled.\n\nOption 3:\n\n\tAllow for a users to specifiy a configmap that matches kueue configuration\n\tOperator will fetch the configmap and assume its valid.", + Type: []string{"object"}, Properties: map[string]spec.Schema{ - "waitForPodsReady": { - SchemaProps: spec.SchemaProps{ - Description: "waitForPodsReady configures gang admission", - Default: map[string]interface{}{}, - Ref: ref("github.com/openshift/api/operator/v1alpha1.WaitForPodsReady"), - }, - }, - "featureGates": { - SchemaProps: spec.SchemaProps{ - Description: "featureGates are advanced features for Kueue if ManagementState is Unmanaged Otherwise we will fail at validation time.", - Type: []string{"object"}, - AdditionalProperties: &spec.SchemaOrBool{ - Allows: true, - Schema: &spec.Schema{ - SchemaProps: spec.SchemaProps{ - Default: "", - Type: []string{"string"}, - Format: "", - }, - }, - }, - }, - }, "integrations": { SchemaProps: spec.SchemaProps{ Description: "integrations are the types of integrations Kueue will manage", @@ -57303,7 +57279,7 @@ func schema_openshift_api_operator_v1alpha1_KueueConfiguration(ref common.Refere }, }, Dependencies: []string{ - "github.com/openshift/api/operator/v1alpha1.FairSharing", "github.com/openshift/api/operator/v1alpha1.Integrations", "github.com/openshift/api/operator/v1alpha1.Resources", "github.com/openshift/api/operator/v1alpha1.WaitForPodsReady", "k8s.io/apimachinery/pkg/apis/meta/v1.LabelSelector"}, + "github.com/openshift/api/operator/v1alpha1.FairSharing", "github.com/openshift/api/operator/v1alpha1.Integrations", "github.com/openshift/api/operator/v1alpha1.Resources", "k8s.io/apimachinery/pkg/apis/meta/v1.LabelSelector"}, } } @@ -58040,46 +58016,6 @@ func schema_openshift_api_operator_v1alpha1_RepositoryDigestMirrors(ref common.R } } -func schema_openshift_api_operator_v1alpha1_RequeuingStrategy(ref common.ReferenceCallback) common.OpenAPIDefinition { - return common.OpenAPIDefinition{ - Schema: spec.Schema{ - SchemaProps: spec.SchemaProps{ - Type: []string{"object"}, - Properties: map[string]spec.Schema{ - "timestamp": { - SchemaProps: spec.SchemaProps{ - Description: "timestamp defines the timestamp used for re-queuing a Workload that was evicted due to Pod readiness. The possible values are:\n\n- `Eviction` (default) indicates from Workload `Evicted` condition with `PodsReadyTimeout` reason. - `Creation` indicates from Workload .metadata.creationTimestamp.", - Type: []string{"string"}, - Format: "", - }, - }, - "backoffLimitCount": { - SchemaProps: spec.SchemaProps{ - Description: "backoffLimitCount defines the maximum number of re-queuing retries. Once the number is reached, the workload is deactivated (`.spec.activate`=`false`). When it is null, the workloads will repeatedly and endless re-queueing.\n\nEvery backoff duration is about \"b*2^(n-1)+Rand\" where: - \"b\" represents the base set by \"BackoffBaseSeconds\" parameter, - \"n\" represents the \"workloadStatus.requeueState.count\", - \"Rand\" represents the random jitter. During this time, the workload is taken as an inadmissible and other workloads will have a chance to be admitted. By default, the consecutive requeue delays are around: (60s, 120s, 240s, ...).\n\nDefaults to null.", - Type: []string{"integer"}, - Format: "int32", - }, - }, - "backoffBaseSeconds": { - SchemaProps: spec.SchemaProps{ - Description: "backoffBaseSeconds defines the base for the exponential backoff for re-queuing an evicted workload.\n\nDefaults to 60.", - Type: []string{"integer"}, - Format: "int32", - }, - }, - "backoffMaxSeconds": { - SchemaProps: spec.SchemaProps{ - Description: "backoffMaxSeconds defines the maximum backoff time to re-queue an evicted workload.\n\nDefaults to 3600.", - Type: []string{"integer"}, - Format: "int32", - }, - }, - }, - }, - }, - } -} - func schema_openshift_api_operator_v1alpha1_ResourceTransformation(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ @@ -58088,7 +58024,7 @@ func schema_openshift_api_operator_v1alpha1_ResourceTransformation(ref common.Re Properties: map[string]spec.Schema{ "input": { SchemaProps: spec.SchemaProps{ - Description: "input is the name of the input resource.", + Description: "input is the name of the input resource. resources are pod spec resources like cpu, memory, gpus", Default: "", Type: []string{"string"}, Format: "", @@ -58322,46 +58258,6 @@ func schema_openshift_api_operator_v1alpha1_VersionAvailability(ref common.Refer } } -func schema_openshift_api_operator_v1alpha1_WaitForPodsReady(ref common.ReferenceCallback) common.OpenAPIDefinition { - return common.OpenAPIDefinition{ - Schema: spec.Schema{ - SchemaProps: spec.SchemaProps{ - Description: "WaitForPodsReady defines configuration for the Wait For Pods Ready feature, which is used to ensure that all Pods are ready within the specified time.", - Type: []string{"object"}, - Properties: map[string]spec.Schema{ - "timeout": { - SchemaProps: spec.SchemaProps{ - Description: "timeout defines the time for an admitted workload to reach the PodsReady=true condition. When the timeout is exceeded, the workload evicted and requeued in the same cluster queue. Defaults to 5min.", - Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.Duration"), - }, - }, - "blockAdmission": { - SchemaProps: spec.SchemaProps{ - Description: "blockAdmission when true, cluster queue will block admissions for all subsequent jobs until the jobs reach the PodsReady=true condition. This setting is only honored when `Enable` is set to true.", - Type: []string{"string"}, - Format: "", - }, - }, - "requeuingStrategy": { - SchemaProps: spec.SchemaProps{ - Description: "requeuingStrategy defines the strategy for requeuing a Workload.", - Ref: ref("github.com/openshift/api/operator/v1alpha1.RequeuingStrategy"), - }, - }, - "recoveryTimeout": { - SchemaProps: spec.SchemaProps{ - Description: "recoveryTimeout defines an opt-in timeout, measured since the last transition to the PodsReady=false condition after a Workload is Admitted and running. Such a transition may happen when a Pod failed and the replacement Pod is awaited to be scheduled. After exceeding the timeout the corresponding job gets suspended again and requeued after the backoff delay. The timeout is enforced only if waitForPodsReady.enable=true. If not set, there is no timeout.", - Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.Duration"), - }, - }, - }, - }, - }, - Dependencies: []string{ - "github.com/openshift/api/operator/v1alpha1.RequeuingStrategy", "k8s.io/apimachinery/pkg/apis/meta/v1.Duration"}, - } -} - func schema_openshift_api_operatorcontrolplane_v1alpha1_LogEntry(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ diff --git a/openapi/openapi.json b/openapi/openapi.json index 5421bf61a87..558194cd108 100644 --- a/openapi/openapi.json +++ b/openapi/openapi.json @@ -33421,7 +33421,7 @@ "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta" }, "spec": { - "description": "spec holds user settable values for configuration", + "description": "spec holds user settable values for configuration kueue configuration must not be changed once the object exists to change the configuration, one can delete the object and create a new object.", "default": {}, "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.KueueOperandSpec" }, @@ -33433,6 +33433,7 @@ } }, "com.github.openshift.api.operator.v1alpha1.KueueConfiguration": { + "description": "Feature gates is unresolved Option 1:\n\n\tDrop the feature gate entirely\n\tFeature gate modification is not supported.\n\nOption 2:\n\n\tAllow for modifications of feature gates if operator is an Unmanaged state\n\tWe could validate that user set operator in \"Unmanaged\" state as part of operator v1 API\n\tWhen this is set, specific experimental fields will be enabled.\n\nOption 3:\n\n\tAllow for a users to specifiy a configmap that matches kueue configuration\n\tOperator will fetch the configmap and assume its valid.", "type": "object", "required": [ "integrations" @@ -33443,14 +33444,6 @@ "default": {}, "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.FairSharing" }, - "featureGates": { - "description": "featureGates are advanced features for Kueue if ManagementState is Unmanaged Otherwise we will fail at validation time.", - "type": "object", - "additionalProperties": { - "type": "string", - "default": "" - } - }, "integrations": { "description": "integrations are the types of integrations Kueue will manage", "default": {}, @@ -33472,11 +33465,6 @@ "description": "resources provides additional configuration options for handling the resources. Supports https://github.com/kubernetes-sigs/kueue/blob/release-0.10/keps/2937-resource-transformer/README.md", "default": {}, "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.Resources" - }, - "waitForPodsReady": { - "description": "waitForPodsReady configures gang admission", - "default": {}, - "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.WaitForPodsReady" } } }, @@ -33935,30 +33923,6 @@ } } }, - "com.github.openshift.api.operator.v1alpha1.RequeuingStrategy": { - "type": "object", - "properties": { - "backoffBaseSeconds": { - "description": "backoffBaseSeconds defines the base for the exponential backoff for re-queuing an evicted workload.\n\nDefaults to 60.", - "type": "integer", - "format": "int32" - }, - "backoffLimitCount": { - "description": "backoffLimitCount defines the maximum number of re-queuing retries. Once the number is reached, the workload is deactivated (`.spec.activate`=`false`). When it is null, the workloads will repeatedly and endless re-queueing.\n\nEvery backoff duration is about \"b*2^(n-1)+Rand\" where: - \"b\" represents the base set by \"BackoffBaseSeconds\" parameter, - \"n\" represents the \"workloadStatus.requeueState.count\", - \"Rand\" represents the random jitter. During this time, the workload is taken as an inadmissible and other workloads will have a chance to be admitted. By default, the consecutive requeue delays are around: (60s, 120s, 240s, ...).\n\nDefaults to null.", - "type": "integer", - "format": "int32" - }, - "backoffMaxSeconds": { - "description": "backoffMaxSeconds defines the maximum backoff time to re-queue an evicted workload.\n\nDefaults to 3600.", - "type": "integer", - "format": "int32" - }, - "timestamp": { - "description": "timestamp defines the timestamp used for re-queuing a Workload that was evicted due to Pod readiness. The possible values are:\n\n- `Eviction` (default) indicates from Workload `Evicted` condition with `PodsReadyTimeout` reason. - `Creation` indicates from Workload .metadata.creationTimestamp.", - "type": "string" - } - } - }, "com.github.openshift.api.operator.v1alpha1.ResourceTransformation": { "type": "object", "required": [ @@ -33966,7 +33930,7 @@ ], "properties": { "input": { - "description": "input is the name of the input resource.", + "description": "input is the name of the input resource. resources are pod spec resources like cpu, memory, gpus", "type": "string", "default": "" }, @@ -34104,28 +34068,6 @@ } } }, - "com.github.openshift.api.operator.v1alpha1.WaitForPodsReady": { - "description": "WaitForPodsReady defines configuration for the Wait For Pods Ready feature, which is used to ensure that all Pods are ready within the specified time.", - "type": "object", - "properties": { - "blockAdmission": { - "description": "blockAdmission when true, cluster queue will block admissions for all subsequent jobs until the jobs reach the PodsReady=true condition. This setting is only honored when `Enable` is set to true.", - "type": "string" - }, - "recoveryTimeout": { - "description": "recoveryTimeout defines an opt-in timeout, measured since the last transition to the PodsReady=false condition after a Workload is Admitted and running. Such a transition may happen when a Pod failed and the replacement Pod is awaited to be scheduled. After exceeding the timeout the corresponding job gets suspended again and requeued after the backoff delay. The timeout is enforced only if waitForPodsReady.enable=true. If not set, there is no timeout.", - "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.Duration" - }, - "requeuingStrategy": { - "description": "requeuingStrategy defines the strategy for requeuing a Workload.", - "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.RequeuingStrategy" - }, - "timeout": { - "description": "timeout defines the time for an admitted workload to reach the PodsReady=true condition. When the timeout is exceeded, the workload evicted and requeued in the same cluster queue. Defaults to 5min.", - "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.Duration" - } - } - }, "com.github.openshift.api.operatorcontrolplane.v1alpha1.LogEntry": { "description": "LogEntry records events", "type": "object", diff --git a/operator/v1alpha1/types_kueue.go b/operator/v1alpha1/types_kueue.go index fe85a7486ae..1515b98e7f7 100644 --- a/operator/v1alpha1/types_kueue.go +++ b/operator/v1alpha1/types_kueue.go @@ -27,6 +27,9 @@ type Kueue struct { metav1.ObjectMeta `json:"metadata,omitempty"` // spec holds user settable values for configuration + // kueue configuration must not be changed once the object exists + // to change the configuration, one can delete the object and create a new object. + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="values are immutable once set" // +required Spec KueueOperandSpec `json:"spec"` // status holds observed values from the cluster. They may not be overridden. @@ -66,32 +69,7 @@ const ( Disabled EnabledOrDisabled = "Disabled" ) -// Feature gates is unresolved -// Option 1: -// -// Drop the feature gate entirely -// Feature gate modification is not supported. -// -// Option 2: -// -// Allow for modifications of feature gates if operator is an Unmanaged state -// We could validate that user set operator in "Unmanaged" state as part of operator v1 API -// When this is set, specific experimental fields will be enabled. -// -// Option 3: -// -// Allow for a users to specifiy a configmap that matches kueue configuration -// Operator will fetch the configmap and assume its valid. type KueueConfiguration struct { - // waitForPodsReady configures gang admission - // +optional - WaitForPodsReady WaitForPodsReady `json:"waitForPodsReady,omitempty"` - // featureGates are advanced features for Kueue - // TODO: we could only allow features - // if ManagementState is Unmanaged - // Otherwise we will fail at validation time. - // +optional - FeatureGates map[string]EnabledOrDisabled `json:"featureGates,omitempty"` // integrations are the types of integrations Kueue will manage // +required Integrations Integrations `json:"integrations"` @@ -153,19 +131,24 @@ type Resources struct { // transformations defines how to transform PodSpec resources into Workload resource requests. // This is intended to be a map with Input as the key (enforced by validation code) // +optional - // +kubebuilder:validation:items:MaxLength=64 // +kubebuilder:validation:MaxItems=16 Transformations []ResourceTransformation `json:"transformations,omitempty"` } -// +kubebuilder:validation:Enum="Retain";"Replace" +// +kubebuilder:validation:Enum=Retain;Replace type ResourceTransformationStrategy string +// ResourceTransformations apply transformation to pod spec resources +// Retain means that we will keep the original resources and +// apply a transformation. +// Replace means that the original resources will be replaced +// after the transformation is done. const Retain ResourceTransformationStrategy = "Retain" const Replace ResourceTransformationStrategy = "Replace" type ResourceTransformation struct { // input is the name of the input resource. + // resources are pod spec resources like cpu, memory, gpus // +required Input corev1.ResourceName `json:"input"` @@ -190,7 +173,6 @@ const ( type FairSharing struct { // enable indicates whether to enable fair sharing for all cohorts. - // +kubebuilder:default=Disabled // +optional Enable EnabledOrDisabled `json:"enable"` @@ -215,90 +197,6 @@ type FairSharing struct { PreemptionStrategies []PreemptionStrategy `json:"preemptionStrategies,omitempty"` } -// WaitForPodsReady defines configuration for the Wait For Pods Ready feature, -// which is used to ensure that all Pods are ready within the specified time. -type WaitForPodsReady struct { - // timeout defines the time for an admitted workload to reach the - // PodsReady=true condition. When the timeout is exceeded, the workload - // evicted and requeued in the same cluster queue. - // Defaults to 5min. - // +optional - Timeout *metav1.Duration `json:"timeout,omitempty"` - - // blockAdmission when true, cluster queue will block admissions for all - // subsequent jobs until the jobs reach the PodsReady=true condition. - // This setting is only honored when `Enable` is set to true. - // +optional - BlockAdmission *EnabledOrDisabled `json:"blockAdmission,omitempty"` - - // requeuingStrategy defines the strategy for requeuing a Workload. - // +optional - RequeuingStrategy *RequeuingStrategy `json:"requeuingStrategy,omitempty"` - - // recoveryTimeout defines an opt-in timeout, measured since the - // last transition to the PodsReady=false condition after a Workload is Admitted and running. - // Such a transition may happen when a Pod failed and the replacement Pod - // is awaited to be scheduled. - // After exceeding the timeout the corresponding job gets suspended again - // and requeued after the backoff delay. The timeout is enforced only if waitForPodsReady.enable=true. - // If not set, there is no timeout. - // +optional - RecoveryTimeout *metav1.Duration `json:"recoveryTimeout,omitempty"` -} - -type RequeuingStrategy struct { - // timestamp defines the timestamp used for re-queuing a Workload - // that was evicted due to Pod readiness. The possible values are: - // - // - `Eviction` (default) indicates from Workload `Evicted` condition with `PodsReadyTimeout` reason. - // - `Creation` indicates from Workload .metadata.creationTimestamp. - // - // +kubebuilder:validation:MaxLength=8 - // +optional - Timestamp *RequeuingTimestamp `json:"timestamp,omitempty"` - - // backoffLimitCount defines the maximum number of re-queuing retries. - // Once the number is reached, the workload is deactivated (`.spec.activate`=`false`). - // When it is null, the workloads will repeatedly and endless re-queueing. - // - // Every backoff duration is about "b*2^(n-1)+Rand" where: - // - "b" represents the base set by "BackoffBaseSeconds" parameter, - // - "n" represents the "workloadStatus.requeueState.count", - // - "Rand" represents the random jitter. - // During this time, the workload is taken as an inadmissible and - // other workloads will have a chance to be admitted. - // By default, the consecutive requeue delays are around: (60s, 120s, 240s, ...). - // - // Defaults to null. - // +optional - BackoffLimitCount *int32 `json:"backoffLimitCount,omitempty"` - - // backoffBaseSeconds defines the base for the exponential backoff for - // re-queuing an evicted workload. - // - // Defaults to 60. - // +kubebuilder:default=60 - // +optional - BackoffBaseSeconds *int32 `json:"backoffBaseSeconds,omitempty"` - - // backoffMaxSeconds defines the maximum backoff time to re-queue an evicted workload. - // - // Defaults to 3600. - // +kubebuilder:default=3600 - // +optional - BackoffMaxSeconds *int32 `json:"backoffMaxSeconds,omitempty"` -} - -type RequeuingTimestamp string - -const ( - // CreationTimestamp timestamp (from Workload .metadata.creationTimestamp). - CreationTimestamp RequeuingTimestamp = "Creation" - - // EvictionTimestamp timestamp (from Workload .status.conditions). - EvictionTimestamp RequeuingTimestamp = "Eviction" -) - type Integrations struct { // frameworks are a list of names to be enabled. // Possible options: diff --git a/operator/v1alpha1/zz_generated.deepcopy.go b/operator/v1alpha1/zz_generated.deepcopy.go index 4ee68793b67..237ae83c09b 100644 --- a/operator/v1alpha1/zz_generated.deepcopy.go +++ b/operator/v1alpha1/zz_generated.deepcopy.go @@ -468,14 +468,6 @@ func (in *Kueue) DeepCopyObject() runtime.Object { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KueueConfiguration) DeepCopyInto(out *KueueConfiguration) { *out = *in - in.WaitForPodsReady.DeepCopyInto(&out.WaitForPodsReady) - if in.FeatureGates != nil { - in, out := &in.FeatureGates, &out.FeatureGates - *out = make(map[string]EnabledOrDisabled, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } in.Integrations.DeepCopyInto(&out.Integrations) in.Resources.DeepCopyInto(&out.Resources) if in.ManageJobsWithoutQueueName != nil { @@ -795,42 +787,6 @@ func (in *RepositoryDigestMirrors) DeepCopy() *RepositoryDigestMirrors { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *RequeuingStrategy) DeepCopyInto(out *RequeuingStrategy) { - *out = *in - if in.Timestamp != nil { - in, out := &in.Timestamp, &out.Timestamp - *out = new(RequeuingTimestamp) - **out = **in - } - if in.BackoffLimitCount != nil { - in, out := &in.BackoffLimitCount, &out.BackoffLimitCount - *out = new(int32) - **out = **in - } - if in.BackoffBaseSeconds != nil { - in, out := &in.BackoffBaseSeconds, &out.BackoffBaseSeconds - *out = new(int32) - **out = **in - } - if in.BackoffMaxSeconds != nil { - in, out := &in.BackoffMaxSeconds, &out.BackoffMaxSeconds - *out = new(int32) - **out = **in - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RequeuingStrategy. -func (in *RequeuingStrategy) DeepCopy() *RequeuingStrategy { - if in == nil { - return nil - } - out := new(RequeuingStrategy) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ResourceTransformation) DeepCopyInto(out *ResourceTransformation) { *out = *in @@ -936,39 +892,3 @@ func (in *VersionAvailability) DeepCopy() *VersionAvailability { in.DeepCopyInto(out) return out } - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *WaitForPodsReady) DeepCopyInto(out *WaitForPodsReady) { - *out = *in - if in.Timeout != nil { - in, out := &in.Timeout, &out.Timeout - *out = new(v1.Duration) - **out = **in - } - if in.BlockAdmission != nil { - in, out := &in.BlockAdmission, &out.BlockAdmission - *out = new(EnabledOrDisabled) - **out = **in - } - if in.RequeuingStrategy != nil { - in, out := &in.RequeuingStrategy, &out.RequeuingStrategy - *out = new(RequeuingStrategy) - (*in).DeepCopyInto(*out) - } - if in.RecoveryTimeout != nil { - in, out := &in.RecoveryTimeout, &out.RecoveryTimeout - *out = new(v1.Duration) - **out = **in - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WaitForPodsReady. -func (in *WaitForPodsReady) DeepCopy() *WaitForPodsReady { - if in == nil { - return nil - } - out := new(WaitForPodsReady) - in.DeepCopyInto(out) - return out -} diff --git a/operator/v1alpha1/zz_generated.swagger_doc_generated.go b/operator/v1alpha1/zz_generated.swagger_doc_generated.go index 3b7a1ba7740..39c81a510b3 100644 --- a/operator/v1alpha1/zz_generated.swagger_doc_generated.go +++ b/operator/v1alpha1/zz_generated.swagger_doc_generated.go @@ -279,7 +279,7 @@ func (Integrations) SwaggerDoc() map[string]string { var map_Kueue = map[string]string{ "": "Kueue is the CRD to represent the kueue operator This CRD defines the configuration that the Kueue Compatibility level 4: No compatibility is provided, the API can change at any point for any reason. These capabilities should not be used by applications needing long term support.", "metadata": "metadata for kueue", - "spec": "spec holds user settable values for configuration", + "spec": "spec holds user settable values for configuration kueue configuration must not be changed once the object exists to change the configuration, one can delete the object and create a new object.", "status": "status holds observed values from the cluster. They may not be overridden.", } @@ -288,8 +288,6 @@ func (Kueue) SwaggerDoc() map[string]string { } var map_KueueConfiguration = map[string]string{ - "waitForPodsReady": "waitForPodsReady configures gang admission", - "featureGates": "featureGates are advanced features for Kueue if ManagementState is Unmanaged Otherwise we will fail at validation time.", "integrations": "integrations are the types of integrations Kueue will manage", "resources": "resources provides additional configuration options for handling the resources. Supports https://github.com/kubernetes-sigs/kueue/blob/release-0.10/keps/2937-resource-transformer/README.md", "manageJobsWithoutQueueName": "manageJobsWithoutQueueName controls whether or not Kueue reconciles jobs that don't set the annotation kueue.x-k8s.io/queue-name.", @@ -328,19 +326,8 @@ func (KueueStatus) SwaggerDoc() map[string]string { return map_KueueStatus } -var map_RequeuingStrategy = map[string]string{ - "timestamp": "timestamp defines the timestamp used for re-queuing a Workload that was evicted due to Pod readiness. The possible values are:\n\n- `Eviction` (default) indicates from Workload `Evicted` condition with `PodsReadyTimeout` reason. - `Creation` indicates from Workload .metadata.creationTimestamp.", - "backoffLimitCount": "backoffLimitCount defines the maximum number of re-queuing retries. Once the number is reached, the workload is deactivated (`.spec.activate`=`false`). When it is null, the workloads will repeatedly and endless re-queueing.\n\nEvery backoff duration is about \"b*2^(n-1)+Rand\" where: - \"b\" represents the base set by \"BackoffBaseSeconds\" parameter, - \"n\" represents the \"workloadStatus.requeueState.count\", - \"Rand\" represents the random jitter. During this time, the workload is taken as an inadmissible and other workloads will have a chance to be admitted. By default, the consecutive requeue delays are around: (60s, 120s, 240s, ...).\n\nDefaults to null.", - "backoffBaseSeconds": "backoffBaseSeconds defines the base for the exponential backoff for re-queuing an evicted workload.\n\nDefaults to 60.", - "backoffMaxSeconds": "backoffMaxSeconds defines the maximum backoff time to re-queue an evicted workload.\n\nDefaults to 3600.", -} - -func (RequeuingStrategy) SwaggerDoc() map[string]string { - return map_RequeuingStrategy -} - var map_ResourceTransformation = map[string]string{ - "input": "input is the name of the input resource.", + "input": "input is the name of the input resource. resources are pod spec resources like cpu, memory, gpus", "strategy": "strategy specifies if the input resource should be replaced or retained.", "outputs": "outputs specifies the output resources and quantities per unit of input resource. An empty Outputs combined with a `Replace` Strategy causes the Input resource to be ignored by Kueue.", } @@ -359,18 +346,6 @@ func (Resources) SwaggerDoc() map[string]string { return map_Resources } -var map_WaitForPodsReady = map[string]string{ - "": "WaitForPodsReady defines configuration for the Wait For Pods Ready feature, which is used to ensure that all Pods are ready within the specified time.", - "timeout": "timeout defines the time for an admitted workload to reach the PodsReady=true condition. When the timeout is exceeded, the workload evicted and requeued in the same cluster queue. Defaults to 5min.", - "blockAdmission": "blockAdmission when true, cluster queue will block admissions for all subsequent jobs until the jobs reach the PodsReady=true condition. This setting is only honored when `Enable` is set to true.", - "requeuingStrategy": "requeuingStrategy defines the strategy for requeuing a Workload.", - "recoveryTimeout": "recoveryTimeout defines an opt-in timeout, measured since the last transition to the PodsReady=false condition after a Workload is Admitted and running. Such a transition may happen when a Pod failed and the replacement Pod is awaited to be scheduled. After exceeding the timeout the corresponding job gets suspended again and requeued after the backoff delay. The timeout is enforced only if waitForPodsReady.enable=true. If not set, there is no timeout.", -} - -func (WaitForPodsReady) SwaggerDoc() map[string]string { - return map_WaitForPodsReady -} - var map_OLM = map[string]string{ "": "OLM provides information to configure an operator to manage the OLM controllers\n\nCompatibility level 4: No compatibility is provided, the API can change at any point for any reason. These capabilities should not be used by applications needing long term support.", "metadata": "metadata is the standard object's metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata", From 4a5e0d030ff5bffc037bbfe06da1761d4fbc2b09 Mon Sep 17 00:00:00 2001 From: Kevin Hannon Date: Fri, 14 Mar 2025 15:16:48 -0400 Subject: [PATCH 10/42] api updates for unsupported --- .../generated_openapi/zz_generated.openapi.go | 154 ++++-- openapi/openapi.json | 96 +++- operator/v1alpha1/types_kueue.go | 96 +++- .../zz_generated.crd-manifests/kueue.crd.yaml | 514 ++++++++++++++++++ operator/v1alpha1/zz_generated.deepcopy.go | 72 ++- .../AAA_ungated.yaml | 492 +++++++++++++++++ .../zz_generated.swagger_doc_generated.go | 42 +- 7 files changed, 1364 insertions(+), 102 deletions(-) create mode 100644 operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml diff --git a/openapi/generated_openapi/zz_generated.openapi.go b/openapi/generated_openapi/zz_generated.openapi.go index 6c2ac02f752..cdb6c0950df 100644 --- a/openapi/generated_openapi/zz_generated.openapi.go +++ b/openapi/generated_openapi/zz_generated.openapi.go @@ -1110,6 +1110,8 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "github.com/openshift/api/operator/v1alpha1.EtcdBackupList": schema_openshift_api_operator_v1alpha1_EtcdBackupList(ref), "github.com/openshift/api/operator/v1alpha1.EtcdBackupSpec": schema_openshift_api_operator_v1alpha1_EtcdBackupSpec(ref), "github.com/openshift/api/operator/v1alpha1.EtcdBackupStatus": schema_openshift_api_operator_v1alpha1_EtcdBackupStatus(ref), + "github.com/openshift/api/operator/v1alpha1.ExpermentalFeatures": schema_openshift_api_operator_v1alpha1_ExpermentalFeatures(ref), + "github.com/openshift/api/operator/v1alpha1.ExternalFramework": schema_openshift_api_operator_v1alpha1_ExternalFramework(ref), "github.com/openshift/api/operator/v1alpha1.FairSharing": schema_openshift_api_operator_v1alpha1_FairSharing(ref), "github.com/openshift/api/operator/v1alpha1.GenerationHistory": schema_openshift_api_operator_v1alpha1_GenerationHistory(ref), "github.com/openshift/api/operator/v1alpha1.GenericOperatorConfig": schema_openshift_api_operator_v1alpha1_GenericOperatorConfig(ref), @@ -56848,6 +56850,102 @@ func schema_openshift_api_operator_v1alpha1_EtcdBackupStatus(ref common.Referenc } } +func schema_openshift_api_operator_v1alpha1_ExpermentalFeatures(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "These are more advanced features.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "resources": { + SchemaProps: spec.SchemaProps{ + Description: "resources provides additional configuration options for handling the resources. Supports https://github.com/kubernetes-sigs/kueue/blob/release-0.10/keps/2937-resource-transformer/README.md", + Default: map[string]interface{}{}, + Ref: ref("github.com/openshift/api/operator/v1alpha1.Resources"), + }, + }, + "featureGates": { + SchemaProps: spec.SchemaProps{ + Description: "featureGates are advanced gates that can control the feature gates that kueue sets in the configuration.", + Type: []string{"object"}, + AdditionalProperties: &spec.SchemaOrBool{ + Allows: true, + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + "manageJobsWithoutQueueName": { + SchemaProps: spec.SchemaProps{ + Description: "manageJobsWithoutQueueName controls whether or not Kueue reconciles jobs that don't set the annotation kueue.x-k8s.io/queue-name.", + Type: []string{"string"}, + Format: "", + }, + }, + "managedJobsNamespaceSelector": { + SchemaProps: spec.SchemaProps{ + Description: "managedJobsNamespaceSelector can be used to omit some namespaces from ManagedJobsWithoutQueueName Only valid if ManagedJobsWithoutQueueName is QueueNameOptional", + Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.LabelSelector"), + }, + }, + "fairSharing": { + SchemaProps: spec.SchemaProps{ + Description: "fairSharing controls the fair sharing semantics across the cluster.", + Default: map[string]interface{}{}, + Ref: ref("github.com/openshift/api/operator/v1alpha1.FairSharing"), + }, + }, + }, + }, + }, + Dependencies: []string{ + "github.com/openshift/api/operator/v1alpha1.FairSharing", "github.com/openshift/api/operator/v1alpha1.Resources", "k8s.io/apimachinery/pkg/apis/meta/v1.LabelSelector"}, + } +} + +func schema_openshift_api_operator_v1alpha1_ExternalFramework(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "This is the GVK for an external framework. Controller runtime requires this in this format for api discoverability.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "group": { + SchemaProps: spec.SchemaProps{ + Description: "group of externalFramework", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "resourceType": { + SchemaProps: spec.SchemaProps{ + Description: "resourceType of external framework this is the same as Kind in the GVK settings", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "version": { + SchemaProps: spec.SchemaProps{ + Description: "version is the version of the api", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + Required: []string{"group", "resourceType", "version"}, + }, + }, + } +} + func schema_openshift_api_operator_v1alpha1_FairSharing(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ @@ -56856,7 +56954,7 @@ func schema_openshift_api_operator_v1alpha1_FairSharing(ref common.ReferenceCall Properties: map[string]spec.Schema{ "enable": { SchemaProps: spec.SchemaProps{ - Description: "enable indicates whether to enable fair sharing for all cohorts.", + Description: "enable indicates whether to enable fair sharing for all cohorts. this is disabled by default.", Default: "", Type: []string{"string"}, Format: "", @@ -57125,7 +57223,7 @@ func schema_openshift_api_operator_v1alpha1_Integrations(ref common.ReferenceCal Properties: map[string]spec.Schema{ "frameworks": { SchemaProps: spec.SchemaProps{ - Description: "frameworks are a list of names to be enabled. Possible options:\n - \"batch/job\"\n - \"kubeflow.org/mpijob\"\n - \"ray.io/rayjob\"\n - \"ray.io/raycluster\"\n - \"jobset.x-k8s.io/jobset\"\n - \"kubeflow.org/paddlejob\"\n - \"kubeflow.org/pytorchjob\"\n - \"kubeflow.org/tfjob\"\n - \"kubeflow.org/xgboostjob\"\n - \"workload.codeflare.dev/appwrapper\"\n - \"pod\"\n - \"deployment\" (requires enabling pod integration)\n - \"statefulset\" (requires enabling pod integration)\n - \"leaderworkerset.x-k8s.io/leaderworkerset\" (requires enabling pod integration)", + Description: "frameworks are a list of names to be enabled. Possible options:\n - \"batch/job\"\n - \"kubeflow.org/mpijob\"\n - \"ray.io/rayjob\"\n - \"ray.io/raycluster\"\n - \"jobset.x-k8s.io/jobset\"\n - \"kubeflow.org/paddlejob\"\n - \"kubeflow.org/pytorchjob\"\n - \"kubeflow.org/tfjob\"\n - \"kubeflow.org/xgboostjob\"\n - \"workload.codeflare.dev/appwrapper\"\n - \"pod\"\n - \"deployment\" (requires enabling pod integration)\n - \"statefulset\" (requires enabling pod integration)\n - \"leaderworkerset.x-k8s.io/leaderworkerset\" (requires enabling pod integration)\nThis is required and must have at least one element. The frameworks are jobs that Kueue will manage.", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -57140,14 +57238,13 @@ func schema_openshift_api_operator_v1alpha1_Integrations(ref common.ReferenceCal }, "externalFrameworks": { SchemaProps: spec.SchemaProps{ - Description: "externalFrameworks are a list of GroupVersionKinds that are managed for Kueue by external controllers; the expected format is `Kind.version.group.com`. As far as", + Description: "externalFrameworks are a list of GroupVersionKinds that are managed for Kueue by external controllers; the expected format is `Kind.version.group.com`. These are optional and should only be used if you have an external controller that integrations with kueue.", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ SchemaProps: spec.SchemaProps{ - Default: "", - Type: []string{"string"}, - Format: "", + Default: map[string]interface{}{}, + Ref: ref("github.com/openshift/api/operator/v1alpha1.ExternalFramework"), }, }, }, @@ -57172,6 +57269,8 @@ func schema_openshift_api_operator_v1alpha1_Integrations(ref common.ReferenceCal Required: []string{"frameworks"}, }, }, + Dependencies: []string{ + "github.com/openshift/api/operator/v1alpha1.ExternalFramework"}, } } @@ -57230,48 +57329,26 @@ func schema_openshift_api_operator_v1alpha1_KueueConfiguration(ref common.Refere return common.OpenAPIDefinition{ Schema: spec.Schema{ SchemaProps: spec.SchemaProps{ - Description: "Feature gates is unresolved Option 1:\n\n\tDrop the feature gate entirely\n\tFeature gate modification is not supported.\n\nOption 2:\n\n\tAllow for modifications of feature gates if operator is an Unmanaged state\n\tWe could validate that user set operator in \"Unmanaged\" state as part of operator v1 API\n\tWhen this is set, specific experimental fields will be enabled.\n\nOption 3:\n\n\tAllow for a users to specifiy a configmap that matches kueue configuration\n\tOperator will fetch the configmap and assume its valid.", - Type: []string{"object"}, + Type: []string{"object"}, Properties: map[string]spec.Schema{ "integrations": { SchemaProps: spec.SchemaProps{ - Description: "integrations are the types of integrations Kueue will manage", + Description: "integrations are the workloads Kueue will manage kueue has integrations in the codebase and it also allows for external frameworks", Default: map[string]interface{}{}, Ref: ref("github.com/openshift/api/operator/v1alpha1.Integrations"), }, }, - "resources": { - SchemaProps: spec.SchemaProps{ - Description: "resources provides additional configuration options for handling the resources. Supports https://github.com/kubernetes-sigs/kueue/blob/release-0.10/keps/2937-resource-transformer/README.md", - Default: map[string]interface{}{}, - Ref: ref("github.com/openshift/api/operator/v1alpha1.Resources"), - }, - }, - "manageJobsWithoutQueueName": { + "metrics": { SchemaProps: spec.SchemaProps{ - Description: "manageJobsWithoutQueueName controls whether or not Kueue reconciles jobs that don't set the annotation kueue.x-k8s.io/queue-name.", + Description: "metrics allows one to configure if kueue metrics will be exposed to monitoring. Kueue provides a series of metrics to monitor the status of LocalQueues and resource flavors See https://kueue.sigs.k8s.io/docs/reference/metrics/ for a detailed list", Type: []string{"string"}, Format: "", }, }, - "managedJobsNamespaceSelector": { - SchemaProps: spec.SchemaProps{ - Description: "managedJobsNamespaceSelector can be used to omit some namespaces from ManagedJobsWithoutQueueName Only valid if ManagedJobsWithoutQueueName is QueueNameOptional", - Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.LabelSelector"), - }, - }, - "fairSharing": { + "expermentalFeatures": { SchemaProps: spec.SchemaProps{ - Description: "fairSharing controls the fair sharing semantics across the cluster.", - Default: map[string]interface{}{}, - Ref: ref("github.com/openshift/api/operator/v1alpha1.FairSharing"), - }, - }, - "metrics": { - SchemaProps: spec.SchemaProps{ - Description: "metrics allows one to change if metrics are enabled or disabled. Microshift does not enable metrics by default Default will assume metrics are enabled.", - Type: []string{"string"}, - Format: "", + Description: "unsupportedConfigOverrides are more expermental features that users can use to configure kueue. We do not guarantee that these features will yet be supported Once we are comfortable with these features, we will move this out of this list.", + Ref: ref("github.com/openshift/api/operator/v1alpha1.ExpermentalFeatures"), }, }, }, @@ -57279,7 +57356,7 @@ func schema_openshift_api_operator_v1alpha1_KueueConfiguration(ref common.Refere }, }, Dependencies: []string{ - "github.com/openshift/api/operator/v1alpha1.FairSharing", "github.com/openshift/api/operator/v1alpha1.Integrations", "github.com/openshift/api/operator/v1alpha1.Resources", "k8s.io/apimachinery/pkg/apis/meta/v1.LabelSelector"}, + "github.com/openshift/api/operator/v1alpha1.ExpermentalFeatures", "github.com/openshift/api/operator/v1alpha1.Integrations"}, } } @@ -58020,7 +58097,8 @@ func schema_openshift_api_operator_v1alpha1_ResourceTransformation(ref common.Re return common.OpenAPIDefinition{ Schema: spec.Schema{ SchemaProps: spec.SchemaProps{ - Type: []string{"object"}, + Description: "ResourceTransformations apply transformation to pod spec resources Retain means that we will keep the original resources and apply a transformation. Replace means that the original resources will be replaced after the transformation is done.", + Type: []string{"object"}, Properties: map[string]spec.Schema{ "input": { SchemaProps: spec.SchemaProps{ @@ -58032,7 +58110,7 @@ func schema_openshift_api_operator_v1alpha1_ResourceTransformation(ref common.Re }, "strategy": { SchemaProps: spec.SchemaProps{ - Description: "strategy specifies if the input resource should be replaced or retained.", + Description: "strategy specifies if the input resource should be replaced or retained. retain means that we will keep the original resources and apply a transformation. replace means that the original resources will be replaced after the transformation is done.", Type: []string{"string"}, Format: "", }, diff --git a/openapi/openapi.json b/openapi/openapi.json index 558194cd108..9bce58d2bad 100644 --- a/openapi/openapi.json +++ b/openapi/openapi.json @@ -33206,11 +33206,69 @@ } } }, + "com.github.openshift.api.operator.v1alpha1.ExpermentalFeatures": { + "description": "These are more advanced features.", + "type": "object", + "properties": { + "fairSharing": { + "description": "fairSharing controls the fair sharing semantics across the cluster.", + "default": {}, + "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.FairSharing" + }, + "featureGates": { + "description": "featureGates are advanced gates that can control the feature gates that kueue sets in the configuration.", + "type": "object", + "additionalProperties": { + "type": "string", + "default": "" + } + }, + "manageJobsWithoutQueueName": { + "description": "manageJobsWithoutQueueName controls whether or not Kueue reconciles jobs that don't set the annotation kueue.x-k8s.io/queue-name.", + "type": "string" + }, + "managedJobsNamespaceSelector": { + "description": "managedJobsNamespaceSelector can be used to omit some namespaces from ManagedJobsWithoutQueueName Only valid if ManagedJobsWithoutQueueName is QueueNameOptional", + "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.LabelSelector" + }, + "resources": { + "description": "resources provides additional configuration options for handling the resources. Supports https://github.com/kubernetes-sigs/kueue/blob/release-0.10/keps/2937-resource-transformer/README.md", + "default": {}, + "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.Resources" + } + } + }, + "com.github.openshift.api.operator.v1alpha1.ExternalFramework": { + "description": "This is the GVK for an external framework. Controller runtime requires this in this format for api discoverability.", + "type": "object", + "required": [ + "group", + "resourceType", + "version" + ], + "properties": { + "group": { + "description": "group of externalFramework", + "type": "string", + "default": "" + }, + "resourceType": { + "description": "resourceType of external framework this is the same as Kind in the GVK settings", + "type": "string", + "default": "" + }, + "version": { + "description": "version is the version of the api", + "type": "string", + "default": "" + } + } + }, "com.github.openshift.api.operator.v1alpha1.FairSharing": { "type": "object", "properties": { "enable": { - "description": "enable indicates whether to enable fair sharing for all cohorts.", + "description": "enable indicates whether to enable fair sharing for all cohorts. this is disabled by default.", "type": "string", "default": "" }, @@ -33375,15 +33433,15 @@ ], "properties": { "externalFrameworks": { - "description": "externalFrameworks are a list of GroupVersionKinds that are managed for Kueue by external controllers; the expected format is `Kind.version.group.com`. As far as", + "description": "externalFrameworks are a list of GroupVersionKinds that are managed for Kueue by external controllers; the expected format is `Kind.version.group.com`. These are optional and should only be used if you have an external controller that integrations with kueue.", "type": "array", "items": { - "type": "string", - "default": "" + "default": {}, + "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.ExternalFramework" } }, "frameworks": { - "description": "frameworks are a list of names to be enabled. Possible options:\n - \"batch/job\"\n - \"kubeflow.org/mpijob\"\n - \"ray.io/rayjob\"\n - \"ray.io/raycluster\"\n - \"jobset.x-k8s.io/jobset\"\n - \"kubeflow.org/paddlejob\"\n - \"kubeflow.org/pytorchjob\"\n - \"kubeflow.org/tfjob\"\n - \"kubeflow.org/xgboostjob\"\n - \"workload.codeflare.dev/appwrapper\"\n - \"pod\"\n - \"deployment\" (requires enabling pod integration)\n - \"statefulset\" (requires enabling pod integration)\n - \"leaderworkerset.x-k8s.io/leaderworkerset\" (requires enabling pod integration)", + "description": "frameworks are a list of names to be enabled. Possible options:\n - \"batch/job\"\n - \"kubeflow.org/mpijob\"\n - \"ray.io/rayjob\"\n - \"ray.io/raycluster\"\n - \"jobset.x-k8s.io/jobset\"\n - \"kubeflow.org/paddlejob\"\n - \"kubeflow.org/pytorchjob\"\n - \"kubeflow.org/tfjob\"\n - \"kubeflow.org/xgboostjob\"\n - \"workload.codeflare.dev/appwrapper\"\n - \"pod\"\n - \"deployment\" (requires enabling pod integration)\n - \"statefulset\" (requires enabling pod integration)\n - \"leaderworkerset.x-k8s.io/leaderworkerset\" (requires enabling pod integration)\nThis is required and must have at least one element. The frameworks are jobs that Kueue will manage.", "type": "array", "items": { "type": "string", @@ -33433,38 +33491,23 @@ } }, "com.github.openshift.api.operator.v1alpha1.KueueConfiguration": { - "description": "Feature gates is unresolved Option 1:\n\n\tDrop the feature gate entirely\n\tFeature gate modification is not supported.\n\nOption 2:\n\n\tAllow for modifications of feature gates if operator is an Unmanaged state\n\tWe could validate that user set operator in \"Unmanaged\" state as part of operator v1 API\n\tWhen this is set, specific experimental fields will be enabled.\n\nOption 3:\n\n\tAllow for a users to specifiy a configmap that matches kueue configuration\n\tOperator will fetch the configmap and assume its valid.", "type": "object", "required": [ "integrations" ], "properties": { - "fairSharing": { - "description": "fairSharing controls the fair sharing semantics across the cluster.", - "default": {}, - "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.FairSharing" + "expermentalFeatures": { + "description": "unsupportedConfigOverrides are more expermental features that users can use to configure kueue. We do not guarantee that these features will yet be supported Once we are comfortable with these features, we will move this out of this list.", + "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.ExpermentalFeatures" }, "integrations": { - "description": "integrations are the types of integrations Kueue will manage", + "description": "integrations are the workloads Kueue will manage kueue has integrations in the codebase and it also allows for external frameworks", "default": {}, "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.Integrations" }, - "manageJobsWithoutQueueName": { - "description": "manageJobsWithoutQueueName controls whether or not Kueue reconciles jobs that don't set the annotation kueue.x-k8s.io/queue-name.", - "type": "string" - }, - "managedJobsNamespaceSelector": { - "description": "managedJobsNamespaceSelector can be used to omit some namespaces from ManagedJobsWithoutQueueName Only valid if ManagedJobsWithoutQueueName is QueueNameOptional", - "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.LabelSelector" - }, "metrics": { - "description": "metrics allows one to change if metrics are enabled or disabled. Microshift does not enable metrics by default Default will assume metrics are enabled.", + "description": "metrics allows one to configure if kueue metrics will be exposed to monitoring. Kueue provides a series of metrics to monitor the status of LocalQueues and resource flavors See https://kueue.sigs.k8s.io/docs/reference/metrics/ for a detailed list", "type": "string" - }, - "resources": { - "description": "resources provides additional configuration options for handling the resources. Supports https://github.com/kubernetes-sigs/kueue/blob/release-0.10/keps/2937-resource-transformer/README.md", - "default": {}, - "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.Resources" } } }, @@ -33924,6 +33967,7 @@ } }, "com.github.openshift.api.operator.v1alpha1.ResourceTransformation": { + "description": "ResourceTransformations apply transformation to pod spec resources Retain means that we will keep the original resources and apply a transformation. Replace means that the original resources will be replaced after the transformation is done.", "type": "object", "required": [ "input" @@ -33942,7 +33986,7 @@ } }, "strategy": { - "description": "strategy specifies if the input resource should be replaced or retained.", + "description": "strategy specifies if the input resource should be replaced or retained. retain means that we will keep the original resources and apply a transformation. replace means that the original resources will be replaced after the transformation is done.", "type": "string" } } diff --git a/operator/v1alpha1/types_kueue.go b/operator/v1alpha1/types_kueue.go index 1515b98e7f7..5f65056e0b6 100644 --- a/operator/v1alpha1/types_kueue.go +++ b/operator/v1alpha1/types_kueue.go @@ -19,7 +19,7 @@ import ( // +genclient:nonNamespaced // +kubebuilder:storageversion // +kubebuilder:subresource:status -// +kubebuilder:validation:XValidation:rule="self.metadata.name == 'cluster'",message="olm is a singleton, .metadata.name must be 'cluster'" +// +kubebuilder:validation:XValidation:rule="self.metadata.name == 'cluster'",message="kueue is a singleton, .metadata.name must be 'cluster'" type Kueue struct { metav1.TypeMeta `json:",inline"` // metadata for kueue @@ -70,13 +70,36 @@ const ( ) type KueueConfiguration struct { - // integrations are the types of integrations Kueue will manage + // integrations are the workloads Kueue will manage + // kueue has integrations in the codebase and it also allows + // for external frameworks // +required Integrations Integrations `json:"integrations"` + // metrics allows one to configure if kueue metrics + // will be exposed to monitoring. + // Kueue provides a series of metrics to monitor + // the status of LocalQueues and resource flavors + // See https://kueue.sigs.k8s.io/docs/reference/metrics/ for a detailed list + // +optional + Metrics *EnabledOrDisabled `json:"metrics,omitempty"` + // expermentalFeatures are more expermental features + // that users can use to configure kueue. + // We do not guarantee that these features will yet be supported + // Once we are comfortable with these features, we will move this out of this list. + // +optional + ExpermentalFeatures *ExpermentalFeatures `json:"expermentalFeatures,omitempty"` +} + +// These are more advanced features. +type ExpermentalFeatures struct { // resources provides additional configuration options for handling the resources. // Supports https://github.com/kubernetes-sigs/kueue/blob/release-0.10/keps/2937-resource-transformer/README.md // +optional Resources Resources `json:"resources,omitempty"` + // featureGates are advanced gates that can control the feature gates that kueue sets + // in the configuration. + // +optional + FeatureGates map[string]EnabledOrDisabled `json:"featureGates,omitempty"` // manageJobsWithoutQueueName controls whether or not Kueue reconciles // jobs that don't set the annotation kueue.x-k8s.io/queue-name. // +kubebuilder:default=QueueNameRequired @@ -84,17 +107,12 @@ type KueueConfiguration struct { ManageJobsWithoutQueueName *ManageJobsWithoutQueueNameOption `json:"manageJobsWithoutQueueName,omitempty"` // managedJobsNamespaceSelector can be used to omit some namespaces from ManagedJobsWithoutQueueName // Only valid if ManagedJobsWithoutQueueName is QueueNameOptional + // +kubebuilder:validation:XValidation:rule="self.manageJobsWithoutQueueName==QueueNameOptional",message="managedJobsNamespaceSelector is only valid if manageJobsWithoutQueueName is QueueNameOptional" // +optional ManagedJobsNamespaceSelector *metav1.LabelSelector `json:"managedJobsNamespaceSelector,omitempty"` // fairSharing controls the fair sharing semantics across the cluster. // +optional FairSharing FairSharing `json:"fairSharing,omitempty"` - // metrics allows one to change if metrics - // are enabled or disabled. - // Microshift does not enable metrics by default - // Default will assume metrics are enabled. - // +optional - Metrics *EnabledOrDisabled `json:"metrics,omitempty"` } // KueueStatus defines the observed state of Kueue @@ -146,6 +164,11 @@ type ResourceTransformationStrategy string const Retain ResourceTransformationStrategy = "Retain" const Replace ResourceTransformationStrategy = "Replace" +// ResourceTransformations apply transformation to pod spec resources +// Retain means that we will keep the original resources and +// apply a transformation. +// Replace means that the original resources will be replaced +// after the transformation is done. type ResourceTransformation struct { // input is the name of the input resource. // resources are pod spec resources like cpu, memory, gpus @@ -153,7 +176,10 @@ type ResourceTransformation struct { Input corev1.ResourceName `json:"input"` // strategy specifies if the input resource should be replaced or retained. - // +kubebuilder:default=Retain + // retain means that we will keep the original resources and + // apply a transformation. + // replace means that the original resources will be replaced + // after the transformation is done. // +optional Strategy *ResourceTransformationStrategy `json:"strategy,omitempty"` @@ -173,6 +199,7 @@ const ( type FairSharing struct { // enable indicates whether to enable fair sharing for all cohorts. + // this is disabled by default. // +optional Enable EnabledOrDisabled `json:"enable"` @@ -197,6 +224,45 @@ type FairSharing struct { PreemptionStrategies []PreemptionStrategy `json:"preemptionStrategies,omitempty"` } +// +kubebuilder:validation:Enum=batch/job;ray.io/rayjob;ray.io/raycluster;jobset.x-k8s.io/jobset;kubeflow.org/mpijob;kubeflow.org/paddlejob;kubeflow.org/pytorchjob;kubeflow.org/tfjob;kubeflow.org/xgboostjob;workload.codeflare.dev/appwrapper;pod;deployment;statefulset;leaderworkerset.x-k8s.io/leaderworkerset +type KueueIntegrations string + +const ( + BatchJob KueueIntegrations = "batch/job" + RayJob KueueIntegrations = "ray.io/rayjob" + RayCluster KueueIntegrations = "ray.io/raycluster" + JobSet KueueIntegrations = "jobset.x-k8s.io/jobset" + MPIJob KueueIntegrations = "kubeflow.org/mpijob" + PaddeJob KueueIntegrations = "kubeflow.org/paddlejob" + PyTorchJob KueueIntegrations = "kubeflow.org/pytorchjob" + TfJob KueueIntegrations = "kubeflow.org/tfjob" + XGBoostJob KueueIntegrations = "kubeflow.org/xgboostjob" + AppWrappers KueueIntegrations = "workload.codeflare.dev/appwrapper" + Pod KueueIntegrations = "pod" + Deployment KueueIntegrations = "deployment" + Statefulset KueueIntegrations = "statefulset" + LeaderWorkerSet KueueIntegrations = "leaderworkerset.x-k8s.io/leaderworkerset" +) + +// This is the GVK for an external framework. +// Controller runtime requires this in this format +// for api discoverability. +type ExternalFramework struct { + // group of externalFramework + // +kubebuilder:validation:MaxLength=256 + // +required + Group string `json:"group"` + // resourceType of external framework + // this is the same as Kind in the GVK settings + // +required + // +kubebuilder:validation:MaxLength=256 + ResourceType string `json:"resourceType"` + // version is the version of the api + // +required + // +kubebuilder:validation:MaxLength=256 + Version string `json:"version"` +} + type Integrations struct { // frameworks are a list of names to be enabled. // Possible options: @@ -215,17 +281,19 @@ type Integrations struct { // - "statefulset" (requires enabling pod integration) // - "leaderworkerset.x-k8s.io/leaderworkerset" (requires enabling pod integration) // +kubebuilder:validation:MaxItems=14 - // +kubebuilder:validation:items:MaxLength=64 + // +kubebuilder:validation:MinItems=1 + // This is required and must have at least one element. + // The frameworks are jobs that Kueue will manage. // +required - Frameworks []string `json:"frameworks"` + Frameworks []KueueIntegrations `json:"frameworks"` // externalFrameworks are a list of GroupVersionKinds // that are managed for Kueue by external controllers; // the expected format is `Kind.version.group.com`. - // As far as + // These are optional and should only be used if you have an external controller + // that integrations with kueue. // +optional - // +kubebuilder:validation:items:MaxLength=64 // +kubebuilder:validation:MaxItems=4 - ExternalFrameworks []string `json:"externalFrameworks,omitempty"` + ExternalFrameworks []ExternalFramework `json:"externalFrameworks,omitempty"` // labelKeysToCopy is a list of label keys that should be copied from the job into the // workload object. It is not required for the job to have all the labels from this diff --git a/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml b/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml new file mode 100644 index 00000000000..a8d27e30bf8 --- /dev/null +++ b/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml @@ -0,0 +1,514 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + api.openshift.io/merged-by-featuregates: "true" + include.release.openshift.io/ibm-cloud-managed: "true" + include.release.openshift.io/self-managed-high-availability: "true" + name: kueue.operator.openshift.io +spec: + group: operator.openshift.io + names: + kind: Kueue + listKind: KueueList + plural: kueue + singular: kueue + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + Kueue is the CRD to represent the kueue operator + This CRD defines the configuration that the Kueue + Compatibility level 4: No compatibility is provided, the API can change at any point for any reason. These capabilities should not be used by applications needing long term support. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + spec holds user settable values for configuration + kueue configuration must not be changed once the object exists + to change the configuration, one can delete the object and create a new object. + properties: + config: + description: |- + config is the desired configuration + for the kueue operator. + properties: + expermentalFeatures: + description: |- + unsupportedConfigOverrides are more expermental features + that users can use to configure kueue. + We do not guarantee that these features will yet be supported + Once we are comfortable with these features, we will move this out of this list. + properties: + fairSharing: + description: fairSharing controls the fair sharing semantics + across the cluster. + properties: + enable: + description: |- + enable indicates whether to enable fair sharing for all cohorts. + this is disabled by default. + enum: + - Enabled + - Disabled + type: string + preemptionStrategies: + description: |- + preemptionStrategies indicates which constraints should a preemption satisfy. + The preemption algorithm will only use the next strategy in the list if the + incoming workload (preemptor) doesn't fit after using the previous strategies. + Possible values are: + - LessThanOrEqualToFinalShare: Only preempt a workload if the share of the preemptor CQ + with the preemptor workload is less than or equal to the share of the preemptee CQ + without the workload to be preempted. + This strategy might favor preemption of smaller workloads in the preemptee CQ, + regardless of priority or start time, in an effort to keep the share of the CQ + as high as possible. + - LessThanInitialShare: Only preempt a workload if the share of the preemptor CQ + with the incoming workload is strictly less than the share of the preemptee CQ. + This strategy doesn't depend on the share usage of the workload being preempted. + As a result, the strategy chooses to preempt workloads with the lowest priority and + newest start time first. + The default strategy is ["LessThanOrEqualToFinalShare", "LessThanInitialShare"]. + items: + enum: + - LessThanOrEqualToFinalShare + - LessThanInitialShare + type: string + maxItems: 2 + type: array + type: object + featureGates: + additionalProperties: + enum: + - Enabled + - Disabled + type: string + description: |- + featureGates are advanced gates that can control the feature gates that kueue sets + in the configuration. + type: object + manageJobsWithoutQueueName: + default: QueueNameRequired + description: |- + manageJobsWithoutQueueName controls whether or not Kueue reconciles + jobs that don't set the annotation kueue.x-k8s.io/queue-name. + enum: + - QueueNameOptional + - QueueNameRequired + type: string + managedJobsNamespaceSelector: + description: |- + managedJobsNamespaceSelector can be used to omit some namespaces from ManagedJobsWithoutQueueName + Only valid if ManagedJobsWithoutQueueName is QueueNameOptional + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: managedJobsNamespaceSelector is only valid if manageJobsWithoutQueueName + is QueueNameOptional + rule: self.manageJobsWithoutQueueName==QueueNameOptional + resources: + description: |- + resources provides additional configuration options for handling the resources. + Supports https://github.com/kubernetes-sigs/kueue/blob/release-0.10/keps/2937-resource-transformer/README.md + properties: + excludeResourcePrefixes: + description: excludeResourcePrefixes defines which resources + should be ignored by Kueue + items: + maxLength: 64 + type: string + maxItems: 16 + type: array + transformations: + description: |- + transformations defines how to transform PodSpec resources into Workload resource requests. + This is intended to be a map with Input as the key (enforced by validation code) + items: + description: |- + ResourceTransformations apply transformation to pod spec resources + Retain means that we will keep the original resources and + apply a transformation. + Replace means that the original resources will be replaced + after the transformation is done. + properties: + input: + description: |- + input is the name of the input resource. + resources are pod spec resources like cpu, memory, gpus + type: string + outputs: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + outputs specifies the output resources and quantities per unit of input resource. + An empty Outputs combined with a `Replace` Strategy causes the Input resource to be ignored by Kueue. + type: object + strategy: + description: |- + strategy specifies if the input resource should be replaced or retained. + retain means that we will keep the original resources and + apply a transformation. + replace means that the original resources will be replaced + after the transformation is done. + enum: + - Retain + - Replace + type: string + required: + - input + type: object + maxItems: 16 + type: array + type: object + type: object + integrations: + description: |- + integrations are the workloads Kueue will manage + kueue has integrations in the codebase and it also allows + for external frameworks + properties: + externalFrameworks: + description: |- + externalFrameworks are a list of GroupVersionKinds + that are managed for Kueue by external controllers; + the expected format is `Kind.version.group.com`. + These are optional and should only be used if you have an external controller + that integrations with kueue. + items: + description: |- + This is the GVK for an external framework. + Controller runtime requires this in this format + for api discoverability. + properties: + group: + description: group of externalFramework + maxLength: 256 + type: string + resourceType: + description: |- + resourceType of external framework + this is the same as Kind in the GVK settings + maxLength: 256 + type: string + version: + description: version is the version of the api + maxLength: 256 + type: string + required: + - group + - resourceType + - version + type: object + maxItems: 4 + type: array + frameworks: + description: |- + frameworks are a list of names to be enabled. + Possible options: + - "batch/job" + - "kubeflow.org/mpijob" + - "ray.io/rayjob" + - "ray.io/raycluster" + - "jobset.x-k8s.io/jobset" + - "kubeflow.org/paddlejob" + - "kubeflow.org/pytorchjob" + - "kubeflow.org/tfjob" + - "kubeflow.org/xgboostjob" + - "workload.codeflare.dev/appwrapper" + - "pod" + - "deployment" (requires enabling pod integration) + - "statefulset" (requires enabling pod integration) + - "leaderworkerset.x-k8s.io/leaderworkerset" (requires enabling pod integration) + This is required and must have at least one element. + The frameworks are jobs that Kueue will manage. + items: + enum: + - batch/job + - ray.io/rayjob + - ray.io/raycluster + - jobset.x-k8s.io/jobset + - kubeflow.org/mpijob + - kubeflow.org/paddlejob + - kubeflow.org/pytorchjob + - kubeflow.org/tfjob + - kubeflow.org/xgboostjob + - workload.codeflare.dev/appwrapper + - pod + - deployment + - statefulset + - leaderworkerset.x-k8s.io/leaderworkerset + type: string + maxItems: 14 + minItems: 1 + type: array + labelKeysToCopy: + description: |- + labelKeysToCopy is a list of label keys that should be copied from the job into the + workload object. It is not required for the job to have all the labels from this + list. If a job does not have some label with the given key from this list, the + constructed workload object will be created without this label. In the case + of creating a workload from a composable job (pod group), if multiple objects + have labels with some key from the list, the values of these labels must + match or otherwise the workload creation would fail. The labels are copied only + during the workload creation and are not updated even if the labels of the + underlying job are changed. + items: + maxLength: 64 + type: string + maxItems: 64 + type: array + required: + - frameworks + type: object + metrics: + description: |- + metrics allows one to configure if kueue metrics + will be exposed to monitoring. + Kueue provides a series of metrics to monitor + the status of LocalQueues and resource flavors + See https://kueue.sigs.k8s.io/docs/reference/metrics/ for a detailed list + enum: + - Enabled + - Disabled + type: string + required: + - integrations + type: object + logLevel: + default: Normal + description: |- + logLevel is an intent based logging for an overall component. It does not give fine grained control, but it is a + simple way to manage coarse grained logging choices that operators have to interpret for their operands. + + Valid values are: "Normal", "Debug", "Trace", "TraceAll". + Defaults to "Normal". + enum: + - "" + - Normal + - Debug + - Trace + - TraceAll + type: string + managementState: + description: managementState indicates whether and how the operator + should manage the component + pattern: ^(Managed|Unmanaged|Force|Removed)$ + type: string + observedConfig: + description: |- + observedConfig holds a sparse config that controller has observed from the cluster state. It exists in spec because + it is an input to the level for the operator + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + operatorLogLevel: + default: Normal + description: |- + operatorLogLevel is an intent based logging for the operator itself. It does not give fine grained control, but it is a + simple way to manage coarse grained logging choices that operators have to interpret for themselves. + + Valid values are: "Normal", "Debug", "Trace", "TraceAll". + Defaults to "Normal". + enum: + - "" + - Normal + - Debug + - Trace + - TraceAll + type: string + unsupportedConfigOverrides: + description: |- + unsupportedConfigOverrides overrides the final configuration that was computed by the operator. + Red Hat does not support the use of this field. + Misuse of this field could lead to unexpected behavior or conflict with other configuration options. + Seek guidance from the Red Hat support before using this field. + Use of this property blocks cluster upgrades, it must be removed before upgrading your cluster. + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - config + type: object + x-kubernetes-validations: + - message: values are immutable once set + rule: self == oldSelf + status: + description: status holds observed values from the cluster. They may not + be overridden. + properties: + conditions: + description: conditions is a list of conditions and their status + items: + description: OperatorCondition is just the standard condition fields. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + type: string + reason: + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + generations: + description: generations are used to determine when an item needs + to be reconciled or has changed in a way that needs a reaction. + items: + description: GenerationStatus keeps track of the generation for + a given resource so that decisions about forced updates can be + made. + properties: + group: + description: group is the group of the thing you're tracking + type: string + hash: + description: hash is an optional field set for resources without + generation that are content sensitive like secrets and configmaps + type: string + lastGeneration: + description: lastGeneration is the last generation of the workload + controller involved + format: int64 + type: integer + name: + description: name is the name of the thing you're tracking + type: string + namespace: + description: namespace is where the thing you're tracking is + type: string + resource: + description: resource is the resource type of the thing you're + tracking + type: string + required: + - group + - name + - namespace + - resource + type: object + type: array + x-kubernetes-list-map-keys: + - group + - resource + - namespace + - name + x-kubernetes-list-type: map + latestAvailableRevision: + description: latestAvailableRevision is the deploymentID of the most + recent deployment + format: int32 + type: integer + x-kubernetes-validations: + - message: must only increase + rule: self >= oldSelf + observedGeneration: + description: observedGeneration is the last generation change you've + dealt with + format: int64 + type: integer + readyReplicas: + description: readyReplicas indicates how many replicas are ready and + at the desired state + format: int32 + type: integer + version: + description: version is the level this availability applies to + type: string + type: object + required: + - spec + type: object + x-kubernetes-validations: + - message: kueue is a singleton, .metadata.name must be 'cluster' + rule: self.metadata.name == 'cluster' + served: true + storage: true + subresources: + status: {} diff --git a/operator/v1alpha1/zz_generated.deepcopy.go b/operator/v1alpha1/zz_generated.deepcopy.go index 237ae83c09b..0e8348cf2af 100644 --- a/operator/v1alpha1/zz_generated.deepcopy.go +++ b/operator/v1alpha1/zz_generated.deepcopy.go @@ -257,6 +257,57 @@ func (in *EtcdBackupStatus) DeepCopy() *EtcdBackupStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ExpermentalFeatures) DeepCopyInto(out *ExpermentalFeatures) { + *out = *in + in.Resources.DeepCopyInto(&out.Resources) + if in.FeatureGates != nil { + in, out := &in.FeatureGates, &out.FeatureGates + *out = make(map[string]EnabledOrDisabled, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.ManageJobsWithoutQueueName != nil { + in, out := &in.ManageJobsWithoutQueueName, &out.ManageJobsWithoutQueueName + *out = new(ManageJobsWithoutQueueNameOption) + **out = **in + } + if in.ManagedJobsNamespaceSelector != nil { + in, out := &in.ManagedJobsNamespaceSelector, &out.ManagedJobsNamespaceSelector + *out = new(v1.LabelSelector) + (*in).DeepCopyInto(*out) + } + in.FairSharing.DeepCopyInto(&out.FairSharing) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExpermentalFeatures. +func (in *ExpermentalFeatures) DeepCopy() *ExpermentalFeatures { + if in == nil { + return nil + } + out := new(ExpermentalFeatures) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ExternalFramework) DeepCopyInto(out *ExternalFramework) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExternalFramework. +func (in *ExternalFramework) DeepCopy() *ExternalFramework { + if in == nil { + return nil + } + out := new(ExternalFramework) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *FairSharing) DeepCopyInto(out *FairSharing) { *out = *in @@ -411,12 +462,12 @@ func (in *Integrations) DeepCopyInto(out *Integrations) { *out = *in if in.Frameworks != nil { in, out := &in.Frameworks, &out.Frameworks - *out = make([]string, len(*in)) + *out = make([]KueueIntegrations, len(*in)) copy(*out, *in) } if in.ExternalFrameworks != nil { in, out := &in.ExternalFrameworks, &out.ExternalFrameworks - *out = make([]string, len(*in)) + *out = make([]ExternalFramework, len(*in)) copy(*out, *in) } if in.LabelKeysToCopy != nil { @@ -469,23 +520,16 @@ func (in *Kueue) DeepCopyObject() runtime.Object { func (in *KueueConfiguration) DeepCopyInto(out *KueueConfiguration) { *out = *in in.Integrations.DeepCopyInto(&out.Integrations) - in.Resources.DeepCopyInto(&out.Resources) - if in.ManageJobsWithoutQueueName != nil { - in, out := &in.ManageJobsWithoutQueueName, &out.ManageJobsWithoutQueueName - *out = new(ManageJobsWithoutQueueNameOption) - **out = **in - } - if in.ManagedJobsNamespaceSelector != nil { - in, out := &in.ManagedJobsNamespaceSelector, &out.ManagedJobsNamespaceSelector - *out = new(v1.LabelSelector) - (*in).DeepCopyInto(*out) - } - in.FairSharing.DeepCopyInto(&out.FairSharing) if in.Metrics != nil { in, out := &in.Metrics, &out.Metrics *out = new(EnabledOrDisabled) **out = **in } + if in.ExpermentalFeatures != nil { + in, out := &in.ExpermentalFeatures, &out.ExpermentalFeatures + *out = new(ExpermentalFeatures) + (*in).DeepCopyInto(*out) + } return } diff --git a/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml b/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml index 69425f4f5e5..fb01dd1e418 100644 --- a/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml +++ b/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml @@ -14,6 +14,498 @@ spec: scope: Cluster versions: - name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + Kueue is the CRD to represent the kueue operator + This CRD defines the configuration that the Kueue + Compatibility level 4: No compatibility is provided, the API can change at any point for any reason. These capabilities should not be used by applications needing long term support. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: |- + spec holds user settable values for configuration + kueue configuration must not be changed once the object exists + to change the configuration, one can delete the object and create a new object. + properties: + config: + description: |- + config is the desired configuration + for the kueue operator. + properties: + expermentalFeatures: + description: |- + unsupportedConfigOverrides are more expermental features + that users can use to configure kueue. + We do not guarantee that these features will yet be supported + Once we are comfortable with these features, we will move this out of this list. + properties: + fairSharing: + description: fairSharing controls the fair sharing semantics + across the cluster. + properties: + enable: + description: |- + enable indicates whether to enable fair sharing for all cohorts. + this is disabled by default. + enum: + - Enabled + - Disabled + type: string + preemptionStrategies: + description: |- + preemptionStrategies indicates which constraints should a preemption satisfy. + The preemption algorithm will only use the next strategy in the list if the + incoming workload (preemptor) doesn't fit after using the previous strategies. + Possible values are: + - LessThanOrEqualToFinalShare: Only preempt a workload if the share of the preemptor CQ + with the preemptor workload is less than or equal to the share of the preemptee CQ + without the workload to be preempted. + This strategy might favor preemption of smaller workloads in the preemptee CQ, + regardless of priority or start time, in an effort to keep the share of the CQ + as high as possible. + - LessThanInitialShare: Only preempt a workload if the share of the preemptor CQ + with the incoming workload is strictly less than the share of the preemptee CQ. + This strategy doesn't depend on the share usage of the workload being preempted. + As a result, the strategy chooses to preempt workloads with the lowest priority and + newest start time first. + The default strategy is ["LessThanOrEqualToFinalShare", "LessThanInitialShare"]. + items: + enum: + - LessThanOrEqualToFinalShare + - LessThanInitialShare + type: string + maxItems: 2 + type: array + type: object + featureGates: + additionalProperties: + enum: + - Enabled + - Disabled + type: string + description: |- + featureGates are advanced gates that can control the feature gates that kueue sets + in the configuration. + type: object + manageJobsWithoutQueueName: + default: QueueNameRequired + description: |- + manageJobsWithoutQueueName controls whether or not Kueue reconciles + jobs that don't set the annotation kueue.x-k8s.io/queue-name. + enum: + - QueueNameOptional + - QueueNameRequired + type: string + managedJobsNamespaceSelector: + description: |- + managedJobsNamespaceSelector can be used to omit some namespaces from ManagedJobsWithoutQueueName + Only valid if ManagedJobsWithoutQueueName is QueueNameOptional + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + x-kubernetes-validations: + - message: managedJobsNamespaceSelector is only valid if manageJobsWithoutQueueName + is QueueNameOptional + rule: self.manageJobsWithoutQueueName==QueueNameOptional + resources: + description: |- + resources provides additional configuration options for handling the resources. + Supports https://github.com/kubernetes-sigs/kueue/blob/release-0.10/keps/2937-resource-transformer/README.md + properties: + excludeResourcePrefixes: + description: excludeResourcePrefixes defines which resources + should be ignored by Kueue + items: + maxLength: 64 + type: string + maxItems: 16 + type: array + transformations: + description: |- + transformations defines how to transform PodSpec resources into Workload resource requests. + This is intended to be a map with Input as the key (enforced by validation code) + items: + description: |- + ResourceTransformations apply transformation to pod spec resources + Retain means that we will keep the original resources and + apply a transformation. + Replace means that the original resources will be replaced + after the transformation is done. + properties: + input: + description: |- + input is the name of the input resource. + resources are pod spec resources like cpu, memory, gpus + type: string + outputs: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + outputs specifies the output resources and quantities per unit of input resource. + An empty Outputs combined with a `Replace` Strategy causes the Input resource to be ignored by Kueue. + type: object + strategy: + description: |- + strategy specifies if the input resource should be replaced or retained. + retain means that we will keep the original resources and + apply a transformation. + replace means that the original resources will be replaced + after the transformation is done. + enum: + - Retain + - Replace + type: string + required: + - input + type: object + maxItems: 16 + type: array + type: object + type: object + integrations: + description: |- + integrations are the workloads Kueue will manage + kueue has integrations in the codebase and it also allows + for external frameworks + properties: + externalFrameworks: + description: |- + externalFrameworks are a list of GroupVersionKinds + that are managed for Kueue by external controllers; + the expected format is `Kind.version.group.com`. + These are optional and should only be used if you have an external controller + that integrations with kueue. + items: + description: |- + This is the GVK for an external framework. + Controller runtime requires this in this format + for api discoverability. + properties: + group: + description: group of externalFramework + maxLength: 256 + type: string + resourceType: + description: |- + resourceType of external framework + this is the same as Kind in the GVK settings + maxLength: 256 + type: string + version: + description: version is the version of the api + maxLength: 256 + type: string + required: + - group + - resourceType + - version + type: object + maxItems: 4 + type: array + frameworks: + description: |- + frameworks are a list of names to be enabled. + Possible options: + - "batch/job" + - "kubeflow.org/mpijob" + - "ray.io/rayjob" + - "ray.io/raycluster" + - "jobset.x-k8s.io/jobset" + - "kubeflow.org/paddlejob" + - "kubeflow.org/pytorchjob" + - "kubeflow.org/tfjob" + - "kubeflow.org/xgboostjob" + - "workload.codeflare.dev/appwrapper" + - "pod" + - "deployment" (requires enabling pod integration) + - "statefulset" (requires enabling pod integration) + - "leaderworkerset.x-k8s.io/leaderworkerset" (requires enabling pod integration) + This is required and must have at least one element. + The frameworks are jobs that Kueue will manage. + items: + enum: + - batch/job + - ray.io/rayjob + - ray.io/raycluster + - jobset.x-k8s.io/jobset + - kubeflow.org/mpijob + - kubeflow.org/paddlejob + - kubeflow.org/pytorchjob + - kubeflow.org/tfjob + - kubeflow.org/xgboostjob + - workload.codeflare.dev/appwrapper + - pod + - deployment + - statefulset + - leaderworkerset.x-k8s.io/leaderworkerset + type: string + maxItems: 14 + minItems: 1 + type: array + labelKeysToCopy: + description: |- + labelKeysToCopy is a list of label keys that should be copied from the job into the + workload object. It is not required for the job to have all the labels from this + list. If a job does not have some label with the given key from this list, the + constructed workload object will be created without this label. In the case + of creating a workload from a composable job (pod group), if multiple objects + have labels with some key from the list, the values of these labels must + match or otherwise the workload creation would fail. The labels are copied only + during the workload creation and are not updated even if the labels of the + underlying job are changed. + items: + maxLength: 64 + type: string + maxItems: 64 + type: array + required: + - frameworks + type: object + metrics: + description: |- + metrics allows one to configure if kueue metrics + will be exposed to monitoring. + Kueue provides a series of metrics to monitor + the status of LocalQueues and resource flavors + See https://kueue.sigs.k8s.io/docs/reference/metrics/ for a detailed list + enum: + - Enabled + - Disabled + type: string + required: + - integrations + type: object + logLevel: + default: Normal + description: |- + logLevel is an intent based logging for an overall component. It does not give fine grained control, but it is a + simple way to manage coarse grained logging choices that operators have to interpret for their operands. + + Valid values are: "Normal", "Debug", "Trace", "TraceAll". + Defaults to "Normal". + enum: + - "" + - Normal + - Debug + - Trace + - TraceAll + type: string + managementState: + description: managementState indicates whether and how the operator + should manage the component + pattern: ^(Managed|Unmanaged|Force|Removed)$ + type: string + observedConfig: + description: |- + observedConfig holds a sparse config that controller has observed from the cluster state. It exists in spec because + it is an input to the level for the operator + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + operatorLogLevel: + default: Normal + description: |- + operatorLogLevel is an intent based logging for the operator itself. It does not give fine grained control, but it is a + simple way to manage coarse grained logging choices that operators have to interpret for themselves. + + Valid values are: "Normal", "Debug", "Trace", "TraceAll". + Defaults to "Normal". + enum: + - "" + - Normal + - Debug + - Trace + - TraceAll + type: string + unsupportedConfigOverrides: + description: |- + unsupportedConfigOverrides overrides the final configuration that was computed by the operator. + Red Hat does not support the use of this field. + Misuse of this field could lead to unexpected behavior or conflict with other configuration options. + Seek guidance from the Red Hat support before using this field. + Use of this property blocks cluster upgrades, it must be removed before upgrading your cluster. + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - config + type: object + x-kubernetes-validations: + - message: values are immutable once set + rule: self == oldSelf + status: + description: status holds observed values from the cluster. They may not + be overridden. + properties: + conditions: + description: conditions is a list of conditions and their status + items: + description: OperatorCondition is just the standard condition fields. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + type: string + reason: + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + generations: + description: generations are used to determine when an item needs + to be reconciled or has changed in a way that needs a reaction. + items: + description: GenerationStatus keeps track of the generation for + a given resource so that decisions about forced updates can be + made. + properties: + group: + description: group is the group of the thing you're tracking + type: string + hash: + description: hash is an optional field set for resources without + generation that are content sensitive like secrets and configmaps + type: string + lastGeneration: + description: lastGeneration is the last generation of the workload + controller involved + format: int64 + type: integer + name: + description: name is the name of the thing you're tracking + type: string + namespace: + description: namespace is where the thing you're tracking is + type: string + resource: + description: resource is the resource type of the thing you're + tracking + type: string + required: + - group + - name + - namespace + - resource + type: object + type: array + x-kubernetes-list-map-keys: + - group + - resource + - namespace + - name + x-kubernetes-list-type: map + latestAvailableRevision: + description: latestAvailableRevision is the deploymentID of the most + recent deployment + format: int32 + type: integer + x-kubernetes-validations: + - message: must only increase + rule: self >= oldSelf + observedGeneration: + description: observedGeneration is the last generation change you've + dealt with + format: int64 + type: integer + readyReplicas: + description: readyReplicas indicates how many replicas are ready and + at the desired state + format: int32 + type: integer + version: + description: version is the level this availability applies to + type: string + type: object + required: + - spec + type: object + x-kubernetes-validations: + - message: kueue is a singleton, .metadata.name must be 'cluster' + rule: self.metadata.name == 'cluster' served: true storage: true subresources: diff --git a/operator/v1alpha1/zz_generated.swagger_doc_generated.go b/operator/v1alpha1/zz_generated.swagger_doc_generated.go index 39c81a510b3..15f4b684e3e 100644 --- a/operator/v1alpha1/zz_generated.swagger_doc_generated.go +++ b/operator/v1alpha1/zz_generated.swagger_doc_generated.go @@ -257,8 +257,32 @@ func (RepositoryDigestMirrors) SwaggerDoc() map[string]string { return map_RepositoryDigestMirrors } +var map_ExpermentalFeatures = map[string]string{ + "": "These are more advanced features.", + "resources": "resources provides additional configuration options for handling the resources. Supports https://github.com/kubernetes-sigs/kueue/blob/release-0.10/keps/2937-resource-transformer/README.md", + "featureGates": "featureGates are advanced gates that can control the feature gates that kueue sets in the configuration.", + "manageJobsWithoutQueueName": "manageJobsWithoutQueueName controls whether or not Kueue reconciles jobs that don't set the annotation kueue.x-k8s.io/queue-name.", + "managedJobsNamespaceSelector": "managedJobsNamespaceSelector can be used to omit some namespaces from ManagedJobsWithoutQueueName Only valid if ManagedJobsWithoutQueueName is QueueNameOptional", + "fairSharing": "fairSharing controls the fair sharing semantics across the cluster.", +} + +func (ExpermentalFeatures) SwaggerDoc() map[string]string { + return map_ExpermentalFeatures +} + +var map_ExternalFramework = map[string]string{ + "": "This is the GVK for an external framework. Controller runtime requires this in this format for api discoverability.", + "group": "group of externalFramework", + "resourceType": "resourceType of external framework this is the same as Kind in the GVK settings", + "version": "version is the version of the api", +} + +func (ExternalFramework) SwaggerDoc() map[string]string { + return map_ExternalFramework +} + var map_FairSharing = map[string]string{ - "enable": "enable indicates whether to enable fair sharing for all cohorts.", + "enable": "enable indicates whether to enable fair sharing for all cohorts. this is disabled by default.", "preemptionStrategies": "preemptionStrategies indicates which constraints should a preemption satisfy. The preemption algorithm will only use the next strategy in the list if the incoming workload (preemptor) doesn't fit after using the previous strategies. Possible values are: - LessThanOrEqualToFinalShare: Only preempt a workload if the share of the preemptor CQ\n with the preemptor workload is less than or equal to the share of the preemptee CQ\n without the workload to be preempted.\n This strategy might favor preemption of smaller workloads in the preemptee CQ,\n regardless of priority or start time, in an effort to keep the share of the CQ\n as high as possible.\n- LessThanInitialShare: Only preempt a workload if the share of the preemptor CQ\n with the incoming workload is strictly less than the share of the preemptee CQ.\n This strategy doesn't depend on the share usage of the workload being preempted.\n As a result, the strategy chooses to preempt workloads with the lowest priority and\n newest start time first.\nThe default strategy is [\"LessThanOrEqualToFinalShare\", \"LessThanInitialShare\"].", } @@ -267,8 +291,8 @@ func (FairSharing) SwaggerDoc() map[string]string { } var map_Integrations = map[string]string{ - "frameworks": "frameworks are a list of names to be enabled. Possible options:\n - \"batch/job\"\n - \"kubeflow.org/mpijob\"\n - \"ray.io/rayjob\"\n - \"ray.io/raycluster\"\n - \"jobset.x-k8s.io/jobset\"\n - \"kubeflow.org/paddlejob\"\n - \"kubeflow.org/pytorchjob\"\n - \"kubeflow.org/tfjob\"\n - \"kubeflow.org/xgboostjob\"\n - \"workload.codeflare.dev/appwrapper\"\n - \"pod\"\n - \"deployment\" (requires enabling pod integration)\n - \"statefulset\" (requires enabling pod integration)\n - \"leaderworkerset.x-k8s.io/leaderworkerset\" (requires enabling pod integration)", - "externalFrameworks": "externalFrameworks are a list of GroupVersionKinds that are managed for Kueue by external controllers; the expected format is `Kind.version.group.com`. As far as", + "frameworks": "frameworks are a list of names to be enabled. Possible options:\n - \"batch/job\"\n - \"kubeflow.org/mpijob\"\n - \"ray.io/rayjob\"\n - \"ray.io/raycluster\"\n - \"jobset.x-k8s.io/jobset\"\n - \"kubeflow.org/paddlejob\"\n - \"kubeflow.org/pytorchjob\"\n - \"kubeflow.org/tfjob\"\n - \"kubeflow.org/xgboostjob\"\n - \"workload.codeflare.dev/appwrapper\"\n - \"pod\"\n - \"deployment\" (requires enabling pod integration)\n - \"statefulset\" (requires enabling pod integration)\n - \"leaderworkerset.x-k8s.io/leaderworkerset\" (requires enabling pod integration)\nThis is required and must have at least one element. The frameworks are jobs that Kueue will manage.", + "externalFrameworks": "externalFrameworks are a list of GroupVersionKinds that are managed for Kueue by external controllers; the expected format is `Kind.version.group.com`. These are optional and should only be used if you have an external controller that integrations with kueue.", "labelKeysToCopy": "labelKeysToCopy is a list of label keys that should be copied from the job into the workload object. It is not required for the job to have all the labels from this list. If a job does not have some label with the given key from this list, the constructed workload object will be created without this label. In the case of creating a workload from a composable job (pod group), if multiple objects have labels with some key from the list, the values of these labels must match or otherwise the workload creation would fail. The labels are copied only during the workload creation and are not updated even if the labels of the underlying job are changed.", } @@ -288,12 +312,9 @@ func (Kueue) SwaggerDoc() map[string]string { } var map_KueueConfiguration = map[string]string{ - "integrations": "integrations are the types of integrations Kueue will manage", - "resources": "resources provides additional configuration options for handling the resources. Supports https://github.com/kubernetes-sigs/kueue/blob/release-0.10/keps/2937-resource-transformer/README.md", - "manageJobsWithoutQueueName": "manageJobsWithoutQueueName controls whether or not Kueue reconciles jobs that don't set the annotation kueue.x-k8s.io/queue-name.", - "managedJobsNamespaceSelector": "managedJobsNamespaceSelector can be used to omit some namespaces from ManagedJobsWithoutQueueName Only valid if ManagedJobsWithoutQueueName is QueueNameOptional", - "fairSharing": "fairSharing controls the fair sharing semantics across the cluster.", - "metrics": "metrics allows one to change if metrics are enabled or disabled. Microshift does not enable metrics by default Default will assume metrics are enabled.", + "integrations": "integrations are the workloads Kueue will manage kueue has integrations in the codebase and it also allows for external frameworks", + "metrics": "metrics allows one to configure if kueue metrics will be exposed to monitoring. Kueue provides a series of metrics to monitor the status of LocalQueues and resource flavors See https://kueue.sigs.k8s.io/docs/reference/metrics/ for a detailed list", + "expermentalFeatures": "unsupportedConfigOverrides are more expermental features that users can use to configure kueue. We do not guarantee that these features will yet be supported Once we are comfortable with these features, we will move this out of this list.", } func (KueueConfiguration) SwaggerDoc() map[string]string { @@ -327,8 +348,9 @@ func (KueueStatus) SwaggerDoc() map[string]string { } var map_ResourceTransformation = map[string]string{ + "": "ResourceTransformations apply transformation to pod spec resources Retain means that we will keep the original resources and apply a transformation. Replace means that the original resources will be replaced after the transformation is done.", "input": "input is the name of the input resource. resources are pod spec resources like cpu, memory, gpus", - "strategy": "strategy specifies if the input resource should be replaced or retained.", + "strategy": "strategy specifies if the input resource should be replaced or retained. retain means that we will keep the original resources and apply a transformation. replace means that the original resources will be replaced after the transformation is done.", "outputs": "outputs specifies the output resources and quantities per unit of input resource. An empty Outputs combined with a `Replace` Strategy causes the Input resource to be ignored by Kueue.", } From 23e19e4a6b082e134f255d6b70082ece46fb2454 Mon Sep 17 00:00:00 2001 From: Kevin Hannon Date: Fri, 14 Mar 2025 16:49:30 -0400 Subject: [PATCH 11/42] drop metrics from api as we can detect without api --- operator/v1alpha1/types_kueue.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/operator/v1alpha1/types_kueue.go b/operator/v1alpha1/types_kueue.go index 5f65056e0b6..73d381d444e 100644 --- a/operator/v1alpha1/types_kueue.go +++ b/operator/v1alpha1/types_kueue.go @@ -75,13 +75,6 @@ type KueueConfiguration struct { // for external frameworks // +required Integrations Integrations `json:"integrations"` - // metrics allows one to configure if kueue metrics - // will be exposed to monitoring. - // Kueue provides a series of metrics to monitor - // the status of LocalQueues and resource flavors - // See https://kueue.sigs.k8s.io/docs/reference/metrics/ for a detailed list - // +optional - Metrics *EnabledOrDisabled `json:"metrics,omitempty"` // expermentalFeatures are more expermental features // that users can use to configure kueue. // We do not guarantee that these features will yet be supported From 84aeac65ab0134ffb024c2e2d356ac2907afe1f8 Mon Sep 17 00:00:00 2001 From: Kevin Hannon Date: Mon, 17 Mar 2025 11:24:03 -0400 Subject: [PATCH 12/42] Drop experimental options --- .../generated_openapi/zz_generated.openapi.go | 201 +----------------- openapi/openapi.json | 105 --------- operator/v1alpha1/types_kueue.go | 143 +------------ .../zz_generated.crd-manifests/kueue.crd.yaml | 186 ---------------- operator/v1alpha1/zz_generated.deepcopy.go | 123 ----------- .../AAA_ungated.yaml | 186 ---------------- .../zz_generated.swagger_doc_generated.go | 47 +--- 7 files changed, 3 insertions(+), 988 deletions(-) diff --git a/openapi/generated_openapi/zz_generated.openapi.go b/openapi/generated_openapi/zz_generated.openapi.go index cdb6c0950df..29a80e45ec6 100644 --- a/openapi/generated_openapi/zz_generated.openapi.go +++ b/openapi/generated_openapi/zz_generated.openapi.go @@ -1110,9 +1110,7 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "github.com/openshift/api/operator/v1alpha1.EtcdBackupList": schema_openshift_api_operator_v1alpha1_EtcdBackupList(ref), "github.com/openshift/api/operator/v1alpha1.EtcdBackupSpec": schema_openshift_api_operator_v1alpha1_EtcdBackupSpec(ref), "github.com/openshift/api/operator/v1alpha1.EtcdBackupStatus": schema_openshift_api_operator_v1alpha1_EtcdBackupStatus(ref), - "github.com/openshift/api/operator/v1alpha1.ExpermentalFeatures": schema_openshift_api_operator_v1alpha1_ExpermentalFeatures(ref), "github.com/openshift/api/operator/v1alpha1.ExternalFramework": schema_openshift_api_operator_v1alpha1_ExternalFramework(ref), - "github.com/openshift/api/operator/v1alpha1.FairSharing": schema_openshift_api_operator_v1alpha1_FairSharing(ref), "github.com/openshift/api/operator/v1alpha1.GenerationHistory": schema_openshift_api_operator_v1alpha1_GenerationHistory(ref), "github.com/openshift/api/operator/v1alpha1.GenericOperatorConfig": schema_openshift_api_operator_v1alpha1_GenericOperatorConfig(ref), "github.com/openshift/api/operator/v1alpha1.ImageContentSourcePolicy": schema_openshift_api_operator_v1alpha1_ImageContentSourcePolicy(ref), @@ -1134,8 +1132,6 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "github.com/openshift/api/operator/v1alpha1.OperatorSpec": schema_openshift_api_operator_v1alpha1_OperatorSpec(ref), "github.com/openshift/api/operator/v1alpha1.OperatorStatus": schema_openshift_api_operator_v1alpha1_OperatorStatus(ref), "github.com/openshift/api/operator/v1alpha1.RepositoryDigestMirrors": schema_openshift_api_operator_v1alpha1_RepositoryDigestMirrors(ref), - "github.com/openshift/api/operator/v1alpha1.ResourceTransformation": schema_openshift_api_operator_v1alpha1_ResourceTransformation(ref), - "github.com/openshift/api/operator/v1alpha1.Resources": schema_openshift_api_operator_v1alpha1_Resources(ref), "github.com/openshift/api/operator/v1alpha1.StaticPodOperatorStatus": schema_openshift_api_operator_v1alpha1_StaticPodOperatorStatus(ref), "github.com/openshift/api/operator/v1alpha1.VersionAvailability": schema_openshift_api_operator_v1alpha1_VersionAvailability(ref), "github.com/openshift/api/operatorcontrolplane/v1alpha1.LogEntry": schema_openshift_api_operatorcontrolplane_v1alpha1_LogEntry(ref), @@ -56850,64 +56846,6 @@ func schema_openshift_api_operator_v1alpha1_EtcdBackupStatus(ref common.Referenc } } -func schema_openshift_api_operator_v1alpha1_ExpermentalFeatures(ref common.ReferenceCallback) common.OpenAPIDefinition { - return common.OpenAPIDefinition{ - Schema: spec.Schema{ - SchemaProps: spec.SchemaProps{ - Description: "These are more advanced features.", - Type: []string{"object"}, - Properties: map[string]spec.Schema{ - "resources": { - SchemaProps: spec.SchemaProps{ - Description: "resources provides additional configuration options for handling the resources. Supports https://github.com/kubernetes-sigs/kueue/blob/release-0.10/keps/2937-resource-transformer/README.md", - Default: map[string]interface{}{}, - Ref: ref("github.com/openshift/api/operator/v1alpha1.Resources"), - }, - }, - "featureGates": { - SchemaProps: spec.SchemaProps{ - Description: "featureGates are advanced gates that can control the feature gates that kueue sets in the configuration.", - Type: []string{"object"}, - AdditionalProperties: &spec.SchemaOrBool{ - Allows: true, - Schema: &spec.Schema{ - SchemaProps: spec.SchemaProps{ - Default: "", - Type: []string{"string"}, - Format: "", - }, - }, - }, - }, - }, - "manageJobsWithoutQueueName": { - SchemaProps: spec.SchemaProps{ - Description: "manageJobsWithoutQueueName controls whether or not Kueue reconciles jobs that don't set the annotation kueue.x-k8s.io/queue-name.", - Type: []string{"string"}, - Format: "", - }, - }, - "managedJobsNamespaceSelector": { - SchemaProps: spec.SchemaProps{ - Description: "managedJobsNamespaceSelector can be used to omit some namespaces from ManagedJobsWithoutQueueName Only valid if ManagedJobsWithoutQueueName is QueueNameOptional", - Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.LabelSelector"), - }, - }, - "fairSharing": { - SchemaProps: spec.SchemaProps{ - Description: "fairSharing controls the fair sharing semantics across the cluster.", - Default: map[string]interface{}{}, - Ref: ref("github.com/openshift/api/operator/v1alpha1.FairSharing"), - }, - }, - }, - }, - }, - Dependencies: []string{ - "github.com/openshift/api/operator/v1alpha1.FairSharing", "github.com/openshift/api/operator/v1alpha1.Resources", "k8s.io/apimachinery/pkg/apis/meta/v1.LabelSelector"}, - } -} - func schema_openshift_api_operator_v1alpha1_ExternalFramework(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ @@ -56946,41 +56884,6 @@ func schema_openshift_api_operator_v1alpha1_ExternalFramework(ref common.Referen } } -func schema_openshift_api_operator_v1alpha1_FairSharing(ref common.ReferenceCallback) common.OpenAPIDefinition { - return common.OpenAPIDefinition{ - Schema: spec.Schema{ - SchemaProps: spec.SchemaProps{ - Type: []string{"object"}, - Properties: map[string]spec.Schema{ - "enable": { - SchemaProps: spec.SchemaProps{ - Description: "enable indicates whether to enable fair sharing for all cohorts. this is disabled by default.", - Default: "", - Type: []string{"string"}, - Format: "", - }, - }, - "preemptionStrategies": { - SchemaProps: spec.SchemaProps{ - Description: "preemptionStrategies indicates which constraints should a preemption satisfy. The preemption algorithm will only use the next strategy in the list if the incoming workload (preemptor) doesn't fit after using the previous strategies. Possible values are: - LessThanOrEqualToFinalShare: Only preempt a workload if the share of the preemptor CQ\n with the preemptor workload is less than or equal to the share of the preemptee CQ\n without the workload to be preempted.\n This strategy might favor preemption of smaller workloads in the preemptee CQ,\n regardless of priority or start time, in an effort to keep the share of the CQ\n as high as possible.\n- LessThanInitialShare: Only preempt a workload if the share of the preemptor CQ\n with the incoming workload is strictly less than the share of the preemptee CQ.\n This strategy doesn't depend on the share usage of the workload being preempted.\n As a result, the strategy chooses to preempt workloads with the lowest priority and\n newest start time first.\nThe default strategy is [\"LessThanOrEqualToFinalShare\", \"LessThanInitialShare\"].", - Type: []string{"array"}, - Items: &spec.SchemaOrArray{ - Schema: &spec.Schema{ - SchemaProps: spec.SchemaProps{ - Default: "", - Type: []string{"string"}, - Format: "", - }, - }, - }, - }, - }, - }, - }, - }, - } -} - func schema_openshift_api_operator_v1alpha1_GenerationHistory(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ @@ -57338,25 +57241,12 @@ func schema_openshift_api_operator_v1alpha1_KueueConfiguration(ref common.Refere Ref: ref("github.com/openshift/api/operator/v1alpha1.Integrations"), }, }, - "metrics": { - SchemaProps: spec.SchemaProps{ - Description: "metrics allows one to configure if kueue metrics will be exposed to monitoring. Kueue provides a series of metrics to monitor the status of LocalQueues and resource flavors See https://kueue.sigs.k8s.io/docs/reference/metrics/ for a detailed list", - Type: []string{"string"}, - Format: "", - }, - }, - "expermentalFeatures": { - SchemaProps: spec.SchemaProps{ - Description: "unsupportedConfigOverrides are more expermental features that users can use to configure kueue. We do not guarantee that these features will yet be supported Once we are comfortable with these features, we will move this out of this list.", - Ref: ref("github.com/openshift/api/operator/v1alpha1.ExpermentalFeatures"), - }, - }, }, Required: []string{"integrations"}, }, }, Dependencies: []string{ - "github.com/openshift/api/operator/v1alpha1.ExpermentalFeatures", "github.com/openshift/api/operator/v1alpha1.Integrations"}, + "github.com/openshift/api/operator/v1alpha1.Integrations"}, } } @@ -58093,95 +57983,6 @@ func schema_openshift_api_operator_v1alpha1_RepositoryDigestMirrors(ref common.R } } -func schema_openshift_api_operator_v1alpha1_ResourceTransformation(ref common.ReferenceCallback) common.OpenAPIDefinition { - return common.OpenAPIDefinition{ - Schema: spec.Schema{ - SchemaProps: spec.SchemaProps{ - Description: "ResourceTransformations apply transformation to pod spec resources Retain means that we will keep the original resources and apply a transformation. Replace means that the original resources will be replaced after the transformation is done.", - Type: []string{"object"}, - Properties: map[string]spec.Schema{ - "input": { - SchemaProps: spec.SchemaProps{ - Description: "input is the name of the input resource. resources are pod spec resources like cpu, memory, gpus", - Default: "", - Type: []string{"string"}, - Format: "", - }, - }, - "strategy": { - SchemaProps: spec.SchemaProps{ - Description: "strategy specifies if the input resource should be replaced or retained. retain means that we will keep the original resources and apply a transformation. replace means that the original resources will be replaced after the transformation is done.", - Type: []string{"string"}, - Format: "", - }, - }, - "outputs": { - SchemaProps: spec.SchemaProps{ - Description: "outputs specifies the output resources and quantities per unit of input resource. An empty Outputs combined with a `Replace` Strategy causes the Input resource to be ignored by Kueue.", - Type: []string{"object"}, - AdditionalProperties: &spec.SchemaOrBool{ - Allows: true, - Schema: &spec.Schema{ - SchemaProps: spec.SchemaProps{ - Ref: ref("k8s.io/apimachinery/pkg/api/resource.Quantity"), - }, - }, - }, - }, - }, - }, - Required: []string{"input"}, - }, - }, - Dependencies: []string{ - "k8s.io/apimachinery/pkg/api/resource.Quantity"}, - } -} - -func schema_openshift_api_operator_v1alpha1_Resources(ref common.ReferenceCallback) common.OpenAPIDefinition { - return common.OpenAPIDefinition{ - Schema: spec.Schema{ - SchemaProps: spec.SchemaProps{ - Description: "These structs come directly from Kueue.", - Type: []string{"object"}, - Properties: map[string]spec.Schema{ - "excludeResourcePrefixes": { - SchemaProps: spec.SchemaProps{ - Description: "excludeResourcePrefixes defines which resources should be ignored by Kueue", - Type: []string{"array"}, - Items: &spec.SchemaOrArray{ - Schema: &spec.Schema{ - SchemaProps: spec.SchemaProps{ - Default: "", - Type: []string{"string"}, - Format: "", - }, - }, - }, - }, - }, - "transformations": { - SchemaProps: spec.SchemaProps{ - Description: "transformations defines how to transform PodSpec resources into Workload resource requests. This is intended to be a map with Input as the key (enforced by validation code)", - Type: []string{"array"}, - Items: &spec.SchemaOrArray{ - Schema: &spec.Schema{ - SchemaProps: spec.SchemaProps{ - Default: map[string]interface{}{}, - Ref: ref("github.com/openshift/api/operator/v1alpha1.ResourceTransformation"), - }, - }, - }, - }, - }, - }, - }, - }, - Dependencies: []string{ - "github.com/openshift/api/operator/v1alpha1.ResourceTransformation"}, - } -} - func schema_openshift_api_operator_v1alpha1_StaticPodOperatorStatus(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ diff --git a/openapi/openapi.json b/openapi/openapi.json index 9bce58d2bad..21d7fce3ed6 100644 --- a/openapi/openapi.json +++ b/openapi/openapi.json @@ -33206,38 +33206,6 @@ } } }, - "com.github.openshift.api.operator.v1alpha1.ExpermentalFeatures": { - "description": "These are more advanced features.", - "type": "object", - "properties": { - "fairSharing": { - "description": "fairSharing controls the fair sharing semantics across the cluster.", - "default": {}, - "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.FairSharing" - }, - "featureGates": { - "description": "featureGates are advanced gates that can control the feature gates that kueue sets in the configuration.", - "type": "object", - "additionalProperties": { - "type": "string", - "default": "" - } - }, - "manageJobsWithoutQueueName": { - "description": "manageJobsWithoutQueueName controls whether or not Kueue reconciles jobs that don't set the annotation kueue.x-k8s.io/queue-name.", - "type": "string" - }, - "managedJobsNamespaceSelector": { - "description": "managedJobsNamespaceSelector can be used to omit some namespaces from ManagedJobsWithoutQueueName Only valid if ManagedJobsWithoutQueueName is QueueNameOptional", - "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.LabelSelector" - }, - "resources": { - "description": "resources provides additional configuration options for handling the resources. Supports https://github.com/kubernetes-sigs/kueue/blob/release-0.10/keps/2937-resource-transformer/README.md", - "default": {}, - "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.Resources" - } - } - }, "com.github.openshift.api.operator.v1alpha1.ExternalFramework": { "description": "This is the GVK for an external framework. Controller runtime requires this in this format for api discoverability.", "type": "object", @@ -33264,24 +33232,6 @@ } } }, - "com.github.openshift.api.operator.v1alpha1.FairSharing": { - "type": "object", - "properties": { - "enable": { - "description": "enable indicates whether to enable fair sharing for all cohorts. this is disabled by default.", - "type": "string", - "default": "" - }, - "preemptionStrategies": { - "description": "preemptionStrategies indicates which constraints should a preemption satisfy. The preemption algorithm will only use the next strategy in the list if the incoming workload (preemptor) doesn't fit after using the previous strategies. Possible values are: - LessThanOrEqualToFinalShare: Only preempt a workload if the share of the preemptor CQ\n with the preemptor workload is less than or equal to the share of the preemptee CQ\n without the workload to be preempted.\n This strategy might favor preemption of smaller workloads in the preemptee CQ,\n regardless of priority or start time, in an effort to keep the share of the CQ\n as high as possible.\n- LessThanInitialShare: Only preempt a workload if the share of the preemptor CQ\n with the incoming workload is strictly less than the share of the preemptee CQ.\n This strategy doesn't depend on the share usage of the workload being preempted.\n As a result, the strategy chooses to preempt workloads with the lowest priority and\n newest start time first.\nThe default strategy is [\"LessThanOrEqualToFinalShare\", \"LessThanInitialShare\"].", - "type": "array", - "items": { - "type": "string", - "default": "" - } - } - } - }, "com.github.openshift.api.operator.v1alpha1.GenerationHistory": { "description": "GenerationHistory keeps track of the generation for a given resource so that decisions about forced updated can be made. DEPRECATED: Use fields in v1.GenerationStatus instead", "type": "object", @@ -33496,18 +33446,10 @@ "integrations" ], "properties": { - "expermentalFeatures": { - "description": "unsupportedConfigOverrides are more expermental features that users can use to configure kueue. We do not guarantee that these features will yet be supported Once we are comfortable with these features, we will move this out of this list.", - "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.ExpermentalFeatures" - }, "integrations": { "description": "integrations are the workloads Kueue will manage kueue has integrations in the codebase and it also allows for external frameworks", "default": {}, "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.Integrations" - }, - "metrics": { - "description": "metrics allows one to configure if kueue metrics will be exposed to monitoring. Kueue provides a series of metrics to monitor the status of LocalQueues and resource flavors See https://kueue.sigs.k8s.io/docs/reference/metrics/ for a detailed list", - "type": "string" } } }, @@ -33966,53 +33908,6 @@ } } }, - "com.github.openshift.api.operator.v1alpha1.ResourceTransformation": { - "description": "ResourceTransformations apply transformation to pod spec resources Retain means that we will keep the original resources and apply a transformation. Replace means that the original resources will be replaced after the transformation is done.", - "type": "object", - "required": [ - "input" - ], - "properties": { - "input": { - "description": "input is the name of the input resource. resources are pod spec resources like cpu, memory, gpus", - "type": "string", - "default": "" - }, - "outputs": { - "description": "outputs specifies the output resources and quantities per unit of input resource. An empty Outputs combined with a `Replace` Strategy causes the Input resource to be ignored by Kueue.", - "type": "object", - "additionalProperties": { - "$ref": "#/definitions/io.k8s.apimachinery.pkg.api.resource.Quantity" - } - }, - "strategy": { - "description": "strategy specifies if the input resource should be replaced or retained. retain means that we will keep the original resources and apply a transformation. replace means that the original resources will be replaced after the transformation is done.", - "type": "string" - } - } - }, - "com.github.openshift.api.operator.v1alpha1.Resources": { - "description": "These structs come directly from Kueue.", - "type": "object", - "properties": { - "excludeResourcePrefixes": { - "description": "excludeResourcePrefixes defines which resources should be ignored by Kueue", - "type": "array", - "items": { - "type": "string", - "default": "" - } - }, - "transformations": { - "description": "transformations defines how to transform PodSpec resources into Workload resource requests. This is intended to be a map with Input as the key (enforced by validation code)", - "type": "array", - "items": { - "default": {}, - "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.ResourceTransformation" - } - } - } - }, "com.github.openshift.api.operator.v1alpha1.StaticPodOperatorStatus": { "description": "StaticPodOperatorStatus is status for controllers that manage static pods. There are different needs because individual node status must be tracked. DEPRECATED: Use v1.StaticPodOperatorStatus instead", "type": "object", diff --git a/operator/v1alpha1/types_kueue.go b/operator/v1alpha1/types_kueue.go index 73d381d444e..ea2ea1e09ca 100644 --- a/operator/v1alpha1/types_kueue.go +++ b/operator/v1alpha1/types_kueue.go @@ -2,7 +2,6 @@ package v1alpha1 import ( operatorv1 "github.com/openshift/api/operator/v1" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -45,67 +44,12 @@ type KueueOperandSpec struct { Config KueueConfiguration `json:"config"` } -// +kubebuilder:validation:Enum=QueueNameOptional;QueueNameRequired -type ManageJobsWithoutQueueNameOption string - -// ManageJobsWithoutQueueName allows to control what kind of -// jobs kueue will manage. -// Kueue jobs usually require kueue.x-k8s.io/queue-name on each job -// to be opt-in for Kueue. -const ( - // QueueNameOptional means kueue will assume all workloads - // are to be gated by Kueue. - // This must be used with ManagedJobsNamespaceSelector. - QueueNameOptional ManageJobsWithoutQueueNameOption = "QueueNameOptional" - // QueueNameRequired means that the jobs require a queue label. - QueueNameRequired ManageJobsWithoutQueueNameOption = "QueueNameRequired" -) - -// +kubebuilder:validation:Enum=Enabled;Disabled -type EnabledOrDisabled string - -const ( - Enabled EnabledOrDisabled = "Enabled" - Disabled EnabledOrDisabled = "Disabled" -) - type KueueConfiguration struct { // integrations are the workloads Kueue will manage // kueue has integrations in the codebase and it also allows // for external frameworks // +required Integrations Integrations `json:"integrations"` - // expermentalFeatures are more expermental features - // that users can use to configure kueue. - // We do not guarantee that these features will yet be supported - // Once we are comfortable with these features, we will move this out of this list. - // +optional - ExpermentalFeatures *ExpermentalFeatures `json:"expermentalFeatures,omitempty"` -} - -// These are more advanced features. -type ExpermentalFeatures struct { - // resources provides additional configuration options for handling the resources. - // Supports https://github.com/kubernetes-sigs/kueue/blob/release-0.10/keps/2937-resource-transformer/README.md - // +optional - Resources Resources `json:"resources,omitempty"` - // featureGates are advanced gates that can control the feature gates that kueue sets - // in the configuration. - // +optional - FeatureGates map[string]EnabledOrDisabled `json:"featureGates,omitempty"` - // manageJobsWithoutQueueName controls whether or not Kueue reconciles - // jobs that don't set the annotation kueue.x-k8s.io/queue-name. - // +kubebuilder:default=QueueNameRequired - // +optional - ManageJobsWithoutQueueName *ManageJobsWithoutQueueNameOption `json:"manageJobsWithoutQueueName,omitempty"` - // managedJobsNamespaceSelector can be used to omit some namespaces from ManagedJobsWithoutQueueName - // Only valid if ManagedJobsWithoutQueueName is QueueNameOptional - // +kubebuilder:validation:XValidation:rule="self.manageJobsWithoutQueueName==QueueNameOptional",message="managedJobsNamespaceSelector is only valid if manageJobsWithoutQueueName is QueueNameOptional" - // +optional - ManagedJobsNamespaceSelector *metav1.LabelSelector `json:"managedJobsNamespaceSelector,omitempty"` - // fairSharing controls the fair sharing semantics across the cluster. - // +optional - FairSharing FairSharing `json:"fairSharing,omitempty"` } // KueueStatus defines the observed state of Kueue @@ -131,92 +75,6 @@ type KueueList struct { Items []Kueue `json:"items"` } -// These structs come directly from Kueue. -type Resources struct { - // excludeResourcePrefixes defines which resources should be ignored by Kueue - // +optional - // +kubebuilder:validation:items:MaxLength=64 - // +kubebuilder:validation:MaxItems=16 - ExcludeResourcePrefixes []string `json:"excludeResourcePrefixes,omitempty"` - - // transformations defines how to transform PodSpec resources into Workload resource requests. - // This is intended to be a map with Input as the key (enforced by validation code) - // +optional - // +kubebuilder:validation:MaxItems=16 - Transformations []ResourceTransformation `json:"transformations,omitempty"` -} - -// +kubebuilder:validation:Enum=Retain;Replace -type ResourceTransformationStrategy string - -// ResourceTransformations apply transformation to pod spec resources -// Retain means that we will keep the original resources and -// apply a transformation. -// Replace means that the original resources will be replaced -// after the transformation is done. -const Retain ResourceTransformationStrategy = "Retain" -const Replace ResourceTransformationStrategy = "Replace" - -// ResourceTransformations apply transformation to pod spec resources -// Retain means that we will keep the original resources and -// apply a transformation. -// Replace means that the original resources will be replaced -// after the transformation is done. -type ResourceTransformation struct { - // input is the name of the input resource. - // resources are pod spec resources like cpu, memory, gpus - // +required - Input corev1.ResourceName `json:"input"` - - // strategy specifies if the input resource should be replaced or retained. - // retain means that we will keep the original resources and - // apply a transformation. - // replace means that the original resources will be replaced - // after the transformation is done. - // +optional - Strategy *ResourceTransformationStrategy `json:"strategy,omitempty"` - - // outputs specifies the output resources and quantities per unit of input resource. - // An empty Outputs combined with a `Replace` Strategy causes the Input resource to be ignored by Kueue. - // +optional - Outputs corev1.ResourceList `json:"outputs,omitempty"` -} - -// +kubebuilder:validation:Enum=LessThanOrEqualToFinalShare;LessThanInitialShare -type PreemptionStrategy string - -const ( - LessThanOrEqualToFinalShare PreemptionStrategy = "LessThanOrEqualToFinalShare" - LessThanInitialShare PreemptionStrategy = "LessThanInitialShare" -) - -type FairSharing struct { - // enable indicates whether to enable fair sharing for all cohorts. - // this is disabled by default. - // +optional - Enable EnabledOrDisabled `json:"enable"` - - // preemptionStrategies indicates which constraints should a preemption satisfy. - // The preemption algorithm will only use the next strategy in the list if the - // incoming workload (preemptor) doesn't fit after using the previous strategies. - // Possible values are: - // - LessThanOrEqualToFinalShare: Only preempt a workload if the share of the preemptor CQ - // with the preemptor workload is less than or equal to the share of the preemptee CQ - // without the workload to be preempted. - // This strategy might favor preemption of smaller workloads in the preemptee CQ, - // regardless of priority or start time, in an effort to keep the share of the CQ - // as high as possible. - // - LessThanInitialShare: Only preempt a workload if the share of the preemptor CQ - // with the incoming workload is strictly less than the share of the preemptee CQ. - // This strategy doesn't depend on the share usage of the workload being preempted. - // As a result, the strategy chooses to preempt workloads with the lowest priority and - // newest start time first. - // The default strategy is ["LessThanOrEqualToFinalShare", "LessThanInitialShare"]. - // +optional - // +kubebuilder:validation:MaxItems=2 - PreemptionStrategies []PreemptionStrategy `json:"preemptionStrategies,omitempty"` -} - // +kubebuilder:validation:Enum=batch/job;ray.io/rayjob;ray.io/raycluster;jobset.x-k8s.io/jobset;kubeflow.org/mpijob;kubeflow.org/paddlejob;kubeflow.org/pytorchjob;kubeflow.org/tfjob;kubeflow.org/xgboostjob;workload.codeflare.dev/appwrapper;pod;deployment;statefulset;leaderworkerset.x-k8s.io/leaderworkerset type KueueIntegrations string @@ -275,6 +133,7 @@ type Integrations struct { // - "leaderworkerset.x-k8s.io/leaderworkerset" (requires enabling pod integration) // +kubebuilder:validation:MaxItems=14 // +kubebuilder:validation:MinItems=1 + // kubebuilder:validation:UniqueItems=true // This is required and must have at least one element. // The frameworks are jobs that Kueue will manage. // +required diff --git a/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml b/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml index a8d27e30bf8..f93954a990f 100644 --- a/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml +++ b/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml @@ -51,181 +51,6 @@ spec: config is the desired configuration for the kueue operator. properties: - expermentalFeatures: - description: |- - unsupportedConfigOverrides are more expermental features - that users can use to configure kueue. - We do not guarantee that these features will yet be supported - Once we are comfortable with these features, we will move this out of this list. - properties: - fairSharing: - description: fairSharing controls the fair sharing semantics - across the cluster. - properties: - enable: - description: |- - enable indicates whether to enable fair sharing for all cohorts. - this is disabled by default. - enum: - - Enabled - - Disabled - type: string - preemptionStrategies: - description: |- - preemptionStrategies indicates which constraints should a preemption satisfy. - The preemption algorithm will only use the next strategy in the list if the - incoming workload (preemptor) doesn't fit after using the previous strategies. - Possible values are: - - LessThanOrEqualToFinalShare: Only preempt a workload if the share of the preemptor CQ - with the preemptor workload is less than or equal to the share of the preemptee CQ - without the workload to be preempted. - This strategy might favor preemption of smaller workloads in the preemptee CQ, - regardless of priority or start time, in an effort to keep the share of the CQ - as high as possible. - - LessThanInitialShare: Only preempt a workload if the share of the preemptor CQ - with the incoming workload is strictly less than the share of the preemptee CQ. - This strategy doesn't depend on the share usage of the workload being preempted. - As a result, the strategy chooses to preempt workloads with the lowest priority and - newest start time first. - The default strategy is ["LessThanOrEqualToFinalShare", "LessThanInitialShare"]. - items: - enum: - - LessThanOrEqualToFinalShare - - LessThanInitialShare - type: string - maxItems: 2 - type: array - type: object - featureGates: - additionalProperties: - enum: - - Enabled - - Disabled - type: string - description: |- - featureGates are advanced gates that can control the feature gates that kueue sets - in the configuration. - type: object - manageJobsWithoutQueueName: - default: QueueNameRequired - description: |- - manageJobsWithoutQueueName controls whether or not Kueue reconciles - jobs that don't set the annotation kueue.x-k8s.io/queue-name. - enum: - - QueueNameOptional - - QueueNameRequired - type: string - managedJobsNamespaceSelector: - description: |- - managedJobsNamespaceSelector can be used to omit some namespaces from ManagedJobsWithoutQueueName - Only valid if ManagedJobsWithoutQueueName is QueueNameOptional - properties: - matchExpressions: - description: matchExpressions is a list of label selector - requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector - applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - x-kubernetes-validations: - - message: managedJobsNamespaceSelector is only valid if manageJobsWithoutQueueName - is QueueNameOptional - rule: self.manageJobsWithoutQueueName==QueueNameOptional - resources: - description: |- - resources provides additional configuration options for handling the resources. - Supports https://github.com/kubernetes-sigs/kueue/blob/release-0.10/keps/2937-resource-transformer/README.md - properties: - excludeResourcePrefixes: - description: excludeResourcePrefixes defines which resources - should be ignored by Kueue - items: - maxLength: 64 - type: string - maxItems: 16 - type: array - transformations: - description: |- - transformations defines how to transform PodSpec resources into Workload resource requests. - This is intended to be a map with Input as the key (enforced by validation code) - items: - description: |- - ResourceTransformations apply transformation to pod spec resources - Retain means that we will keep the original resources and - apply a transformation. - Replace means that the original resources will be replaced - after the transformation is done. - properties: - input: - description: |- - input is the name of the input resource. - resources are pod spec resources like cpu, memory, gpus - type: string - outputs: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - outputs specifies the output resources and quantities per unit of input resource. - An empty Outputs combined with a `Replace` Strategy causes the Input resource to be ignored by Kueue. - type: object - strategy: - description: |- - strategy specifies if the input resource should be replaced or retained. - retain means that we will keep the original resources and - apply a transformation. - replace means that the original resources will be replaced - after the transformation is done. - enum: - - Retain - - Replace - type: string - required: - - input - type: object - maxItems: 16 - type: array - type: object - type: object integrations: description: |- integrations are the workloads Kueue will manage @@ -325,17 +150,6 @@ spec: required: - frameworks type: object - metrics: - description: |- - metrics allows one to configure if kueue metrics - will be exposed to monitoring. - Kueue provides a series of metrics to monitor - the status of LocalQueues and resource flavors - See https://kueue.sigs.k8s.io/docs/reference/metrics/ for a detailed list - enum: - - Enabled - - Disabled - type: string required: - integrations type: object diff --git a/operator/v1alpha1/zz_generated.deepcopy.go b/operator/v1alpha1/zz_generated.deepcopy.go index 0e8348cf2af..31533e23661 100644 --- a/operator/v1alpha1/zz_generated.deepcopy.go +++ b/operator/v1alpha1/zz_generated.deepcopy.go @@ -6,7 +6,6 @@ package v1alpha1 import ( - corev1 "k8s.io/api/core/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) @@ -257,41 +256,6 @@ func (in *EtcdBackupStatus) DeepCopy() *EtcdBackupStatus { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ExpermentalFeatures) DeepCopyInto(out *ExpermentalFeatures) { - *out = *in - in.Resources.DeepCopyInto(&out.Resources) - if in.FeatureGates != nil { - in, out := &in.FeatureGates, &out.FeatureGates - *out = make(map[string]EnabledOrDisabled, len(*in)) - for key, val := range *in { - (*out)[key] = val - } - } - if in.ManageJobsWithoutQueueName != nil { - in, out := &in.ManageJobsWithoutQueueName, &out.ManageJobsWithoutQueueName - *out = new(ManageJobsWithoutQueueNameOption) - **out = **in - } - if in.ManagedJobsNamespaceSelector != nil { - in, out := &in.ManagedJobsNamespaceSelector, &out.ManagedJobsNamespaceSelector - *out = new(v1.LabelSelector) - (*in).DeepCopyInto(*out) - } - in.FairSharing.DeepCopyInto(&out.FairSharing) - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExpermentalFeatures. -func (in *ExpermentalFeatures) DeepCopy() *ExpermentalFeatures { - if in == nil { - return nil - } - out := new(ExpermentalFeatures) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ExternalFramework) DeepCopyInto(out *ExternalFramework) { *out = *in @@ -308,27 +272,6 @@ func (in *ExternalFramework) DeepCopy() *ExternalFramework { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *FairSharing) DeepCopyInto(out *FairSharing) { - *out = *in - if in.PreemptionStrategies != nil { - in, out := &in.PreemptionStrategies, &out.PreemptionStrategies - *out = make([]PreemptionStrategy, len(*in)) - copy(*out, *in) - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FairSharing. -func (in *FairSharing) DeepCopy() *FairSharing { - if in == nil { - return nil - } - out := new(FairSharing) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GenerationHistory) DeepCopyInto(out *GenerationHistory) { *out = *in @@ -520,16 +463,6 @@ func (in *Kueue) DeepCopyObject() runtime.Object { func (in *KueueConfiguration) DeepCopyInto(out *KueueConfiguration) { *out = *in in.Integrations.DeepCopyInto(&out.Integrations) - if in.Metrics != nil { - in, out := &in.Metrics, &out.Metrics - *out = new(EnabledOrDisabled) - **out = **in - } - if in.ExpermentalFeatures != nil { - in, out := &in.ExpermentalFeatures, &out.ExpermentalFeatures - *out = new(ExpermentalFeatures) - (*in).DeepCopyInto(*out) - } return } @@ -831,62 +764,6 @@ func (in *RepositoryDigestMirrors) DeepCopy() *RepositoryDigestMirrors { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ResourceTransformation) DeepCopyInto(out *ResourceTransformation) { - *out = *in - if in.Strategy != nil { - in, out := &in.Strategy, &out.Strategy - *out = new(ResourceTransformationStrategy) - **out = **in - } - if in.Outputs != nil { - in, out := &in.Outputs, &out.Outputs - *out = make(corev1.ResourceList, len(*in)) - for key, val := range *in { - (*out)[key] = val.DeepCopy() - } - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ResourceTransformation. -func (in *ResourceTransformation) DeepCopy() *ResourceTransformation { - if in == nil { - return nil - } - out := new(ResourceTransformation) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Resources) DeepCopyInto(out *Resources) { - *out = *in - if in.ExcludeResourcePrefixes != nil { - in, out := &in.ExcludeResourcePrefixes, &out.ExcludeResourcePrefixes - *out = make([]string, len(*in)) - copy(*out, *in) - } - if in.Transformations != nil { - in, out := &in.Transformations, &out.Transformations - *out = make([]ResourceTransformation, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Resources. -func (in *Resources) DeepCopy() *Resources { - if in == nil { - return nil - } - out := new(Resources) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *StaticPodOperatorStatus) DeepCopyInto(out *StaticPodOperatorStatus) { *out = *in diff --git a/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml b/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml index fb01dd1e418..f930a92814a 100644 --- a/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml +++ b/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml @@ -49,181 +49,6 @@ spec: config is the desired configuration for the kueue operator. properties: - expermentalFeatures: - description: |- - unsupportedConfigOverrides are more expermental features - that users can use to configure kueue. - We do not guarantee that these features will yet be supported - Once we are comfortable with these features, we will move this out of this list. - properties: - fairSharing: - description: fairSharing controls the fair sharing semantics - across the cluster. - properties: - enable: - description: |- - enable indicates whether to enable fair sharing for all cohorts. - this is disabled by default. - enum: - - Enabled - - Disabled - type: string - preemptionStrategies: - description: |- - preemptionStrategies indicates which constraints should a preemption satisfy. - The preemption algorithm will only use the next strategy in the list if the - incoming workload (preemptor) doesn't fit after using the previous strategies. - Possible values are: - - LessThanOrEqualToFinalShare: Only preempt a workload if the share of the preemptor CQ - with the preemptor workload is less than or equal to the share of the preemptee CQ - without the workload to be preempted. - This strategy might favor preemption of smaller workloads in the preemptee CQ, - regardless of priority or start time, in an effort to keep the share of the CQ - as high as possible. - - LessThanInitialShare: Only preempt a workload if the share of the preemptor CQ - with the incoming workload is strictly less than the share of the preemptee CQ. - This strategy doesn't depend on the share usage of the workload being preempted. - As a result, the strategy chooses to preempt workloads with the lowest priority and - newest start time first. - The default strategy is ["LessThanOrEqualToFinalShare", "LessThanInitialShare"]. - items: - enum: - - LessThanOrEqualToFinalShare - - LessThanInitialShare - type: string - maxItems: 2 - type: array - type: object - featureGates: - additionalProperties: - enum: - - Enabled - - Disabled - type: string - description: |- - featureGates are advanced gates that can control the feature gates that kueue sets - in the configuration. - type: object - manageJobsWithoutQueueName: - default: QueueNameRequired - description: |- - manageJobsWithoutQueueName controls whether or not Kueue reconciles - jobs that don't set the annotation kueue.x-k8s.io/queue-name. - enum: - - QueueNameOptional - - QueueNameRequired - type: string - managedJobsNamespaceSelector: - description: |- - managedJobsNamespaceSelector can be used to omit some namespaces from ManagedJobsWithoutQueueName - Only valid if ManagedJobsWithoutQueueName is QueueNameOptional - properties: - matchExpressions: - description: matchExpressions is a list of label selector - requirements. The requirements are ANDed. - items: - description: |- - A label selector requirement is a selector that contains values, a key, and an operator that - relates the key and values. - properties: - key: - description: key is the label key that the selector - applies to. - type: string - operator: - description: |- - operator represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists and DoesNotExist. - type: string - values: - description: |- - values is an array of string values. If the operator is In or NotIn, - the values array must be non-empty. If the operator is Exists or DoesNotExist, - the values array must be empty. This array is replaced during a strategic - merge patch. - items: - type: string - type: array - x-kubernetes-list-type: atomic - required: - - key - - operator - type: object - type: array - x-kubernetes-list-type: atomic - matchLabels: - additionalProperties: - type: string - description: |- - matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels - map is equivalent to an element of matchExpressions, whose key field is "key", the - operator is "In", and the values array contains only "value". The requirements are ANDed. - type: object - type: object - x-kubernetes-map-type: atomic - x-kubernetes-validations: - - message: managedJobsNamespaceSelector is only valid if manageJobsWithoutQueueName - is QueueNameOptional - rule: self.manageJobsWithoutQueueName==QueueNameOptional - resources: - description: |- - resources provides additional configuration options for handling the resources. - Supports https://github.com/kubernetes-sigs/kueue/blob/release-0.10/keps/2937-resource-transformer/README.md - properties: - excludeResourcePrefixes: - description: excludeResourcePrefixes defines which resources - should be ignored by Kueue - items: - maxLength: 64 - type: string - maxItems: 16 - type: array - transformations: - description: |- - transformations defines how to transform PodSpec resources into Workload resource requests. - This is intended to be a map with Input as the key (enforced by validation code) - items: - description: |- - ResourceTransformations apply transformation to pod spec resources - Retain means that we will keep the original resources and - apply a transformation. - Replace means that the original resources will be replaced - after the transformation is done. - properties: - input: - description: |- - input is the name of the input resource. - resources are pod spec resources like cpu, memory, gpus - type: string - outputs: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: |- - outputs specifies the output resources and quantities per unit of input resource. - An empty Outputs combined with a `Replace` Strategy causes the Input resource to be ignored by Kueue. - type: object - strategy: - description: |- - strategy specifies if the input resource should be replaced or retained. - retain means that we will keep the original resources and - apply a transformation. - replace means that the original resources will be replaced - after the transformation is done. - enum: - - Retain - - Replace - type: string - required: - - input - type: object - maxItems: 16 - type: array - type: object - type: object integrations: description: |- integrations are the workloads Kueue will manage @@ -323,17 +148,6 @@ spec: required: - frameworks type: object - metrics: - description: |- - metrics allows one to configure if kueue metrics - will be exposed to monitoring. - Kueue provides a series of metrics to monitor - the status of LocalQueues and resource flavors - See https://kueue.sigs.k8s.io/docs/reference/metrics/ for a detailed list - enum: - - Enabled - - Disabled - type: string required: - integrations type: object diff --git a/operator/v1alpha1/zz_generated.swagger_doc_generated.go b/operator/v1alpha1/zz_generated.swagger_doc_generated.go index 15f4b684e3e..11d0a27f3a2 100644 --- a/operator/v1alpha1/zz_generated.swagger_doc_generated.go +++ b/operator/v1alpha1/zz_generated.swagger_doc_generated.go @@ -257,19 +257,6 @@ func (RepositoryDigestMirrors) SwaggerDoc() map[string]string { return map_RepositoryDigestMirrors } -var map_ExpermentalFeatures = map[string]string{ - "": "These are more advanced features.", - "resources": "resources provides additional configuration options for handling the resources. Supports https://github.com/kubernetes-sigs/kueue/blob/release-0.10/keps/2937-resource-transformer/README.md", - "featureGates": "featureGates are advanced gates that can control the feature gates that kueue sets in the configuration.", - "manageJobsWithoutQueueName": "manageJobsWithoutQueueName controls whether or not Kueue reconciles jobs that don't set the annotation kueue.x-k8s.io/queue-name.", - "managedJobsNamespaceSelector": "managedJobsNamespaceSelector can be used to omit some namespaces from ManagedJobsWithoutQueueName Only valid if ManagedJobsWithoutQueueName is QueueNameOptional", - "fairSharing": "fairSharing controls the fair sharing semantics across the cluster.", -} - -func (ExpermentalFeatures) SwaggerDoc() map[string]string { - return map_ExpermentalFeatures -} - var map_ExternalFramework = map[string]string{ "": "This is the GVK for an external framework. Controller runtime requires this in this format for api discoverability.", "group": "group of externalFramework", @@ -281,15 +268,6 @@ func (ExternalFramework) SwaggerDoc() map[string]string { return map_ExternalFramework } -var map_FairSharing = map[string]string{ - "enable": "enable indicates whether to enable fair sharing for all cohorts. this is disabled by default.", - "preemptionStrategies": "preemptionStrategies indicates which constraints should a preemption satisfy. The preemption algorithm will only use the next strategy in the list if the incoming workload (preemptor) doesn't fit after using the previous strategies. Possible values are: - LessThanOrEqualToFinalShare: Only preempt a workload if the share of the preemptor CQ\n with the preemptor workload is less than or equal to the share of the preemptee CQ\n without the workload to be preempted.\n This strategy might favor preemption of smaller workloads in the preemptee CQ,\n regardless of priority or start time, in an effort to keep the share of the CQ\n as high as possible.\n- LessThanInitialShare: Only preempt a workload if the share of the preemptor CQ\n with the incoming workload is strictly less than the share of the preemptee CQ.\n This strategy doesn't depend on the share usage of the workload being preempted.\n As a result, the strategy chooses to preempt workloads with the lowest priority and\n newest start time first.\nThe default strategy is [\"LessThanOrEqualToFinalShare\", \"LessThanInitialShare\"].", -} - -func (FairSharing) SwaggerDoc() map[string]string { - return map_FairSharing -} - var map_Integrations = map[string]string{ "frameworks": "frameworks are a list of names to be enabled. Possible options:\n - \"batch/job\"\n - \"kubeflow.org/mpijob\"\n - \"ray.io/rayjob\"\n - \"ray.io/raycluster\"\n - \"jobset.x-k8s.io/jobset\"\n - \"kubeflow.org/paddlejob\"\n - \"kubeflow.org/pytorchjob\"\n - \"kubeflow.org/tfjob\"\n - \"kubeflow.org/xgboostjob\"\n - \"workload.codeflare.dev/appwrapper\"\n - \"pod\"\n - \"deployment\" (requires enabling pod integration)\n - \"statefulset\" (requires enabling pod integration)\n - \"leaderworkerset.x-k8s.io/leaderworkerset\" (requires enabling pod integration)\nThis is required and must have at least one element. The frameworks are jobs that Kueue will manage.", "externalFrameworks": "externalFrameworks are a list of GroupVersionKinds that are managed for Kueue by external controllers; the expected format is `Kind.version.group.com`. These are optional and should only be used if you have an external controller that integrations with kueue.", @@ -312,9 +290,7 @@ func (Kueue) SwaggerDoc() map[string]string { } var map_KueueConfiguration = map[string]string{ - "integrations": "integrations are the workloads Kueue will manage kueue has integrations in the codebase and it also allows for external frameworks", - "metrics": "metrics allows one to configure if kueue metrics will be exposed to monitoring. Kueue provides a series of metrics to monitor the status of LocalQueues and resource flavors See https://kueue.sigs.k8s.io/docs/reference/metrics/ for a detailed list", - "expermentalFeatures": "unsupportedConfigOverrides are more expermental features that users can use to configure kueue. We do not guarantee that these features will yet be supported Once we are comfortable with these features, we will move this out of this list.", + "integrations": "integrations are the workloads Kueue will manage kueue has integrations in the codebase and it also allows for external frameworks", } func (KueueConfiguration) SwaggerDoc() map[string]string { @@ -347,27 +323,6 @@ func (KueueStatus) SwaggerDoc() map[string]string { return map_KueueStatus } -var map_ResourceTransformation = map[string]string{ - "": "ResourceTransformations apply transformation to pod spec resources Retain means that we will keep the original resources and apply a transformation. Replace means that the original resources will be replaced after the transformation is done.", - "input": "input is the name of the input resource. resources are pod spec resources like cpu, memory, gpus", - "strategy": "strategy specifies if the input resource should be replaced or retained. retain means that we will keep the original resources and apply a transformation. replace means that the original resources will be replaced after the transformation is done.", - "outputs": "outputs specifies the output resources and quantities per unit of input resource. An empty Outputs combined with a `Replace` Strategy causes the Input resource to be ignored by Kueue.", -} - -func (ResourceTransformation) SwaggerDoc() map[string]string { - return map_ResourceTransformation -} - -var map_Resources = map[string]string{ - "": "These structs come directly from Kueue.", - "excludeResourcePrefixes": "excludeResourcePrefixes defines which resources should be ignored by Kueue", - "transformations": "transformations defines how to transform PodSpec resources into Workload resource requests. This is intended to be a map with Input as the key (enforced by validation code)", -} - -func (Resources) SwaggerDoc() map[string]string { - return map_Resources -} - var map_OLM = map[string]string{ "": "OLM provides information to configure an operator to manage the OLM controllers\n\nCompatibility level 4: No compatibility is provided, the API can change at any point for any reason. These capabilities should not be used by applications needing long term support.", "metadata": "metadata is the standard object's metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata", From d515c668a019532ed3d41a50904064f103c601e1 Mon Sep 17 00:00:00 2001 From: Kevin Hannon Date: Mon, 17 Mar 2025 17:09:04 -0400 Subject: [PATCH 13/42] shorten framework names --- .../generated_openapi/zz_generated.openapi.go | 10 +-- openapi/openapi.json | 10 +-- operator/v1alpha1/types_kueue.go | 64 +++++++--------- .../zz_generated.crd-manifests/kueue.crd.yaml | 76 +++++++++---------- .../AAA_ungated.yaml | 76 +++++++++---------- .../zz_generated.swagger_doc_generated.go | 12 +-- 6 files changed, 116 insertions(+), 132 deletions(-) diff --git a/openapi/generated_openapi/zz_generated.openapi.go b/openapi/generated_openapi/zz_generated.openapi.go index 29a80e45ec6..770e083f269 100644 --- a/openapi/generated_openapi/zz_generated.openapi.go +++ b/openapi/generated_openapi/zz_generated.openapi.go @@ -56861,9 +56861,9 @@ func schema_openshift_api_operator_v1alpha1_ExternalFramework(ref common.Referen Format: "", }, }, - "resourceType": { + "resource": { SchemaProps: spec.SchemaProps{ - Description: "resourceType of external framework this is the same as Kind in the GVK settings", + Description: "resource of external framework", Default: "", Type: []string{"string"}, Format: "", @@ -56878,7 +56878,7 @@ func schema_openshift_api_operator_v1alpha1_ExternalFramework(ref common.Referen }, }, }, - Required: []string{"group", "resourceType", "version"}, + Required: []string{"group", "resource", "version"}, }, }, } @@ -57126,7 +57126,7 @@ func schema_openshift_api_operator_v1alpha1_Integrations(ref common.ReferenceCal Properties: map[string]spec.Schema{ "frameworks": { SchemaProps: spec.SchemaProps{ - Description: "frameworks are a list of names to be enabled. Possible options:\n - \"batch/job\"\n - \"kubeflow.org/mpijob\"\n - \"ray.io/rayjob\"\n - \"ray.io/raycluster\"\n - \"jobset.x-k8s.io/jobset\"\n - \"kubeflow.org/paddlejob\"\n - \"kubeflow.org/pytorchjob\"\n - \"kubeflow.org/tfjob\"\n - \"kubeflow.org/xgboostjob\"\n - \"workload.codeflare.dev/appwrapper\"\n - \"pod\"\n - \"deployment\" (requires enabling pod integration)\n - \"statefulset\" (requires enabling pod integration)\n - \"leaderworkerset.x-k8s.io/leaderworkerset\" (requires enabling pod integration)\nThis is required and must have at least one element. The frameworks are jobs that Kueue will manage.", + Description: "frameworks are a list of names to be enabled. This is required and must have at least one element. The frameworks are jobs that Kueue will manage. kubebuilder:validation:UniqueItems=true", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -57141,7 +57141,7 @@ func schema_openshift_api_operator_v1alpha1_Integrations(ref common.ReferenceCal }, "externalFrameworks": { SchemaProps: spec.SchemaProps{ - Description: "externalFrameworks are a list of GroupVersionKinds that are managed for Kueue by external controllers; the expected format is `Kind.version.group.com`. These are optional and should only be used if you have an external controller that integrations with kueue.", + Description: "externalFrameworks are a list of GroupVersionResources that are managed for Kueue by external controllers; These are optional and should only be used if you have an external controller that integrations with kueue.", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ diff --git a/openapi/openapi.json b/openapi/openapi.json index 21d7fce3ed6..2940e39a04d 100644 --- a/openapi/openapi.json +++ b/openapi/openapi.json @@ -33211,7 +33211,7 @@ "type": "object", "required": [ "group", - "resourceType", + "resource", "version" ], "properties": { @@ -33220,8 +33220,8 @@ "type": "string", "default": "" }, - "resourceType": { - "description": "resourceType of external framework this is the same as Kind in the GVK settings", + "resource": { + "description": "resource of external framework", "type": "string", "default": "" }, @@ -33383,7 +33383,7 @@ ], "properties": { "externalFrameworks": { - "description": "externalFrameworks are a list of GroupVersionKinds that are managed for Kueue by external controllers; the expected format is `Kind.version.group.com`. These are optional and should only be used if you have an external controller that integrations with kueue.", + "description": "externalFrameworks are a list of GroupVersionResources that are managed for Kueue by external controllers; These are optional and should only be used if you have an external controller that integrations with kueue.", "type": "array", "items": { "default": {}, @@ -33391,7 +33391,7 @@ } }, "frameworks": { - "description": "frameworks are a list of names to be enabled. Possible options:\n - \"batch/job\"\n - \"kubeflow.org/mpijob\"\n - \"ray.io/rayjob\"\n - \"ray.io/raycluster\"\n - \"jobset.x-k8s.io/jobset\"\n - \"kubeflow.org/paddlejob\"\n - \"kubeflow.org/pytorchjob\"\n - \"kubeflow.org/tfjob\"\n - \"kubeflow.org/xgboostjob\"\n - \"workload.codeflare.dev/appwrapper\"\n - \"pod\"\n - \"deployment\" (requires enabling pod integration)\n - \"statefulset\" (requires enabling pod integration)\n - \"leaderworkerset.x-k8s.io/leaderworkerset\" (requires enabling pod integration)\nThis is required and must have at least one element. The frameworks are jobs that Kueue will manage.", + "description": "frameworks are a list of names to be enabled. This is required and must have at least one element. The frameworks are jobs that Kueue will manage. kubebuilder:validation:UniqueItems=true", "type": "array", "items": { "type": "string", diff --git a/operator/v1alpha1/types_kueue.go b/operator/v1alpha1/types_kueue.go index ea2ea1e09ca..dea7a96a2b1 100644 --- a/operator/v1alpha1/types_kueue.go +++ b/operator/v1alpha1/types_kueue.go @@ -28,7 +28,6 @@ type Kueue struct { // spec holds user settable values for configuration // kueue configuration must not be changed once the object exists // to change the configuration, one can delete the object and create a new object. - // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="values are immutable once set" // +required Spec KueueOperandSpec `json:"spec"` // status holds observed values from the cluster. They may not be overridden. @@ -75,24 +74,24 @@ type KueueList struct { Items []Kueue `json:"items"` } -// +kubebuilder:validation:Enum=batch/job;ray.io/rayjob;ray.io/raycluster;jobset.x-k8s.io/jobset;kubeflow.org/mpijob;kubeflow.org/paddlejob;kubeflow.org/pytorchjob;kubeflow.org/tfjob;kubeflow.org/xgboostjob;workload.codeflare.dev/appwrapper;pod;deployment;statefulset;leaderworkerset.x-k8s.io/leaderworkerset +// +kubebuilder:validation:Enum=batchjob;rayjob;raycluster;jobset;mpijob;paddlejob;pytorchjob;tfjob;xgboostjob;appwrapper;pod;deployment;statefulset;leaderworkerset type KueueIntegrations string const ( - BatchJob KueueIntegrations = "batch/job" - RayJob KueueIntegrations = "ray.io/rayjob" - RayCluster KueueIntegrations = "ray.io/raycluster" - JobSet KueueIntegrations = "jobset.x-k8s.io/jobset" - MPIJob KueueIntegrations = "kubeflow.org/mpijob" - PaddeJob KueueIntegrations = "kubeflow.org/paddlejob" - PyTorchJob KueueIntegrations = "kubeflow.org/pytorchjob" - TfJob KueueIntegrations = "kubeflow.org/tfjob" - XGBoostJob KueueIntegrations = "kubeflow.org/xgboostjob" - AppWrappers KueueIntegrations = "workload.codeflare.dev/appwrapper" + BatchJob KueueIntegrations = "batchjob" + RayJob KueueIntegrations = "rayjob" + RayCluster KueueIntegrations = "raycluster" + JobSet KueueIntegrations = "jobset" + MPIJob KueueIntegrations = "mpijob" + PaddeJob KueueIntegrations = "paddlejob" + PyTorchJob KueueIntegrations = "pytorchjob" + TfJob KueueIntegrations = "tfjob" + XGBoostJob KueueIntegrations = "xgboostjob" + AppWrappers KueueIntegrations = "appwrapper" Pod KueueIntegrations = "pod" Deployment KueueIntegrations = "deployment" Statefulset KueueIntegrations = "statefulset" - LeaderWorkerSet KueueIntegrations = "leaderworkerset.x-k8s.io/leaderworkerset" + LeaderWorkerSet KueueIntegrations = "leaderworkerset" ) // This is the GVK for an external framework. @@ -100,51 +99,39 @@ const ( // for api discoverability. type ExternalFramework struct { // group of externalFramework - // +kubebuilder:validation:MaxLength=256 + // +kubebuilder:validation:MaxLength=253 + // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:XValidation:rule="self.size() == 0 || !format.qualifiedName().validate(self).hasValue()",message="a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character." // +required Group string `json:"group"` - // resourceType of external framework - // this is the same as Kind in the GVK settings + // resource of external framework // +required // +kubebuilder:validation:MaxLength=256 - ResourceType string `json:"resourceType"` + // +kubebuilder:validation:XValidation:rule="self.size() == 0 || !format.qualifiedName().validate(self).hasValue()",message="a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character." + // +kubebuilder:validation:MinLength=1 + Resource string `json:"resource"` // version is the version of the api // +required // +kubebuilder:validation:MaxLength=256 + // +kubebuilder:validation:MinLength=1 Version string `json:"version"` } type Integrations struct { // frameworks are a list of names to be enabled. - // Possible options: - // - "batch/job" - // - "kubeflow.org/mpijob" - // - "ray.io/rayjob" - // - "ray.io/raycluster" - // - "jobset.x-k8s.io/jobset" - // - "kubeflow.org/paddlejob" - // - "kubeflow.org/pytorchjob" - // - "kubeflow.org/tfjob" - // - "kubeflow.org/xgboostjob" - // - "workload.codeflare.dev/appwrapper" - // - "pod" - // - "deployment" (requires enabling pod integration) - // - "statefulset" (requires enabling pod integration) - // - "leaderworkerset.x-k8s.io/leaderworkerset" (requires enabling pod integration) + // This is required and must have at least one element. + // The frameworks are jobs that Kueue will manage. // +kubebuilder:validation:MaxItems=14 // +kubebuilder:validation:MinItems=1 // kubebuilder:validation:UniqueItems=true - // This is required and must have at least one element. - // The frameworks are jobs that Kueue will manage. // +required Frameworks []KueueIntegrations `json:"frameworks"` - // externalFrameworks are a list of GroupVersionKinds + // externalFrameworks are a list of GroupVersionResources // that are managed for Kueue by external controllers; - // the expected format is `Kind.version.group.com`. // These are optional and should only be used if you have an external controller // that integrations with kueue. // +optional - // +kubebuilder:validation:MaxItems=4 + // +kubebuilder:validation:MaxItems=32 ExternalFrameworks []ExternalFramework `json:"externalFrameworks,omitempty"` // labelKeysToCopy is a list of label keys that should be copied from the job into the @@ -156,8 +143,9 @@ type Integrations struct { // match or otherwise the workload creation would fail. The labels are copied only // during the workload creation and are not updated even if the labels of the // underlying job are changed. - // +kubebuilder:validation:items:MaxLength=64 + // +kubebuilder:validation:items:MaxLength=317 // +kubebuilder:validation:MaxItems=64 + // +kubebuilder:validation:XValidation:rule="self.size() == 0 || !format.qualifiedName().validate(self).hasValue()",message="a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character." // +optional LabelKeysToCopy []string `json:"labelKeysToCopy,omitempty"` } diff --git a/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml b/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml index f93954a990f..c4b75e65709 100644 --- a/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml +++ b/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml @@ -59,9 +59,8 @@ spec: properties: externalFrameworks: description: |- - externalFrameworks are a list of GroupVersionKinds + externalFrameworks are a list of GroupVersionResources that are managed for Kueue by external controllers; - the expected format is `Kind.version.group.com`. These are optional and should only be used if you have an external controller that integrations with kueue. items: @@ -72,61 +71,58 @@ spec: properties: group: description: group of externalFramework - maxLength: 256 + maxLength: 253 + minLength: 1 type: string - resourceType: - description: |- - resourceType of external framework - this is the same as Kind in the GVK settings + x-kubernetes-validations: + - message: a lowercase RFC 1123 subdomain must consist + of lower case alphanumeric characters, '-' or '.', + and must start and end with an alphanumeric character. + rule: self.size() == 0 || !format.qualifiedName().validate(self).hasValue() + resource: + description: resource of external framework maxLength: 256 + minLength: 1 type: string + x-kubernetes-validations: + - message: a lowercase RFC 1123 subdomain must consist + of lower case alphanumeric characters, '-' or '.', + and must start and end with an alphanumeric character. + rule: self.size() == 0 || !format.qualifiedName().validate(self).hasValue() version: description: version is the version of the api maxLength: 256 + minLength: 1 type: string required: - group - - resourceType + - resource - version type: object - maxItems: 4 + maxItems: 32 type: array frameworks: description: |- frameworks are a list of names to be enabled. - Possible options: - - "batch/job" - - "kubeflow.org/mpijob" - - "ray.io/rayjob" - - "ray.io/raycluster" - - "jobset.x-k8s.io/jobset" - - "kubeflow.org/paddlejob" - - "kubeflow.org/pytorchjob" - - "kubeflow.org/tfjob" - - "kubeflow.org/xgboostjob" - - "workload.codeflare.dev/appwrapper" - - "pod" - - "deployment" (requires enabling pod integration) - - "statefulset" (requires enabling pod integration) - - "leaderworkerset.x-k8s.io/leaderworkerset" (requires enabling pod integration) This is required and must have at least one element. The frameworks are jobs that Kueue will manage. + kubebuilder:validation:UniqueItems=true items: enum: - - batch/job - - ray.io/rayjob - - ray.io/raycluster - - jobset.x-k8s.io/jobset - - kubeflow.org/mpijob - - kubeflow.org/paddlejob - - kubeflow.org/pytorchjob - - kubeflow.org/tfjob - - kubeflow.org/xgboostjob - - workload.codeflare.dev/appwrapper + - batchjob + - rayjob + - raycluster + - jobset + - mpijob + - paddlejob + - pytorchjob + - tfjob + - xgboostjob + - appwrapper - pod - deployment - statefulset - - leaderworkerset.x-k8s.io/leaderworkerset + - leaderworkerset type: string maxItems: 14 minItems: 1 @@ -143,10 +139,15 @@ spec: during the workload creation and are not updated even if the labels of the underlying job are changed. items: - maxLength: 64 + maxLength: 317 type: string maxItems: 64 type: array + x-kubernetes-validations: + - message: a lowercase RFC 1123 subdomain must consist of + lower case alphanumeric characters, '-' or '.', and must + start and end with an alphanumeric character. + rule: self.size() == 0 || !format.qualifiedName().validate(self).hasValue() required: - frameworks type: object @@ -208,9 +209,6 @@ spec: required: - config type: object - x-kubernetes-validations: - - message: values are immutable once set - rule: self == oldSelf status: description: status holds observed values from the cluster. They may not be overridden. diff --git a/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml b/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml index f930a92814a..42956ab9a69 100644 --- a/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml +++ b/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml @@ -57,9 +57,8 @@ spec: properties: externalFrameworks: description: |- - externalFrameworks are a list of GroupVersionKinds + externalFrameworks are a list of GroupVersionResources that are managed for Kueue by external controllers; - the expected format is `Kind.version.group.com`. These are optional and should only be used if you have an external controller that integrations with kueue. items: @@ -70,61 +69,58 @@ spec: properties: group: description: group of externalFramework - maxLength: 256 + maxLength: 253 + minLength: 1 type: string - resourceType: - description: |- - resourceType of external framework - this is the same as Kind in the GVK settings + x-kubernetes-validations: + - message: a lowercase RFC 1123 subdomain must consist + of lower case alphanumeric characters, '-' or '.', + and must start and end with an alphanumeric character. + rule: self.size() == 0 || !format.qualifiedName().validate(self).hasValue() + resource: + description: resource of external framework maxLength: 256 + minLength: 1 type: string + x-kubernetes-validations: + - message: a lowercase RFC 1123 subdomain must consist + of lower case alphanumeric characters, '-' or '.', + and must start and end with an alphanumeric character. + rule: self.size() == 0 || !format.qualifiedName().validate(self).hasValue() version: description: version is the version of the api maxLength: 256 + minLength: 1 type: string required: - group - - resourceType + - resource - version type: object - maxItems: 4 + maxItems: 32 type: array frameworks: description: |- frameworks are a list of names to be enabled. - Possible options: - - "batch/job" - - "kubeflow.org/mpijob" - - "ray.io/rayjob" - - "ray.io/raycluster" - - "jobset.x-k8s.io/jobset" - - "kubeflow.org/paddlejob" - - "kubeflow.org/pytorchjob" - - "kubeflow.org/tfjob" - - "kubeflow.org/xgboostjob" - - "workload.codeflare.dev/appwrapper" - - "pod" - - "deployment" (requires enabling pod integration) - - "statefulset" (requires enabling pod integration) - - "leaderworkerset.x-k8s.io/leaderworkerset" (requires enabling pod integration) This is required and must have at least one element. The frameworks are jobs that Kueue will manage. + kubebuilder:validation:UniqueItems=true items: enum: - - batch/job - - ray.io/rayjob - - ray.io/raycluster - - jobset.x-k8s.io/jobset - - kubeflow.org/mpijob - - kubeflow.org/paddlejob - - kubeflow.org/pytorchjob - - kubeflow.org/tfjob - - kubeflow.org/xgboostjob - - workload.codeflare.dev/appwrapper + - batchjob + - rayjob + - raycluster + - jobset + - mpijob + - paddlejob + - pytorchjob + - tfjob + - xgboostjob + - appwrapper - pod - deployment - statefulset - - leaderworkerset.x-k8s.io/leaderworkerset + - leaderworkerset type: string maxItems: 14 minItems: 1 @@ -141,10 +137,15 @@ spec: during the workload creation and are not updated even if the labels of the underlying job are changed. items: - maxLength: 64 + maxLength: 317 type: string maxItems: 64 type: array + x-kubernetes-validations: + - message: a lowercase RFC 1123 subdomain must consist of + lower case alphanumeric characters, '-' or '.', and must + start and end with an alphanumeric character. + rule: self.size() == 0 || !format.qualifiedName().validate(self).hasValue() required: - frameworks type: object @@ -206,9 +207,6 @@ spec: required: - config type: object - x-kubernetes-validations: - - message: values are immutable once set - rule: self == oldSelf status: description: status holds observed values from the cluster. They may not be overridden. diff --git a/operator/v1alpha1/zz_generated.swagger_doc_generated.go b/operator/v1alpha1/zz_generated.swagger_doc_generated.go index 11d0a27f3a2..8f6be116544 100644 --- a/operator/v1alpha1/zz_generated.swagger_doc_generated.go +++ b/operator/v1alpha1/zz_generated.swagger_doc_generated.go @@ -258,10 +258,10 @@ func (RepositoryDigestMirrors) SwaggerDoc() map[string]string { } var map_ExternalFramework = map[string]string{ - "": "This is the GVK for an external framework. Controller runtime requires this in this format for api discoverability.", - "group": "group of externalFramework", - "resourceType": "resourceType of external framework this is the same as Kind in the GVK settings", - "version": "version is the version of the api", + "": "This is the GVK for an external framework. Controller runtime requires this in this format for api discoverability.", + "group": "group of externalFramework", + "resource": "resource of external framework", + "version": "version is the version of the api", } func (ExternalFramework) SwaggerDoc() map[string]string { @@ -269,8 +269,8 @@ func (ExternalFramework) SwaggerDoc() map[string]string { } var map_Integrations = map[string]string{ - "frameworks": "frameworks are a list of names to be enabled. Possible options:\n - \"batch/job\"\n - \"kubeflow.org/mpijob\"\n - \"ray.io/rayjob\"\n - \"ray.io/raycluster\"\n - \"jobset.x-k8s.io/jobset\"\n - \"kubeflow.org/paddlejob\"\n - \"kubeflow.org/pytorchjob\"\n - \"kubeflow.org/tfjob\"\n - \"kubeflow.org/xgboostjob\"\n - \"workload.codeflare.dev/appwrapper\"\n - \"pod\"\n - \"deployment\" (requires enabling pod integration)\n - \"statefulset\" (requires enabling pod integration)\n - \"leaderworkerset.x-k8s.io/leaderworkerset\" (requires enabling pod integration)\nThis is required and must have at least one element. The frameworks are jobs that Kueue will manage.", - "externalFrameworks": "externalFrameworks are a list of GroupVersionKinds that are managed for Kueue by external controllers; the expected format is `Kind.version.group.com`. These are optional and should only be used if you have an external controller that integrations with kueue.", + "frameworks": "frameworks are a list of names to be enabled. This is required and must have at least one element. The frameworks are jobs that Kueue will manage. kubebuilder:validation:UniqueItems=true", + "externalFrameworks": "externalFrameworks are a list of GroupVersionResources that are managed for Kueue by external controllers; These are optional and should only be used if you have an external controller that integrations with kueue.", "labelKeysToCopy": "labelKeysToCopy is a list of label keys that should be copied from the job into the workload object. It is not required for the job to have all the labels from this list. If a job does not have some label with the given key from this list, the constructed workload object will be created without this label. In the case of creating a workload from a composable job (pod group), if multiple objects have labels with some key from the list, the values of these labels must match or otherwise the workload creation would fail. The labels are copied only during the workload creation and are not updated even if the labels of the underlying job are changed.", } From c14027aa39a13bcfbb77e7048824eaaf6b47d866 Mon Sep 17 00:00:00 2001 From: Kevin Hannon Date: Mon, 17 Mar 2025 17:32:11 -0400 Subject: [PATCH 14/42] add listmapkey --- .../generated_openapi/zz_generated.openapi.go | 21 +++++++++++++++++++ openapi/openapi.json | 15 ++++++++++--- operator/v1alpha1/types_kueue.go | 5 ++++- 3 files changed, 37 insertions(+), 4 deletions(-) diff --git a/openapi/generated_openapi/zz_generated.openapi.go b/openapi/generated_openapi/zz_generated.openapi.go index 770e083f269..90c51ecefdf 100644 --- a/openapi/generated_openapi/zz_generated.openapi.go +++ b/openapi/generated_openapi/zz_generated.openapi.go @@ -57125,6 +57125,13 @@ func schema_openshift_api_operator_v1alpha1_Integrations(ref common.ReferenceCal Type: []string{"object"}, Properties: map[string]spec.Schema{ "frameworks": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-map-keys": []interface{}{ + "atomic", + }, + }, + }, SchemaProps: spec.SchemaProps{ Description: "frameworks are a list of names to be enabled. This is required and must have at least one element. The frameworks are jobs that Kueue will manage. kubebuilder:validation:UniqueItems=true", Type: []string{"array"}, @@ -57140,6 +57147,13 @@ func schema_openshift_api_operator_v1alpha1_Integrations(ref common.ReferenceCal }, }, "externalFrameworks": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-map-keys": []interface{}{ + "atomic", + }, + }, + }, SchemaProps: spec.SchemaProps{ Description: "externalFrameworks are a list of GroupVersionResources that are managed for Kueue by external controllers; These are optional and should only be used if you have an external controller that integrations with kueue.", Type: []string{"array"}, @@ -57154,6 +57168,13 @@ func schema_openshift_api_operator_v1alpha1_Integrations(ref common.ReferenceCal }, }, "labelKeysToCopy": { + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-list-map-keys": []interface{}{ + "atomic", + }, + }, + }, SchemaProps: spec.SchemaProps{ Description: "labelKeysToCopy is a list of label keys that should be copied from the job into the workload object. It is not required for the job to have all the labels from this list. If a job does not have some label with the given key from this list, the constructed workload object will be created without this label. In the case of creating a workload from a composable job (pod group), if multiple objects have labels with some key from the list, the values of these labels must match or otherwise the workload creation would fail. The labels are copied only during the workload creation and are not updated even if the labels of the underlying job are changed.", Type: []string{"array"}, diff --git a/openapi/openapi.json b/openapi/openapi.json index 2940e39a04d..02eb5d4c6ea 100644 --- a/openapi/openapi.json +++ b/openapi/openapi.json @@ -33388,7 +33388,10 @@ "items": { "default": {}, "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.ExternalFramework" - } + }, + "x-kubernetes-list-map-keys": [ + "atomic" + ] }, "frameworks": { "description": "frameworks are a list of names to be enabled. This is required and must have at least one element. The frameworks are jobs that Kueue will manage. kubebuilder:validation:UniqueItems=true", @@ -33396,7 +33399,10 @@ "items": { "type": "string", "default": "" - } + }, + "x-kubernetes-list-map-keys": [ + "atomic" + ] }, "labelKeysToCopy": { "description": "labelKeysToCopy is a list of label keys that should be copied from the job into the workload object. It is not required for the job to have all the labels from this list. If a job does not have some label with the given key from this list, the constructed workload object will be created without this label. In the case of creating a workload from a composable job (pod group), if multiple objects have labels with some key from the list, the values of these labels must match or otherwise the workload creation would fail. The labels are copied only during the workload creation and are not updated even if the labels of the underlying job are changed.", @@ -33404,7 +33410,10 @@ "items": { "type": "string", "default": "" - } + }, + "x-kubernetes-list-map-keys": [ + "atomic" + ] } } }, diff --git a/operator/v1alpha1/types_kueue.go b/operator/v1alpha1/types_kueue.go index dea7a96a2b1..cd0876f7f0c 100644 --- a/operator/v1alpha1/types_kueue.go +++ b/operator/v1alpha1/types_kueue.go @@ -124,14 +124,16 @@ type Integrations struct { // +kubebuilder:validation:MaxItems=14 // +kubebuilder:validation:MinItems=1 // kubebuilder:validation:UniqueItems=true + // +listMapKey=atomic // +required Frameworks []KueueIntegrations `json:"frameworks"` // externalFrameworks are a list of GroupVersionResources // that are managed for Kueue by external controllers; // These are optional and should only be used if you have an external controller // that integrations with kueue. - // +optional + // +listMapKey=atomic // +kubebuilder:validation:MaxItems=32 + // +optional ExternalFrameworks []ExternalFramework `json:"externalFrameworks,omitempty"` // labelKeysToCopy is a list of label keys that should be copied from the job into the @@ -146,6 +148,7 @@ type Integrations struct { // +kubebuilder:validation:items:MaxLength=317 // +kubebuilder:validation:MaxItems=64 // +kubebuilder:validation:XValidation:rule="self.size() == 0 || !format.qualifiedName().validate(self).hasValue()",message="a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character." + // +listMapKey=atomic // +optional LabelKeysToCopy []string `json:"labelKeysToCopy,omitempty"` } From fd627fb153862052d2b3039f9b57bda95d6b5eb7 Mon Sep 17 00:00:00 2001 From: Kevin Hannon Date: Tue, 18 Mar 2025 11:37:33 -0400 Subject: [PATCH 15/42] add godocs and fix schema generation --- .../generated_openapi/zz_generated.openapi.go | 59 ++++++---- openapi/openapi.json | 45 ++++---- operator/v1alpha1/types_kueue.go | 100 ++++++++++------- .../zz_generated.crd-manifests/kueue.crd.yaml | 105 +++++++++++------- operator/v1alpha1/zz_generated.deepcopy.go | 18 ++- .../AAA_ungated.yaml | 105 +++++++++++------- .../zz_generated.swagger_doc_generated.go | 28 +++-- 7 files changed, 283 insertions(+), 177 deletions(-) diff --git a/openapi/generated_openapi/zz_generated.openapi.go b/openapi/generated_openapi/zz_generated.openapi.go index 90c51ecefdf..18a8c5abf55 100644 --- a/openapi/generated_openapi/zz_generated.openapi.go +++ b/openapi/generated_openapi/zz_generated.openapi.go @@ -1122,6 +1122,7 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "github.com/openshift/api/operator/v1alpha1.KueueList": schema_openshift_api_operator_v1alpha1_KueueList(ref), "github.com/openshift/api/operator/v1alpha1.KueueOperandSpec": schema_openshift_api_operator_v1alpha1_KueueOperandSpec(ref), "github.com/openshift/api/operator/v1alpha1.KueueStatus": schema_openshift_api_operator_v1alpha1_KueueStatus(ref), + "github.com/openshift/api/operator/v1alpha1.LabelKeys": schema_openshift_api_operator_v1alpha1_LabelKeys(ref), "github.com/openshift/api/operator/v1alpha1.LoggingConfig": schema_openshift_api_operator_v1alpha1_LoggingConfig(ref), "github.com/openshift/api/operator/v1alpha1.NodeStatus": schema_openshift_api_operator_v1alpha1_NodeStatus(ref), "github.com/openshift/api/operator/v1alpha1.OLM": schema_openshift_api_operator_v1alpha1_OLM(ref), @@ -56855,7 +56856,7 @@ func schema_openshift_api_operator_v1alpha1_ExternalFramework(ref common.Referen Properties: map[string]spec.Schema{ "group": { SchemaProps: spec.SchemaProps{ - Description: "group of externalFramework", + Description: "group of externalFramework must be a valid qualified name consisting of a lower-case alphanumeric string, and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. The optional prefix and the name are separate by a forward slash (/).", Default: "", Type: []string{"string"}, Format: "", @@ -56863,7 +56864,7 @@ func schema_openshift_api_operator_v1alpha1_ExternalFramework(ref common.Referen }, "resource": { SchemaProps: spec.SchemaProps{ - Description: "resource of external framework", + Description: "resource of external framework must be a valid qualified name consisting of a lower-case alphanumeric string, and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character.", Default: "", Type: []string{"string"}, Format: "", @@ -57127,13 +57128,11 @@ func schema_openshift_api_operator_v1alpha1_Integrations(ref common.ReferenceCal "frameworks": { VendorExtensible: spec.VendorExtensible{ Extensions: spec.Extensions{ - "x-kubernetes-list-map-keys": []interface{}{ - "atomic", - }, + "x-kubernetes-list-type": "atomic", }, }, SchemaProps: spec.SchemaProps{ - Description: "frameworks are a list of names to be enabled. This is required and must have at least one element. The frameworks are jobs that Kueue will manage. kubebuilder:validation:UniqueItems=true", + Description: "frameworks are a list of names to be enabled. This is required and must have at least one element. The frameworks are jobs that Kueue will manage. KueueIntegrations is a list of frameworks that Kueue has support for. The allowed values are BatchJob;RayJob;RayCluster;JobSet;MPIJob;PaddleJob;PytorchJob;TFJob;XGBoostJob;AppWrappers;Pod;Deployment;StatefulSet;LeaderWorkerSet. kubebuilder:validation:UniqueItems=true", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -57149,13 +57148,11 @@ func schema_openshift_api_operator_v1alpha1_Integrations(ref common.ReferenceCal "externalFrameworks": { VendorExtensible: spec.VendorExtensible{ Extensions: spec.Extensions{ - "x-kubernetes-list-map-keys": []interface{}{ - "atomic", - }, + "x-kubernetes-list-type": "atomic", }, }, SchemaProps: spec.SchemaProps{ - Description: "externalFrameworks are a list of GroupVersionResources that are managed for Kueue by external controllers; These are optional and should only be used if you have an external controller that integrations with kueue.", + Description: "externalFrameworks are a list of GroupVersionResources that are managed for Kueue by external controllers; These are optional and should only be used if you have an external controller that integrates with Kueue.", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -57170,20 +57167,17 @@ func schema_openshift_api_operator_v1alpha1_Integrations(ref common.ReferenceCal "labelKeysToCopy": { VendorExtensible: spec.VendorExtensible{ Extensions: spec.Extensions{ - "x-kubernetes-list-map-keys": []interface{}{ - "atomic", - }, + "x-kubernetes-list-type": "atomic", }, }, SchemaProps: spec.SchemaProps{ - Description: "labelKeysToCopy is a list of label keys that should be copied from the job into the workload object. It is not required for the job to have all the labels from this list. If a job does not have some label with the given key from this list, the constructed workload object will be created without this label. In the case of creating a workload from a composable job (pod group), if multiple objects have labels with some key from the list, the values of these labels must match or otherwise the workload creation would fail. The labels are copied only during the workload creation and are not updated even if the labels of the underlying job are changed.", + Description: "labelKeysToCopy are a list of label keys that are copied once a workload is created these keys are persisted to the internal Kueue workload object. otherwise only the Kueue labels will be copied.", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ SchemaProps: spec.SchemaProps{ - Default: "", - Type: []string{"string"}, - Format: "", + Default: map[string]interface{}{}, + Ref: ref("github.com/openshift/api/operator/v1alpha1.LabelKeys"), }, }, }, @@ -57194,7 +57188,7 @@ func schema_openshift_api_operator_v1alpha1_Integrations(ref common.ReferenceCal }, }, Dependencies: []string{ - "github.com/openshift/api/operator/v1alpha1.ExternalFramework"}, + "github.com/openshift/api/operator/v1alpha1.ExternalFramework", "github.com/openshift/api/operator/v1alpha1.LabelKeys"}, } } @@ -57202,7 +57196,7 @@ func schema_openshift_api_operator_v1alpha1_Kueue(ref common.ReferenceCallback) return common.OpenAPIDefinition{ Schema: spec.Schema{ SchemaProps: spec.SchemaProps{ - Description: "Kueue is the CRD to represent the kueue operator This CRD defines the configuration that the Kueue Compatibility level 4: No compatibility is provided, the API can change at any point for any reason. These capabilities should not be used by applications needing long term support.", + Description: "Kueue is the CRD to represent the Kueue operator This CRD defines the configuration that the Kueue Compatibility level 4: No compatibility is provided, the API can change at any point for any reason. These capabilities should not be used by applications needing long term support.", Type: []string{"object"}, Properties: map[string]spec.Schema{ "kind": { @@ -57228,7 +57222,7 @@ func schema_openshift_api_operator_v1alpha1_Kueue(ref common.ReferenceCallback) }, "spec": { SchemaProps: spec.SchemaProps{ - Description: "spec holds user settable values for configuration kueue configuration must not be changed once the object exists to change the configuration, one can delete the object and create a new object.", + Description: "spec holds user settable values for configuration", Default: map[string]interface{}{}, Ref: ref("github.com/openshift/api/operator/v1alpha1.KueueOperandSpec"), }, @@ -57257,7 +57251,7 @@ func schema_openshift_api_operator_v1alpha1_KueueConfiguration(ref common.Refere Properties: map[string]spec.Schema{ "integrations": { SchemaProps: spec.SchemaProps{ - Description: "integrations are the workloads Kueue will manage kueue has integrations in the codebase and it also allows for external frameworks", + Description: "integrations are the workloads Kueue will manage Kueue has integrations in the codebase and it also allows for external frameworks Kueue are an important part to specify for the API as Kueue will only manage the workloads that are specfied in this list. This is a required field.", Default: map[string]interface{}{}, Ref: ref("github.com/openshift/api/operator/v1alpha1.Integrations"), }, @@ -57301,7 +57295,7 @@ func schema_openshift_api_operator_v1alpha1_KueueList(ref common.ReferenceCallba }, "items": { SchemaProps: spec.SchemaProps{ - Description: "items is a slice of kueue this is a cluster scoped resource and there can only be 1 kueue", + Description: "items is a slice of Kueue this is a cluster scoped resource and there can only be 1 Kueue", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -57364,7 +57358,7 @@ func schema_openshift_api_operator_v1alpha1_KueueOperandSpec(ref common.Referenc }, "config": { SchemaProps: spec.SchemaProps{ - Description: "config is the desired configuration for the kueue operator.", + Description: "config is the desired configuration for the Kueue operator.", Default: map[string]interface{}{}, Ref: ref("github.com/openshift/api/operator/v1alpha1.KueueConfiguration"), }, @@ -57470,6 +57464,25 @@ func schema_openshift_api_operator_v1alpha1_KueueStatus(ref common.ReferenceCall } } +func schema_openshift_api_operator_v1alpha1_LabelKeys(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "key": { + SchemaProps: spec.SchemaProps{ + Description: "key is the label key A label key must be a valid qualified name consisting of a lower-case alphanumeric string, and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. The optional prefix and the name are separate by a forward slash (/).", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + } +} + func schema_openshift_api_operator_v1alpha1_LoggingConfig(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ diff --git a/openapi/openapi.json b/openapi/openapi.json index 02eb5d4c6ea..47567594daa 100644 --- a/openapi/openapi.json +++ b/openapi/openapi.json @@ -33216,12 +33216,12 @@ ], "properties": { "group": { - "description": "group of externalFramework", + "description": "group of externalFramework must be a valid qualified name consisting of a lower-case alphanumeric string, and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. The optional prefix and the name are separate by a forward slash (/).", "type": "string", "default": "" }, "resource": { - "description": "resource of external framework", + "description": "resource of external framework must be a valid qualified name consisting of a lower-case alphanumeric string, and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character.", "type": "string", "default": "" }, @@ -33383,42 +33383,36 @@ ], "properties": { "externalFrameworks": { - "description": "externalFrameworks are a list of GroupVersionResources that are managed for Kueue by external controllers; These are optional and should only be used if you have an external controller that integrations with kueue.", + "description": "externalFrameworks are a list of GroupVersionResources that are managed for Kueue by external controllers; These are optional and should only be used if you have an external controller that integrates with Kueue.", "type": "array", "items": { "default": {}, "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.ExternalFramework" }, - "x-kubernetes-list-map-keys": [ - "atomic" - ] + "x-kubernetes-list-type": "atomic" }, "frameworks": { - "description": "frameworks are a list of names to be enabled. This is required and must have at least one element. The frameworks are jobs that Kueue will manage. kubebuilder:validation:UniqueItems=true", + "description": "frameworks are a list of names to be enabled. This is required and must have at least one element. The frameworks are jobs that Kueue will manage. KueueIntegrations is a list of frameworks that Kueue has support for. The allowed values are BatchJob;RayJob;RayCluster;JobSet;MPIJob;PaddleJob;PytorchJob;TFJob;XGBoostJob;AppWrappers;Pod;Deployment;StatefulSet;LeaderWorkerSet. kubebuilder:validation:UniqueItems=true", "type": "array", "items": { "type": "string", "default": "" }, - "x-kubernetes-list-map-keys": [ - "atomic" - ] + "x-kubernetes-list-type": "atomic" }, "labelKeysToCopy": { - "description": "labelKeysToCopy is a list of label keys that should be copied from the job into the workload object. It is not required for the job to have all the labels from this list. If a job does not have some label with the given key from this list, the constructed workload object will be created without this label. In the case of creating a workload from a composable job (pod group), if multiple objects have labels with some key from the list, the values of these labels must match or otherwise the workload creation would fail. The labels are copied only during the workload creation and are not updated even if the labels of the underlying job are changed.", + "description": "labelKeysToCopy are a list of label keys that are copied once a workload is created these keys are persisted to the internal Kueue workload object. otherwise only the Kueue labels will be copied.", "type": "array", "items": { - "type": "string", - "default": "" + "default": {}, + "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.LabelKeys" }, - "x-kubernetes-list-map-keys": [ - "atomic" - ] + "x-kubernetes-list-type": "atomic" } } }, "com.github.openshift.api.operator.v1alpha1.Kueue": { - "description": "Kueue is the CRD to represent the kueue operator This CRD defines the configuration that the Kueue Compatibility level 4: No compatibility is provided, the API can change at any point for any reason. These capabilities should not be used by applications needing long term support.", + "description": "Kueue is the CRD to represent the Kueue operator This CRD defines the configuration that the Kueue Compatibility level 4: No compatibility is provided, the API can change at any point for any reason. These capabilities should not be used by applications needing long term support.", "type": "object", "required": [ "spec" @@ -33438,7 +33432,7 @@ "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta" }, "spec": { - "description": "spec holds user settable values for configuration kueue configuration must not be changed once the object exists to change the configuration, one can delete the object and create a new object.", + "description": "spec holds user settable values for configuration", "default": {}, "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.KueueOperandSpec" }, @@ -33456,7 +33450,7 @@ ], "properties": { "integrations": { - "description": "integrations are the workloads Kueue will manage kueue has integrations in the codebase and it also allows for external frameworks", + "description": "integrations are the workloads Kueue will manage Kueue has integrations in the codebase and it also allows for external frameworks Kueue are an important part to specify for the API as Kueue will only manage the workloads that are specfied in this list. This is a required field.", "default": {}, "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.Integrations" } @@ -33474,7 +33468,7 @@ "type": "string" }, "items": { - "description": "items is a slice of kueue this is a cluster scoped resource and there can only be 1 kueue", + "description": "items is a slice of Kueue this is a cluster scoped resource and there can only be 1 Kueue", "type": "array", "items": { "default": {}, @@ -33500,7 +33494,7 @@ ], "properties": { "config": { - "description": "config is the desired configuration for the kueue operator.", + "description": "config is the desired configuration for the Kueue operator.", "default": {}, "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.KueueConfiguration" }, @@ -33583,6 +33577,15 @@ } } }, + "com.github.openshift.api.operator.v1alpha1.LabelKeys": { + "type": "object", + "properties": { + "key": { + "description": "key is the label key A label key must be a valid qualified name consisting of a lower-case alphanumeric string, and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. The optional prefix and the name are separate by a forward slash (/).", + "type": "string" + } + } + }, "com.github.openshift.api.operator.v1alpha1.LoggingConfig": { "description": "LoggingConfig holds information about configuring logging DEPRECATED: Use v1.LogLevel instead", "type": "object", diff --git a/operator/v1alpha1/types_kueue.go b/operator/v1alpha1/types_kueue.go index cd0876f7f0c..707ca65f65a 100644 --- a/operator/v1alpha1/types_kueue.go +++ b/operator/v1alpha1/types_kueue.go @@ -7,7 +7,7 @@ import ( // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object -// Kueue is the CRD to represent the kueue operator +// Kueue is the CRD to represent the Kueue operator // This CRD defines the configuration that the Kueue // Compatibility level 4: No compatibility is provided, the API can change at any point for any reason. These capabilities should not be used by applications needing long term support. // +openshift:compatibility-gen:level=4 @@ -18,7 +18,7 @@ import ( // +genclient:nonNamespaced // +kubebuilder:storageversion // +kubebuilder:subresource:status -// +kubebuilder:validation:XValidation:rule="self.metadata.name == 'cluster'",message="kueue is a singleton, .metadata.name must be 'cluster'" +// +kubebuilder:validation:XValidation:rule="self.metadata.name == 'cluster'",message="Kueue is a singleton, .metadata.name must be 'cluster'" type Kueue struct { metav1.TypeMeta `json:",inline"` // metadata for kueue @@ -26,8 +26,6 @@ type Kueue struct { metav1.ObjectMeta `json:"metadata,omitempty"` // spec holds user settable values for configuration - // kueue configuration must not be changed once the object exists - // to change the configuration, one can delete the object and create a new object. // +required Spec KueueOperandSpec `json:"spec"` // status holds observed values from the cluster. They may not be overridden. @@ -38,15 +36,18 @@ type Kueue struct { type KueueOperandSpec struct { operatorv1.OperatorSpec `json:",inline"` // config is the desired configuration - // for the kueue operator. + // for the Kueue operator. // +required Config KueueConfiguration `json:"config"` } type KueueConfiguration struct { // integrations are the workloads Kueue will manage - // kueue has integrations in the codebase and it also allows + // Kueue has integrations in the codebase and it also allows // for external frameworks + // Kueue are an important part to specify for the API as Kueue will + // only manage the workloads that are specfied in this list. + // This is a required field. // +required Integrations Integrations `json:"integrations"` } @@ -67,31 +68,31 @@ type KueueList struct { // metadata for the list // +optional metav1.ListMeta `json:"metadata,omitempty"` - // items is a slice of kueue - // this is a cluster scoped resource and there can only be 1 kueue + // items is a slice of Kueue + // this is a cluster scoped resource and there can only be 1 Kueue // +kubebuilder:validation:MaxItems=1 // +required Items []Kueue `json:"items"` } -// +kubebuilder:validation:Enum=batchjob;rayjob;raycluster;jobset;mpijob;paddlejob;pytorchjob;tfjob;xgboostjob;appwrapper;pod;deployment;statefulset;leaderworkerset +// +kubebuilder:validation:Enum=BatchJob;RayJob;RayCluster;JobSet;MPIJob;PaddleJob;PytorchJob;TFJob;XGBoostJob;AppWrappers;Pod;Deployment;StatefulSet;LeaderWorkerSet type KueueIntegrations string const ( - BatchJob KueueIntegrations = "batchjob" - RayJob KueueIntegrations = "rayjob" - RayCluster KueueIntegrations = "raycluster" - JobSet KueueIntegrations = "jobset" - MPIJob KueueIntegrations = "mpijob" - PaddeJob KueueIntegrations = "paddlejob" - PyTorchJob KueueIntegrations = "pytorchjob" - TfJob KueueIntegrations = "tfjob" - XGBoostJob KueueIntegrations = "xgboostjob" - AppWrappers KueueIntegrations = "appwrapper" - Pod KueueIntegrations = "pod" - Deployment KueueIntegrations = "deployment" - Statefulset KueueIntegrations = "statefulset" - LeaderWorkerSet KueueIntegrations = "leaderworkerset" + BatchJob KueueIntegrations = "BatchJob" + RayJob KueueIntegrations = "RayJob" + RayCluster KueueIntegrations = "RayCluster" + JobSet KueueIntegrations = "JobSet" + MPIJob KueueIntegrations = "MPIJob" + PaddeJob KueueIntegrations = "PaddeJob" + PyTorchJob KueueIntegrations = "PyTorchJob" + TFJob KueueIntegrations = "TFJob" + XGBoostJob KueueIntegrations = "XGBoostJob" + AppWrappers KueueIntegrations = "AppWrappers" + Pod KueueIntegrations = "Pod" + Deployment KueueIntegrations = "Deployment" + StatefulSet KueueIntegrations = "StatefulSet" + LeaderWorkerSet KueueIntegrations = "LeaderWorkerSet" ) // This is the GVK for an external framework. @@ -99,21 +100,32 @@ const ( // for api discoverability. type ExternalFramework struct { // group of externalFramework + // must be a valid qualified name consisting of a lower-case alphanumeric string, + // and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. + // The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, + // hyphens and periods, of at most 253 characters in length. + // Each period separated segment within the subdomain must start and end with an alphanumeric character. + // The optional prefix and the name are separate by a forward slash (/). // +kubebuilder:validation:MaxLength=253 // +kubebuilder:validation:MinLength=1 // +kubebuilder:validation:XValidation:rule="self.size() == 0 || !format.qualifiedName().validate(self).hasValue()",message="a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character." // +required Group string `json:"group"` // resource of external framework - // +required + // must be a valid qualified name consisting of a lower-case alphanumeric string, + // and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. + // The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, + // hyphens and periods, of at most 253 characters in length. + // Each period separated segment within the subdomain must start and end with an alphanumeric character. // +kubebuilder:validation:MaxLength=256 // +kubebuilder:validation:XValidation:rule="self.size() == 0 || !format.qualifiedName().validate(self).hasValue()",message="a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character." // +kubebuilder:validation:MinLength=1 + // +required Resource string `json:"resource"` // version is the version of the api - // +required // +kubebuilder:validation:MaxLength=256 // +kubebuilder:validation:MinLength=1 + // +required Version string `json:"version"` } @@ -121,34 +133,42 @@ type Integrations struct { // frameworks are a list of names to be enabled. // This is required and must have at least one element. // The frameworks are jobs that Kueue will manage. + // KueueIntegrations is a list of frameworks that Kueue has support for. + // The allowed values are BatchJob;RayJob;RayCluster;JobSet;MPIJob;PaddleJob;PytorchJob;TFJob;XGBoostJob;AppWrappers;Pod;Deployment;StatefulSet;LeaderWorkerSet. // +kubebuilder:validation:MaxItems=14 // +kubebuilder:validation:MinItems=1 // kubebuilder:validation:UniqueItems=true - // +listMapKey=atomic + // +listType=atomic // +required Frameworks []KueueIntegrations `json:"frameworks"` // externalFrameworks are a list of GroupVersionResources // that are managed for Kueue by external controllers; // These are optional and should only be used if you have an external controller - // that integrations with kueue. - // +listMapKey=atomic + // that integrates with Kueue. + // +listType=atomic // +kubebuilder:validation:MaxItems=32 // +optional ExternalFrameworks []ExternalFramework `json:"externalFrameworks,omitempty"` - // labelKeysToCopy is a list of label keys that should be copied from the job into the - // workload object. It is not required for the job to have all the labels from this - // list. If a job does not have some label with the given key from this list, the - // constructed workload object will be created without this label. In the case - // of creating a workload from a composable job (pod group), if multiple objects - // have labels with some key from the list, the values of these labels must - // match or otherwise the workload creation would fail. The labels are copied only - // during the workload creation and are not updated even if the labels of the - // underlying job are changed. - // +kubebuilder:validation:items:MaxLength=317 + // labelKeysToCopy are a list of label keys that are copied once a workload is created + // these keys are persisted to the internal Kueue workload object. + // otherwise only the Kueue labels will be copied. // +kubebuilder:validation:MaxItems=64 + // +listType=atomic + // +optional + LabelKeysToCopy []LabelKeys `json:"labelKeysToCopy,omitempty"` +} + +type LabelKeys struct { + // key is the label key + // A label key must be a valid qualified name consisting of a lower-case alphanumeric string, + // and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. + // The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, + // hyphens and periods, of at most 253 characters in length. + // Each period separated segment within the subdomain must start and end with an alphanumeric character. + // The optional prefix and the name are separate by a forward slash (/). + // +kubebuilder:validation:MaxLength=317 // +kubebuilder:validation:XValidation:rule="self.size() == 0 || !format.qualifiedName().validate(self).hasValue()",message="a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character." - // +listMapKey=atomic // +optional - LabelKeysToCopy []string `json:"labelKeysToCopy,omitempty"` + Key string `json:"key,omitempty"` } diff --git a/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml b/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml index c4b75e65709..139c59425d3 100644 --- a/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml +++ b/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml @@ -19,7 +19,7 @@ spec: schema: openAPIV3Schema: description: |- - Kueue is the CRD to represent the kueue operator + Kueue is the CRD to represent the Kueue operator This CRD defines the configuration that the Kueue Compatibility level 4: No compatibility is provided, the API can change at any point for any reason. These capabilities should not be used by applications needing long term support. properties: @@ -41,28 +41,28 @@ spec: metadata: type: object spec: - description: |- - spec holds user settable values for configuration - kueue configuration must not be changed once the object exists - to change the configuration, one can delete the object and create a new object. + description: spec holds user settable values for configuration properties: config: description: |- config is the desired configuration - for the kueue operator. + for the Kueue operator. properties: integrations: description: |- integrations are the workloads Kueue will manage - kueue has integrations in the codebase and it also allows + Kueue has integrations in the codebase and it also allows for external frameworks + Kueue are an important part to specify for the API as Kueue will + only manage the workloads that are specfied in this list. + This is a required field. properties: externalFrameworks: description: |- externalFrameworks are a list of GroupVersionResources that are managed for Kueue by external controllers; These are optional and should only be used if you have an external controller - that integrations with kueue. + that integrates with Kueue. items: description: |- This is the GVK for an external framework. @@ -70,7 +70,14 @@ spec: for api discoverability. properties: group: - description: group of externalFramework + description: |- + group of externalFramework + must be a valid qualified name consisting of a lower-case alphanumeric string, + and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. + The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, + hyphens and periods, of at most 253 characters in length. + Each period separated segment within the subdomain must start and end with an alphanumeric character. + The optional prefix and the name are separate by a forward slash (/). maxLength: 253 minLength: 1 type: string @@ -80,7 +87,13 @@ spec: and must start and end with an alphanumeric character. rule: self.size() == 0 || !format.qualifiedName().validate(self).hasValue() resource: - description: resource of external framework + description: |- + resource of external framework + must be a valid qualified name consisting of a lower-case alphanumeric string, + and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. + The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, + hyphens and periods, of at most 253 characters in length. + Each period separated segment within the subdomain must start and end with an alphanumeric character. maxLength: 256 minLength: 1 type: string @@ -101,53 +114,63 @@ spec: type: object maxItems: 32 type: array + x-kubernetes-list-type: atomic frameworks: description: |- frameworks are a list of names to be enabled. This is required and must have at least one element. The frameworks are jobs that Kueue will manage. + KueueIntegrations is a list of frameworks that Kueue has support for. + The allowed values are BatchJob;RayJob;RayCluster;JobSet;MPIJob;PaddleJob;PytorchJob;TFJob;XGBoostJob;AppWrappers;Pod;Deployment;StatefulSet;LeaderWorkerSet. kubebuilder:validation:UniqueItems=true items: enum: - - batchjob - - rayjob - - raycluster - - jobset - - mpijob - - paddlejob - - pytorchjob - - tfjob - - xgboostjob - - appwrapper - - pod - - deployment - - statefulset - - leaderworkerset + - BatchJob + - RayJob + - RayCluster + - JobSet + - MPIJob + - PaddleJob + - PytorchJob + - TFJob + - XGBoostJob + - AppWrappers + - Pod + - Deployment + - StatefulSet + - LeaderWorkerSet type: string maxItems: 14 minItems: 1 type: array + x-kubernetes-list-type: atomic labelKeysToCopy: description: |- - labelKeysToCopy is a list of label keys that should be copied from the job into the - workload object. It is not required for the job to have all the labels from this - list. If a job does not have some label with the given key from this list, the - constructed workload object will be created without this label. In the case - of creating a workload from a composable job (pod group), if multiple objects - have labels with some key from the list, the values of these labels must - match or otherwise the workload creation would fail. The labels are copied only - during the workload creation and are not updated even if the labels of the - underlying job are changed. + labelKeysToCopy are a list of label keys that are copied once a workload is created + these keys are persisted to the internal Kueue workload object. + otherwise only the Kueue labels will be copied. items: - maxLength: 317 - type: string + properties: + key: + description: |- + key is the label key + A label key must be a valid qualified name consisting of a lower-case alphanumeric string, + and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. + The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, + hyphens and periods, of at most 253 characters in length. + Each period separated segment within the subdomain must start and end with an alphanumeric character. + The optional prefix and the name are separate by a forward slash (/). + maxLength: 317 + type: string + x-kubernetes-validations: + - message: a lowercase RFC 1123 subdomain must consist + of lower case alphanumeric characters, '-' or '.', + and must start and end with an alphanumeric character. + rule: self.size() == 0 || !format.qualifiedName().validate(self).hasValue() + type: object maxItems: 64 type: array - x-kubernetes-validations: - - message: a lowercase RFC 1123 subdomain must consist of - lower case alphanumeric characters, '-' or '.', and must - start and end with an alphanumeric character. - rule: self.size() == 0 || !format.qualifiedName().validate(self).hasValue() + x-kubernetes-list-type: atomic required: - frameworks type: object @@ -318,7 +341,7 @@ spec: - spec type: object x-kubernetes-validations: - - message: kueue is a singleton, .metadata.name must be 'cluster' + - message: Kueue is a singleton, .metadata.name must be 'cluster' rule: self.metadata.name == 'cluster' served: true storage: true diff --git a/operator/v1alpha1/zz_generated.deepcopy.go b/operator/v1alpha1/zz_generated.deepcopy.go index 31533e23661..24b510ceae4 100644 --- a/operator/v1alpha1/zz_generated.deepcopy.go +++ b/operator/v1alpha1/zz_generated.deepcopy.go @@ -415,7 +415,7 @@ func (in *Integrations) DeepCopyInto(out *Integrations) { } if in.LabelKeysToCopy != nil { in, out := &in.LabelKeysToCopy, &out.LabelKeysToCopy - *out = make([]string, len(*in)) + *out = make([]LabelKeys, len(*in)) copy(*out, *in) } return @@ -544,6 +544,22 @@ func (in *KueueStatus) DeepCopy() *KueueStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LabelKeys) DeepCopyInto(out *LabelKeys) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LabelKeys. +func (in *LabelKeys) DeepCopy() *LabelKeys { + if in == nil { + return nil + } + out := new(LabelKeys) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *LoggingConfig) DeepCopyInto(out *LoggingConfig) { *out = *in diff --git a/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml b/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml index 42956ab9a69..e859c740189 100644 --- a/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml +++ b/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml @@ -17,7 +17,7 @@ spec: schema: openAPIV3Schema: description: |- - Kueue is the CRD to represent the kueue operator + Kueue is the CRD to represent the Kueue operator This CRD defines the configuration that the Kueue Compatibility level 4: No compatibility is provided, the API can change at any point for any reason. These capabilities should not be used by applications needing long term support. properties: @@ -39,28 +39,28 @@ spec: metadata: type: object spec: - description: |- - spec holds user settable values for configuration - kueue configuration must not be changed once the object exists - to change the configuration, one can delete the object and create a new object. + description: spec holds user settable values for configuration properties: config: description: |- config is the desired configuration - for the kueue operator. + for the Kueue operator. properties: integrations: description: |- integrations are the workloads Kueue will manage - kueue has integrations in the codebase and it also allows + Kueue has integrations in the codebase and it also allows for external frameworks + Kueue are an important part to specify for the API as Kueue will + only manage the workloads that are specfied in this list. + This is a required field. properties: externalFrameworks: description: |- externalFrameworks are a list of GroupVersionResources that are managed for Kueue by external controllers; These are optional and should only be used if you have an external controller - that integrations with kueue. + that integrates with Kueue. items: description: |- This is the GVK for an external framework. @@ -68,7 +68,14 @@ spec: for api discoverability. properties: group: - description: group of externalFramework + description: |- + group of externalFramework + must be a valid qualified name consisting of a lower-case alphanumeric string, + and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. + The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, + hyphens and periods, of at most 253 characters in length. + Each period separated segment within the subdomain must start and end with an alphanumeric character. + The optional prefix and the name are separate by a forward slash (/). maxLength: 253 minLength: 1 type: string @@ -78,7 +85,13 @@ spec: and must start and end with an alphanumeric character. rule: self.size() == 0 || !format.qualifiedName().validate(self).hasValue() resource: - description: resource of external framework + description: |- + resource of external framework + must be a valid qualified name consisting of a lower-case alphanumeric string, + and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. + The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, + hyphens and periods, of at most 253 characters in length. + Each period separated segment within the subdomain must start and end with an alphanumeric character. maxLength: 256 minLength: 1 type: string @@ -99,53 +112,63 @@ spec: type: object maxItems: 32 type: array + x-kubernetes-list-type: atomic frameworks: description: |- frameworks are a list of names to be enabled. This is required and must have at least one element. The frameworks are jobs that Kueue will manage. + KueueIntegrations is a list of frameworks that Kueue has support for. + The allowed values are BatchJob;RayJob;RayCluster;JobSet;MPIJob;PaddleJob;PytorchJob;TFJob;XGBoostJob;AppWrappers;Pod;Deployment;StatefulSet;LeaderWorkerSet. kubebuilder:validation:UniqueItems=true items: enum: - - batchjob - - rayjob - - raycluster - - jobset - - mpijob - - paddlejob - - pytorchjob - - tfjob - - xgboostjob - - appwrapper - - pod - - deployment - - statefulset - - leaderworkerset + - BatchJob + - RayJob + - RayCluster + - JobSet + - MPIJob + - PaddleJob + - PytorchJob + - TFJob + - XGBoostJob + - AppWrappers + - Pod + - Deployment + - StatefulSet + - LeaderWorkerSet type: string maxItems: 14 minItems: 1 type: array + x-kubernetes-list-type: atomic labelKeysToCopy: description: |- - labelKeysToCopy is a list of label keys that should be copied from the job into the - workload object. It is not required for the job to have all the labels from this - list. If a job does not have some label with the given key from this list, the - constructed workload object will be created without this label. In the case - of creating a workload from a composable job (pod group), if multiple objects - have labels with some key from the list, the values of these labels must - match or otherwise the workload creation would fail. The labels are copied only - during the workload creation and are not updated even if the labels of the - underlying job are changed. + labelKeysToCopy are a list of label keys that are copied once a workload is created + these keys are persisted to the internal Kueue workload object. + otherwise only the Kueue labels will be copied. items: - maxLength: 317 - type: string + properties: + key: + description: |- + key is the label key + A label key must be a valid qualified name consisting of a lower-case alphanumeric string, + and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. + The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, + hyphens and periods, of at most 253 characters in length. + Each period separated segment within the subdomain must start and end with an alphanumeric character. + The optional prefix and the name are separate by a forward slash (/). + maxLength: 317 + type: string + x-kubernetes-validations: + - message: a lowercase RFC 1123 subdomain must consist + of lower case alphanumeric characters, '-' or '.', + and must start and end with an alphanumeric character. + rule: self.size() == 0 || !format.qualifiedName().validate(self).hasValue() + type: object maxItems: 64 type: array - x-kubernetes-validations: - - message: a lowercase RFC 1123 subdomain must consist of - lower case alphanumeric characters, '-' or '.', and must - start and end with an alphanumeric character. - rule: self.size() == 0 || !format.qualifiedName().validate(self).hasValue() + x-kubernetes-list-type: atomic required: - frameworks type: object @@ -316,7 +339,7 @@ spec: - spec type: object x-kubernetes-validations: - - message: kueue is a singleton, .metadata.name must be 'cluster' + - message: Kueue is a singleton, .metadata.name must be 'cluster' rule: self.metadata.name == 'cluster' served: true storage: true diff --git a/operator/v1alpha1/zz_generated.swagger_doc_generated.go b/operator/v1alpha1/zz_generated.swagger_doc_generated.go index 8f6be116544..36511bb9eb2 100644 --- a/operator/v1alpha1/zz_generated.swagger_doc_generated.go +++ b/operator/v1alpha1/zz_generated.swagger_doc_generated.go @@ -259,8 +259,8 @@ func (RepositoryDigestMirrors) SwaggerDoc() map[string]string { var map_ExternalFramework = map[string]string{ "": "This is the GVK for an external framework. Controller runtime requires this in this format for api discoverability.", - "group": "group of externalFramework", - "resource": "resource of external framework", + "group": "group of externalFramework must be a valid qualified name consisting of a lower-case alphanumeric string, and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. The optional prefix and the name are separate by a forward slash (/).", + "resource": "resource of external framework must be a valid qualified name consisting of a lower-case alphanumeric string, and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character.", "version": "version is the version of the api", } @@ -269,9 +269,9 @@ func (ExternalFramework) SwaggerDoc() map[string]string { } var map_Integrations = map[string]string{ - "frameworks": "frameworks are a list of names to be enabled. This is required and must have at least one element. The frameworks are jobs that Kueue will manage. kubebuilder:validation:UniqueItems=true", - "externalFrameworks": "externalFrameworks are a list of GroupVersionResources that are managed for Kueue by external controllers; These are optional and should only be used if you have an external controller that integrations with kueue.", - "labelKeysToCopy": "labelKeysToCopy is a list of label keys that should be copied from the job into the workload object. It is not required for the job to have all the labels from this list. If a job does not have some label with the given key from this list, the constructed workload object will be created without this label. In the case of creating a workload from a composable job (pod group), if multiple objects have labels with some key from the list, the values of these labels must match or otherwise the workload creation would fail. The labels are copied only during the workload creation and are not updated even if the labels of the underlying job are changed.", + "frameworks": "frameworks are a list of names to be enabled. This is required and must have at least one element. The frameworks are jobs that Kueue will manage. KueueIntegrations is a list of frameworks that Kueue has support for. The allowed values are BatchJob;RayJob;RayCluster;JobSet;MPIJob;PaddleJob;PytorchJob;TFJob;XGBoostJob;AppWrappers;Pod;Deployment;StatefulSet;LeaderWorkerSet. kubebuilder:validation:UniqueItems=true", + "externalFrameworks": "externalFrameworks are a list of GroupVersionResources that are managed for Kueue by external controllers; These are optional and should only be used if you have an external controller that integrates with Kueue.", + "labelKeysToCopy": "labelKeysToCopy are a list of label keys that are copied once a workload is created these keys are persisted to the internal Kueue workload object. otherwise only the Kueue labels will be copied.", } func (Integrations) SwaggerDoc() map[string]string { @@ -279,9 +279,9 @@ func (Integrations) SwaggerDoc() map[string]string { } var map_Kueue = map[string]string{ - "": "Kueue is the CRD to represent the kueue operator This CRD defines the configuration that the Kueue Compatibility level 4: No compatibility is provided, the API can change at any point for any reason. These capabilities should not be used by applications needing long term support.", + "": "Kueue is the CRD to represent the Kueue operator This CRD defines the configuration that the Kueue Compatibility level 4: No compatibility is provided, the API can change at any point for any reason. These capabilities should not be used by applications needing long term support.", "metadata": "metadata for kueue", - "spec": "spec holds user settable values for configuration kueue configuration must not be changed once the object exists to change the configuration, one can delete the object and create a new object.", + "spec": "spec holds user settable values for configuration", "status": "status holds observed values from the cluster. They may not be overridden.", } @@ -290,7 +290,7 @@ func (Kueue) SwaggerDoc() map[string]string { } var map_KueueConfiguration = map[string]string{ - "integrations": "integrations are the workloads Kueue will manage kueue has integrations in the codebase and it also allows for external frameworks", + "integrations": "integrations are the workloads Kueue will manage Kueue has integrations in the codebase and it also allows for external frameworks Kueue are an important part to specify for the API as Kueue will only manage the workloads that are specfied in this list. This is a required field.", } func (KueueConfiguration) SwaggerDoc() map[string]string { @@ -300,7 +300,7 @@ func (KueueConfiguration) SwaggerDoc() map[string]string { var map_KueueList = map[string]string{ "": "KueueList contains a list of Kueue\n\nCompatibility level 4: No compatibility is provided, the API can change at any point for any reason. These capabilities should not be used by applications needing long term support.", "metadata": "metadata for the list", - "items": "items is a slice of kueue this is a cluster scoped resource and there can only be 1 kueue", + "items": "items is a slice of Kueue this is a cluster scoped resource and there can only be 1 Kueue", } func (KueueList) SwaggerDoc() map[string]string { @@ -308,7 +308,7 @@ func (KueueList) SwaggerDoc() map[string]string { } var map_KueueOperandSpec = map[string]string{ - "config": "config is the desired configuration for the kueue operator.", + "config": "config is the desired configuration for the Kueue operator.", } func (KueueOperandSpec) SwaggerDoc() map[string]string { @@ -323,6 +323,14 @@ func (KueueStatus) SwaggerDoc() map[string]string { return map_KueueStatus } +var map_LabelKeys = map[string]string{ + "key": "key is the label key A label key must be a valid qualified name consisting of a lower-case alphanumeric string, and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. The optional prefix and the name are separate by a forward slash (/).", +} + +func (LabelKeys) SwaggerDoc() map[string]string { + return map_LabelKeys +} + var map_OLM = map[string]string{ "": "OLM provides information to configure an operator to manage the OLM controllers\n\nCompatibility level 4: No compatibility is provided, the API can change at any point for any reason. These capabilities should not be used by applications needing long term support.", "metadata": "metadata is the standard object's metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata", From 0aba7b7c541684632334ed5aaf8f59c9c2e132ed Mon Sep 17 00:00:00 2001 From: Kevin Hannon Date: Wed, 19 Mar 2025 09:41:30 -0400 Subject: [PATCH 16/42] address final comments --- .../generated_openapi/zz_generated.openapi.go | 14 ++-- openapi/openapi.json | 14 ++-- operator/v1alpha1/types_kueue.go | 78 ++++++++++--------- .../zz_generated.crd-manifests/kueue.crd.yaml | 53 +++++++------ operator/v1alpha1/zz_generated.deepcopy.go | 2 +- .../AAA_ungated.yaml | 53 +++++++------ .../zz_generated.swagger_doc_generated.go | 14 ++-- 7 files changed, 124 insertions(+), 104 deletions(-) diff --git a/openapi/generated_openapi/zz_generated.openapi.go b/openapi/generated_openapi/zz_generated.openapi.go index 18a8c5abf55..57d0bd0d906 100644 --- a/openapi/generated_openapi/zz_generated.openapi.go +++ b/openapi/generated_openapi/zz_generated.openapi.go @@ -56851,12 +56851,12 @@ func schema_openshift_api_operator_v1alpha1_ExternalFramework(ref common.Referen return common.OpenAPIDefinition{ Schema: spec.Schema{ SchemaProps: spec.SchemaProps{ - Description: "This is the GVK for an external framework. Controller runtime requires this in this format for api discoverability.", + Description: "This is the GVR for an external framework. Controller runtime requires this in this format for api discoverability.", Type: []string{"object"}, Properties: map[string]spec.Schema{ "group": { SchemaProps: spec.SchemaProps{ - Description: "group of externalFramework must be a valid qualified name consisting of a lower-case alphanumeric string, and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. The optional prefix and the name are separate by a forward slash (/).", + Description: "group of externalFramework group is the API group of the externalFramework. Must be a valid qualified name consisting of a lower-case alphanumeric string, and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. The optional prefix and the name are separate by a forward slash (/).", Default: "", Type: []string{"string"}, Format: "", @@ -56864,7 +56864,7 @@ func schema_openshift_api_operator_v1alpha1_ExternalFramework(ref common.Referen }, "resource": { SchemaProps: spec.SchemaProps{ - Description: "resource of external framework must be a valid qualified name consisting of a lower-case alphanumeric string, and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character.", + Description: "resource is the Resource type of the external framework. Resource types are lowercase and plural (e.g. pods, deployments). Must be a valid qualified name consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character.", Default: "", Type: []string{"string"}, Format: "", @@ -56872,7 +56872,7 @@ func schema_openshift_api_operator_v1alpha1_ExternalFramework(ref common.Referen }, "version": { SchemaProps: spec.SchemaProps{ - Description: "version is the version of the api", + Description: "version is the version of the api (e.g. v1alpha1, v1beta1, v1).", Default: "", Type: []string{"string"}, Format: "", @@ -57132,7 +57132,7 @@ func schema_openshift_api_operator_v1alpha1_Integrations(ref common.ReferenceCal }, }, SchemaProps: spec.SchemaProps{ - Description: "frameworks are a list of names to be enabled. This is required and must have at least one element. The frameworks are jobs that Kueue will manage. KueueIntegrations is a list of frameworks that Kueue has support for. The allowed values are BatchJob;RayJob;RayCluster;JobSet;MPIJob;PaddleJob;PytorchJob;TFJob;XGBoostJob;AppWrappers;Pod;Deployment;StatefulSet;LeaderWorkerSet. kubebuilder:validation:UniqueItems=true", + Description: "frameworks are a unique list of names to be enabled. This is required and must have at least one element. The frameworks are jobs that Kueue will manage. Frameworks are a list of frameworks that Kueue has support for. The allowed values are BatchJob;RayJob;RayCluster;JobSet;MPIJob;PaddleJob;PytorchJob;TFJob;XGBoostJob;AppWrappers;Pod;Deployment;StatefulSet;LeaderWorkerSet.", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -57171,7 +57171,7 @@ func schema_openshift_api_operator_v1alpha1_Integrations(ref common.ReferenceCal }, }, SchemaProps: spec.SchemaProps{ - Description: "labelKeysToCopy are a list of label keys that are copied once a workload is created these keys are persisted to the internal Kueue workload object. otherwise only the Kueue labels will be copied.", + Description: "labelKeysToCopy are a list of label keys that are copied once a workload is created. These keys are persisted to the internal Kueue workload object. If not specified, only the Kueue labels will be copied.", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -57251,7 +57251,7 @@ func schema_openshift_api_operator_v1alpha1_KueueConfiguration(ref common.Refere Properties: map[string]spec.Schema{ "integrations": { SchemaProps: spec.SchemaProps{ - Description: "integrations are the workloads Kueue will manage Kueue has integrations in the codebase and it also allows for external frameworks Kueue are an important part to specify for the API as Kueue will only manage the workloads that are specfied in this list. This is a required field.", + Description: "integrations is a required field that configures the Kueue's workload integrations. Kueue has both standard integrations, known as job frameworks, and external integrations known as external frameworks. Kueue will only manage workloads that correspond to the specified integrations.", Default: map[string]interface{}{}, Ref: ref("github.com/openshift/api/operator/v1alpha1.Integrations"), }, diff --git a/openapi/openapi.json b/openapi/openapi.json index 47567594daa..070b38ce37e 100644 --- a/openapi/openapi.json +++ b/openapi/openapi.json @@ -33207,7 +33207,7 @@ } }, "com.github.openshift.api.operator.v1alpha1.ExternalFramework": { - "description": "This is the GVK for an external framework. Controller runtime requires this in this format for api discoverability.", + "description": "This is the GVR for an external framework. Controller runtime requires this in this format for api discoverability.", "type": "object", "required": [ "group", @@ -33216,17 +33216,17 @@ ], "properties": { "group": { - "description": "group of externalFramework must be a valid qualified name consisting of a lower-case alphanumeric string, and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. The optional prefix and the name are separate by a forward slash (/).", + "description": "group of externalFramework group is the API group of the externalFramework. Must be a valid qualified name consisting of a lower-case alphanumeric string, and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. The optional prefix and the name are separate by a forward slash (/).", "type": "string", "default": "" }, "resource": { - "description": "resource of external framework must be a valid qualified name consisting of a lower-case alphanumeric string, and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character.", + "description": "resource is the Resource type of the external framework. Resource types are lowercase and plural (e.g. pods, deployments). Must be a valid qualified name consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character.", "type": "string", "default": "" }, "version": { - "description": "version is the version of the api", + "description": "version is the version of the api (e.g. v1alpha1, v1beta1, v1).", "type": "string", "default": "" } @@ -33392,7 +33392,7 @@ "x-kubernetes-list-type": "atomic" }, "frameworks": { - "description": "frameworks are a list of names to be enabled. This is required and must have at least one element. The frameworks are jobs that Kueue will manage. KueueIntegrations is a list of frameworks that Kueue has support for. The allowed values are BatchJob;RayJob;RayCluster;JobSet;MPIJob;PaddleJob;PytorchJob;TFJob;XGBoostJob;AppWrappers;Pod;Deployment;StatefulSet;LeaderWorkerSet. kubebuilder:validation:UniqueItems=true", + "description": "frameworks are a unique list of names to be enabled. This is required and must have at least one element. The frameworks are jobs that Kueue will manage. Frameworks are a list of frameworks that Kueue has support for. The allowed values are BatchJob;RayJob;RayCluster;JobSet;MPIJob;PaddleJob;PytorchJob;TFJob;XGBoostJob;AppWrappers;Pod;Deployment;StatefulSet;LeaderWorkerSet.", "type": "array", "items": { "type": "string", @@ -33401,7 +33401,7 @@ "x-kubernetes-list-type": "atomic" }, "labelKeysToCopy": { - "description": "labelKeysToCopy are a list of label keys that are copied once a workload is created these keys are persisted to the internal Kueue workload object. otherwise only the Kueue labels will be copied.", + "description": "labelKeysToCopy are a list of label keys that are copied once a workload is created. These keys are persisted to the internal Kueue workload object. If not specified, only the Kueue labels will be copied.", "type": "array", "items": { "default": {}, @@ -33450,7 +33450,7 @@ ], "properties": { "integrations": { - "description": "integrations are the workloads Kueue will manage Kueue has integrations in the codebase and it also allows for external frameworks Kueue are an important part to specify for the API as Kueue will only manage the workloads that are specfied in this list. This is a required field.", + "description": "integrations is a required field that configures the Kueue's workload integrations. Kueue has both standard integrations, known as job frameworks, and external integrations known as external frameworks. Kueue will only manage workloads that correspond to the specified integrations.", "default": {}, "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.Integrations" } diff --git a/operator/v1alpha1/types_kueue.go b/operator/v1alpha1/types_kueue.go index 707ca65f65a..178da9ef2b3 100644 --- a/operator/v1alpha1/types_kueue.go +++ b/operator/v1alpha1/types_kueue.go @@ -42,12 +42,9 @@ type KueueOperandSpec struct { } type KueueConfiguration struct { - // integrations are the workloads Kueue will manage - // Kueue has integrations in the codebase and it also allows - // for external frameworks - // Kueue are an important part to specify for the API as Kueue will - // only manage the workloads that are specfied in this list. - // This is a required field. + // integrations is a required field that configures the Kueue's workload integrations. + // Kueue has both standard integrations, known as job frameworks, and external integrations known as external frameworks. + // Kueue will only manage workloads that correspond to the specified integrations. // +required Integrations Integrations `json:"integrations"` } @@ -75,32 +72,33 @@ type KueueList struct { Items []Kueue `json:"items"` } -// +kubebuilder:validation:Enum=BatchJob;RayJob;RayCluster;JobSet;MPIJob;PaddleJob;PytorchJob;TFJob;XGBoostJob;AppWrappers;Pod;Deployment;StatefulSet;LeaderWorkerSet -type KueueIntegrations string +// +kubebuilder:validation:Enum=BatchJob;RayJob;RayCluster;JobSet;MPIJob;PaddleJob;PytorchJob;TFJob;XGBoostJob;AppWrapper;Pod;Deployment;StatefulSet;LeaderWorkerSet +type KueueIntegration string const ( - BatchJob KueueIntegrations = "BatchJob" - RayJob KueueIntegrations = "RayJob" - RayCluster KueueIntegrations = "RayCluster" - JobSet KueueIntegrations = "JobSet" - MPIJob KueueIntegrations = "MPIJob" - PaddeJob KueueIntegrations = "PaddeJob" - PyTorchJob KueueIntegrations = "PyTorchJob" - TFJob KueueIntegrations = "TFJob" - XGBoostJob KueueIntegrations = "XGBoostJob" - AppWrappers KueueIntegrations = "AppWrappers" - Pod KueueIntegrations = "Pod" - Deployment KueueIntegrations = "Deployment" - StatefulSet KueueIntegrations = "StatefulSet" - LeaderWorkerSet KueueIntegrations = "LeaderWorkerSet" + KueueIntegrationBatchJob KueueIntegration = "BatchJob" + KueueIntegrationRayJob KueueIntegration = "RayJob" + KueueIntegrationRayCluster KueueIntegration = "RayCluster" + KueueIntegrationJobSet KueueIntegration = "JobSet" + KueueIntegrationMPIJob KueueIntegration = "MPIJob" + KueueIntegrationPaddeJob KueueIntegration = "PaddeJob" + KueueIntegrationPyTorchJob KueueIntegration = "PyTorchJob" + KueueIntegrationTFJob KueueIntegration = "TFJob" + KueueIntegrationXGBoostJob KueueIntegration = "XGBoostJob" + KueueIntegrationAppWrapper KueueIntegration = "AppWrapper" + KueueIntegrationPod KueueIntegration = "Pod" + KueueIntegrationDeployment KueueIntegration = "Deployment" + KueueIntegrationStatefulSet KueueIntegration = "StatefulSet" + KueueIntegrationLeaderWorkerSet KueueIntegration = "LeaderWorkerSet" ) -// This is the GVK for an external framework. +// This is the GVR for an external framework. // Controller runtime requires this in this format // for api discoverability. type ExternalFramework struct { // group of externalFramework - // must be a valid qualified name consisting of a lower-case alphanumeric string, + // group is the API group of the externalFramework. + // Must be a valid qualified name consisting of a lower-case alphanumeric string, // and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. // The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, // hyphens and periods, of at most 253 characters in length. @@ -108,39 +106,42 @@ type ExternalFramework struct { // The optional prefix and the name are separate by a forward slash (/). // +kubebuilder:validation:MaxLength=253 // +kubebuilder:validation:MinLength=1 - // +kubebuilder:validation:XValidation:rule="self.size() == 0 || !format.qualifiedName().validate(self).hasValue()",message="a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character." + // +kubebuilder:validation:XValidation:rule="self.size() == 0 || !format.dns1123Subdomain().validate(self).hasValue()",message="a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character." // +required Group string `json:"group"` - // resource of external framework - // must be a valid qualified name consisting of a lower-case alphanumeric string, - // and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. + // resource is the Resource type of the external framework. + // Resource types are lowercase and plural (e.g. pods, deployments). + // Must be a valid qualified name consisting of a lower-case alphanumeric string + // and hyphens of at most 63 characters in length. + // The name must start and end with an alphanumeric character. // The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, // hyphens and periods, of at most 253 characters in length. // Each period separated segment within the subdomain must start and end with an alphanumeric character. // +kubebuilder:validation:MaxLength=256 - // +kubebuilder:validation:XValidation:rule="self.size() == 0 || !format.qualifiedName().validate(self).hasValue()",message="a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character." // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:XValidation:rule="self.size() == 0 || !format.dns1123Subdomain().validate(self).hasValue()",message="a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character." // +required Resource string `json:"resource"` - // version is the version of the api + // version is the version of the api (e.g. v1alpha1, v1beta1, v1). // +kubebuilder:validation:MaxLength=256 // +kubebuilder:validation:MinLength=1 + // +kubebuilder:validation:XValidation:rule="self.size() == 0 || !format.dns1123Subdomain().validate(self).hasValue()",message="a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character." // +required Version string `json:"version"` } type Integrations struct { - // frameworks are a list of names to be enabled. + // frameworks are a unique list of names to be enabled. // This is required and must have at least one element. // The frameworks are jobs that Kueue will manage. - // KueueIntegrations is a list of frameworks that Kueue has support for. + // Frameworks are a list of frameworks that Kueue has support for. // The allowed values are BatchJob;RayJob;RayCluster;JobSet;MPIJob;PaddleJob;PytorchJob;TFJob;XGBoostJob;AppWrappers;Pod;Deployment;StatefulSet;LeaderWorkerSet. // +kubebuilder:validation:MaxItems=14 // +kubebuilder:validation:MinItems=1 - // kubebuilder:validation:UniqueItems=true + // +kubebuilder:validation:XValidation:rule="self.all(x, self.exists_one(y, x == y))",message="each item in frameworks must be unique" // +listType=atomic // +required - Frameworks []KueueIntegrations `json:"frameworks"` + Frameworks []KueueIntegration `json:"frameworks"` // externalFrameworks are a list of GroupVersionResources // that are managed for Kueue by external controllers; // These are optional and should only be used if you have an external controller @@ -150,9 +151,9 @@ type Integrations struct { // +optional ExternalFrameworks []ExternalFramework `json:"externalFrameworks,omitempty"` - // labelKeysToCopy are a list of label keys that are copied once a workload is created - // these keys are persisted to the internal Kueue workload object. - // otherwise only the Kueue labels will be copied. + // labelKeysToCopy are a list of label keys that are copied once a workload is created. + // These keys are persisted to the internal Kueue workload object. + // If not specified, only the Kueue labels will be copied. // +kubebuilder:validation:MaxItems=64 // +listType=atomic // +optional @@ -162,7 +163,8 @@ type Integrations struct { type LabelKeys struct { // key is the label key // A label key must be a valid qualified name consisting of a lower-case alphanumeric string, - // and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. + // and hyphens of at most 63 characters in length. + // The name must start and end with an alphanumeric character. // The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, // hyphens and periods, of at most 253 characters in length. // Each period separated segment within the subdomain must start and end with an alphanumeric character. diff --git a/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml b/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml index 139c59425d3..1b3b34b2852 100644 --- a/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml +++ b/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml @@ -50,12 +50,9 @@ spec: properties: integrations: description: |- - integrations are the workloads Kueue will manage - Kueue has integrations in the codebase and it also allows - for external frameworks - Kueue are an important part to specify for the API as Kueue will - only manage the workloads that are specfied in this list. - This is a required field. + integrations is a required field that configures the Kueue's workload integrations. + Kueue has both standard integrations, known as job frameworks, and external integrations known as external frameworks. + Kueue will only manage workloads that correspond to the specified integrations. properties: externalFrameworks: description: |- @@ -65,14 +62,15 @@ spec: that integrates with Kueue. items: description: |- - This is the GVK for an external framework. + This is the GVR for an external framework. Controller runtime requires this in this format for api discoverability. properties: group: description: |- group of externalFramework - must be a valid qualified name consisting of a lower-case alphanumeric string, + group is the API group of the externalFramework. + Must be a valid qualified name consisting of a lower-case alphanumeric string, and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. @@ -85,12 +83,14 @@ spec: - message: a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character. - rule: self.size() == 0 || !format.qualifiedName().validate(self).hasValue() + rule: self.size() == 0 || !format.dns1123Subdomain().validate(self).hasValue() resource: description: |- - resource of external framework - must be a valid qualified name consisting of a lower-case alphanumeric string, - and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. + resource is the Resource type of the external framework. + Resource types are lowercase and plural (e.g. pods, deployments). + Must be a valid qualified name consisting of a lower-case alphanumeric string + and hyphens of at most 63 characters in length. + The name must start and end with an alphanumeric character. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. @@ -101,12 +101,18 @@ spec: - message: a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character. - rule: self.size() == 0 || !format.qualifiedName().validate(self).hasValue() + rule: self.size() == 0 || !format.dns1123Subdomain().validate(self).hasValue() version: - description: version is the version of the api + description: version is the version of the api (e.g. + v1alpha1, v1beta1, v1). maxLength: 256 minLength: 1 type: string + x-kubernetes-validations: + - message: a lowercase RFC 1123 subdomain must consist + of lower case alphanumeric characters, '-' or '.', + and must start and end with an alphanumeric character. + rule: self.size() == 0 || !format.dns1123Subdomain().validate(self).hasValue() required: - group - resource @@ -117,12 +123,11 @@ spec: x-kubernetes-list-type: atomic frameworks: description: |- - frameworks are a list of names to be enabled. + frameworks are a unique list of names to be enabled. This is required and must have at least one element. The frameworks are jobs that Kueue will manage. - KueueIntegrations is a list of frameworks that Kueue has support for. + Frameworks are a list of frameworks that Kueue has support for. The allowed values are BatchJob;RayJob;RayCluster;JobSet;MPIJob;PaddleJob;PytorchJob;TFJob;XGBoostJob;AppWrappers;Pod;Deployment;StatefulSet;LeaderWorkerSet. - kubebuilder:validation:UniqueItems=true items: enum: - BatchJob @@ -134,7 +139,7 @@ spec: - PytorchJob - TFJob - XGBoostJob - - AppWrappers + - AppWrapper - Pod - Deployment - StatefulSet @@ -144,18 +149,22 @@ spec: minItems: 1 type: array x-kubernetes-list-type: atomic + x-kubernetes-validations: + - message: each item in frameworks must be unique + rule: self.all(x, self.exists_one(y, x == y)) labelKeysToCopy: description: |- - labelKeysToCopy are a list of label keys that are copied once a workload is created - these keys are persisted to the internal Kueue workload object. - otherwise only the Kueue labels will be copied. + labelKeysToCopy are a list of label keys that are copied once a workload is created. + These keys are persisted to the internal Kueue workload object. + If not specified, only the Kueue labels will be copied. items: properties: key: description: |- key is the label key A label key must be a valid qualified name consisting of a lower-case alphanumeric string, - and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. + and hyphens of at most 63 characters in length. + The name must start and end with an alphanumeric character. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. diff --git a/operator/v1alpha1/zz_generated.deepcopy.go b/operator/v1alpha1/zz_generated.deepcopy.go index 24b510ceae4..48e924eacf0 100644 --- a/operator/v1alpha1/zz_generated.deepcopy.go +++ b/operator/v1alpha1/zz_generated.deepcopy.go @@ -405,7 +405,7 @@ func (in *Integrations) DeepCopyInto(out *Integrations) { *out = *in if in.Frameworks != nil { in, out := &in.Frameworks, &out.Frameworks - *out = make([]KueueIntegrations, len(*in)) + *out = make([]KueueIntegration, len(*in)) copy(*out, *in) } if in.ExternalFrameworks != nil { diff --git a/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml b/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml index e859c740189..58e98c964ce 100644 --- a/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml +++ b/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml @@ -48,12 +48,9 @@ spec: properties: integrations: description: |- - integrations are the workloads Kueue will manage - Kueue has integrations in the codebase and it also allows - for external frameworks - Kueue are an important part to specify for the API as Kueue will - only manage the workloads that are specfied in this list. - This is a required field. + integrations is a required field that configures the Kueue's workload integrations. + Kueue has both standard integrations, known as job frameworks, and external integrations known as external frameworks. + Kueue will only manage workloads that correspond to the specified integrations. properties: externalFrameworks: description: |- @@ -63,14 +60,15 @@ spec: that integrates with Kueue. items: description: |- - This is the GVK for an external framework. + This is the GVR for an external framework. Controller runtime requires this in this format for api discoverability. properties: group: description: |- group of externalFramework - must be a valid qualified name consisting of a lower-case alphanumeric string, + group is the API group of the externalFramework. + Must be a valid qualified name consisting of a lower-case alphanumeric string, and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. @@ -83,12 +81,14 @@ spec: - message: a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character. - rule: self.size() == 0 || !format.qualifiedName().validate(self).hasValue() + rule: self.size() == 0 || !format.dns1123Subdomain().validate(self).hasValue() resource: description: |- - resource of external framework - must be a valid qualified name consisting of a lower-case alphanumeric string, - and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. + resource is the Resource type of the external framework. + Resource types are lowercase and plural (e.g. pods, deployments). + Must be a valid qualified name consisting of a lower-case alphanumeric string + and hyphens of at most 63 characters in length. + The name must start and end with an alphanumeric character. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. @@ -99,12 +99,18 @@ spec: - message: a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character. - rule: self.size() == 0 || !format.qualifiedName().validate(self).hasValue() + rule: self.size() == 0 || !format.dns1123Subdomain().validate(self).hasValue() version: - description: version is the version of the api + description: version is the version of the api (e.g. + v1alpha1, v1beta1, v1). maxLength: 256 minLength: 1 type: string + x-kubernetes-validations: + - message: a lowercase RFC 1123 subdomain must consist + of lower case alphanumeric characters, '-' or '.', + and must start and end with an alphanumeric character. + rule: self.size() == 0 || !format.dns1123Subdomain().validate(self).hasValue() required: - group - resource @@ -115,12 +121,11 @@ spec: x-kubernetes-list-type: atomic frameworks: description: |- - frameworks are a list of names to be enabled. + frameworks are a unique list of names to be enabled. This is required and must have at least one element. The frameworks are jobs that Kueue will manage. - KueueIntegrations is a list of frameworks that Kueue has support for. + Frameworks are a list of frameworks that Kueue has support for. The allowed values are BatchJob;RayJob;RayCluster;JobSet;MPIJob;PaddleJob;PytorchJob;TFJob;XGBoostJob;AppWrappers;Pod;Deployment;StatefulSet;LeaderWorkerSet. - kubebuilder:validation:UniqueItems=true items: enum: - BatchJob @@ -132,7 +137,7 @@ spec: - PytorchJob - TFJob - XGBoostJob - - AppWrappers + - AppWrapper - Pod - Deployment - StatefulSet @@ -142,18 +147,22 @@ spec: minItems: 1 type: array x-kubernetes-list-type: atomic + x-kubernetes-validations: + - message: each item in frameworks must be unique + rule: self.all(x, self.exists_one(y, x == y)) labelKeysToCopy: description: |- - labelKeysToCopy are a list of label keys that are copied once a workload is created - these keys are persisted to the internal Kueue workload object. - otherwise only the Kueue labels will be copied. + labelKeysToCopy are a list of label keys that are copied once a workload is created. + These keys are persisted to the internal Kueue workload object. + If not specified, only the Kueue labels will be copied. items: properties: key: description: |- key is the label key A label key must be a valid qualified name consisting of a lower-case alphanumeric string, - and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. + and hyphens of at most 63 characters in length. + The name must start and end with an alphanumeric character. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. diff --git a/operator/v1alpha1/zz_generated.swagger_doc_generated.go b/operator/v1alpha1/zz_generated.swagger_doc_generated.go index 36511bb9eb2..11f15ea4d63 100644 --- a/operator/v1alpha1/zz_generated.swagger_doc_generated.go +++ b/operator/v1alpha1/zz_generated.swagger_doc_generated.go @@ -258,10 +258,10 @@ func (RepositoryDigestMirrors) SwaggerDoc() map[string]string { } var map_ExternalFramework = map[string]string{ - "": "This is the GVK for an external framework. Controller runtime requires this in this format for api discoverability.", - "group": "group of externalFramework must be a valid qualified name consisting of a lower-case alphanumeric string, and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. The optional prefix and the name are separate by a forward slash (/).", - "resource": "resource of external framework must be a valid qualified name consisting of a lower-case alphanumeric string, and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character.", - "version": "version is the version of the api", + "": "This is the GVR for an external framework. Controller runtime requires this in this format for api discoverability.", + "group": "group of externalFramework group is the API group of the externalFramework. Must be a valid qualified name consisting of a lower-case alphanumeric string, and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. The optional prefix and the name are separate by a forward slash (/).", + "resource": "resource is the Resource type of the external framework. Resource types are lowercase and plural (e.g. pods, deployments). Must be a valid qualified name consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character.", + "version": "version is the version of the api (e.g. v1alpha1, v1beta1, v1).", } func (ExternalFramework) SwaggerDoc() map[string]string { @@ -269,9 +269,9 @@ func (ExternalFramework) SwaggerDoc() map[string]string { } var map_Integrations = map[string]string{ - "frameworks": "frameworks are a list of names to be enabled. This is required and must have at least one element. The frameworks are jobs that Kueue will manage. KueueIntegrations is a list of frameworks that Kueue has support for. The allowed values are BatchJob;RayJob;RayCluster;JobSet;MPIJob;PaddleJob;PytorchJob;TFJob;XGBoostJob;AppWrappers;Pod;Deployment;StatefulSet;LeaderWorkerSet. kubebuilder:validation:UniqueItems=true", + "frameworks": "frameworks are a unique list of names to be enabled. This is required and must have at least one element. The frameworks are jobs that Kueue will manage. Frameworks are a list of frameworks that Kueue has support for. The allowed values are BatchJob;RayJob;RayCluster;JobSet;MPIJob;PaddleJob;PytorchJob;TFJob;XGBoostJob;AppWrappers;Pod;Deployment;StatefulSet;LeaderWorkerSet.", "externalFrameworks": "externalFrameworks are a list of GroupVersionResources that are managed for Kueue by external controllers; These are optional and should only be used if you have an external controller that integrates with Kueue.", - "labelKeysToCopy": "labelKeysToCopy are a list of label keys that are copied once a workload is created these keys are persisted to the internal Kueue workload object. otherwise only the Kueue labels will be copied.", + "labelKeysToCopy": "labelKeysToCopy are a list of label keys that are copied once a workload is created. These keys are persisted to the internal Kueue workload object. If not specified, only the Kueue labels will be copied.", } func (Integrations) SwaggerDoc() map[string]string { @@ -290,7 +290,7 @@ func (Kueue) SwaggerDoc() map[string]string { } var map_KueueConfiguration = map[string]string{ - "integrations": "integrations are the workloads Kueue will manage Kueue has integrations in the codebase and it also allows for external frameworks Kueue are an important part to specify for the API as Kueue will only manage the workloads that are specfied in this list. This is a required field.", + "integrations": "integrations is a required field that configures the Kueue's workload integrations. Kueue has both standard integrations, known as job frameworks, and external integrations known as external frameworks. Kueue will only manage workloads that correspond to the specified integrations.", } func (KueueConfiguration) SwaggerDoc() map[string]string { From 30a20f136bba2ba9bfd5aabaea438ed861df7a59 Mon Sep 17 00:00:00 2001 From: Kevin Hannon Date: Wed, 19 Mar 2025 13:32:58 -0400 Subject: [PATCH 17/42] joel comments --- openapi/generated_openapi/zz_generated.openapi.go | 5 +++-- openapi/openapi.json | 3 ++- operator/v1alpha1/types_kueue.go | 11 +++++++---- .../zz_generated.crd-manifests/kueue.crd.yaml | 8 ++++---- .../kueue.operator.openshift.io/AAA_ungated.yaml | 8 ++++---- .../v1alpha1/zz_generated.swagger_doc_generated.go | 3 ++- 6 files changed, 22 insertions(+), 16 deletions(-) diff --git a/openapi/generated_openapi/zz_generated.openapi.go b/openapi/generated_openapi/zz_generated.openapi.go index 57d0bd0d906..132e15fc5d4 100644 --- a/openapi/generated_openapi/zz_generated.openapi.go +++ b/openapi/generated_openapi/zz_generated.openapi.go @@ -56856,7 +56856,7 @@ func schema_openshift_api_operator_v1alpha1_ExternalFramework(ref common.Referen Properties: map[string]spec.Schema{ "group": { SchemaProps: spec.SchemaProps{ - Description: "group of externalFramework group is the API group of the externalFramework. Must be a valid qualified name consisting of a lower-case alphanumeric string, and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. The optional prefix and the name are separate by a forward slash (/).", + Description: "group is the API group of the externalFramework. Must be a valid qualified name consisting of a lower-case alphanumeric string, and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. The optional prefix and the name are separate by a forward slash (/).", Default: "", Type: []string{"string"}, Format: "", @@ -57123,7 +57123,8 @@ func schema_openshift_api_operator_v1alpha1_Integrations(ref common.ReferenceCal return common.OpenAPIDefinition{ Schema: spec.Schema{ SchemaProps: spec.SchemaProps{ - Type: []string{"object"}, + Description: "This is the integrations for Kueue. Kueue uses these apis to determine which jobs will be managed by Kueue.", + Type: []string{"object"}, Properties: map[string]spec.Schema{ "frameworks": { VendorExtensible: spec.VendorExtensible{ diff --git a/openapi/openapi.json b/openapi/openapi.json index 070b38ce37e..863b9913a28 100644 --- a/openapi/openapi.json +++ b/openapi/openapi.json @@ -33216,7 +33216,7 @@ ], "properties": { "group": { - "description": "group of externalFramework group is the API group of the externalFramework. Must be a valid qualified name consisting of a lower-case alphanumeric string, and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. The optional prefix and the name are separate by a forward slash (/).", + "description": "group is the API group of the externalFramework. Must be a valid qualified name consisting of a lower-case alphanumeric string, and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. The optional prefix and the name are separate by a forward slash (/).", "type": "string", "default": "" }, @@ -33377,6 +33377,7 @@ } }, "com.github.openshift.api.operator.v1alpha1.Integrations": { + "description": "This is the integrations for Kueue. Kueue uses these apis to determine which jobs will be managed by Kueue.", "type": "object", "required": [ "frameworks" diff --git a/operator/v1alpha1/types_kueue.go b/operator/v1alpha1/types_kueue.go index 178da9ef2b3..08b3c359db6 100644 --- a/operator/v1alpha1/types_kueue.go +++ b/operator/v1alpha1/types_kueue.go @@ -96,7 +96,6 @@ const ( // Controller runtime requires this in this format // for api discoverability. type ExternalFramework struct { - // group of externalFramework // group is the API group of the externalFramework. // Must be a valid qualified name consisting of a lower-case alphanumeric string, // and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. @@ -106,7 +105,7 @@ type ExternalFramework struct { // The optional prefix and the name are separate by a forward slash (/). // +kubebuilder:validation:MaxLength=253 // +kubebuilder:validation:MinLength=1 - // +kubebuilder:validation:XValidation:rule="self.size() == 0 || !format.dns1123Subdomain().validate(self).hasValue()",message="a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character." + // +kubebuilder:validation:XValidation:rule="self.size() == 0 || !format.dns1123Label().validate(self).hasValue()",message="a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character." // +required Group string `json:"group"` // resource is the Resource type of the external framework. @@ -119,17 +118,20 @@ type ExternalFramework struct { // Each period separated segment within the subdomain must start and end with an alphanumeric character. // +kubebuilder:validation:MaxLength=256 // +kubebuilder:validation:MinLength=1 - // +kubebuilder:validation:XValidation:rule="self.size() == 0 || !format.dns1123Subdomain().validate(self).hasValue()",message="a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character." + // +kubebuilder:validation:XValidation:rule="self.size() == 0 || !format.dns1123Label().validate(self).hasValue()",message="a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character." // +required Resource string `json:"resource"` // version is the version of the api (e.g. v1alpha1, v1beta1, v1). // +kubebuilder:validation:MaxLength=256 // +kubebuilder:validation:MinLength=1 - // +kubebuilder:validation:XValidation:rule="self.size() == 0 || !format.dns1123Subdomain().validate(self).hasValue()",message="a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character." + // +kubebuilder:validation:XValidation:rule="self.size() == 0 || !format.dns1123Label().validate(self).hasValue()",message="a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character." // +required Version string `json:"version"` } +// This is the integrations for Kueue. +// Kueue uses these apis to determine +// which jobs will be managed by Kueue. type Integrations struct { // frameworks are a unique list of names to be enabled. // This is required and must have at least one element. @@ -170,6 +172,7 @@ type LabelKeys struct { // Each period separated segment within the subdomain must start and end with an alphanumeric character. // The optional prefix and the name are separate by a forward slash (/). // +kubebuilder:validation:MaxLength=317 + // +kubebuilder:validation:MinLength=1 // +kubebuilder:validation:XValidation:rule="self.size() == 0 || !format.qualifiedName().validate(self).hasValue()",message="a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character." // +optional Key string `json:"key,omitempty"` diff --git a/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml b/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml index 1b3b34b2852..93c46bf57eb 100644 --- a/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml +++ b/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml @@ -68,7 +68,6 @@ spec: properties: group: description: |- - group of externalFramework group is the API group of the externalFramework. Must be a valid qualified name consisting of a lower-case alphanumeric string, and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. @@ -83,7 +82,7 @@ spec: - message: a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character. - rule: self.size() == 0 || !format.dns1123Subdomain().validate(self).hasValue() + rule: self.size() == 0 || !format.dns1123Label().validate(self).hasValue() resource: description: |- resource is the Resource type of the external framework. @@ -101,7 +100,7 @@ spec: - message: a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character. - rule: self.size() == 0 || !format.dns1123Subdomain().validate(self).hasValue() + rule: self.size() == 0 || !format.dns1123Label().validate(self).hasValue() version: description: version is the version of the api (e.g. v1alpha1, v1beta1, v1). @@ -112,7 +111,7 @@ spec: - message: a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character. - rule: self.size() == 0 || !format.dns1123Subdomain().validate(self).hasValue() + rule: self.size() == 0 || !format.dns1123Label().validate(self).hasValue() required: - group - resource @@ -170,6 +169,7 @@ spec: Each period separated segment within the subdomain must start and end with an alphanumeric character. The optional prefix and the name are separate by a forward slash (/). maxLength: 317 + minLength: 1 type: string x-kubernetes-validations: - message: a lowercase RFC 1123 subdomain must consist diff --git a/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml b/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml index 58e98c964ce..ce2208298ee 100644 --- a/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml +++ b/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml @@ -66,7 +66,6 @@ spec: properties: group: description: |- - group of externalFramework group is the API group of the externalFramework. Must be a valid qualified name consisting of a lower-case alphanumeric string, and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. @@ -81,7 +80,7 @@ spec: - message: a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character. - rule: self.size() == 0 || !format.dns1123Subdomain().validate(self).hasValue() + rule: self.size() == 0 || !format.dns1123Label().validate(self).hasValue() resource: description: |- resource is the Resource type of the external framework. @@ -99,7 +98,7 @@ spec: - message: a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character. - rule: self.size() == 0 || !format.dns1123Subdomain().validate(self).hasValue() + rule: self.size() == 0 || !format.dns1123Label().validate(self).hasValue() version: description: version is the version of the api (e.g. v1alpha1, v1beta1, v1). @@ -110,7 +109,7 @@ spec: - message: a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character. - rule: self.size() == 0 || !format.dns1123Subdomain().validate(self).hasValue() + rule: self.size() == 0 || !format.dns1123Label().validate(self).hasValue() required: - group - resource @@ -168,6 +167,7 @@ spec: Each period separated segment within the subdomain must start and end with an alphanumeric character. The optional prefix and the name are separate by a forward slash (/). maxLength: 317 + minLength: 1 type: string x-kubernetes-validations: - message: a lowercase RFC 1123 subdomain must consist diff --git a/operator/v1alpha1/zz_generated.swagger_doc_generated.go b/operator/v1alpha1/zz_generated.swagger_doc_generated.go index 11f15ea4d63..90bff09ec54 100644 --- a/operator/v1alpha1/zz_generated.swagger_doc_generated.go +++ b/operator/v1alpha1/zz_generated.swagger_doc_generated.go @@ -259,7 +259,7 @@ func (RepositoryDigestMirrors) SwaggerDoc() map[string]string { var map_ExternalFramework = map[string]string{ "": "This is the GVR for an external framework. Controller runtime requires this in this format for api discoverability.", - "group": "group of externalFramework group is the API group of the externalFramework. Must be a valid qualified name consisting of a lower-case alphanumeric string, and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. The optional prefix and the name are separate by a forward slash (/).", + "group": "group is the API group of the externalFramework. Must be a valid qualified name consisting of a lower-case alphanumeric string, and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. The optional prefix and the name are separate by a forward slash (/).", "resource": "resource is the Resource type of the external framework. Resource types are lowercase and plural (e.g. pods, deployments). Must be a valid qualified name consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character.", "version": "version is the version of the api (e.g. v1alpha1, v1beta1, v1).", } @@ -269,6 +269,7 @@ func (ExternalFramework) SwaggerDoc() map[string]string { } var map_Integrations = map[string]string{ + "": "This is the integrations for Kueue. Kueue uses these apis to determine which jobs will be managed by Kueue.", "frameworks": "frameworks are a unique list of names to be enabled. This is required and must have at least one element. The frameworks are jobs that Kueue will manage. Frameworks are a list of frameworks that Kueue has support for. The allowed values are BatchJob;RayJob;RayCluster;JobSet;MPIJob;PaddleJob;PytorchJob;TFJob;XGBoostJob;AppWrappers;Pod;Deployment;StatefulSet;LeaderWorkerSet.", "externalFrameworks": "externalFrameworks are a list of GroupVersionResources that are managed for Kueue by external controllers; These are optional and should only be used if you have an external controller that integrates with Kueue.", "labelKeysToCopy": "labelKeysToCopy are a list of label keys that are copied once a workload is created. These keys are persisted to the internal Kueue workload object. If not specified, only the Kueue labels will be copied.", From b295d9b91ace8a0015e6671833414a2169b7583b Mon Sep 17 00:00:00 2001 From: Kevin Hannon Date: Thu, 20 Mar 2025 14:32:44 -0400 Subject: [PATCH 18/42] address joel feedback --- .../generated_openapi/zz_generated.openapi.go | 10 ++-- openapi/openapi.json | 10 ++-- operator/v1alpha1/types_kueue.go | 32 +++++----- .../zz_generated.crd-manifests/kueue.crd.yaml | 58 ++++++++++--------- .../AAA_ungated.yaml | 58 ++++++++++--------- .../zz_generated.swagger_doc_generated.go | 10 ++-- 6 files changed, 91 insertions(+), 87 deletions(-) diff --git a/openapi/generated_openapi/zz_generated.openapi.go b/openapi/generated_openapi/zz_generated.openapi.go index 132e15fc5d4..1f1cf260c2e 100644 --- a/openapi/generated_openapi/zz_generated.openapi.go +++ b/openapi/generated_openapi/zz_generated.openapi.go @@ -56856,7 +56856,7 @@ func schema_openshift_api_operator_v1alpha1_ExternalFramework(ref common.Referen Properties: map[string]spec.Schema{ "group": { SchemaProps: spec.SchemaProps{ - Description: "group is the API group of the externalFramework. Must be a valid qualified name consisting of a lower-case alphanumeric string, and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. The optional prefix and the name are separate by a forward slash (/).", + Description: "group is the API group of the externalFramework. Must be a valid DNS 1123 subdomain consisting of of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character.", Default: "", Type: []string{"string"}, Format: "", @@ -56864,7 +56864,7 @@ func schema_openshift_api_operator_v1alpha1_ExternalFramework(ref common.Referen }, "resource": { SchemaProps: spec.SchemaProps{ - Description: "resource is the Resource type of the external framework. Resource types are lowercase and plural (e.g. pods, deployments). Must be a valid qualified name consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character.", + Description: "resource is the Resource type of the external framework. Resource types are lowercase and plural (e.g. pods, deployments). Must be a valid DNS 1123 label consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. The value must start and end with an alphanumeric character.", Default: "", Type: []string{"string"}, Format: "", @@ -56872,7 +56872,7 @@ func schema_openshift_api_operator_v1alpha1_ExternalFramework(ref common.Referen }, "version": { SchemaProps: spec.SchemaProps{ - Description: "version is the version of the api (e.g. v1alpha1, v1beta1, v1).", + Description: "version is the version of the api (e.g. v1alpha1, v1beta1, v1). Must be a valid DNS 1035 label consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. The value must start with an alphabetic character and end with an alphanumeric character.", Default: "", Type: []string{"string"}, Format: "", @@ -57133,7 +57133,7 @@ func schema_openshift_api_operator_v1alpha1_Integrations(ref common.ReferenceCal }, }, SchemaProps: spec.SchemaProps{ - Description: "frameworks are a unique list of names to be enabled. This is required and must have at least one element. The frameworks are jobs that Kueue will manage. Frameworks are a list of frameworks that Kueue has support for. The allowed values are BatchJob;RayJob;RayCluster;JobSet;MPIJob;PaddleJob;PytorchJob;TFJob;XGBoostJob;AppWrappers;Pod;Deployment;StatefulSet;LeaderWorkerSet.", + Description: "frameworks are a unique list of names to be enabled. This is required and must have at least one element. Each framework represents a type of job that Kueue will manage. Frameworks are a list of frameworks that Kueue has support for. The allowed values are BatchJob, RayJob, RayCluster, JobSet, MPIJob, PaddleJob, PytorchJob, TFJob, XGBoostJob, AppWrapper, Pod, Deployment, StatefulSet and LeaderWorkerSet.", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -57153,7 +57153,7 @@ func schema_openshift_api_operator_v1alpha1_Integrations(ref common.ReferenceCal }, }, SchemaProps: spec.SchemaProps{ - Description: "externalFrameworks are a list of GroupVersionResources that are managed for Kueue by external controllers; These are optional and should only be used if you have an external controller that integrates with Kueue.", + Description: "externalFrameworks are a list of GroupVersionResources that are managed for Kueue by external controllers. These are optional and should only be used if you have an external controller that integrates with Kueue.", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ diff --git a/openapi/openapi.json b/openapi/openapi.json index 863b9913a28..731374033f9 100644 --- a/openapi/openapi.json +++ b/openapi/openapi.json @@ -33216,17 +33216,17 @@ ], "properties": { "group": { - "description": "group is the API group of the externalFramework. Must be a valid qualified name consisting of a lower-case alphanumeric string, and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. The optional prefix and the name are separate by a forward slash (/).", + "description": "group is the API group of the externalFramework. Must be a valid DNS 1123 subdomain consisting of of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character.", "type": "string", "default": "" }, "resource": { - "description": "resource is the Resource type of the external framework. Resource types are lowercase and plural (e.g. pods, deployments). Must be a valid qualified name consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character.", + "description": "resource is the Resource type of the external framework. Resource types are lowercase and plural (e.g. pods, deployments). Must be a valid DNS 1123 label consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. The value must start and end with an alphanumeric character.", "type": "string", "default": "" }, "version": { - "description": "version is the version of the api (e.g. v1alpha1, v1beta1, v1).", + "description": "version is the version of the api (e.g. v1alpha1, v1beta1, v1). Must be a valid DNS 1035 label consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. The value must start with an alphabetic character and end with an alphanumeric character.", "type": "string", "default": "" } @@ -33384,7 +33384,7 @@ ], "properties": { "externalFrameworks": { - "description": "externalFrameworks are a list of GroupVersionResources that are managed for Kueue by external controllers; These are optional and should only be used if you have an external controller that integrates with Kueue.", + "description": "externalFrameworks are a list of GroupVersionResources that are managed for Kueue by external controllers. These are optional and should only be used if you have an external controller that integrates with Kueue.", "type": "array", "items": { "default": {}, @@ -33393,7 +33393,7 @@ "x-kubernetes-list-type": "atomic" }, "frameworks": { - "description": "frameworks are a unique list of names to be enabled. This is required and must have at least one element. The frameworks are jobs that Kueue will manage. Frameworks are a list of frameworks that Kueue has support for. The allowed values are BatchJob;RayJob;RayCluster;JobSet;MPIJob;PaddleJob;PytorchJob;TFJob;XGBoostJob;AppWrappers;Pod;Deployment;StatefulSet;LeaderWorkerSet.", + "description": "frameworks are a unique list of names to be enabled. This is required and must have at least one element. Each framework represents a type of job that Kueue will manage. Frameworks are a list of frameworks that Kueue has support for. The allowed values are BatchJob, RayJob, RayCluster, JobSet, MPIJob, PaddleJob, PytorchJob, TFJob, XGBoostJob, AppWrapper, Pod, Deployment, StatefulSet and LeaderWorkerSet.", "type": "array", "items": { "type": "string", diff --git a/operator/v1alpha1/types_kueue.go b/operator/v1alpha1/types_kueue.go index 08b3c359db6..b37c731c456 100644 --- a/operator/v1alpha1/types_kueue.go +++ b/operator/v1alpha1/types_kueue.go @@ -97,12 +97,9 @@ const ( // for api discoverability. type ExternalFramework struct { // group is the API group of the externalFramework. - // Must be a valid qualified name consisting of a lower-case alphanumeric string, - // and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. - // The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, + // Must be a valid DNS 1123 subdomain consisting of of lower-case alphanumeric characters, // hyphens and periods, of at most 253 characters in length. // Each period separated segment within the subdomain must start and end with an alphanumeric character. - // The optional prefix and the name are separate by a forward slash (/). // +kubebuilder:validation:MaxLength=253 // +kubebuilder:validation:MinLength=1 // +kubebuilder:validation:XValidation:rule="self.size() == 0 || !format.dns1123Label().validate(self).hasValue()",message="a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character." @@ -110,21 +107,21 @@ type ExternalFramework struct { Group string `json:"group"` // resource is the Resource type of the external framework. // Resource types are lowercase and plural (e.g. pods, deployments). - // Must be a valid qualified name consisting of a lower-case alphanumeric string + // Must be a valid DNS 1123 label consisting of a lower-case alphanumeric string // and hyphens of at most 63 characters in length. - // The name must start and end with an alphanumeric character. - // The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, - // hyphens and periods, of at most 253 characters in length. - // Each period separated segment within the subdomain must start and end with an alphanumeric character. - // +kubebuilder:validation:MaxLength=256 + // The value must start and end with an alphanumeric character. + // +kubebuilder:validation:MaxLength=63 // +kubebuilder:validation:MinLength=1 - // +kubebuilder:validation:XValidation:rule="self.size() == 0 || !format.dns1123Label().validate(self).hasValue()",message="a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character." + // +kubebuilder:validation:XValidation:rule="self.size() == 0 || !format.dns1123Label().validate(self).hasValue()",message="a lowercase RFC 1123 label must consist of lower case alphanumeric characters and '-', and must start and end with an alphanumeric character." // +required Resource string `json:"resource"` // version is the version of the api (e.g. v1alpha1, v1beta1, v1). - // +kubebuilder:validation:MaxLength=256 + // Must be a valid DNS 1035 label consisting of a lower-case alphanumeric string + // and hyphens of at most 63 characters in length. + // The value must start with an alphabetic character and end with an alphanumeric character. + // +kubebuilder:validation:MaxLength=63 // +kubebuilder:validation:MinLength=1 - // +kubebuilder:validation:XValidation:rule="self.size() == 0 || !format.dns1123Label().validate(self).hasValue()",message="a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character." + // +kubebuilder:validation:XValidation:rule="self.size() == 0 || !format.dns1035Label().validate(self).hasValue()",message="a lowercase RFC 1035 label must consist of lower case alphanumeric characters, '-' or '.', and must start with an alphabetic character and end with an alphanumeric character." // +required Version string `json:"version"` } @@ -135,9 +132,9 @@ type ExternalFramework struct { type Integrations struct { // frameworks are a unique list of names to be enabled. // This is required and must have at least one element. - // The frameworks are jobs that Kueue will manage. + // Each framework represents a type of job that Kueue will manage. // Frameworks are a list of frameworks that Kueue has support for. - // The allowed values are BatchJob;RayJob;RayCluster;JobSet;MPIJob;PaddleJob;PytorchJob;TFJob;XGBoostJob;AppWrappers;Pod;Deployment;StatefulSet;LeaderWorkerSet. + // The allowed values are BatchJob, RayJob, RayCluster, JobSet, MPIJob, PaddleJob, PytorchJob, TFJob, XGBoostJob, AppWrapper, Pod, Deployment, StatefulSet and LeaderWorkerSet. // +kubebuilder:validation:MaxItems=14 // +kubebuilder:validation:MinItems=1 // +kubebuilder:validation:XValidation:rule="self.all(x, self.exists_one(y, x == y))",message="each item in frameworks must be unique" @@ -145,14 +142,13 @@ type Integrations struct { // +required Frameworks []KueueIntegration `json:"frameworks"` // externalFrameworks are a list of GroupVersionResources - // that are managed for Kueue by external controllers; + // that are managed for Kueue by external controllers. // These are optional and should only be used if you have an external controller // that integrates with Kueue. // +listType=atomic // +kubebuilder:validation:MaxItems=32 // +optional ExternalFrameworks []ExternalFramework `json:"externalFrameworks,omitempty"` - // labelKeysToCopy are a list of label keys that are copied once a workload is created. // These keys are persisted to the internal Kueue workload object. // If not specified, only the Kueue labels will be copied. @@ -173,7 +169,7 @@ type LabelKeys struct { // The optional prefix and the name are separate by a forward slash (/). // +kubebuilder:validation:MaxLength=317 // +kubebuilder:validation:MinLength=1 - // +kubebuilder:validation:XValidation:rule="self.size() == 0 || !format.qualifiedName().validate(self).hasValue()",message="a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character." + // +kubebuilder:validation:XValidation:rule="!format.qualifiedName().validate(self).hasValue()",message="a qualified name must consist of a lower-case alphanumeric and hyphenated string of at most 63 characters in length, starting and ending with alphanumeric chracters. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character." // +optional Key string `json:"key,omitempty"` } diff --git a/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml b/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml index 93c46bf57eb..af8c9965ac9 100644 --- a/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml +++ b/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml @@ -57,7 +57,7 @@ spec: externalFrameworks: description: |- externalFrameworks are a list of GroupVersionResources - that are managed for Kueue by external controllers; + that are managed for Kueue by external controllers. These are optional and should only be used if you have an external controller that integrates with Kueue. items: @@ -69,12 +69,9 @@ spec: group: description: |- group is the API group of the externalFramework. - Must be a valid qualified name consisting of a lower-case alphanumeric string, - and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. - The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, + Must be a valid DNS 1123 subdomain consisting of of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. - The optional prefix and the name are separate by a forward slash (/). maxLength: 253 minLength: 1 type: string @@ -87,31 +84,32 @@ spec: description: |- resource is the Resource type of the external framework. Resource types are lowercase and plural (e.g. pods, deployments). - Must be a valid qualified name consisting of a lower-case alphanumeric string + Must be a valid DNS 1123 label consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. - The name must start and end with an alphanumeric character. - The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, - hyphens and periods, of at most 253 characters in length. - Each period separated segment within the subdomain must start and end with an alphanumeric character. - maxLength: 256 + The value must start and end with an alphanumeric character. + maxLength: 63 minLength: 1 type: string x-kubernetes-validations: - - message: a lowercase RFC 1123 subdomain must consist - of lower case alphanumeric characters, '-' or '.', - and must start and end with an alphanumeric character. + - message: a lowercase RFC 1123 label must consist of + lower case alphanumeric characters and '-', and + must start and end with an alphanumeric character. rule: self.size() == 0 || !format.dns1123Label().validate(self).hasValue() version: - description: version is the version of the api (e.g. - v1alpha1, v1beta1, v1). - maxLength: 256 + description: |- + version is the version of the api (e.g. v1alpha1, v1beta1, v1). + Must be a valid DNS 1035 label consisting of a lower-case alphanumeric string + and hyphens of at most 63 characters in length. + The value must start with an alphabetic character and end with an alphanumeric character. + maxLength: 63 minLength: 1 type: string x-kubernetes-validations: - - message: a lowercase RFC 1123 subdomain must consist - of lower case alphanumeric characters, '-' or '.', - and must start and end with an alphanumeric character. - rule: self.size() == 0 || !format.dns1123Label().validate(self).hasValue() + - message: a lowercase RFC 1035 label must consist of + lower case alphanumeric characters, '-' or '.', + and must start with an alphabetic character and + end with an alphanumeric character. + rule: self.size() == 0 || !format.dns1035Label().validate(self).hasValue() required: - group - resource @@ -124,9 +122,9 @@ spec: description: |- frameworks are a unique list of names to be enabled. This is required and must have at least one element. - The frameworks are jobs that Kueue will manage. + Each framework represents a type of job that Kueue will manage. Frameworks are a list of frameworks that Kueue has support for. - The allowed values are BatchJob;RayJob;RayCluster;JobSet;MPIJob;PaddleJob;PytorchJob;TFJob;XGBoostJob;AppWrappers;Pod;Deployment;StatefulSet;LeaderWorkerSet. + The allowed values are BatchJob, RayJob, RayCluster, JobSet, MPIJob, PaddleJob, PytorchJob, TFJob, XGBoostJob, AppWrapper, Pod, Deployment, StatefulSet and LeaderWorkerSet. items: enum: - BatchJob @@ -172,10 +170,16 @@ spec: minLength: 1 type: string x-kubernetes-validations: - - message: a lowercase RFC 1123 subdomain must consist - of lower case alphanumeric characters, '-' or '.', - and must start and end with an alphanumeric character. - rule: self.size() == 0 || !format.qualifiedName().validate(self).hasValue() + - message: a qualified name must consist of a lower-case + alphanumeric and hyphenated string of at most 63 + characters in length, starting and ending with alphanumeric + chracters. The name may be optionally prefixed with + a subdomain consisting of lower-case alphanumeric + characters, hyphens and periods, of at most 253 + characters in length. Each period separated segment + within the subdomain must start and end with an + alphanumeric character. + rule: '!format.qualifiedName().validate(self).hasValue()' type: object maxItems: 64 type: array diff --git a/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml b/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml index ce2208298ee..5b51737365b 100644 --- a/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml +++ b/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml @@ -55,7 +55,7 @@ spec: externalFrameworks: description: |- externalFrameworks are a list of GroupVersionResources - that are managed for Kueue by external controllers; + that are managed for Kueue by external controllers. These are optional and should only be used if you have an external controller that integrates with Kueue. items: @@ -67,12 +67,9 @@ spec: group: description: |- group is the API group of the externalFramework. - Must be a valid qualified name consisting of a lower-case alphanumeric string, - and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. - The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, + Must be a valid DNS 1123 subdomain consisting of of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. - The optional prefix and the name are separate by a forward slash (/). maxLength: 253 minLength: 1 type: string @@ -85,31 +82,32 @@ spec: description: |- resource is the Resource type of the external framework. Resource types are lowercase and plural (e.g. pods, deployments). - Must be a valid qualified name consisting of a lower-case alphanumeric string + Must be a valid DNS 1123 label consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. - The name must start and end with an alphanumeric character. - The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, - hyphens and periods, of at most 253 characters in length. - Each period separated segment within the subdomain must start and end with an alphanumeric character. - maxLength: 256 + The value must start and end with an alphanumeric character. + maxLength: 63 minLength: 1 type: string x-kubernetes-validations: - - message: a lowercase RFC 1123 subdomain must consist - of lower case alphanumeric characters, '-' or '.', - and must start and end with an alphanumeric character. + - message: a lowercase RFC 1123 label must consist of + lower case alphanumeric characters and '-', and + must start and end with an alphanumeric character. rule: self.size() == 0 || !format.dns1123Label().validate(self).hasValue() version: - description: version is the version of the api (e.g. - v1alpha1, v1beta1, v1). - maxLength: 256 + description: |- + version is the version of the api (e.g. v1alpha1, v1beta1, v1). + Must be a valid DNS 1035 label consisting of a lower-case alphanumeric string + and hyphens of at most 63 characters in length. + The value must start with an alphabetic character and end with an alphanumeric character. + maxLength: 63 minLength: 1 type: string x-kubernetes-validations: - - message: a lowercase RFC 1123 subdomain must consist - of lower case alphanumeric characters, '-' or '.', - and must start and end with an alphanumeric character. - rule: self.size() == 0 || !format.dns1123Label().validate(self).hasValue() + - message: a lowercase RFC 1035 label must consist of + lower case alphanumeric characters, '-' or '.', + and must start with an alphabetic character and + end with an alphanumeric character. + rule: self.size() == 0 || !format.dns1035Label().validate(self).hasValue() required: - group - resource @@ -122,9 +120,9 @@ spec: description: |- frameworks are a unique list of names to be enabled. This is required and must have at least one element. - The frameworks are jobs that Kueue will manage. + Each framework represents a type of job that Kueue will manage. Frameworks are a list of frameworks that Kueue has support for. - The allowed values are BatchJob;RayJob;RayCluster;JobSet;MPIJob;PaddleJob;PytorchJob;TFJob;XGBoostJob;AppWrappers;Pod;Deployment;StatefulSet;LeaderWorkerSet. + The allowed values are BatchJob, RayJob, RayCluster, JobSet, MPIJob, PaddleJob, PytorchJob, TFJob, XGBoostJob, AppWrapper, Pod, Deployment, StatefulSet and LeaderWorkerSet. items: enum: - BatchJob @@ -170,10 +168,16 @@ spec: minLength: 1 type: string x-kubernetes-validations: - - message: a lowercase RFC 1123 subdomain must consist - of lower case alphanumeric characters, '-' or '.', - and must start and end with an alphanumeric character. - rule: self.size() == 0 || !format.qualifiedName().validate(self).hasValue() + - message: a qualified name must consist of a lower-case + alphanumeric and hyphenated string of at most 63 + characters in length, starting and ending with alphanumeric + chracters. The name may be optionally prefixed with + a subdomain consisting of lower-case alphanumeric + characters, hyphens and periods, of at most 253 + characters in length. Each period separated segment + within the subdomain must start and end with an + alphanumeric character. + rule: '!format.qualifiedName().validate(self).hasValue()' type: object maxItems: 64 type: array diff --git a/operator/v1alpha1/zz_generated.swagger_doc_generated.go b/operator/v1alpha1/zz_generated.swagger_doc_generated.go index 90bff09ec54..f4a3023a200 100644 --- a/operator/v1alpha1/zz_generated.swagger_doc_generated.go +++ b/operator/v1alpha1/zz_generated.swagger_doc_generated.go @@ -259,9 +259,9 @@ func (RepositoryDigestMirrors) SwaggerDoc() map[string]string { var map_ExternalFramework = map[string]string{ "": "This is the GVR for an external framework. Controller runtime requires this in this format for api discoverability.", - "group": "group is the API group of the externalFramework. Must be a valid qualified name consisting of a lower-case alphanumeric string, and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. The optional prefix and the name are separate by a forward slash (/).", - "resource": "resource is the Resource type of the external framework. Resource types are lowercase and plural (e.g. pods, deployments). Must be a valid qualified name consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character.", - "version": "version is the version of the api (e.g. v1alpha1, v1beta1, v1).", + "group": "group is the API group of the externalFramework. Must be a valid DNS 1123 subdomain consisting of of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character.", + "resource": "resource is the Resource type of the external framework. Resource types are lowercase and plural (e.g. pods, deployments). Must be a valid DNS 1123 label consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. The value must start and end with an alphanumeric character.", + "version": "version is the version of the api (e.g. v1alpha1, v1beta1, v1). Must be a valid DNS 1035 label consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. The value must start with an alphabetic character and end with an alphanumeric character.", } func (ExternalFramework) SwaggerDoc() map[string]string { @@ -270,8 +270,8 @@ func (ExternalFramework) SwaggerDoc() map[string]string { var map_Integrations = map[string]string{ "": "This is the integrations for Kueue. Kueue uses these apis to determine which jobs will be managed by Kueue.", - "frameworks": "frameworks are a unique list of names to be enabled. This is required and must have at least one element. The frameworks are jobs that Kueue will manage. Frameworks are a list of frameworks that Kueue has support for. The allowed values are BatchJob;RayJob;RayCluster;JobSet;MPIJob;PaddleJob;PytorchJob;TFJob;XGBoostJob;AppWrappers;Pod;Deployment;StatefulSet;LeaderWorkerSet.", - "externalFrameworks": "externalFrameworks are a list of GroupVersionResources that are managed for Kueue by external controllers; These are optional and should only be used if you have an external controller that integrates with Kueue.", + "frameworks": "frameworks are a unique list of names to be enabled. This is required and must have at least one element. Each framework represents a type of job that Kueue will manage. Frameworks are a list of frameworks that Kueue has support for. The allowed values are BatchJob, RayJob, RayCluster, JobSet, MPIJob, PaddleJob, PytorchJob, TFJob, XGBoostJob, AppWrapper, Pod, Deployment, StatefulSet and LeaderWorkerSet.", + "externalFrameworks": "externalFrameworks are a list of GroupVersionResources that are managed for Kueue by external controllers. These are optional and should only be used if you have an external controller that integrates with Kueue.", "labelKeysToCopy": "labelKeysToCopy are a list of label keys that are copied once a workload is created. These keys are persisted to the internal Kueue workload object. If not specified, only the Kueue labels will be copied.", } From 89f746cf0183b8d02c0b2c4cc98cf0ab5f29982d Mon Sep 17 00:00:00 2001 From: Kevin Hannon Date: Wed, 26 Mar 2025 17:15:26 -0400 Subject: [PATCH 19/42] add two apis, one left --- .../generated_openapi/zz_generated.openapi.go | 50 ++++++++++++- openapi/openapi.json | 25 +++++++ operator/v1alpha1/types_kueue.go | 73 +++++++++++++++++++ operator/v1alpha1/zz_generated.deepcopy.go | 36 +++++++++ 4 files changed, 183 insertions(+), 1 deletion(-) diff --git a/openapi/generated_openapi/zz_generated.openapi.go b/openapi/generated_openapi/zz_generated.openapi.go index 1f1cf260c2e..59ab841ef13 100644 --- a/openapi/generated_openapi/zz_generated.openapi.go +++ b/openapi/generated_openapi/zz_generated.openapi.go @@ -1111,6 +1111,7 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "github.com/openshift/api/operator/v1alpha1.EtcdBackupSpec": schema_openshift_api_operator_v1alpha1_EtcdBackupSpec(ref), "github.com/openshift/api/operator/v1alpha1.EtcdBackupStatus": schema_openshift_api_operator_v1alpha1_EtcdBackupStatus(ref), "github.com/openshift/api/operator/v1alpha1.ExternalFramework": schema_openshift_api_operator_v1alpha1_ExternalFramework(ref), + "github.com/openshift/api/operator/v1alpha1.FairSharing": schema_openshift_api_operator_v1alpha1_FairSharing(ref), "github.com/openshift/api/operator/v1alpha1.GenerationHistory": schema_openshift_api_operator_v1alpha1_GenerationHistory(ref), "github.com/openshift/api/operator/v1alpha1.GenericOperatorConfig": schema_openshift_api_operator_v1alpha1_GenericOperatorConfig(ref), "github.com/openshift/api/operator/v1alpha1.ImageContentSourcePolicy": schema_openshift_api_operator_v1alpha1_ImageContentSourcePolicy(ref), @@ -56885,6 +56886,33 @@ func schema_openshift_api_operator_v1alpha1_ExternalFramework(ref common.Referen } } +func schema_openshift_api_operator_v1alpha1_FairSharing(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "preemptionStrategies": { + SchemaProps: spec.SchemaProps{ + Description: "preemptionStrategies indicates which constraints should a preemption satisfy. The preemption algorithm will only use the next strategy in the list if the incoming workload (preemptor) doesn't fit after using the previous strategies. Possible values are: - LessThanOrEqualToFinalShare: Only preempt a workload if the share of the preemptor CQ\n with the preemptor workload is less than or equal to the share of the preemptee CQ\n without the workload to be preempted.\n This strategy might favor preemption of smaller workloads in the preemptee CQ,\n regardless of priority or start time, in an effort to keep the share of the CQ\n as high as possible.\n- LessThanInitialShare: Only preempt a workload if the share of the preemptor CQ\n with the incoming workload is strictly less than the share of the preemptee CQ.\n This strategy doesn't depend on the share usage of the workload being preempted.\n As a result, the strategy chooses to preempt workloads with the lowest priority and\n newest start time first.\nThe default strategy is [\"LessThanOrEqualToFinalShare\", \"LessThanInitialShare\"].", + Type: []string{"array"}, + Items: &spec.SchemaOrArray{ + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + }, + }, + }, + } +} + func schema_openshift_api_operator_v1alpha1_GenerationHistory(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ @@ -57257,12 +57285,32 @@ func schema_openshift_api_operator_v1alpha1_KueueConfiguration(ref common.Refere Ref: ref("github.com/openshift/api/operator/v1alpha1.Integrations"), }, }, + "queueLabelPolicy": { + SchemaProps: spec.SchemaProps{ + Description: "queueLabelPolicy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. The allowed values are QueueNameRequired and QueueNameOptional. If set to QueueNameRequired, then those jobs will be suspended and never started unless they are assigned a queue and eventually admitted. This also applies to jobs created before starting the kueue controller. Defaults to QueueNameOptional; therefore, those jobs are not managed and if they are created unsuspended, they will start immediately.", + Type: []string{"string"}, + Format: "", + }, + }, + "kueueGangSchedulingPolicy": { + SchemaProps: spec.SchemaProps{ + Description: "kueueGangSchedulingPolicy controls how Kueue admits workloads Kueue provides the ability to admit workloads all in one (gang admission) and evicts workloads if they are not ready within a specific time. This is an optional field. The allowed values are BlockAdmission, EvictNotReadyWorkloads, and Disabled. The default value will be Disabled. BlockAdmission serializes the admission process for Kueue Workloads will be admitted one at a time and will wait for the workloads to be ready before going on to the next one in the list. EvictNotReadyWorkloads will not block admission but will evict Workloads that are not ready within a defaulted timelimit.", + Type: []string{"string"}, + Format: "", + }, + }, + "fairSharing": { + SchemaProps: spec.SchemaProps{ + Description: "fairSharing TODO not done yet", + Ref: ref("github.com/openshift/api/operator/v1alpha1.FairSharing"), + }, + }, }, Required: []string{"integrations"}, }, }, Dependencies: []string{ - "github.com/openshift/api/operator/v1alpha1.Integrations"}, + "github.com/openshift/api/operator/v1alpha1.FairSharing", "github.com/openshift/api/operator/v1alpha1.Integrations"}, } } diff --git a/openapi/openapi.json b/openapi/openapi.json index 731374033f9..2b9ac19bd05 100644 --- a/openapi/openapi.json +++ b/openapi/openapi.json @@ -33232,6 +33232,19 @@ } } }, + "com.github.openshift.api.operator.v1alpha1.FairSharing": { + "type": "object", + "properties": { + "preemptionStrategies": { + "description": "preemptionStrategies indicates which constraints should a preemption satisfy. The preemption algorithm will only use the next strategy in the list if the incoming workload (preemptor) doesn't fit after using the previous strategies. Possible values are: - LessThanOrEqualToFinalShare: Only preempt a workload if the share of the preemptor CQ\n with the preemptor workload is less than or equal to the share of the preemptee CQ\n without the workload to be preempted.\n This strategy might favor preemption of smaller workloads in the preemptee CQ,\n regardless of priority or start time, in an effort to keep the share of the CQ\n as high as possible.\n- LessThanInitialShare: Only preempt a workload if the share of the preemptor CQ\n with the incoming workload is strictly less than the share of the preemptee CQ.\n This strategy doesn't depend on the share usage of the workload being preempted.\n As a result, the strategy chooses to preempt workloads with the lowest priority and\n newest start time first.\nThe default strategy is [\"LessThanOrEqualToFinalShare\", \"LessThanInitialShare\"].", + "type": "array", + "items": { + "type": "string", + "default": "" + } + } + } + }, "com.github.openshift.api.operator.v1alpha1.GenerationHistory": { "description": "GenerationHistory keeps track of the generation for a given resource so that decisions about forced updated can be made. DEPRECATED: Use fields in v1.GenerationStatus instead", "type": "object", @@ -33450,10 +33463,22 @@ "integrations" ], "properties": { + "fairSharing": { + "description": "fairSharing TODO not done yet", + "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.FairSharing" + }, "integrations": { "description": "integrations is a required field that configures the Kueue's workload integrations. Kueue has both standard integrations, known as job frameworks, and external integrations known as external frameworks. Kueue will only manage workloads that correspond to the specified integrations.", "default": {}, "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.Integrations" + }, + "kueueGangSchedulingPolicy": { + "description": "kueueGangSchedulingPolicy controls how Kueue admits workloads Kueue provides the ability to admit workloads all in one (gang admission) and evicts workloads if they are not ready within a specific time. This is an optional field. The allowed values are BlockAdmission, EvictNotReadyWorkloads, and Disabled. The default value will be Disabled. BlockAdmission serializes the admission process for Kueue Workloads will be admitted one at a time and will wait for the workloads to be ready before going on to the next one in the list. EvictNotReadyWorkloads will not block admission but will evict Workloads that are not ready within a defaulted timelimit.", + "type": "string" + }, + "queueLabelPolicy": { + "description": "queueLabelPolicy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. The allowed values are QueueNameRequired and QueueNameOptional. If set to QueueNameRequired, then those jobs will be suspended and never started unless they are assigned a queue and eventually admitted. This also applies to jobs created before starting the kueue controller. Defaults to QueueNameOptional; therefore, those jobs are not managed and if they are created unsuspended, they will start immediately.", + "type": "string" } } }, diff --git a/operator/v1alpha1/types_kueue.go b/operator/v1alpha1/types_kueue.go index b37c731c456..6e37af00853 100644 --- a/operator/v1alpha1/types_kueue.go +++ b/operator/v1alpha1/types_kueue.go @@ -47,6 +47,32 @@ type KueueConfiguration struct { // Kueue will only manage workloads that correspond to the specified integrations. // +required Integrations Integrations `json:"integrations"` + // queueLabelPolicy controls whether or not Kueue reconciles + // jobs that don't set the label kueue.x-k8s.io/queue-name. + // The allowed values are QueueNameRequired and QueueNameOptional. + // If set to QueueNameRequired, then those jobs will be suspended and never started unless + // they are assigned a queue and eventually admitted. This also applies to + // jobs created before starting the kueue controller. + // Defaults to QueueNameOptional; therefore, those jobs are not managed and if they are created + // unsuspended, they will start immediately. + // +optional + QueueLabelPolicy *QueueLabelNamePolicy `json:"queueLabelPolicy,omitempty"` + // kueueGangSchedulingPolicy controls how Kueue admits workloads + // Kueue provides the ability to admit workloads all in one (gang admission) + // and evicts workloads if they are not ready within a specific time. + // This is an optional field. + // The allowed values are BlockAdmission, EvictNotReadyWorkloads, and Disabled. + // The default value will be Disabled. + // BlockAdmission serializes the admission process for Kueue + // Workloads will be admitted one at a time and will wait + // for the workloads to be ready before going on to the next one in the list. + // EvictNotReadyWorkloads will not block admission but will evict Workloads that are not ready + // within a defaulted timelimit. + // +optional + KueueGangSchedulingPolicy *KueueGangSchedulingPolicyOptions `json:"kueueGangSchedulingPolicy,omitempty"` + // fairSharing TODO not done yet + // +optional + FairSharing *FairSharing `json:"fairSharing,omitemoty"` } // KueueStatus defines the observed state of Kueue @@ -173,3 +199,50 @@ type LabelKeys struct { // +optional Key string `json:"key,omitempty"` } + +// +kubebuilder:validation:Enum=EvictNotReadyWorkloads;Disabled +type KueueGangSchedulingPolicyOptions string + +const ( + KueueGangSchedulingPolicyBlockAdmission KueueGangSchedulingPolicyOptions = "BlockAdmission" + KueueGangSchedulingPolicyEvictNotReadyWorkloads KueueGangSchedulingPolicyOptions = "EvictNotReadyWorkloads" + KueueGangSchedulingPolicyDisabled KueueGangSchedulingPolicyOptions = "Disabled" +) + +// +kubebuilder:validation:Enum=QueueNameRequired;QueueNameOptional +type QueueLabelNamePolicy string + +const ( + QueueLabelNamePolicyRequired QueueLabelNamePolicy = "QueueNameRequired" + QueueLabelNamePolicyOptional QueueLabelNamePolicy = "QueueNameOptional" +) + +// +kubebuilder:validation:Enum=LessThanOrEqualToFinalShare;LessThanInitialShare +type PreemptionStrategy string + +const ( + LessThanOrEqualToFinalShare PreemptionStrategy = "LessThanOrEqualToFinalShare" + LessThanInitialShare PreemptionStrategy = "LessThanInitialShare" +) + +type FairSharing struct { + // preemptionStrategies indicates which constraints should a preemption satisfy. + // The preemption algorithm will only use the next strategy in the list if the + // incoming workload (preemptor) doesn't fit after using the previous strategies. + // Possible values are: + // - LessThanOrEqualToFinalShare: Only preempt a workload if the share of the preemptor CQ + // with the preemptor workload is less than or equal to the share of the preemptee CQ + // without the workload to be preempted. + // This strategy might favor preemption of smaller workloads in the preemptee CQ, + // regardless of priority or start time, in an effort to keep the share of the CQ + // as high as possible. + // - LessThanInitialShare: Only preempt a workload if the share of the preemptor CQ + // with the incoming workload is strictly less than the share of the preemptee CQ. + // This strategy doesn't depend on the share usage of the workload being preempted. + // As a result, the strategy chooses to preempt workloads with the lowest priority and + // newest start time first. + // The default strategy is ["LessThanOrEqualToFinalShare", "LessThanInitialShare"]. + // +kubebuilder:validation:MaxItems=2 + // +optional + PreemptionStrategies []PreemptionStrategy `json:"preemptionStrategies,omitempty"` +} diff --git a/operator/v1alpha1/zz_generated.deepcopy.go b/operator/v1alpha1/zz_generated.deepcopy.go index 48e924eacf0..ad6a337c5b9 100644 --- a/operator/v1alpha1/zz_generated.deepcopy.go +++ b/operator/v1alpha1/zz_generated.deepcopy.go @@ -272,6 +272,27 @@ func (in *ExternalFramework) DeepCopy() *ExternalFramework { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *FairSharing) DeepCopyInto(out *FairSharing) { + *out = *in + if in.PreemptionStrategies != nil { + in, out := &in.PreemptionStrategies, &out.PreemptionStrategies + *out = make([]PreemptionStrategy, len(*in)) + copy(*out, *in) + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FairSharing. +func (in *FairSharing) DeepCopy() *FairSharing { + if in == nil { + return nil + } + out := new(FairSharing) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GenerationHistory) DeepCopyInto(out *GenerationHistory) { *out = *in @@ -463,6 +484,21 @@ func (in *Kueue) DeepCopyObject() runtime.Object { func (in *KueueConfiguration) DeepCopyInto(out *KueueConfiguration) { *out = *in in.Integrations.DeepCopyInto(&out.Integrations) + if in.QueueLabelPolicy != nil { + in, out := &in.QueueLabelPolicy, &out.QueueLabelPolicy + *out = new(QueueLabelNamePolicy) + **out = **in + } + if in.KueueGangSchedulingPolicy != nil { + in, out := &in.KueueGangSchedulingPolicy, &out.KueueGangSchedulingPolicy + *out = new(KueueGangSchedulingPolicyOptions) + **out = **in + } + if in.FairSharing != nil { + in, out := &in.FairSharing, &out.FairSharing + *out = new(FairSharing) + (*in).DeepCopyInto(*out) + } return } From ededf97ac110a198f05e5c7feb513916d1bed0bc Mon Sep 17 00:00:00 2001 From: Kevin Hannon Date: Thu, 27 Mar 2025 22:10:59 -0400 Subject: [PATCH 20/42] update api --- .../generated_openapi/zz_generated.openapi.go | 66 +++++++++++++-- openapi/openapi.json | 37 ++++++++- operator/v1alpha1/types_kueue.go | 74 +++++++++++------ .../zz_generated.crd-manifests/kueue.crd.yaml | 81 +++++++++++++++++++ operator/v1alpha1/zz_generated.deepcopy.go | 50 ++++++++---- .../AAA_ungated.yaml | 81 +++++++++++++++++++ .../zz_generated.swagger_doc_generated.go | 31 ++++++- 7 files changed, 369 insertions(+), 51 deletions(-) diff --git a/openapi/generated_openapi/zz_generated.openapi.go b/openapi/generated_openapi/zz_generated.openapi.go index 59ab841ef13..8ffd8efb281 100644 --- a/openapi/generated_openapi/zz_generated.openapi.go +++ b/openapi/generated_openapi/zz_generated.openapi.go @@ -1120,6 +1120,7 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "github.com/openshift/api/operator/v1alpha1.Integrations": schema_openshift_api_operator_v1alpha1_Integrations(ref), "github.com/openshift/api/operator/v1alpha1.Kueue": schema_openshift_api_operator_v1alpha1_Kueue(ref), "github.com/openshift/api/operator/v1alpha1.KueueConfiguration": schema_openshift_api_operator_v1alpha1_KueueConfiguration(ref), + "github.com/openshift/api/operator/v1alpha1.KueueGangSchedulingPolicy": schema_openshift_api_operator_v1alpha1_KueueGangSchedulingPolicy(ref), "github.com/openshift/api/operator/v1alpha1.KueueList": schema_openshift_api_operator_v1alpha1_KueueList(ref), "github.com/openshift/api/operator/v1alpha1.KueueOperandSpec": schema_openshift_api_operator_v1alpha1_KueueOperandSpec(ref), "github.com/openshift/api/operator/v1alpha1.KueueStatus": schema_openshift_api_operator_v1alpha1_KueueStatus(ref), @@ -1133,6 +1134,7 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "github.com/openshift/api/operator/v1alpha1.OperatorCondition": schema_openshift_api_operator_v1alpha1_OperatorCondition(ref), "github.com/openshift/api/operator/v1alpha1.OperatorSpec": schema_openshift_api_operator_v1alpha1_OperatorSpec(ref), "github.com/openshift/api/operator/v1alpha1.OperatorStatus": schema_openshift_api_operator_v1alpha1_OperatorStatus(ref), + "github.com/openshift/api/operator/v1alpha1.QueueLabelPolicy": schema_openshift_api_operator_v1alpha1_QueueLabelPolicy(ref), "github.com/openshift/api/operator/v1alpha1.RepositoryDigestMirrors": schema_openshift_api_operator_v1alpha1_RepositoryDigestMirrors(ref), "github.com/openshift/api/operator/v1alpha1.StaticPodOperatorStatus": schema_openshift_api_operator_v1alpha1_StaticPodOperatorStatus(ref), "github.com/openshift/api/operator/v1alpha1.VersionAvailability": schema_openshift_api_operator_v1alpha1_VersionAvailability(ref), @@ -57287,21 +57289,22 @@ func schema_openshift_api_operator_v1alpha1_KueueConfiguration(ref common.Refere }, "queueLabelPolicy": { SchemaProps: spec.SchemaProps{ - Description: "queueLabelPolicy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. The allowed values are QueueNameRequired and QueueNameOptional. If set to QueueNameRequired, then those jobs will be suspended and never started unless they are assigned a queue and eventually admitted. This also applies to jobs created before starting the kueue controller. Defaults to QueueNameOptional; therefore, those jobs are not managed and if they are created unsuspended, they will start immediately.", - Type: []string{"string"}, - Format: "", + Description: "queueLabelPolicy controls how kueue manages workloads The default behavior of Kueue will manage workloads that have a queue-name label.", + Default: map[string]interface{}{}, + Ref: ref("github.com/openshift/api/operator/v1alpha1.QueueLabelPolicy"), }, }, "kueueGangSchedulingPolicy": { SchemaProps: spec.SchemaProps{ - Description: "kueueGangSchedulingPolicy controls how Kueue admits workloads Kueue provides the ability to admit workloads all in one (gang admission) and evicts workloads if they are not ready within a specific time. This is an optional field. The allowed values are BlockAdmission, EvictNotReadyWorkloads, and Disabled. The default value will be Disabled. BlockAdmission serializes the admission process for Kueue Workloads will be admitted one at a time and will wait for the workloads to be ready before going on to the next one in the list. EvictNotReadyWorkloads will not block admission but will evict Workloads that are not ready within a defaulted timelimit.", - Type: []string{"string"}, - Format: "", + Description: "kueueGangSchedulingPolicy controls how Kueue admits workloads", + Default: map[string]interface{}{}, + Ref: ref("github.com/openshift/api/operator/v1alpha1.KueueGangSchedulingPolicy"), }, }, "fairSharing": { SchemaProps: spec.SchemaProps{ Description: "fairSharing TODO not done yet", + Default: map[string]interface{}{}, Ref: ref("github.com/openshift/api/operator/v1alpha1.FairSharing"), }, }, @@ -57310,7 +57313,36 @@ func schema_openshift_api_operator_v1alpha1_KueueConfiguration(ref common.Refere }, }, Dependencies: []string{ - "github.com/openshift/api/operator/v1alpha1.FairSharing", "github.com/openshift/api/operator/v1alpha1.Integrations"}, + "github.com/openshift/api/operator/v1alpha1.FairSharing", "github.com/openshift/api/operator/v1alpha1.Integrations", "github.com/openshift/api/operator/v1alpha1.KueueGangSchedulingPolicy", "github.com/openshift/api/operator/v1alpha1.QueueLabelPolicy"}, + } +} + +func schema_openshift_api_operator_v1alpha1_KueueGangSchedulingPolicy(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "Kueue provides the ability to admit workloads all in one (gang admission) and evicts workloads if they are not ready within a specific time.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "policy": { + SchemaProps: spec.SchemaProps{ + Description: "policy allows for changing the kinds of gang scheduling Kueue does. This is an optional field. The allowed values are ByWorkload and Disabled. The default value will be Disabled. ByWorkload allows for configuration how admission is performed for Kueue.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + "byWorkload": { + SchemaProps: spec.SchemaProps{ + Description: "byWorkload controls how admission is done. The options are Sequential and Parallel. Sequential means workloads are admitted in sequential order and Kueue waits for those workloads to be ready. Parallel admits workloads in parallel and does not wait on these workloads to be ready. Workloads can be evicted in the background if they are not ready within default settings.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, } } @@ -58029,6 +58061,26 @@ func schema_openshift_api_operator_v1alpha1_OperatorStatus(ref common.ReferenceC } } +func schema_openshift_api_operator_v1alpha1_QueueLabelPolicy(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "queueLabelPolicy": { + SchemaProps: spec.SchemaProps{ + Description: "queueLabelPolicy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. The allowed values are QueueNameRequired and QueueNameOptional. If set to QueueNameRequired, then those jobs will be suspended and never started unless they are assigned a queue and eventually admitted. This also applies to jobs created before starting the kueue controller. Defaults to QueueNameOptional; therefore, those jobs are not managed and if they are created unsuspended, they will start immediately.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + } +} + func schema_openshift_api_operator_v1alpha1_RepositoryDigestMirrors(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ diff --git a/openapi/openapi.json b/openapi/openapi.json index 2b9ac19bd05..4f2de019c49 100644 --- a/openapi/openapi.json +++ b/openapi/openapi.json @@ -33465,6 +33465,7 @@ "properties": { "fairSharing": { "description": "fairSharing TODO not done yet", + "default": {}, "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.FairSharing" }, "integrations": { @@ -33473,12 +33474,30 @@ "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.Integrations" }, "kueueGangSchedulingPolicy": { - "description": "kueueGangSchedulingPolicy controls how Kueue admits workloads Kueue provides the ability to admit workloads all in one (gang admission) and evicts workloads if they are not ready within a specific time. This is an optional field. The allowed values are BlockAdmission, EvictNotReadyWorkloads, and Disabled. The default value will be Disabled. BlockAdmission serializes the admission process for Kueue Workloads will be admitted one at a time and will wait for the workloads to be ready before going on to the next one in the list. EvictNotReadyWorkloads will not block admission but will evict Workloads that are not ready within a defaulted timelimit.", - "type": "string" + "description": "kueueGangSchedulingPolicy controls how Kueue admits workloads", + "default": {}, + "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.KueueGangSchedulingPolicy" }, "queueLabelPolicy": { - "description": "queueLabelPolicy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. The allowed values are QueueNameRequired and QueueNameOptional. If set to QueueNameRequired, then those jobs will be suspended and never started unless they are assigned a queue and eventually admitted. This also applies to jobs created before starting the kueue controller. Defaults to QueueNameOptional; therefore, those jobs are not managed and if they are created unsuspended, they will start immediately.", - "type": "string" + "description": "queueLabelPolicy controls how kueue manages workloads The default behavior of Kueue will manage workloads that have a queue-name label.", + "default": {}, + "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.QueueLabelPolicy" + } + } + }, + "com.github.openshift.api.operator.v1alpha1.KueueGangSchedulingPolicy": { + "description": "Kueue provides the ability to admit workloads all in one (gang admission) and evicts workloads if they are not ready within a specific time.", + "type": "object", + "properties": { + "byWorkload": { + "description": "byWorkload controls how admission is done. The options are Sequential and Parallel. Sequential means workloads are admitted in sequential order and Kueue waits for those workloads to be ready. Parallel admits workloads in parallel and does not wait on these workloads to be ready. Workloads can be evicted in the background if they are not ready within default settings.", + "type": "string", + "default": "" + }, + "policy": { + "description": "policy allows for changing the kinds of gang scheduling Kueue does. This is an optional field. The allowed values are ByWorkload and Disabled. The default value will be Disabled. ByWorkload allows for configuration how admission is performed for Kueue.", + "type": "string", + "default": "" } } }, @@ -33924,6 +33943,16 @@ } } }, + "com.github.openshift.api.operator.v1alpha1.QueueLabelPolicy": { + "type": "object", + "properties": { + "queueLabelPolicy": { + "description": "queueLabelPolicy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. The allowed values are QueueNameRequired and QueueNameOptional. If set to QueueNameRequired, then those jobs will be suspended and never started unless they are assigned a queue and eventually admitted. This also applies to jobs created before starting the kueue controller. Defaults to QueueNameOptional; therefore, those jobs are not managed and if they are created unsuspended, they will start immediately.", + "type": "string", + "default": "" + } + } + }, "com.github.openshift.api.operator.v1alpha1.RepositoryDigestMirrors": { "description": "RepositoryDigestMirrors holds cluster-wide information about how to handle mirros in the registries config. Note: the mirrors only work when pulling the images that are referenced by their digests.", "type": "object", diff --git a/operator/v1alpha1/types_kueue.go b/operator/v1alpha1/types_kueue.go index 6e37af00853..c3de5eb06cb 100644 --- a/operator/v1alpha1/types_kueue.go +++ b/operator/v1alpha1/types_kueue.go @@ -47,32 +47,16 @@ type KueueConfiguration struct { // Kueue will only manage workloads that correspond to the specified integrations. // +required Integrations Integrations `json:"integrations"` - // queueLabelPolicy controls whether or not Kueue reconciles - // jobs that don't set the label kueue.x-k8s.io/queue-name. - // The allowed values are QueueNameRequired and QueueNameOptional. - // If set to QueueNameRequired, then those jobs will be suspended and never started unless - // they are assigned a queue and eventually admitted. This also applies to - // jobs created before starting the kueue controller. - // Defaults to QueueNameOptional; therefore, those jobs are not managed and if they are created - // unsuspended, they will start immediately. + // queueLabelPolicy controls how kueue manages workloads + // The default behavior of Kueue will manage workloads that have a queue-name label. // +optional - QueueLabelPolicy *QueueLabelNamePolicy `json:"queueLabelPolicy,omitempty"` + QueueLabelPolicy QueueLabelPolicy `json:"queueLabelPolicy,omitempty"` // kueueGangSchedulingPolicy controls how Kueue admits workloads - // Kueue provides the ability to admit workloads all in one (gang admission) - // and evicts workloads if they are not ready within a specific time. - // This is an optional field. - // The allowed values are BlockAdmission, EvictNotReadyWorkloads, and Disabled. - // The default value will be Disabled. - // BlockAdmission serializes the admission process for Kueue - // Workloads will be admitted one at a time and will wait - // for the workloads to be ready before going on to the next one in the list. - // EvictNotReadyWorkloads will not block admission but will evict Workloads that are not ready - // within a defaulted timelimit. // +optional - KueueGangSchedulingPolicy *KueueGangSchedulingPolicyOptions `json:"kueueGangSchedulingPolicy,omitempty"` + KueueGangSchedulingPolicy KueueGangSchedulingPolicy `json:"kueueGangSchedulingPolicy,omitempty"` // fairSharing TODO not done yet // +optional - FairSharing *FairSharing `json:"fairSharing,omitemoty"` + FairSharing FairSharing `json:"fairSharing,omitempty"` } // KueueStatus defines the observed state of Kueue @@ -200,15 +184,44 @@ type LabelKeys struct { Key string `json:"key,omitempty"` } -// +kubebuilder:validation:Enum=EvictNotReadyWorkloads;Disabled +// +kubebuilder:validation:Enum=ByWorkload;Disabled type KueueGangSchedulingPolicyOptions string const ( - KueueGangSchedulingPolicyBlockAdmission KueueGangSchedulingPolicyOptions = "BlockAdmission" - KueueGangSchedulingPolicyEvictNotReadyWorkloads KueueGangSchedulingPolicyOptions = "EvictNotReadyWorkloads" + KueueGangSchedulingPolicyEvictNotReadyWorkloads KueueGangSchedulingPolicyOptions = "ByWorkload" KueueGangSchedulingPolicyDisabled KueueGangSchedulingPolicyOptions = "Disabled" ) +// +kubebuilder:validation:Enum=Parallel;Sequential +type KueueGangSchedulingAdmissionOptions string + +const ( + KueueGangSchedulingAdmissionOptionsSequential KueueGangSchedulingAdmissionOptions = "Sequential" + KueueGangSchedulingAdmissionOptionsParallel KueueGangSchedulingAdmissionOptions = "Parallel" +) + +// Kueue provides the ability to admit workloads all in one (gang admission) +// and evicts workloads if they are not ready within a specific time. +type KueueGangSchedulingPolicy struct { + // policy allows for changing the kinds of gang scheduling Kueue does. + // This is an optional field. + // The allowed values are ByWorkload and Disabled. + // The default value will be Disabled. + // ByWorkload allows for configuration how admission is performed + // for Kueue. + // +optional + Policy KueueGangSchedulingPolicyOptions `json:"policy"` + // byWorkload controls how admission is done. + // The options are Sequential and Parallel. + // Sequential means workloads are admitted in sequential order and Kueue waits + // for those workloads to be ready. + // Parallel admits workloads in parallel and does not wait on these workloads to be ready. + // Workloads can be evicted in the background if they are not ready within default settings. + // +kubebuilder:validation:XValidation:rule="self.policy==ByWorkload",message="byWorkload is only valid if policy equals ByWorkload" + // +optional + ByWorkload KueueGangSchedulingAdmissionOptions `json:"byWorkload"` +} + // +kubebuilder:validation:Enum=QueueNameRequired;QueueNameOptional type QueueLabelNamePolicy string @@ -217,6 +230,19 @@ const ( QueueLabelNamePolicyOptional QueueLabelNamePolicy = "QueueNameOptional" ) +type QueueLabelPolicy struct { + // queueLabelPolicy controls whether or not Kueue reconciles + // jobs that don't set the label kueue.x-k8s.io/queue-name. + // The allowed values are QueueNameRequired and QueueNameOptional. + // If set to QueueNameRequired, then those jobs will be suspended and never started unless + // they are assigned a queue and eventually admitted. This also applies to + // jobs created before starting the kueue controller. + // Defaults to QueueNameOptional; therefore, those jobs are not managed and if they are created + // unsuspended, they will start immediately. + // +optional + QueueLabelPolicy QueueLabelNamePolicy `json:"queueLabelPolicy"` +} + // +kubebuilder:validation:Enum=LessThanOrEqualToFinalShare;LessThanInitialShare type PreemptionStrategy string diff --git a/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml b/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml index af8c9965ac9..a6b2658893f 100644 --- a/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml +++ b/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml @@ -48,6 +48,35 @@ spec: config is the desired configuration for the Kueue operator. properties: + fairSharing: + description: fairSharing TODO not done yet + properties: + preemptionStrategies: + description: |- + preemptionStrategies indicates which constraints should a preemption satisfy. + The preemption algorithm will only use the next strategy in the list if the + incoming workload (preemptor) doesn't fit after using the previous strategies. + Possible values are: + - LessThanOrEqualToFinalShare: Only preempt a workload if the share of the preemptor CQ + with the preemptor workload is less than or equal to the share of the preemptee CQ + without the workload to be preempted. + This strategy might favor preemption of smaller workloads in the preemptee CQ, + regardless of priority or start time, in an effort to keep the share of the CQ + as high as possible. + - LessThanInitialShare: Only preempt a workload if the share of the preemptor CQ + with the incoming workload is strictly less than the share of the preemptee CQ. + This strategy doesn't depend on the share usage of the workload being preempted. + As a result, the strategy chooses to preempt workloads with the lowest priority and + newest start time first. + The default strategy is ["LessThanOrEqualToFinalShare", "LessThanInitialShare"]. + items: + enum: + - LessThanOrEqualToFinalShare + - LessThanInitialShare + type: string + maxItems: 2 + type: array + type: object integrations: description: |- integrations is a required field that configures the Kueue's workload integrations. @@ -187,6 +216,58 @@ spec: required: - frameworks type: object + kueueGangSchedulingPolicy: + description: kueueGangSchedulingPolicy controls how Kueue admits + workloads + properties: + byWorkload: + description: |- + byWorkload controls how admission is done. + The options are Sequential and Parallel. + Sequential means workloads are admitted in sequential order and Kueue waits + for those workloads to be ready. + Parallel admits workloads in parallel and does not wait on these workloads to be ready. + Workloads can be evicted in the background if they are not ready within default settings. + enum: + - Parallel + - Sequential + type: string + x-kubernetes-validations: + - message: byWorkload is only valid if policy equals ByWorkload + rule: self.policy==ByWorkload + policy: + description: |- + policy allows for changing the kinds of gang scheduling Kueue does. + This is an optional field. + The allowed values are ByWorkload and Disabled. + The default value will be Disabled. + ByWorkload allows for configuration how admission is performed + for Kueue. + enum: + - ByWorkload + - Disabled + type: string + type: object + queueLabelPolicy: + description: |- + queueLabelPolicy controls how kueue manages workloads + The default behavior of Kueue will manage workloads that have a queue-name label. + properties: + queueLabelPolicy: + description: |- + queueLabelPolicy controls whether or not Kueue reconciles + jobs that don't set the label kueue.x-k8s.io/queue-name. + The allowed values are QueueNameRequired and QueueNameOptional. + If set to QueueNameRequired, then those jobs will be suspended and never started unless + they are assigned a queue and eventually admitted. This also applies to + jobs created before starting the kueue controller. + Defaults to QueueNameOptional; therefore, those jobs are not managed and if they are created + unsuspended, they will start immediately. + enum: + - QueueNameRequired + - QueueNameOptional + type: string + type: object required: - integrations type: object diff --git a/operator/v1alpha1/zz_generated.deepcopy.go b/operator/v1alpha1/zz_generated.deepcopy.go index ad6a337c5b9..8e34878908b 100644 --- a/operator/v1alpha1/zz_generated.deepcopy.go +++ b/operator/v1alpha1/zz_generated.deepcopy.go @@ -484,21 +484,9 @@ func (in *Kueue) DeepCopyObject() runtime.Object { func (in *KueueConfiguration) DeepCopyInto(out *KueueConfiguration) { *out = *in in.Integrations.DeepCopyInto(&out.Integrations) - if in.QueueLabelPolicy != nil { - in, out := &in.QueueLabelPolicy, &out.QueueLabelPolicy - *out = new(QueueLabelNamePolicy) - **out = **in - } - if in.KueueGangSchedulingPolicy != nil { - in, out := &in.KueueGangSchedulingPolicy, &out.KueueGangSchedulingPolicy - *out = new(KueueGangSchedulingPolicyOptions) - **out = **in - } - if in.FairSharing != nil { - in, out := &in.FairSharing, &out.FairSharing - *out = new(FairSharing) - (*in).DeepCopyInto(*out) - } + out.QueueLabelPolicy = in.QueueLabelPolicy + out.KueueGangSchedulingPolicy = in.KueueGangSchedulingPolicy + in.FairSharing.DeepCopyInto(&out.FairSharing) return } @@ -512,6 +500,22 @@ func (in *KueueConfiguration) DeepCopy() *KueueConfiguration { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KueueGangSchedulingPolicy) DeepCopyInto(out *KueueGangSchedulingPolicy) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KueueGangSchedulingPolicy. +func (in *KueueGangSchedulingPolicy) DeepCopy() *KueueGangSchedulingPolicy { + if in == nil { + return nil + } + out := new(KueueGangSchedulingPolicy) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KueueList) DeepCopyInto(out *KueueList) { *out = *in @@ -795,6 +799,22 @@ func (in *OperatorStatus) DeepCopy() *OperatorStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *QueueLabelPolicy) DeepCopyInto(out *QueueLabelPolicy) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new QueueLabelPolicy. +func (in *QueueLabelPolicy) DeepCopy() *QueueLabelPolicy { + if in == nil { + return nil + } + out := new(QueueLabelPolicy) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RepositoryDigestMirrors) DeepCopyInto(out *RepositoryDigestMirrors) { *out = *in diff --git a/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml b/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml index 5b51737365b..37f04ef7de7 100644 --- a/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml +++ b/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml @@ -46,6 +46,35 @@ spec: config is the desired configuration for the Kueue operator. properties: + fairSharing: + description: fairSharing TODO not done yet + properties: + preemptionStrategies: + description: |- + preemptionStrategies indicates which constraints should a preemption satisfy. + The preemption algorithm will only use the next strategy in the list if the + incoming workload (preemptor) doesn't fit after using the previous strategies. + Possible values are: + - LessThanOrEqualToFinalShare: Only preempt a workload if the share of the preemptor CQ + with the preemptor workload is less than or equal to the share of the preemptee CQ + without the workload to be preempted. + This strategy might favor preemption of smaller workloads in the preemptee CQ, + regardless of priority or start time, in an effort to keep the share of the CQ + as high as possible. + - LessThanInitialShare: Only preempt a workload if the share of the preemptor CQ + with the incoming workload is strictly less than the share of the preemptee CQ. + This strategy doesn't depend on the share usage of the workload being preempted. + As a result, the strategy chooses to preempt workloads with the lowest priority and + newest start time first. + The default strategy is ["LessThanOrEqualToFinalShare", "LessThanInitialShare"]. + items: + enum: + - LessThanOrEqualToFinalShare + - LessThanInitialShare + type: string + maxItems: 2 + type: array + type: object integrations: description: |- integrations is a required field that configures the Kueue's workload integrations. @@ -185,6 +214,58 @@ spec: required: - frameworks type: object + kueueGangSchedulingPolicy: + description: kueueGangSchedulingPolicy controls how Kueue admits + workloads + properties: + byWorkload: + description: |- + byWorkload controls how admission is done. + The options are Sequential and Parallel. + Sequential means workloads are admitted in sequential order and Kueue waits + for those workloads to be ready. + Parallel admits workloads in parallel and does not wait on these workloads to be ready. + Workloads can be evicted in the background if they are not ready within default settings. + enum: + - Parallel + - Sequential + type: string + x-kubernetes-validations: + - message: byWorkload is only valid if policy equals ByWorkload + rule: self.policy==ByWorkload + policy: + description: |- + policy allows for changing the kinds of gang scheduling Kueue does. + This is an optional field. + The allowed values are ByWorkload and Disabled. + The default value will be Disabled. + ByWorkload allows for configuration how admission is performed + for Kueue. + enum: + - ByWorkload + - Disabled + type: string + type: object + queueLabelPolicy: + description: |- + queueLabelPolicy controls how kueue manages workloads + The default behavior of Kueue will manage workloads that have a queue-name label. + properties: + queueLabelPolicy: + description: |- + queueLabelPolicy controls whether or not Kueue reconciles + jobs that don't set the label kueue.x-k8s.io/queue-name. + The allowed values are QueueNameRequired and QueueNameOptional. + If set to QueueNameRequired, then those jobs will be suspended and never started unless + they are assigned a queue and eventually admitted. This also applies to + jobs created before starting the kueue controller. + Defaults to QueueNameOptional; therefore, those jobs are not managed and if they are created + unsuspended, they will start immediately. + enum: + - QueueNameRequired + - QueueNameOptional + type: string + type: object required: - integrations type: object diff --git a/operator/v1alpha1/zz_generated.swagger_doc_generated.go b/operator/v1alpha1/zz_generated.swagger_doc_generated.go index f4a3023a200..18b270ef876 100644 --- a/operator/v1alpha1/zz_generated.swagger_doc_generated.go +++ b/operator/v1alpha1/zz_generated.swagger_doc_generated.go @@ -268,6 +268,14 @@ func (ExternalFramework) SwaggerDoc() map[string]string { return map_ExternalFramework } +var map_FairSharing = map[string]string{ + "preemptionStrategies": "preemptionStrategies indicates which constraints should a preemption satisfy. The preemption algorithm will only use the next strategy in the list if the incoming workload (preemptor) doesn't fit after using the previous strategies. Possible values are: - LessThanOrEqualToFinalShare: Only preempt a workload if the share of the preemptor CQ\n with the preemptor workload is less than or equal to the share of the preemptee CQ\n without the workload to be preempted.\n This strategy might favor preemption of smaller workloads in the preemptee CQ,\n regardless of priority or start time, in an effort to keep the share of the CQ\n as high as possible.\n- LessThanInitialShare: Only preempt a workload if the share of the preemptor CQ\n with the incoming workload is strictly less than the share of the preemptee CQ.\n This strategy doesn't depend on the share usage of the workload being preempted.\n As a result, the strategy chooses to preempt workloads with the lowest priority and\n newest start time first.\nThe default strategy is [\"LessThanOrEqualToFinalShare\", \"LessThanInitialShare\"].", +} + +func (FairSharing) SwaggerDoc() map[string]string { + return map_FairSharing +} + var map_Integrations = map[string]string{ "": "This is the integrations for Kueue. Kueue uses these apis to determine which jobs will be managed by Kueue.", "frameworks": "frameworks are a unique list of names to be enabled. This is required and must have at least one element. Each framework represents a type of job that Kueue will manage. Frameworks are a list of frameworks that Kueue has support for. The allowed values are BatchJob, RayJob, RayCluster, JobSet, MPIJob, PaddleJob, PytorchJob, TFJob, XGBoostJob, AppWrapper, Pod, Deployment, StatefulSet and LeaderWorkerSet.", @@ -291,13 +299,26 @@ func (Kueue) SwaggerDoc() map[string]string { } var map_KueueConfiguration = map[string]string{ - "integrations": "integrations is a required field that configures the Kueue's workload integrations. Kueue has both standard integrations, known as job frameworks, and external integrations known as external frameworks. Kueue will only manage workloads that correspond to the specified integrations.", + "integrations": "integrations is a required field that configures the Kueue's workload integrations. Kueue has both standard integrations, known as job frameworks, and external integrations known as external frameworks. Kueue will only manage workloads that correspond to the specified integrations.", + "queueLabelPolicy": "queueLabelPolicy controls how kueue manages workloads The default behavior of Kueue will manage workloads that have a queue-name label.", + "kueueGangSchedulingPolicy": "kueueGangSchedulingPolicy controls how Kueue admits workloads", + "fairSharing": "fairSharing TODO not done yet", } func (KueueConfiguration) SwaggerDoc() map[string]string { return map_KueueConfiguration } +var map_KueueGangSchedulingPolicy = map[string]string{ + "": "Kueue provides the ability to admit workloads all in one (gang admission) and evicts workloads if they are not ready within a specific time.", + "policy": "policy allows for changing the kinds of gang scheduling Kueue does. This is an optional field. The allowed values are ByWorkload and Disabled. The default value will be Disabled. ByWorkload allows for configuration how admission is performed for Kueue.", + "byWorkload": "byWorkload controls how admission is done. The options are Sequential and Parallel. Sequential means workloads are admitted in sequential order and Kueue waits for those workloads to be ready. Parallel admits workloads in parallel and does not wait on these workloads to be ready. Workloads can be evicted in the background if they are not ready within default settings.", +} + +func (KueueGangSchedulingPolicy) SwaggerDoc() map[string]string { + return map_KueueGangSchedulingPolicy +} + var map_KueueList = map[string]string{ "": "KueueList contains a list of Kueue\n\nCompatibility level 4: No compatibility is provided, the API can change at any point for any reason. These capabilities should not be used by applications needing long term support.", "metadata": "metadata for the list", @@ -332,6 +353,14 @@ func (LabelKeys) SwaggerDoc() map[string]string { return map_LabelKeys } +var map_QueueLabelPolicy = map[string]string{ + "queueLabelPolicy": "queueLabelPolicy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. The allowed values are QueueNameRequired and QueueNameOptional. If set to QueueNameRequired, then those jobs will be suspended and never started unless they are assigned a queue and eventually admitted. This also applies to jobs created before starting the kueue controller. Defaults to QueueNameOptional; therefore, those jobs are not managed and if they are created unsuspended, they will start immediately.", +} + +func (QueueLabelPolicy) SwaggerDoc() map[string]string { + return map_QueueLabelPolicy +} + var map_OLM = map[string]string{ "": "OLM provides information to configure an operator to manage the OLM controllers\n\nCompatibility level 4: No compatibility is provided, the API can change at any point for any reason. These capabilities should not be used by applications needing long term support.", "metadata": "metadata is the standard object's metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata", From 929d0112b16b4d15def0516a03ad623e35832159 Mon Sep 17 00:00:00 2001 From: Kevin Hannon Date: Fri, 28 Mar 2025 10:29:56 -0400 Subject: [PATCH 21/42] api comment updates --- .../generated_openapi/zz_generated.openapi.go | 4 +-- openapi/openapi.json | 4 +-- operator/v1alpha1/types_kueue.go | 27 +++++++++++-------- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/openapi/generated_openapi/zz_generated.openapi.go b/openapi/generated_openapi/zz_generated.openapi.go index 8ffd8efb281..0b794dd25b0 100644 --- a/openapi/generated_openapi/zz_generated.openapi.go +++ b/openapi/generated_openapi/zz_generated.openapi.go @@ -56896,7 +56896,7 @@ func schema_openshift_api_operator_v1alpha1_FairSharing(ref common.ReferenceCall Properties: map[string]spec.Schema{ "preemptionStrategies": { SchemaProps: spec.SchemaProps{ - Description: "preemptionStrategies indicates which constraints should a preemption satisfy. The preemption algorithm will only use the next strategy in the list if the incoming workload (preemptor) doesn't fit after using the previous strategies. Possible values are: - LessThanOrEqualToFinalShare: Only preempt a workload if the share of the preemptor CQ\n with the preemptor workload is less than or equal to the share of the preemptee CQ\n without the workload to be preempted.\n This strategy might favor preemption of smaller workloads in the preemptee CQ,\n regardless of priority or start time, in an effort to keep the share of the CQ\n as high as possible.\n- LessThanInitialShare: Only preempt a workload if the share of the preemptor CQ\n with the incoming workload is strictly less than the share of the preemptee CQ.\n This strategy doesn't depend on the share usage of the workload being preempted.\n As a result, the strategy chooses to preempt workloads with the lowest priority and\n newest start time first.\nThe default strategy is [\"LessThanOrEqualToFinalShare\", \"LessThanInitialShare\"].", + Description: "preemptionStrategies indicates which constraints should a preemption satisfy. The preemption algorithm will only use the next strategy in the list if the incoming workload (preemptor) doesn't fit after using the previous strategies. Possible values are: - LessThanOrEqualToFinalShare: Only preempt a workload\n if the share of the preemptor ClusterQueue\n with the preemptor workload is less than\n or equal to the share of the preemptee ClusterQueue\n without the workload to be preempted.\n This strategy might favor preemption of smaller workloads\n in the preemptee ClusterQueue,\n regardless of priority or start time, in an effort to keep the share of the ClusterQueue\n as high as possible.\n- LessThanInitialShare: Only preempt a workload if the share of the preemptor ClusterQueue\n with the incoming workload is strictly less than the share of the preemptee ClusterQueue.\n This strategy doesn't depend on the share usage of the workload being preempted.\n As a result, the strategy chooses to preempt workloads with the lowest priority and\n newest start time first.\nThe default strategy is [\"LessThanOrEqualToFinalShare\", \"LessThanInitialShare\"].", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -57334,7 +57334,7 @@ func schema_openshift_api_operator_v1alpha1_KueueGangSchedulingPolicy(ref common }, "byWorkload": { SchemaProps: spec.SchemaProps{ - Description: "byWorkload controls how admission is done. The options are Sequential and Parallel. Sequential means workloads are admitted in sequential order and Kueue waits for those workloads to be ready. Parallel admits workloads in parallel and does not wait on these workloads to be ready. Workloads can be evicted in the background if they are not ready within default settings.", + Description: "byWorkload controls how admission is done. When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling.", Default: "", Type: []string{"string"}, Format: "", diff --git a/openapi/openapi.json b/openapi/openapi.json index 4f2de019c49..775d0ff2ebe 100644 --- a/openapi/openapi.json +++ b/openapi/openapi.json @@ -33236,7 +33236,7 @@ "type": "object", "properties": { "preemptionStrategies": { - "description": "preemptionStrategies indicates which constraints should a preemption satisfy. The preemption algorithm will only use the next strategy in the list if the incoming workload (preemptor) doesn't fit after using the previous strategies. Possible values are: - LessThanOrEqualToFinalShare: Only preempt a workload if the share of the preemptor CQ\n with the preemptor workload is less than or equal to the share of the preemptee CQ\n without the workload to be preempted.\n This strategy might favor preemption of smaller workloads in the preemptee CQ,\n regardless of priority or start time, in an effort to keep the share of the CQ\n as high as possible.\n- LessThanInitialShare: Only preempt a workload if the share of the preemptor CQ\n with the incoming workload is strictly less than the share of the preemptee CQ.\n This strategy doesn't depend on the share usage of the workload being preempted.\n As a result, the strategy chooses to preempt workloads with the lowest priority and\n newest start time first.\nThe default strategy is [\"LessThanOrEqualToFinalShare\", \"LessThanInitialShare\"].", + "description": "preemptionStrategies indicates which constraints should a preemption satisfy. The preemption algorithm will only use the next strategy in the list if the incoming workload (preemptor) doesn't fit after using the previous strategies. Possible values are: - LessThanOrEqualToFinalShare: Only preempt a workload\n if the share of the preemptor ClusterQueue\n with the preemptor workload is less than\n or equal to the share of the preemptee ClusterQueue\n without the workload to be preempted.\n This strategy might favor preemption of smaller workloads\n in the preemptee ClusterQueue,\n regardless of priority or start time, in an effort to keep the share of the ClusterQueue\n as high as possible.\n- LessThanInitialShare: Only preempt a workload if the share of the preemptor ClusterQueue\n with the incoming workload is strictly less than the share of the preemptee ClusterQueue.\n This strategy doesn't depend on the share usage of the workload being preempted.\n As a result, the strategy chooses to preempt workloads with the lowest priority and\n newest start time first.\nThe default strategy is [\"LessThanOrEqualToFinalShare\", \"LessThanInitialShare\"].", "type": "array", "items": { "type": "string", @@ -33490,7 +33490,7 @@ "type": "object", "properties": { "byWorkload": { - "description": "byWorkload controls how admission is done. The options are Sequential and Parallel. Sequential means workloads are admitted in sequential order and Kueue waits for those workloads to be ready. Parallel admits workloads in parallel and does not wait on these workloads to be ready. Workloads can be evicted in the background if they are not ready within default settings.", + "description": "byWorkload controls how admission is done. When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling.", "type": "string", "default": "" }, diff --git a/operator/v1alpha1/types_kueue.go b/operator/v1alpha1/types_kueue.go index c3de5eb06cb..9fde390d4a4 100644 --- a/operator/v1alpha1/types_kueue.go +++ b/operator/v1alpha1/types_kueue.go @@ -212,11 +212,13 @@ type KueueGangSchedulingPolicy struct { // +optional Policy KueueGangSchedulingPolicyOptions `json:"policy"` // byWorkload controls how admission is done. - // The options are Sequential and Parallel. - // Sequential means workloads are admitted in sequential order and Kueue waits - // for those workloads to be ready. - // Parallel admits workloads in parallel and does not wait on these workloads to be ready. - // Workloads can be evicted in the background if they are not ready within default settings. + // When admission is set to Sequential, only pods from the currently processing workload will be admitted. + // Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. + // Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, + // but provides a higher guarantee of workloads scheduling all pods together successfully. + // When set to Parallel, pods from any workload will be admitted at any time. + // This may lead to a deadlock where workloads are in contention for cluster capacity and + // pods from another workload having successfully scheduled prevent pods from the current workload scheduling. // +kubebuilder:validation:XValidation:rule="self.policy==ByWorkload",message="byWorkload is only valid if policy equals ByWorkload" // +optional ByWorkload KueueGangSchedulingAdmissionOptions `json:"byWorkload"` @@ -256,14 +258,17 @@ type FairSharing struct { // The preemption algorithm will only use the next strategy in the list if the // incoming workload (preemptor) doesn't fit after using the previous strategies. // Possible values are: - // - LessThanOrEqualToFinalShare: Only preempt a workload if the share of the preemptor CQ - // with the preemptor workload is less than or equal to the share of the preemptee CQ + // - LessThanOrEqualToFinalShare: Only preempt a workload + // if the share of the preemptor ClusterQueue + // with the preemptor workload is less than + // or equal to the share of the preemptee ClusterQueue // without the workload to be preempted. - // This strategy might favor preemption of smaller workloads in the preemptee CQ, - // regardless of priority or start time, in an effort to keep the share of the CQ + // This strategy might favor preemption of smaller workloads + // in the preemptee ClusterQueue, + // regardless of priority or start time, in an effort to keep the share of the ClusterQueue // as high as possible. - // - LessThanInitialShare: Only preempt a workload if the share of the preemptor CQ - // with the incoming workload is strictly less than the share of the preemptee CQ. + // - LessThanInitialShare: Only preempt a workload if the share of the preemptor ClusterQueue + // with the incoming workload is strictly less than the share of the preemptee ClusterQueue. // This strategy doesn't depend on the share usage of the workload being preempted. // As a result, the strategy chooses to preempt workloads with the lowest priority and // newest start time first. From 1acd09a20eadef126c167400bccbd1babec8eace Mon Sep 17 00:00:00 2001 From: Kevin Hannon Date: Fri, 28 Mar 2025 17:04:34 -0400 Subject: [PATCH 22/42] fair sharing classical --- .../generated_openapi/zz_generated.openapi.go | 63 ++++++++---------- openapi/openapi.json | 39 +++++------ operator/v1alpha1/types_kueue.go | 45 +++++-------- .../zz_generated.crd-manifests/kueue.crd.yaml | 65 ++++++++----------- operator/v1alpha1/zz_generated.deepcopy.go | 39 +++++------ .../AAA_ungated.yaml | 65 ++++++++----------- .../zz_generated.swagger_doc_generated.go | 26 ++++---- 7 files changed, 148 insertions(+), 194 deletions(-) diff --git a/openapi/generated_openapi/zz_generated.openapi.go b/openapi/generated_openapi/zz_generated.openapi.go index 0b794dd25b0..e23d80b659b 100644 --- a/openapi/generated_openapi/zz_generated.openapi.go +++ b/openapi/generated_openapi/zz_generated.openapi.go @@ -1111,7 +1111,6 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "github.com/openshift/api/operator/v1alpha1.EtcdBackupSpec": schema_openshift_api_operator_v1alpha1_EtcdBackupSpec(ref), "github.com/openshift/api/operator/v1alpha1.EtcdBackupStatus": schema_openshift_api_operator_v1alpha1_EtcdBackupStatus(ref), "github.com/openshift/api/operator/v1alpha1.ExternalFramework": schema_openshift_api_operator_v1alpha1_ExternalFramework(ref), - "github.com/openshift/api/operator/v1alpha1.FairSharing": schema_openshift_api_operator_v1alpha1_FairSharing(ref), "github.com/openshift/api/operator/v1alpha1.GenerationHistory": schema_openshift_api_operator_v1alpha1_GenerationHistory(ref), "github.com/openshift/api/operator/v1alpha1.GenericOperatorConfig": schema_openshift_api_operator_v1alpha1_GenericOperatorConfig(ref), "github.com/openshift/api/operator/v1alpha1.ImageContentSourcePolicy": schema_openshift_api_operator_v1alpha1_ImageContentSourcePolicy(ref), @@ -1134,6 +1133,7 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "github.com/openshift/api/operator/v1alpha1.OperatorCondition": schema_openshift_api_operator_v1alpha1_OperatorCondition(ref), "github.com/openshift/api/operator/v1alpha1.OperatorSpec": schema_openshift_api_operator_v1alpha1_OperatorSpec(ref), "github.com/openshift/api/operator/v1alpha1.OperatorStatus": schema_openshift_api_operator_v1alpha1_OperatorStatus(ref), + "github.com/openshift/api/operator/v1alpha1.Premption": schema_openshift_api_operator_v1alpha1_Premption(ref), "github.com/openshift/api/operator/v1alpha1.QueueLabelPolicy": schema_openshift_api_operator_v1alpha1_QueueLabelPolicy(ref), "github.com/openshift/api/operator/v1alpha1.RepositoryDigestMirrors": schema_openshift_api_operator_v1alpha1_RepositoryDigestMirrors(ref), "github.com/openshift/api/operator/v1alpha1.StaticPodOperatorStatus": schema_openshift_api_operator_v1alpha1_StaticPodOperatorStatus(ref), @@ -56888,33 +56888,6 @@ func schema_openshift_api_operator_v1alpha1_ExternalFramework(ref common.Referen } } -func schema_openshift_api_operator_v1alpha1_FairSharing(ref common.ReferenceCallback) common.OpenAPIDefinition { - return common.OpenAPIDefinition{ - Schema: spec.Schema{ - SchemaProps: spec.SchemaProps{ - Type: []string{"object"}, - Properties: map[string]spec.Schema{ - "preemptionStrategies": { - SchemaProps: spec.SchemaProps{ - Description: "preemptionStrategies indicates which constraints should a preemption satisfy. The preemption algorithm will only use the next strategy in the list if the incoming workload (preemptor) doesn't fit after using the previous strategies. Possible values are: - LessThanOrEqualToFinalShare: Only preempt a workload\n if the share of the preemptor ClusterQueue\n with the preemptor workload is less than\n or equal to the share of the preemptee ClusterQueue\n without the workload to be preempted.\n This strategy might favor preemption of smaller workloads\n in the preemptee ClusterQueue,\n regardless of priority or start time, in an effort to keep the share of the ClusterQueue\n as high as possible.\n- LessThanInitialShare: Only preempt a workload if the share of the preemptor ClusterQueue\n with the incoming workload is strictly less than the share of the preemptee ClusterQueue.\n This strategy doesn't depend on the share usage of the workload being preempted.\n As a result, the strategy chooses to preempt workloads with the lowest priority and\n newest start time first.\nThe default strategy is [\"LessThanOrEqualToFinalShare\", \"LessThanInitialShare\"].", - Type: []string{"array"}, - Items: &spec.SchemaOrArray{ - Schema: &spec.Schema{ - SchemaProps: spec.SchemaProps{ - Default: "", - Type: []string{"string"}, - Format: "", - }, - }, - }, - }, - }, - }, - }, - }, - } -} - func schema_openshift_api_operator_v1alpha1_GenerationHistory(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ @@ -57289,23 +57262,23 @@ func schema_openshift_api_operator_v1alpha1_KueueConfiguration(ref common.Refere }, "queueLabelPolicy": { SchemaProps: spec.SchemaProps{ - Description: "queueLabelPolicy controls how kueue manages workloads The default behavior of Kueue will manage workloads that have a queue-name label.", + Description: "queueLabelPolicy controls how kueue manages workloads The default behavior of Kueue will manage workloads that have a queue-name label. This field is optional.", Default: map[string]interface{}{}, Ref: ref("github.com/openshift/api/operator/v1alpha1.QueueLabelPolicy"), }, }, "kueueGangSchedulingPolicy": { SchemaProps: spec.SchemaProps{ - Description: "kueueGangSchedulingPolicy controls how Kueue admits workloads", + Description: "kueueGangSchedulingPolicy controls how Kueue admits workloads. Gang Scheduling is the act of all or nothing scheduling. Kueue provides this ability. This field is optional.", Default: map[string]interface{}{}, Ref: ref("github.com/openshift/api/operator/v1alpha1.KueueGangSchedulingPolicy"), }, }, - "fairSharing": { + "premption": { SchemaProps: spec.SchemaProps{ - Description: "fairSharing TODO not done yet", + Description: "premption is the process of evicting one or more admitted Workloads to accommodate another Workload. Kueue has classical premption and preemption via fair sharing.", Default: map[string]interface{}{}, - Ref: ref("github.com/openshift/api/operator/v1alpha1.FairSharing"), + Ref: ref("github.com/openshift/api/operator/v1alpha1.Premption"), }, }, }, @@ -57313,7 +57286,7 @@ func schema_openshift_api_operator_v1alpha1_KueueConfiguration(ref common.Refere }, }, Dependencies: []string{ - "github.com/openshift/api/operator/v1alpha1.FairSharing", "github.com/openshift/api/operator/v1alpha1.Integrations", "github.com/openshift/api/operator/v1alpha1.KueueGangSchedulingPolicy", "github.com/openshift/api/operator/v1alpha1.QueueLabelPolicy"}, + "github.com/openshift/api/operator/v1alpha1.Integrations", "github.com/openshift/api/operator/v1alpha1.KueueGangSchedulingPolicy", "github.com/openshift/api/operator/v1alpha1.Premption", "github.com/openshift/api/operator/v1alpha1.QueueLabelPolicy"}, } } @@ -58061,6 +58034,26 @@ func schema_openshift_api_operator_v1alpha1_OperatorStatus(ref common.ReferenceC } } +func schema_openshift_api_operator_v1alpha1_Premption(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "preemptionStrategies": { + SchemaProps: spec.SchemaProps{ + Description: "preemptionStrategies are the types of preemption kueue allows. Kueue has two types of preemption: classical and fair sharing.", + Default: "", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + } +} + func schema_openshift_api_operator_v1alpha1_QueueLabelPolicy(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ @@ -58069,7 +58062,7 @@ func schema_openshift_api_operator_v1alpha1_QueueLabelPolicy(ref common.Referenc Properties: map[string]spec.Schema{ "queueLabelPolicy": { SchemaProps: spec.SchemaProps{ - Description: "queueLabelPolicy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. The allowed values are QueueNameRequired and QueueNameOptional. If set to QueueNameRequired, then those jobs will be suspended and never started unless they are assigned a queue and eventually admitted. This also applies to jobs created before starting the kueue controller. Defaults to QueueNameOptional; therefore, those jobs are not managed and if they are created unsuspended, they will start immediately.", + Description: "queueLabelPolicy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. The allowed values are QueueNameRequired and QueueNameOptional. If set to QueueNameRequired, then those jobs will be suspended and never started unless they are assigned a queue and eventually admitted. This also applies to jobs created before starting the kueue controller. Defaults to QueueNameRequired; therefore, those jobs are not managed and if they are created unsuspended, they will start immediately.", Default: "", Type: []string{"string"}, Format: "", diff --git a/openapi/openapi.json b/openapi/openapi.json index 775d0ff2ebe..c32a7626b25 100644 --- a/openapi/openapi.json +++ b/openapi/openapi.json @@ -33232,19 +33232,6 @@ } } }, - "com.github.openshift.api.operator.v1alpha1.FairSharing": { - "type": "object", - "properties": { - "preemptionStrategies": { - "description": "preemptionStrategies indicates which constraints should a preemption satisfy. The preemption algorithm will only use the next strategy in the list if the incoming workload (preemptor) doesn't fit after using the previous strategies. Possible values are: - LessThanOrEqualToFinalShare: Only preempt a workload\n if the share of the preemptor ClusterQueue\n with the preemptor workload is less than\n or equal to the share of the preemptee ClusterQueue\n without the workload to be preempted.\n This strategy might favor preemption of smaller workloads\n in the preemptee ClusterQueue,\n regardless of priority or start time, in an effort to keep the share of the ClusterQueue\n as high as possible.\n- LessThanInitialShare: Only preempt a workload if the share of the preemptor ClusterQueue\n with the incoming workload is strictly less than the share of the preemptee ClusterQueue.\n This strategy doesn't depend on the share usage of the workload being preempted.\n As a result, the strategy chooses to preempt workloads with the lowest priority and\n newest start time first.\nThe default strategy is [\"LessThanOrEqualToFinalShare\", \"LessThanInitialShare\"].", - "type": "array", - "items": { - "type": "string", - "default": "" - } - } - } - }, "com.github.openshift.api.operator.v1alpha1.GenerationHistory": { "description": "GenerationHistory keeps track of the generation for a given resource so that decisions about forced updated can be made. DEPRECATED: Use fields in v1.GenerationStatus instead", "type": "object", @@ -33463,23 +33450,23 @@ "integrations" ], "properties": { - "fairSharing": { - "description": "fairSharing TODO not done yet", - "default": {}, - "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.FairSharing" - }, "integrations": { "description": "integrations is a required field that configures the Kueue's workload integrations. Kueue has both standard integrations, known as job frameworks, and external integrations known as external frameworks. Kueue will only manage workloads that correspond to the specified integrations.", "default": {}, "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.Integrations" }, "kueueGangSchedulingPolicy": { - "description": "kueueGangSchedulingPolicy controls how Kueue admits workloads", + "description": "kueueGangSchedulingPolicy controls how Kueue admits workloads. Gang Scheduling is the act of all or nothing scheduling. Kueue provides this ability. This field is optional.", "default": {}, "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.KueueGangSchedulingPolicy" }, + "premption": { + "description": "premption is the process of evicting one or more admitted Workloads to accommodate another Workload. Kueue has classical premption and preemption via fair sharing.", + "default": {}, + "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.Premption" + }, "queueLabelPolicy": { - "description": "queueLabelPolicy controls how kueue manages workloads The default behavior of Kueue will manage workloads that have a queue-name label.", + "description": "queueLabelPolicy controls how kueue manages workloads The default behavior of Kueue will manage workloads that have a queue-name label. This field is optional.", "default": {}, "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.QueueLabelPolicy" } @@ -33943,11 +33930,21 @@ } } }, + "com.github.openshift.api.operator.v1alpha1.Premption": { + "type": "object", + "properties": { + "preemptionStrategies": { + "description": "preemptionStrategies are the types of preemption kueue allows. Kueue has two types of preemption: classical and fair sharing.", + "type": "string", + "default": "" + } + } + }, "com.github.openshift.api.operator.v1alpha1.QueueLabelPolicy": { "type": "object", "properties": { "queueLabelPolicy": { - "description": "queueLabelPolicy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. The allowed values are QueueNameRequired and QueueNameOptional. If set to QueueNameRequired, then those jobs will be suspended and never started unless they are assigned a queue and eventually admitted. This also applies to jobs created before starting the kueue controller. Defaults to QueueNameOptional; therefore, those jobs are not managed and if they are created unsuspended, they will start immediately.", + "description": "queueLabelPolicy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. The allowed values are QueueNameRequired and QueueNameOptional. If set to QueueNameRequired, then those jobs will be suspended and never started unless they are assigned a queue and eventually admitted. This also applies to jobs created before starting the kueue controller. Defaults to QueueNameRequired; therefore, those jobs are not managed and if they are created unsuspended, they will start immediately.", "type": "string", "default": "" } diff --git a/operator/v1alpha1/types_kueue.go b/operator/v1alpha1/types_kueue.go index 9fde390d4a4..4a189a6d2a0 100644 --- a/operator/v1alpha1/types_kueue.go +++ b/operator/v1alpha1/types_kueue.go @@ -49,14 +49,19 @@ type KueueConfiguration struct { Integrations Integrations `json:"integrations"` // queueLabelPolicy controls how kueue manages workloads // The default behavior of Kueue will manage workloads that have a queue-name label. + // This field is optional. // +optional QueueLabelPolicy QueueLabelPolicy `json:"queueLabelPolicy,omitempty"` - // kueueGangSchedulingPolicy controls how Kueue admits workloads + // kueueGangSchedulingPolicy controls how Kueue admits workloads. + // Gang Scheduling is the act of all or nothing scheduling. + // Kueue provides this ability. + // This field is optional. // +optional KueueGangSchedulingPolicy KueueGangSchedulingPolicy `json:"kueueGangSchedulingPolicy,omitempty"` - // fairSharing TODO not done yet + // premption is the process of evicting one or more admitted Workloads to accommodate another Workload. + // Kueue has classical premption and preemption via fair sharing. // +optional - FairSharing FairSharing `json:"fairSharing,omitempty"` + Premption Premption `json:"premption,omitempty"` } // KueueStatus defines the observed state of Kueue @@ -239,41 +244,23 @@ type QueueLabelPolicy struct { // If set to QueueNameRequired, then those jobs will be suspended and never started unless // they are assigned a queue and eventually admitted. This also applies to // jobs created before starting the kueue controller. - // Defaults to QueueNameOptional; therefore, those jobs are not managed and if they are created + // Defaults to QueueNameRequired; therefore, those jobs are not managed and if they are created // unsuspended, they will start immediately. // +optional QueueLabelPolicy QueueLabelNamePolicy `json:"queueLabelPolicy"` } -// +kubebuilder:validation:Enum=LessThanOrEqualToFinalShare;LessThanInitialShare +// +kubebuilder:validation:Enum=Classical;FairSharing type PreemptionStrategy string const ( - LessThanOrEqualToFinalShare PreemptionStrategy = "LessThanOrEqualToFinalShare" - LessThanInitialShare PreemptionStrategy = "LessThanInitialShare" + PreemeptionStrategyClassical PreemptionStrategy = "Classical" + PreemeptionStrategyFairsharing PreemptionStrategy = "FairSharing" ) -type FairSharing struct { - // preemptionStrategies indicates which constraints should a preemption satisfy. - // The preemption algorithm will only use the next strategy in the list if the - // incoming workload (preemptor) doesn't fit after using the previous strategies. - // Possible values are: - // - LessThanOrEqualToFinalShare: Only preempt a workload - // if the share of the preemptor ClusterQueue - // with the preemptor workload is less than - // or equal to the share of the preemptee ClusterQueue - // without the workload to be preempted. - // This strategy might favor preemption of smaller workloads - // in the preemptee ClusterQueue, - // regardless of priority or start time, in an effort to keep the share of the ClusterQueue - // as high as possible. - // - LessThanInitialShare: Only preempt a workload if the share of the preemptor ClusterQueue - // with the incoming workload is strictly less than the share of the preemptee ClusterQueue. - // This strategy doesn't depend on the share usage of the workload being preempted. - // As a result, the strategy chooses to preempt workloads with the lowest priority and - // newest start time first. - // The default strategy is ["LessThanOrEqualToFinalShare", "LessThanInitialShare"]. - // +kubebuilder:validation:MaxItems=2 +type Premption struct { + // preemptionStrategies are the types of preemption kueue allows. + // Kueue has two types of preemption: classical and fair sharing. // +optional - PreemptionStrategies []PreemptionStrategy `json:"preemptionStrategies,omitempty"` + PreemptionStrategies PreemptionStrategy `json:"preemptionStrategies"` } diff --git a/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml b/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml index a6b2658893f..a57b92e9145 100644 --- a/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml +++ b/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml @@ -48,35 +48,6 @@ spec: config is the desired configuration for the Kueue operator. properties: - fairSharing: - description: fairSharing TODO not done yet - properties: - preemptionStrategies: - description: |- - preemptionStrategies indicates which constraints should a preemption satisfy. - The preemption algorithm will only use the next strategy in the list if the - incoming workload (preemptor) doesn't fit after using the previous strategies. - Possible values are: - - LessThanOrEqualToFinalShare: Only preempt a workload if the share of the preemptor CQ - with the preemptor workload is less than or equal to the share of the preemptee CQ - without the workload to be preempted. - This strategy might favor preemption of smaller workloads in the preemptee CQ, - regardless of priority or start time, in an effort to keep the share of the CQ - as high as possible. - - LessThanInitialShare: Only preempt a workload if the share of the preemptor CQ - with the incoming workload is strictly less than the share of the preemptee CQ. - This strategy doesn't depend on the share usage of the workload being preempted. - As a result, the strategy chooses to preempt workloads with the lowest priority and - newest start time first. - The default strategy is ["LessThanOrEqualToFinalShare", "LessThanInitialShare"]. - items: - enum: - - LessThanOrEqualToFinalShare - - LessThanInitialShare - type: string - maxItems: 2 - type: array - type: object integrations: description: |- integrations is a required field that configures the Kueue's workload integrations. @@ -217,17 +188,22 @@ spec: - frameworks type: object kueueGangSchedulingPolicy: - description: kueueGangSchedulingPolicy controls how Kueue admits - workloads + description: |- + kueueGangSchedulingPolicy controls how Kueue admits workloads. + Gang Scheduling is the act of all or nothing scheduling. + Kueue provides this ability. + This field is optional. properties: byWorkload: description: |- byWorkload controls how admission is done. - The options are Sequential and Parallel. - Sequential means workloads are admitted in sequential order and Kueue waits - for those workloads to be ready. - Parallel admits workloads in parallel and does not wait on these workloads to be ready. - Workloads can be evicted in the background if they are not ready within default settings. + When admission is set to Sequential, only pods from the currently processing workload will be admitted. + Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. + Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, + but provides a higher guarantee of workloads scheduling all pods together successfully. + When set to Parallel, pods from any workload will be admitted at any time. + This may lead to a deadlock where workloads are in contention for cluster capacity and + pods from another workload having successfully scheduled prevent pods from the current workload scheduling. enum: - Parallel - Sequential @@ -248,10 +224,25 @@ spec: - Disabled type: string type: object + premption: + description: |- + premption is the process of evicting one or more admitted Workloads to accommodate another Workload. + Kueue has classical premption and preemption via fair sharing. + properties: + preemptionStrategies: + description: |- + preemptionStrategies are the types of preemption kueue allows. + Kueue has two types of preemption: classical and fair sharing. + enum: + - Classical + - FairSharing + type: string + type: object queueLabelPolicy: description: |- queueLabelPolicy controls how kueue manages workloads The default behavior of Kueue will manage workloads that have a queue-name label. + This field is optional. properties: queueLabelPolicy: description: |- @@ -261,7 +252,7 @@ spec: If set to QueueNameRequired, then those jobs will be suspended and never started unless they are assigned a queue and eventually admitted. This also applies to jobs created before starting the kueue controller. - Defaults to QueueNameOptional; therefore, those jobs are not managed and if they are created + Defaults to QueueNameRequired; therefore, those jobs are not managed and if they are created unsuspended, they will start immediately. enum: - QueueNameRequired diff --git a/operator/v1alpha1/zz_generated.deepcopy.go b/operator/v1alpha1/zz_generated.deepcopy.go index 8e34878908b..47c3462dfa7 100644 --- a/operator/v1alpha1/zz_generated.deepcopy.go +++ b/operator/v1alpha1/zz_generated.deepcopy.go @@ -272,27 +272,6 @@ func (in *ExternalFramework) DeepCopy() *ExternalFramework { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *FairSharing) DeepCopyInto(out *FairSharing) { - *out = *in - if in.PreemptionStrategies != nil { - in, out := &in.PreemptionStrategies, &out.PreemptionStrategies - *out = make([]PreemptionStrategy, len(*in)) - copy(*out, *in) - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FairSharing. -func (in *FairSharing) DeepCopy() *FairSharing { - if in == nil { - return nil - } - out := new(FairSharing) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GenerationHistory) DeepCopyInto(out *GenerationHistory) { *out = *in @@ -486,7 +465,7 @@ func (in *KueueConfiguration) DeepCopyInto(out *KueueConfiguration) { in.Integrations.DeepCopyInto(&out.Integrations) out.QueueLabelPolicy = in.QueueLabelPolicy out.KueueGangSchedulingPolicy = in.KueueGangSchedulingPolicy - in.FairSharing.DeepCopyInto(&out.FairSharing) + out.Premption = in.Premption return } @@ -799,6 +778,22 @@ func (in *OperatorStatus) DeepCopy() *OperatorStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *Premption) DeepCopyInto(out *Premption) { + *out = *in + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Premption. +func (in *Premption) DeepCopy() *Premption { + if in == nil { + return nil + } + out := new(Premption) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *QueueLabelPolicy) DeepCopyInto(out *QueueLabelPolicy) { *out = *in diff --git a/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml b/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml index 37f04ef7de7..1eeb16f5e99 100644 --- a/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml +++ b/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml @@ -46,35 +46,6 @@ spec: config is the desired configuration for the Kueue operator. properties: - fairSharing: - description: fairSharing TODO not done yet - properties: - preemptionStrategies: - description: |- - preemptionStrategies indicates which constraints should a preemption satisfy. - The preemption algorithm will only use the next strategy in the list if the - incoming workload (preemptor) doesn't fit after using the previous strategies. - Possible values are: - - LessThanOrEqualToFinalShare: Only preempt a workload if the share of the preemptor CQ - with the preemptor workload is less than or equal to the share of the preemptee CQ - without the workload to be preempted. - This strategy might favor preemption of smaller workloads in the preemptee CQ, - regardless of priority or start time, in an effort to keep the share of the CQ - as high as possible. - - LessThanInitialShare: Only preempt a workload if the share of the preemptor CQ - with the incoming workload is strictly less than the share of the preemptee CQ. - This strategy doesn't depend on the share usage of the workload being preempted. - As a result, the strategy chooses to preempt workloads with the lowest priority and - newest start time first. - The default strategy is ["LessThanOrEqualToFinalShare", "LessThanInitialShare"]. - items: - enum: - - LessThanOrEqualToFinalShare - - LessThanInitialShare - type: string - maxItems: 2 - type: array - type: object integrations: description: |- integrations is a required field that configures the Kueue's workload integrations. @@ -215,17 +186,22 @@ spec: - frameworks type: object kueueGangSchedulingPolicy: - description: kueueGangSchedulingPolicy controls how Kueue admits - workloads + description: |- + kueueGangSchedulingPolicy controls how Kueue admits workloads. + Gang Scheduling is the act of all or nothing scheduling. + Kueue provides this ability. + This field is optional. properties: byWorkload: description: |- byWorkload controls how admission is done. - The options are Sequential and Parallel. - Sequential means workloads are admitted in sequential order and Kueue waits - for those workloads to be ready. - Parallel admits workloads in parallel and does not wait on these workloads to be ready. - Workloads can be evicted in the background if they are not ready within default settings. + When admission is set to Sequential, only pods from the currently processing workload will be admitted. + Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. + Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, + but provides a higher guarantee of workloads scheduling all pods together successfully. + When set to Parallel, pods from any workload will be admitted at any time. + This may lead to a deadlock where workloads are in contention for cluster capacity and + pods from another workload having successfully scheduled prevent pods from the current workload scheduling. enum: - Parallel - Sequential @@ -246,10 +222,25 @@ spec: - Disabled type: string type: object + premption: + description: |- + premption is the process of evicting one or more admitted Workloads to accommodate another Workload. + Kueue has classical premption and preemption via fair sharing. + properties: + preemptionStrategies: + description: |- + preemptionStrategies are the types of preemption kueue allows. + Kueue has two types of preemption: classical and fair sharing. + enum: + - Classical + - FairSharing + type: string + type: object queueLabelPolicy: description: |- queueLabelPolicy controls how kueue manages workloads The default behavior of Kueue will manage workloads that have a queue-name label. + This field is optional. properties: queueLabelPolicy: description: |- @@ -259,7 +250,7 @@ spec: If set to QueueNameRequired, then those jobs will be suspended and never started unless they are assigned a queue and eventually admitted. This also applies to jobs created before starting the kueue controller. - Defaults to QueueNameOptional; therefore, those jobs are not managed and if they are created + Defaults to QueueNameRequired; therefore, those jobs are not managed and if they are created unsuspended, they will start immediately. enum: - QueueNameRequired diff --git a/operator/v1alpha1/zz_generated.swagger_doc_generated.go b/operator/v1alpha1/zz_generated.swagger_doc_generated.go index 18b270ef876..c1ba7460583 100644 --- a/operator/v1alpha1/zz_generated.swagger_doc_generated.go +++ b/operator/v1alpha1/zz_generated.swagger_doc_generated.go @@ -268,14 +268,6 @@ func (ExternalFramework) SwaggerDoc() map[string]string { return map_ExternalFramework } -var map_FairSharing = map[string]string{ - "preemptionStrategies": "preemptionStrategies indicates which constraints should a preemption satisfy. The preemption algorithm will only use the next strategy in the list if the incoming workload (preemptor) doesn't fit after using the previous strategies. Possible values are: - LessThanOrEqualToFinalShare: Only preempt a workload if the share of the preemptor CQ\n with the preemptor workload is less than or equal to the share of the preemptee CQ\n without the workload to be preempted.\n This strategy might favor preemption of smaller workloads in the preemptee CQ,\n regardless of priority or start time, in an effort to keep the share of the CQ\n as high as possible.\n- LessThanInitialShare: Only preempt a workload if the share of the preemptor CQ\n with the incoming workload is strictly less than the share of the preemptee CQ.\n This strategy doesn't depend on the share usage of the workload being preempted.\n As a result, the strategy chooses to preempt workloads with the lowest priority and\n newest start time first.\nThe default strategy is [\"LessThanOrEqualToFinalShare\", \"LessThanInitialShare\"].", -} - -func (FairSharing) SwaggerDoc() map[string]string { - return map_FairSharing -} - var map_Integrations = map[string]string{ "": "This is the integrations for Kueue. Kueue uses these apis to determine which jobs will be managed by Kueue.", "frameworks": "frameworks are a unique list of names to be enabled. This is required and must have at least one element. Each framework represents a type of job that Kueue will manage. Frameworks are a list of frameworks that Kueue has support for. The allowed values are BatchJob, RayJob, RayCluster, JobSet, MPIJob, PaddleJob, PytorchJob, TFJob, XGBoostJob, AppWrapper, Pod, Deployment, StatefulSet and LeaderWorkerSet.", @@ -300,9 +292,9 @@ func (Kueue) SwaggerDoc() map[string]string { var map_KueueConfiguration = map[string]string{ "integrations": "integrations is a required field that configures the Kueue's workload integrations. Kueue has both standard integrations, known as job frameworks, and external integrations known as external frameworks. Kueue will only manage workloads that correspond to the specified integrations.", - "queueLabelPolicy": "queueLabelPolicy controls how kueue manages workloads The default behavior of Kueue will manage workloads that have a queue-name label.", - "kueueGangSchedulingPolicy": "kueueGangSchedulingPolicy controls how Kueue admits workloads", - "fairSharing": "fairSharing TODO not done yet", + "queueLabelPolicy": "queueLabelPolicy controls how kueue manages workloads The default behavior of Kueue will manage workloads that have a queue-name label. This field is optional.", + "kueueGangSchedulingPolicy": "kueueGangSchedulingPolicy controls how Kueue admits workloads. Gang Scheduling is the act of all or nothing scheduling. Kueue provides this ability. This field is optional.", + "premption": "premption is the process of evicting one or more admitted Workloads to accommodate another Workload. Kueue has classical premption and preemption via fair sharing.", } func (KueueConfiguration) SwaggerDoc() map[string]string { @@ -312,7 +304,7 @@ func (KueueConfiguration) SwaggerDoc() map[string]string { var map_KueueGangSchedulingPolicy = map[string]string{ "": "Kueue provides the ability to admit workloads all in one (gang admission) and evicts workloads if they are not ready within a specific time.", "policy": "policy allows for changing the kinds of gang scheduling Kueue does. This is an optional field. The allowed values are ByWorkload and Disabled. The default value will be Disabled. ByWorkload allows for configuration how admission is performed for Kueue.", - "byWorkload": "byWorkload controls how admission is done. The options are Sequential and Parallel. Sequential means workloads are admitted in sequential order and Kueue waits for those workloads to be ready. Parallel admits workloads in parallel and does not wait on these workloads to be ready. Workloads can be evicted in the background if they are not ready within default settings.", + "byWorkload": "byWorkload controls how admission is done. When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling.", } func (KueueGangSchedulingPolicy) SwaggerDoc() map[string]string { @@ -353,8 +345,16 @@ func (LabelKeys) SwaggerDoc() map[string]string { return map_LabelKeys } +var map_Premption = map[string]string{ + "preemptionStrategies": "preemptionStrategies are the types of preemption kueue allows. Kueue has two types of preemption: classical and fair sharing.", +} + +func (Premption) SwaggerDoc() map[string]string { + return map_Premption +} + var map_QueueLabelPolicy = map[string]string{ - "queueLabelPolicy": "queueLabelPolicy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. The allowed values are QueueNameRequired and QueueNameOptional. If set to QueueNameRequired, then those jobs will be suspended and never started unless they are assigned a queue and eventually admitted. This also applies to jobs created before starting the kueue controller. Defaults to QueueNameOptional; therefore, those jobs are not managed and if they are created unsuspended, they will start immediately.", + "queueLabelPolicy": "queueLabelPolicy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. The allowed values are QueueNameRequired and QueueNameOptional. If set to QueueNameRequired, then those jobs will be suspended and never started unless they are assigned a queue and eventually admitted. This also applies to jobs created before starting the kueue controller. Defaults to QueueNameRequired; therefore, those jobs are not managed and if they are created unsuspended, they will start immediately.", } func (QueueLabelPolicy) SwaggerDoc() map[string]string { From 6f95ae79d5cec244e955157d68d5c7b424e34585 Mon Sep 17 00:00:00 2001 From: Kevin Hannon Date: Mon, 31 Mar 2025 16:11:37 -0400 Subject: [PATCH 23/42] update and lint --- openapi/generated_openapi/zz_generated.openapi.go | 4 ++-- openapi/openapi.json | 4 ++-- operator/v1alpha1/types_kueue.go | 4 ++-- operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml | 4 ++-- .../kueue.operator.openshift.io/AAA_ungated.yaml | 4 ++-- operator/v1alpha1/zz_generated.swagger_doc_generated.go | 2 +- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/openapi/generated_openapi/zz_generated.openapi.go b/openapi/generated_openapi/zz_generated.openapi.go index e23d80b659b..9c371a119cf 100644 --- a/openapi/generated_openapi/zz_generated.openapi.go +++ b/openapi/generated_openapi/zz_generated.openapi.go @@ -58040,9 +58040,9 @@ func schema_openshift_api_operator_v1alpha1_Premption(ref common.ReferenceCallba SchemaProps: spec.SchemaProps{ Type: []string{"object"}, Properties: map[string]spec.Schema{ - "preemptionStrategies": { + "preemptionStrategy": { SchemaProps: spec.SchemaProps{ - Description: "preemptionStrategies are the types of preemption kueue allows. Kueue has two types of preemption: classical and fair sharing.", + Description: "preemptionStrategy are the types of preemption kueue allows. Kueue has two types of preemption: classical and fair sharing.", Default: "", Type: []string{"string"}, Format: "", diff --git a/openapi/openapi.json b/openapi/openapi.json index c32a7626b25..7071322cd75 100644 --- a/openapi/openapi.json +++ b/openapi/openapi.json @@ -33933,8 +33933,8 @@ "com.github.openshift.api.operator.v1alpha1.Premption": { "type": "object", "properties": { - "preemptionStrategies": { - "description": "preemptionStrategies are the types of preemption kueue allows. Kueue has two types of preemption: classical and fair sharing.", + "preemptionStrategy": { + "description": "preemptionStrategy are the types of preemption kueue allows. Kueue has two types of preemption: classical and fair sharing.", "type": "string", "default": "" } diff --git a/operator/v1alpha1/types_kueue.go b/operator/v1alpha1/types_kueue.go index 4a189a6d2a0..fcb4c957a62 100644 --- a/operator/v1alpha1/types_kueue.go +++ b/operator/v1alpha1/types_kueue.go @@ -259,8 +259,8 @@ const ( ) type Premption struct { - // preemptionStrategies are the types of preemption kueue allows. + // preemptionStrategy are the types of preemption kueue allows. // Kueue has two types of preemption: classical and fair sharing. // +optional - PreemptionStrategies PreemptionStrategy `json:"preemptionStrategies"` + PreemptionStrategy PreemptionStrategy `json:"preemptionStrategy"` } diff --git a/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml b/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml index a57b92e9145..8f47b2bc090 100644 --- a/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml +++ b/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml @@ -229,9 +229,9 @@ spec: premption is the process of evicting one or more admitted Workloads to accommodate another Workload. Kueue has classical premption and preemption via fair sharing. properties: - preemptionStrategies: + preemptionStrategy: description: |- - preemptionStrategies are the types of preemption kueue allows. + preemptionStrategy are the types of preemption kueue allows. Kueue has two types of preemption: classical and fair sharing. enum: - Classical diff --git a/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml b/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml index 1eeb16f5e99..79f015b5b60 100644 --- a/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml +++ b/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml @@ -227,9 +227,9 @@ spec: premption is the process of evicting one or more admitted Workloads to accommodate another Workload. Kueue has classical premption and preemption via fair sharing. properties: - preemptionStrategies: + preemptionStrategy: description: |- - preemptionStrategies are the types of preemption kueue allows. + preemptionStrategy are the types of preemption kueue allows. Kueue has two types of preemption: classical and fair sharing. enum: - Classical diff --git a/operator/v1alpha1/zz_generated.swagger_doc_generated.go b/operator/v1alpha1/zz_generated.swagger_doc_generated.go index c1ba7460583..c7439c52e74 100644 --- a/operator/v1alpha1/zz_generated.swagger_doc_generated.go +++ b/operator/v1alpha1/zz_generated.swagger_doc_generated.go @@ -346,7 +346,7 @@ func (LabelKeys) SwaggerDoc() map[string]string { } var map_Premption = map[string]string{ - "preemptionStrategies": "preemptionStrategies are the types of preemption kueue allows. Kueue has two types of preemption: classical and fair sharing.", + "preemptionStrategy": "preemptionStrategy are the types of preemption kueue allows. Kueue has two types of preemption: classical and fair sharing.", } func (Premption) SwaggerDoc() map[string]string { From 64c3bd7c6a09bff7726806c191df7555753a0d3d Mon Sep 17 00:00:00 2001 From: Kevin Hannon Date: Tue, 1 Apr 2025 11:27:44 -0400 Subject: [PATCH 24/42] api typo updates --- .../generated_openapi/zz_generated.openapi.go | 14 +++++++------- openapi/openapi.json | 10 +++++----- operator/v1alpha1/types_kueue.go | 17 ++++++++++++++--- .../zz_generated.crd-manifests/kueue.crd.yaml | 15 +++++++++++++-- operator/v1alpha1/zz_generated.deepcopy.go | 10 +++++----- .../AAA_ungated.yaml | 15 +++++++++++++-- .../zz_generated.swagger_doc_generated.go | 10 +++++----- 7 files changed, 62 insertions(+), 29 deletions(-) diff --git a/openapi/generated_openapi/zz_generated.openapi.go b/openapi/generated_openapi/zz_generated.openapi.go index 9c371a119cf..4caa467bf81 100644 --- a/openapi/generated_openapi/zz_generated.openapi.go +++ b/openapi/generated_openapi/zz_generated.openapi.go @@ -1133,7 +1133,7 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "github.com/openshift/api/operator/v1alpha1.OperatorCondition": schema_openshift_api_operator_v1alpha1_OperatorCondition(ref), "github.com/openshift/api/operator/v1alpha1.OperatorSpec": schema_openshift_api_operator_v1alpha1_OperatorSpec(ref), "github.com/openshift/api/operator/v1alpha1.OperatorStatus": schema_openshift_api_operator_v1alpha1_OperatorStatus(ref), - "github.com/openshift/api/operator/v1alpha1.Premption": schema_openshift_api_operator_v1alpha1_Premption(ref), + "github.com/openshift/api/operator/v1alpha1.Preemption": schema_openshift_api_operator_v1alpha1_Preemption(ref), "github.com/openshift/api/operator/v1alpha1.QueueLabelPolicy": schema_openshift_api_operator_v1alpha1_QueueLabelPolicy(ref), "github.com/openshift/api/operator/v1alpha1.RepositoryDigestMirrors": schema_openshift_api_operator_v1alpha1_RepositoryDigestMirrors(ref), "github.com/openshift/api/operator/v1alpha1.StaticPodOperatorStatus": schema_openshift_api_operator_v1alpha1_StaticPodOperatorStatus(ref), @@ -57274,11 +57274,11 @@ func schema_openshift_api_operator_v1alpha1_KueueConfiguration(ref common.Refere Ref: ref("github.com/openshift/api/operator/v1alpha1.KueueGangSchedulingPolicy"), }, }, - "premption": { + "preemption": { SchemaProps: spec.SchemaProps{ - Description: "premption is the process of evicting one or more admitted Workloads to accommodate another Workload. Kueue has classical premption and preemption via fair sharing.", + Description: "preemption is the process of evicting one or more admitted Workloads to accommodate another Workload. Kueue has classical premption and preemption via fair sharing.", Default: map[string]interface{}{}, - Ref: ref("github.com/openshift/api/operator/v1alpha1.Premption"), + Ref: ref("github.com/openshift/api/operator/v1alpha1.Preemption"), }, }, }, @@ -57286,7 +57286,7 @@ func schema_openshift_api_operator_v1alpha1_KueueConfiguration(ref common.Refere }, }, Dependencies: []string{ - "github.com/openshift/api/operator/v1alpha1.Integrations", "github.com/openshift/api/operator/v1alpha1.KueueGangSchedulingPolicy", "github.com/openshift/api/operator/v1alpha1.Premption", "github.com/openshift/api/operator/v1alpha1.QueueLabelPolicy"}, + "github.com/openshift/api/operator/v1alpha1.Integrations", "github.com/openshift/api/operator/v1alpha1.KueueGangSchedulingPolicy", "github.com/openshift/api/operator/v1alpha1.Preemption", "github.com/openshift/api/operator/v1alpha1.QueueLabelPolicy"}, } } @@ -58034,7 +58034,7 @@ func schema_openshift_api_operator_v1alpha1_OperatorStatus(ref common.ReferenceC } } -func schema_openshift_api_operator_v1alpha1_Premption(ref common.ReferenceCallback) common.OpenAPIDefinition { +func schema_openshift_api_operator_v1alpha1_Preemption(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ SchemaProps: spec.SchemaProps{ @@ -58042,7 +58042,7 @@ func schema_openshift_api_operator_v1alpha1_Premption(ref common.ReferenceCallba Properties: map[string]spec.Schema{ "preemptionStrategy": { SchemaProps: spec.SchemaProps{ - Description: "preemptionStrategy are the types of preemption kueue allows. Kueue has two types of preemption: classical and fair sharing.", + Description: "preemptionStrategy are the types of preemption kueue allows. Kueue has two types of preemption: classical and fair sharing. Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. FairSharing is a more heavy weight algorithm. ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort.", Default: "", Type: []string{"string"}, Format: "", diff --git a/openapi/openapi.json b/openapi/openapi.json index 7071322cd75..b556d071573 100644 --- a/openapi/openapi.json +++ b/openapi/openapi.json @@ -33460,10 +33460,10 @@ "default": {}, "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.KueueGangSchedulingPolicy" }, - "premption": { - "description": "premption is the process of evicting one or more admitted Workloads to accommodate another Workload. Kueue has classical premption and preemption via fair sharing.", + "preemption": { + "description": "preemption is the process of evicting one or more admitted Workloads to accommodate another Workload. Kueue has classical premption and preemption via fair sharing.", "default": {}, - "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.Premption" + "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.Preemption" }, "queueLabelPolicy": { "description": "queueLabelPolicy controls how kueue manages workloads The default behavior of Kueue will manage workloads that have a queue-name label. This field is optional.", @@ -33930,11 +33930,11 @@ } } }, - "com.github.openshift.api.operator.v1alpha1.Premption": { + "com.github.openshift.api.operator.v1alpha1.Preemption": { "type": "object", "properties": { "preemptionStrategy": { - "description": "preemptionStrategy are the types of preemption kueue allows. Kueue has two types of preemption: classical and fair sharing.", + "description": "preemptionStrategy are the types of preemption kueue allows. Kueue has two types of preemption: classical and fair sharing. Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. FairSharing is a more heavy weight algorithm. ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort.", "type": "string", "default": "" } diff --git a/operator/v1alpha1/types_kueue.go b/operator/v1alpha1/types_kueue.go index fcb4c957a62..a62f0260e9f 100644 --- a/operator/v1alpha1/types_kueue.go +++ b/operator/v1alpha1/types_kueue.go @@ -58,10 +58,10 @@ type KueueConfiguration struct { // This field is optional. // +optional KueueGangSchedulingPolicy KueueGangSchedulingPolicy `json:"kueueGangSchedulingPolicy,omitempty"` - // premption is the process of evicting one or more admitted Workloads to accommodate another Workload. + // preemption is the process of evicting one or more admitted Workloads to accommodate another Workload. // Kueue has classical premption and preemption via fair sharing. // +optional - Premption Premption `json:"premption,omitempty"` + Preemption Preemption `json:"preemption,omitempty"` } // KueueStatus defines the observed state of Kueue @@ -258,9 +258,20 @@ const ( PreemeptionStrategyFairsharing PreemptionStrategy = "FairSharing" ) -type Premption struct { +type Preemption struct { // preemptionStrategy are the types of preemption kueue allows. // Kueue has two types of preemption: classical and fair sharing. + // Classical means that an incoming workload, which does + // not fit within the unusued quota, is eligible to issue preemptions + // when the requests of the workload are below the + // resource flavor's nominal quota or borrowWithinCohort is enabled + // on the Cluster Queue. + // FairSharing is a more heavy weight algorithm. + // ClusterQueues with pending Workloads can preempt other Workloads + // in their cohort until the preempting ClusterQueue + // obtains an equal or weighted share of the borrowable resources. + // The borrowable resources are the unused nominal quota + // of all the ClusterQueues in the cohort. // +optional PreemptionStrategy PreemptionStrategy `json:"preemptionStrategy"` } diff --git a/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml b/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml index 8f47b2bc090..2456ddea696 100644 --- a/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml +++ b/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml @@ -224,15 +224,26 @@ spec: - Disabled type: string type: object - premption: + preemption: description: |- - premption is the process of evicting one or more admitted Workloads to accommodate another Workload. + preemption is the process of evicting one or more admitted Workloads to accommodate another Workload. Kueue has classical premption and preemption via fair sharing. properties: preemptionStrategy: description: |- preemptionStrategy are the types of preemption kueue allows. Kueue has two types of preemption: classical and fair sharing. + Classical means that an incoming workload, which does + not fit within the unusued quota, is eligible to issue preemptions + when the requests of the workload are below the + resource flavor's nominal quota or borrowWithinCohort is enabled + on the Cluster Queue. + FairSharing is a more heavy weight algorithm. + ClusterQueues with pending Workloads can preempt other Workloads + in their cohort until the preempting ClusterQueue + obtains an equal or weighted share of the borrowable resources. + The borrowable resources are the unused nominal quota + of all the ClusterQueues in the cohort. enum: - Classical - FairSharing diff --git a/operator/v1alpha1/zz_generated.deepcopy.go b/operator/v1alpha1/zz_generated.deepcopy.go index 47c3462dfa7..2463b95f25d 100644 --- a/operator/v1alpha1/zz_generated.deepcopy.go +++ b/operator/v1alpha1/zz_generated.deepcopy.go @@ -465,7 +465,7 @@ func (in *KueueConfiguration) DeepCopyInto(out *KueueConfiguration) { in.Integrations.DeepCopyInto(&out.Integrations) out.QueueLabelPolicy = in.QueueLabelPolicy out.KueueGangSchedulingPolicy = in.KueueGangSchedulingPolicy - out.Premption = in.Premption + out.Preemption = in.Preemption return } @@ -779,17 +779,17 @@ func (in *OperatorStatus) DeepCopy() *OperatorStatus { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *Premption) DeepCopyInto(out *Premption) { +func (in *Preemption) DeepCopyInto(out *Preemption) { *out = *in return } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Premption. -func (in *Premption) DeepCopy() *Premption { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Preemption. +func (in *Preemption) DeepCopy() *Preemption { if in == nil { return nil } - out := new(Premption) + out := new(Preemption) in.DeepCopyInto(out) return out } diff --git a/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml b/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml index 79f015b5b60..aa57ea5e27d 100644 --- a/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml +++ b/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml @@ -222,15 +222,26 @@ spec: - Disabled type: string type: object - premption: + preemption: description: |- - premption is the process of evicting one or more admitted Workloads to accommodate another Workload. + preemption is the process of evicting one or more admitted Workloads to accommodate another Workload. Kueue has classical premption and preemption via fair sharing. properties: preemptionStrategy: description: |- preemptionStrategy are the types of preemption kueue allows. Kueue has two types of preemption: classical and fair sharing. + Classical means that an incoming workload, which does + not fit within the unusued quota, is eligible to issue preemptions + when the requests of the workload are below the + resource flavor's nominal quota or borrowWithinCohort is enabled + on the Cluster Queue. + FairSharing is a more heavy weight algorithm. + ClusterQueues with pending Workloads can preempt other Workloads + in their cohort until the preempting ClusterQueue + obtains an equal or weighted share of the borrowable resources. + The borrowable resources are the unused nominal quota + of all the ClusterQueues in the cohort. enum: - Classical - FairSharing diff --git a/operator/v1alpha1/zz_generated.swagger_doc_generated.go b/operator/v1alpha1/zz_generated.swagger_doc_generated.go index c7439c52e74..9e09feca184 100644 --- a/operator/v1alpha1/zz_generated.swagger_doc_generated.go +++ b/operator/v1alpha1/zz_generated.swagger_doc_generated.go @@ -294,7 +294,7 @@ var map_KueueConfiguration = map[string]string{ "integrations": "integrations is a required field that configures the Kueue's workload integrations. Kueue has both standard integrations, known as job frameworks, and external integrations known as external frameworks. Kueue will only manage workloads that correspond to the specified integrations.", "queueLabelPolicy": "queueLabelPolicy controls how kueue manages workloads The default behavior of Kueue will manage workloads that have a queue-name label. This field is optional.", "kueueGangSchedulingPolicy": "kueueGangSchedulingPolicy controls how Kueue admits workloads. Gang Scheduling is the act of all or nothing scheduling. Kueue provides this ability. This field is optional.", - "premption": "premption is the process of evicting one or more admitted Workloads to accommodate another Workload. Kueue has classical premption and preemption via fair sharing.", + "preemption": "preemption is the process of evicting one or more admitted Workloads to accommodate another Workload. Kueue has classical premption and preemption via fair sharing.", } func (KueueConfiguration) SwaggerDoc() map[string]string { @@ -345,12 +345,12 @@ func (LabelKeys) SwaggerDoc() map[string]string { return map_LabelKeys } -var map_Premption = map[string]string{ - "preemptionStrategy": "preemptionStrategy are the types of preemption kueue allows. Kueue has two types of preemption: classical and fair sharing.", +var map_Preemption = map[string]string{ + "preemptionStrategy": "preemptionStrategy are the types of preemption kueue allows. Kueue has two types of preemption: classical and fair sharing. Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. FairSharing is a more heavy weight algorithm. ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort.", } -func (Premption) SwaggerDoc() map[string]string { - return map_Premption +func (Preemption) SwaggerDoc() map[string]string { + return map_Preemption } var map_QueueLabelPolicy = map[string]string{ From b367dd2fc8fc17db205d1ff83fa98857bab85663 Mon Sep 17 00:00:00 2001 From: Kevin Hannon Date: Wed, 2 Apr 2025 15:02:46 -0400 Subject: [PATCH 25/42] use pointers everywhere --- .../generated_openapi/zz_generated.openapi.go | 83 +++++++------- openapi/openapi.json | 63 +++++------ operator/v1alpha1/types_kueue.go | 71 ++++++------ .../zz_generated.crd-manifests/kueue.crd.yaml | 105 ++++++++++-------- operator/v1alpha1/zz_generated.deepcopy.go | 70 ++++++++---- .../AAA_ungated.yaml | 105 ++++++++++-------- .../zz_generated.swagger_doc_generated.go | 36 +++--- 7 files changed, 292 insertions(+), 241 deletions(-) diff --git a/openapi/generated_openapi/zz_generated.openapi.go b/openapi/generated_openapi/zz_generated.openapi.go index 4caa467bf81..28f48db2da5 100644 --- a/openapi/generated_openapi/zz_generated.openapi.go +++ b/openapi/generated_openapi/zz_generated.openapi.go @@ -1111,6 +1111,7 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "github.com/openshift/api/operator/v1alpha1.EtcdBackupSpec": schema_openshift_api_operator_v1alpha1_EtcdBackupSpec(ref), "github.com/openshift/api/operator/v1alpha1.EtcdBackupStatus": schema_openshift_api_operator_v1alpha1_EtcdBackupStatus(ref), "github.com/openshift/api/operator/v1alpha1.ExternalFramework": schema_openshift_api_operator_v1alpha1_ExternalFramework(ref), + "github.com/openshift/api/operator/v1alpha1.GangSchedulingPolicy": schema_openshift_api_operator_v1alpha1_GangSchedulingPolicy(ref), "github.com/openshift/api/operator/v1alpha1.GenerationHistory": schema_openshift_api_operator_v1alpha1_GenerationHistory(ref), "github.com/openshift/api/operator/v1alpha1.GenericOperatorConfig": schema_openshift_api_operator_v1alpha1_GenericOperatorConfig(ref), "github.com/openshift/api/operator/v1alpha1.ImageContentSourcePolicy": schema_openshift_api_operator_v1alpha1_ImageContentSourcePolicy(ref), @@ -1119,7 +1120,6 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "github.com/openshift/api/operator/v1alpha1.Integrations": schema_openshift_api_operator_v1alpha1_Integrations(ref), "github.com/openshift/api/operator/v1alpha1.Kueue": schema_openshift_api_operator_v1alpha1_Kueue(ref), "github.com/openshift/api/operator/v1alpha1.KueueConfiguration": schema_openshift_api_operator_v1alpha1_KueueConfiguration(ref), - "github.com/openshift/api/operator/v1alpha1.KueueGangSchedulingPolicy": schema_openshift_api_operator_v1alpha1_KueueGangSchedulingPolicy(ref), "github.com/openshift/api/operator/v1alpha1.KueueList": schema_openshift_api_operator_v1alpha1_KueueList(ref), "github.com/openshift/api/operator/v1alpha1.KueueOperandSpec": schema_openshift_api_operator_v1alpha1_KueueOperandSpec(ref), "github.com/openshift/api/operator/v1alpha1.KueueStatus": schema_openshift_api_operator_v1alpha1_KueueStatus(ref), @@ -56888,6 +56888,33 @@ func schema_openshift_api_operator_v1alpha1_ExternalFramework(ref common.Referen } } +func schema_openshift_api_operator_v1alpha1_GangSchedulingPolicy(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "Kueue provides the ability to admit workloads all in one (gang admission) and evicts workloads if they are not ready within a specific time.", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "policy": { + SchemaProps: spec.SchemaProps{ + Description: "policy allows you to enable and configure gang scheduling. This is an optional field. The allowed values are ByWorkload and Disabled. The default value will be Disabled. When set to ByWorkload, this means each workload is processed and considered for admission as a single unit. Where workloads do not become ready over time, the entire workload may then be evicted and retried at a later time.", + Type: []string{"string"}, + Format: "", + }, + }, + "byWorkload": { + SchemaProps: spec.SchemaProps{ + Description: "byWorkload controls how admission is done. When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling.", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + } +} + func schema_openshift_api_operator_v1alpha1_GenerationHistory(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ @@ -57200,7 +57227,7 @@ func schema_openshift_api_operator_v1alpha1_Kueue(ref common.ReferenceCallback) return common.OpenAPIDefinition{ Schema: spec.Schema{ SchemaProps: spec.SchemaProps{ - Description: "Kueue is the CRD to represent the Kueue operator This CRD defines the configuration that the Kueue Compatibility level 4: No compatibility is provided, the API can change at any point for any reason. These capabilities should not be used by applications needing long term support.", + Description: "Kueue is the CRD to represent the Kueue operator. This CRD defines the configuration for Kueue. Compatibility level 4: No compatibility is provided, the API can change at any point for any reason. These capabilities should not be used by applications needing long term support.", Type: []string{"object"}, Properties: map[string]spec.Schema{ "kind": { @@ -57263,21 +57290,18 @@ func schema_openshift_api_operator_v1alpha1_KueueConfiguration(ref common.Refere "queueLabelPolicy": { SchemaProps: spec.SchemaProps{ Description: "queueLabelPolicy controls how kueue manages workloads The default behavior of Kueue will manage workloads that have a queue-name label. This field is optional.", - Default: map[string]interface{}{}, Ref: ref("github.com/openshift/api/operator/v1alpha1.QueueLabelPolicy"), }, }, - "kueueGangSchedulingPolicy": { + "gangSchedulingPolicy": { SchemaProps: spec.SchemaProps{ - Description: "kueueGangSchedulingPolicy controls how Kueue admits workloads. Gang Scheduling is the act of all or nothing scheduling. Kueue provides this ability. This field is optional.", - Default: map[string]interface{}{}, - Ref: ref("github.com/openshift/api/operator/v1alpha1.KueueGangSchedulingPolicy"), + Description: "gangSchedulingPolicy controls how Kueue admits workloads. Gang Scheduling is the act of all or nothing scheduling, where workloads do not become ready within a certain period, they may be evicted and later retried. This field is optional.", + Ref: ref("github.com/openshift/api/operator/v1alpha1.GangSchedulingPolicy"), }, }, "preemption": { SchemaProps: spec.SchemaProps{ Description: "preemption is the process of evicting one or more admitted Workloads to accommodate another Workload. Kueue has classical premption and preemption via fair sharing.", - Default: map[string]interface{}{}, Ref: ref("github.com/openshift/api/operator/v1alpha1.Preemption"), }, }, @@ -57286,36 +57310,7 @@ func schema_openshift_api_operator_v1alpha1_KueueConfiguration(ref common.Refere }, }, Dependencies: []string{ - "github.com/openshift/api/operator/v1alpha1.Integrations", "github.com/openshift/api/operator/v1alpha1.KueueGangSchedulingPolicy", "github.com/openshift/api/operator/v1alpha1.Preemption", "github.com/openshift/api/operator/v1alpha1.QueueLabelPolicy"}, - } -} - -func schema_openshift_api_operator_v1alpha1_KueueGangSchedulingPolicy(ref common.ReferenceCallback) common.OpenAPIDefinition { - return common.OpenAPIDefinition{ - Schema: spec.Schema{ - SchemaProps: spec.SchemaProps{ - Description: "Kueue provides the ability to admit workloads all in one (gang admission) and evicts workloads if they are not ready within a specific time.", - Type: []string{"object"}, - Properties: map[string]spec.Schema{ - "policy": { - SchemaProps: spec.SchemaProps{ - Description: "policy allows for changing the kinds of gang scheduling Kueue does. This is an optional field. The allowed values are ByWorkload and Disabled. The default value will be Disabled. ByWorkload allows for configuration how admission is performed for Kueue.", - Default: "", - Type: []string{"string"}, - Format: "", - }, - }, - "byWorkload": { - SchemaProps: spec.SchemaProps{ - Description: "byWorkload controls how admission is done. When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling.", - Default: "", - Type: []string{"string"}, - Format: "", - }, - }, - }, - }, - }, + "github.com/openshift/api/operator/v1alpha1.GangSchedulingPolicy", "github.com/openshift/api/operator/v1alpha1.Integrations", "github.com/openshift/api/operator/v1alpha1.Preemption", "github.com/openshift/api/operator/v1alpha1.QueueLabelPolicy"}, } } @@ -57526,12 +57521,14 @@ func schema_openshift_api_operator_v1alpha1_LabelKeys(ref common.ReferenceCallba Properties: map[string]spec.Schema{ "key": { SchemaProps: spec.SchemaProps{ - Description: "key is the label key A label key must be a valid qualified name consisting of a lower-case alphanumeric string, and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. The optional prefix and the name are separate by a forward slash (/).", + Description: "key is the label key. A label key must be a valid qualified name consisting of a lower-case alphanumeric string, and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. The optional prefix and the name are separate by a forward slash (/).", + Default: "", Type: []string{"string"}, Format: "", }, }, }, + Required: []string{"key"}, }, }, } @@ -58042,8 +58039,7 @@ func schema_openshift_api_operator_v1alpha1_Preemption(ref common.ReferenceCallb Properties: map[string]spec.Schema{ "preemptionStrategy": { SchemaProps: spec.SchemaProps{ - Description: "preemptionStrategy are the types of preemption kueue allows. Kueue has two types of preemption: classical and fair sharing. Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. FairSharing is a more heavy weight algorithm. ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort.", - Default: "", + Description: "preemptionStrategy are the types of preemption kueue allows. Kueue has two types of preemption: classical and fair sharing. Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. Fairsharing means that ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort. FairSharing is a more heavy weight algorithm.", Type: []string{"string"}, Format: "", }, @@ -58060,10 +58056,9 @@ func schema_openshift_api_operator_v1alpha1_QueueLabelPolicy(ref common.Referenc SchemaProps: spec.SchemaProps{ Type: []string{"object"}, Properties: map[string]spec.Schema{ - "queueLabelPolicy": { + "policy": { SchemaProps: spec.SchemaProps{ - Description: "queueLabelPolicy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. The allowed values are QueueNameRequired and QueueNameOptional. If set to QueueNameRequired, then those jobs will be suspended and never started unless they are assigned a queue and eventually admitted. This also applies to jobs created before starting the kueue controller. Defaults to QueueNameRequired; therefore, those jobs are not managed and if they are created unsuspended, they will start immediately.", - Default: "", + Description: "policy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. The allowed values are QueueNameRequired and QueueNameOptional. QueueNameOptional means that workloads will be suspended on creation and a label will be added via a mutating webhook. QueueNameRequired means that workloads that are managed by Kueue must have a the label kueue.x-k8s.io/queue-name. If this label is not present on the workload, then Kueue will ignore this workload. Defaults to QueueNameRequired; therefore, those jobs are not managed and if they are created unsuspended, they will start immediately.", Type: []string{"string"}, Format: "", }, diff --git a/openapi/openapi.json b/openapi/openapi.json index b556d071573..0828d7dd4bd 100644 --- a/openapi/openapi.json +++ b/openapi/openapi.json @@ -33232,6 +33232,20 @@ } } }, + "com.github.openshift.api.operator.v1alpha1.GangSchedulingPolicy": { + "description": "Kueue provides the ability to admit workloads all in one (gang admission) and evicts workloads if they are not ready within a specific time.", + "type": "object", + "properties": { + "byWorkload": { + "description": "byWorkload controls how admission is done. When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling.", + "type": "string" + }, + "policy": { + "description": "policy allows you to enable and configure gang scheduling. This is an optional field. The allowed values are ByWorkload and Disabled. The default value will be Disabled. When set to ByWorkload, this means each workload is processed and considered for admission as a single unit. Where workloads do not become ready over time, the entire workload may then be evicted and retried at a later time.", + "type": "string" + } + } + }, "com.github.openshift.api.operator.v1alpha1.GenerationHistory": { "description": "GenerationHistory keeps track of the generation for a given resource so that decisions about forced updated can be made. DEPRECATED: Use fields in v1.GenerationStatus instead", "type": "object", @@ -33413,7 +33427,7 @@ } }, "com.github.openshift.api.operator.v1alpha1.Kueue": { - "description": "Kueue is the CRD to represent the Kueue operator This CRD defines the configuration that the Kueue Compatibility level 4: No compatibility is provided, the API can change at any point for any reason. These capabilities should not be used by applications needing long term support.", + "description": "Kueue is the CRD to represent the Kueue operator. This CRD defines the configuration for Kueue. Compatibility level 4: No compatibility is provided, the API can change at any point for any reason. These capabilities should not be used by applications needing long term support.", "type": "object", "required": [ "spec" @@ -33450,44 +33464,25 @@ "integrations" ], "properties": { + "gangSchedulingPolicy": { + "description": "gangSchedulingPolicy controls how Kueue admits workloads. Gang Scheduling is the act of all or nothing scheduling, where workloads do not become ready within a certain period, they may be evicted and later retried. This field is optional.", + "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.GangSchedulingPolicy" + }, "integrations": { "description": "integrations is a required field that configures the Kueue's workload integrations. Kueue has both standard integrations, known as job frameworks, and external integrations known as external frameworks. Kueue will only manage workloads that correspond to the specified integrations.", "default": {}, "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.Integrations" }, - "kueueGangSchedulingPolicy": { - "description": "kueueGangSchedulingPolicy controls how Kueue admits workloads. Gang Scheduling is the act of all or nothing scheduling. Kueue provides this ability. This field is optional.", - "default": {}, - "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.KueueGangSchedulingPolicy" - }, "preemption": { "description": "preemption is the process of evicting one or more admitted Workloads to accommodate another Workload. Kueue has classical premption and preemption via fair sharing.", - "default": {}, "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.Preemption" }, "queueLabelPolicy": { "description": "queueLabelPolicy controls how kueue manages workloads The default behavior of Kueue will manage workloads that have a queue-name label. This field is optional.", - "default": {}, "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.QueueLabelPolicy" } } }, - "com.github.openshift.api.operator.v1alpha1.KueueGangSchedulingPolicy": { - "description": "Kueue provides the ability to admit workloads all in one (gang admission) and evicts workloads if they are not ready within a specific time.", - "type": "object", - "properties": { - "byWorkload": { - "description": "byWorkload controls how admission is done. When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling.", - "type": "string", - "default": "" - }, - "policy": { - "description": "policy allows for changing the kinds of gang scheduling Kueue does. This is an optional field. The allowed values are ByWorkload and Disabled. The default value will be Disabled. ByWorkload allows for configuration how admission is performed for Kueue.", - "type": "string", - "default": "" - } - } - }, "com.github.openshift.api.operator.v1alpha1.KueueList": { "description": "KueueList contains a list of Kueue\n\nCompatibility level 4: No compatibility is provided, the API can change at any point for any reason. These capabilities should not be used by applications needing long term support.", "type": "object", @@ -33611,10 +33606,14 @@ }, "com.github.openshift.api.operator.v1alpha1.LabelKeys": { "type": "object", + "required": [ + "key" + ], "properties": { "key": { - "description": "key is the label key A label key must be a valid qualified name consisting of a lower-case alphanumeric string, and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. The optional prefix and the name are separate by a forward slash (/).", - "type": "string" + "description": "key is the label key. A label key must be a valid qualified name consisting of a lower-case alphanumeric string, and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. The optional prefix and the name are separate by a forward slash (/).", + "type": "string", + "default": "" } } }, @@ -33934,19 +33933,17 @@ "type": "object", "properties": { "preemptionStrategy": { - "description": "preemptionStrategy are the types of preemption kueue allows. Kueue has two types of preemption: classical and fair sharing. Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. FairSharing is a more heavy weight algorithm. ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort.", - "type": "string", - "default": "" + "description": "preemptionStrategy are the types of preemption kueue allows. Kueue has two types of preemption: classical and fair sharing. Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. Fairsharing means that ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort. FairSharing is a more heavy weight algorithm.", + "type": "string" } } }, "com.github.openshift.api.operator.v1alpha1.QueueLabelPolicy": { "type": "object", "properties": { - "queueLabelPolicy": { - "description": "queueLabelPolicy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. The allowed values are QueueNameRequired and QueueNameOptional. If set to QueueNameRequired, then those jobs will be suspended and never started unless they are assigned a queue and eventually admitted. This also applies to jobs created before starting the kueue controller. Defaults to QueueNameRequired; therefore, those jobs are not managed and if they are created unsuspended, they will start immediately.", - "type": "string", - "default": "" + "policy": { + "description": "policy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. The allowed values are QueueNameRequired and QueueNameOptional. QueueNameOptional means that workloads will be suspended on creation and a label will be added via a mutating webhook. QueueNameRequired means that workloads that are managed by Kueue must have a the label kueue.x-k8s.io/queue-name. If this label is not present on the workload, then Kueue will ignore this workload. Defaults to QueueNameRequired; therefore, those jobs are not managed and if they are created unsuspended, they will start immediately.", + "type": "string" } } }, diff --git a/operator/v1alpha1/types_kueue.go b/operator/v1alpha1/types_kueue.go index a62f0260e9f..5aa25402fb0 100644 --- a/operator/v1alpha1/types_kueue.go +++ b/operator/v1alpha1/types_kueue.go @@ -7,8 +7,8 @@ import ( // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object -// Kueue is the CRD to represent the Kueue operator -// This CRD defines the configuration that the Kueue +// Kueue is the CRD to represent the Kueue operator. +// This CRD defines the configuration for Kueue. // Compatibility level 4: No compatibility is provided, the API can change at any point for any reason. These capabilities should not be used by applications needing long term support. // +openshift:compatibility-gen:level=4 // +kubebuilder:object:root=true @@ -51,17 +51,17 @@ type KueueConfiguration struct { // The default behavior of Kueue will manage workloads that have a queue-name label. // This field is optional. // +optional - QueueLabelPolicy QueueLabelPolicy `json:"queueLabelPolicy,omitempty"` - // kueueGangSchedulingPolicy controls how Kueue admits workloads. - // Gang Scheduling is the act of all or nothing scheduling. - // Kueue provides this ability. + QueueLabelPolicy *QueueLabelPolicy `json:"queueLabelPolicy,omitempty"` + // gangSchedulingPolicy controls how Kueue admits workloads. + // Gang Scheduling is the act of all or nothing scheduling, + // where workloads do not become ready within a certain period, they may be evicted and later retried. // This field is optional. // +optional - KueueGangSchedulingPolicy KueueGangSchedulingPolicy `json:"kueueGangSchedulingPolicy,omitempty"` + GangSchedulingPolicy *GangSchedulingPolicy `json:"gangSchedulingPolicy,omitempty"` // preemption is the process of evicting one or more admitted Workloads to accommodate another Workload. // Kueue has classical premption and preemption via fair sharing. // +optional - Preemption Preemption `json:"preemption,omitempty"` + Preemption *Preemption `json:"preemption,omitempty"` } // KueueStatus defines the observed state of Kueue @@ -96,7 +96,7 @@ const ( KueueIntegrationRayCluster KueueIntegration = "RayCluster" KueueIntegrationJobSet KueueIntegration = "JobSet" KueueIntegrationMPIJob KueueIntegration = "MPIJob" - KueueIntegrationPaddeJob KueueIntegration = "PaddeJob" + KueueIntegrationPaddleJob KueueIntegration = "PaddleJob" KueueIntegrationPyTorchJob KueueIntegration = "PyTorchJob" KueueIntegrationTFJob KueueIntegration = "TFJob" KueueIntegrationXGBoostJob KueueIntegration = "XGBoostJob" @@ -174,7 +174,7 @@ type Integrations struct { } type LabelKeys struct { - // key is the label key + // key is the label key. // A label key must be a valid qualified name consisting of a lower-case alphanumeric string, // and hyphens of at most 63 characters in length. // The name must start and end with an alphanumeric character. @@ -185,37 +185,39 @@ type LabelKeys struct { // +kubebuilder:validation:MaxLength=317 // +kubebuilder:validation:MinLength=1 // +kubebuilder:validation:XValidation:rule="!format.qualifiedName().validate(self).hasValue()",message="a qualified name must consist of a lower-case alphanumeric and hyphenated string of at most 63 characters in length, starting and ending with alphanumeric chracters. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character." - // +optional - Key string `json:"key,omitempty"` + // +required + Key string `json:"key"` } // +kubebuilder:validation:Enum=ByWorkload;Disabled -type KueueGangSchedulingPolicyOptions string +type GangSchedulingPolicyOptions string const ( - KueueGangSchedulingPolicyEvictNotReadyWorkloads KueueGangSchedulingPolicyOptions = "ByWorkload" - KueueGangSchedulingPolicyDisabled KueueGangSchedulingPolicyOptions = "Disabled" + GangSchedulingPolicyByWorkload GangSchedulingPolicyOptions = "ByWorkload" + GangSchedulingPolicyDisabled GangSchedulingPolicyOptions = "Disabled" ) // +kubebuilder:validation:Enum=Parallel;Sequential -type KueueGangSchedulingAdmissionOptions string +type GangSchedulingAdmissionOptions string const ( - KueueGangSchedulingAdmissionOptionsSequential KueueGangSchedulingAdmissionOptions = "Sequential" - KueueGangSchedulingAdmissionOptionsParallel KueueGangSchedulingAdmissionOptions = "Parallel" + GangSchedulingAdmissionOptionsSequential GangSchedulingAdmissionOptions = "Sequential" + GangSchedulingAdmissionOptionsParallel GangSchedulingAdmissionOptions = "Parallel" ) // Kueue provides the ability to admit workloads all in one (gang admission) // and evicts workloads if they are not ready within a specific time. -type KueueGangSchedulingPolicy struct { - // policy allows for changing the kinds of gang scheduling Kueue does. +// +kubebuilder:validation:XValidation:rule="has(self.policy) && self.policy == 'ByWorkload' ? has(self.byWorkload) : !has(self.byWorkload)",message="byWorkload is required when policy is ByWorkload, and forbidden otherwise" +type GangSchedulingPolicy struct { + // policy allows you to enable and configure gang scheduling. // This is an optional field. // The allowed values are ByWorkload and Disabled. // The default value will be Disabled. - // ByWorkload allows for configuration how admission is performed - // for Kueue. + // When set to ByWorkload, this means each workload is processed and considered + // for admission as a single unit. + // Where workloads do not become ready over time, the entire workload may then be evicted and retried at a later time. // +optional - Policy KueueGangSchedulingPolicyOptions `json:"policy"` + Policy *GangSchedulingPolicyOptions `json:"policy,omitempty"` // byWorkload controls how admission is done. // When admission is set to Sequential, only pods from the currently processing workload will be admitted. // Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. @@ -224,9 +226,9 @@ type KueueGangSchedulingPolicy struct { // When set to Parallel, pods from any workload will be admitted at any time. // This may lead to a deadlock where workloads are in contention for cluster capacity and // pods from another workload having successfully scheduled prevent pods from the current workload scheduling. - // +kubebuilder:validation:XValidation:rule="self.policy==ByWorkload",message="byWorkload is only valid if policy equals ByWorkload" + // +kubebuilder:validation:XValidation:rule="self.policy=='ByWorkload'",message="byWorkload is only valid if policy equals ByWorkload" // +optional - ByWorkload KueueGangSchedulingAdmissionOptions `json:"byWorkload"` + ByWorkload *GangSchedulingAdmissionOptions `json:"byWorkload,omitempty"` } // +kubebuilder:validation:Enum=QueueNameRequired;QueueNameOptional @@ -238,16 +240,19 @@ const ( ) type QueueLabelPolicy struct { - // queueLabelPolicy controls whether or not Kueue reconciles + // policy controls whether or not Kueue reconciles // jobs that don't set the label kueue.x-k8s.io/queue-name. // The allowed values are QueueNameRequired and QueueNameOptional. - // If set to QueueNameRequired, then those jobs will be suspended and never started unless - // they are assigned a queue and eventually admitted. This also applies to - // jobs created before starting the kueue controller. + // QueueNameOptional means that workloads will be suspended on + // creation and a label will be added via a mutating webhook. + // QueueNameRequired means that workloads that are managed + // by Kueue must have a the label kueue.x-k8s.io/queue-name. + // If this label is not present on the workload, then Kueue will + // ignore this workload. // Defaults to QueueNameRequired; therefore, those jobs are not managed and if they are created // unsuspended, they will start immediately. // +optional - QueueLabelPolicy QueueLabelNamePolicy `json:"queueLabelPolicy"` + Policy *QueueLabelNamePolicy `json:"policy,omitempty"` } // +kubebuilder:validation:Enum=Classical;FairSharing @@ -266,12 +271,12 @@ type Preemption struct { // when the requests of the workload are below the // resource flavor's nominal quota or borrowWithinCohort is enabled // on the Cluster Queue. - // FairSharing is a more heavy weight algorithm. - // ClusterQueues with pending Workloads can preempt other Workloads + // Fairsharing means that ClusterQueues with pending Workloads can preempt other Workloads // in their cohort until the preempting ClusterQueue // obtains an equal or weighted share of the borrowable resources. // The borrowable resources are the unused nominal quota // of all the ClusterQueues in the cohort. + // FairSharing is a more heavy weight algorithm. // +optional - PreemptionStrategy PreemptionStrategy `json:"preemptionStrategy"` + PreemptionStrategy *PreemptionStrategy `json:"preemptionStrategy,omitempty"` } diff --git a/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml b/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml index 2456ddea696..4068048e320 100644 --- a/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml +++ b/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml @@ -19,8 +19,8 @@ spec: schema: openAPIV3Schema: description: |- - Kueue is the CRD to represent the Kueue operator - This CRD defines the configuration that the Kueue + Kueue is the CRD to represent the Kueue operator. + This CRD defines the configuration for Kueue. Compatibility level 4: No compatibility is provided, the API can change at any point for any reason. These capabilities should not be used by applications needing long term support. properties: apiVersion: @@ -48,6 +48,49 @@ spec: config is the desired configuration for the Kueue operator. properties: + gangSchedulingPolicy: + description: |- + gangSchedulingPolicy controls how Kueue admits workloads. + Gang Scheduling is the act of all or nothing scheduling, + where workloads do not become ready within a certain period, they may be evicted and later retried. + This field is optional. + properties: + byWorkload: + description: |- + byWorkload controls how admission is done. + When admission is set to Sequential, only pods from the currently processing workload will be admitted. + Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. + Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, + but provides a higher guarantee of workloads scheduling all pods together successfully. + When set to Parallel, pods from any workload will be admitted at any time. + This may lead to a deadlock where workloads are in contention for cluster capacity and + pods from another workload having successfully scheduled prevent pods from the current workload scheduling. + enum: + - Parallel + - Sequential + type: string + x-kubernetes-validations: + - message: byWorkload is only valid if policy equals ByWorkload + rule: self.policy=='ByWorkload' + policy: + description: |- + policy allows you to enable and configure gang scheduling. + This is an optional field. + The allowed values are ByWorkload and Disabled. + The default value will be Disabled. + When set to ByWorkload, this means each workload is processed and considered + for admission as a single unit. + Where workloads do not become ready over time, the entire workload may then be evicted and retried at a later time. + enum: + - ByWorkload + - Disabled + type: string + type: object + x-kubernetes-validations: + - message: byWorkload is required when policy is ByWorkload, and + forbidden otherwise + rule: 'has(self.policy) && self.policy == ''ByWorkload'' ? has(self.byWorkload) + : !has(self.byWorkload)' integrations: description: |- integrations is a required field that configures the Kueue's workload integrations. @@ -158,7 +201,7 @@ spec: properties: key: description: |- - key is the label key + key is the label key. A label key must be a valid qualified name consisting of a lower-case alphanumeric string, and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. @@ -180,6 +223,8 @@ spec: within the subdomain must start and end with an alphanumeric character. rule: '!format.qualifiedName().validate(self).hasValue()' + required: + - key type: object maxItems: 64 type: array @@ -187,43 +232,6 @@ spec: required: - frameworks type: object - kueueGangSchedulingPolicy: - description: |- - kueueGangSchedulingPolicy controls how Kueue admits workloads. - Gang Scheduling is the act of all or nothing scheduling. - Kueue provides this ability. - This field is optional. - properties: - byWorkload: - description: |- - byWorkload controls how admission is done. - When admission is set to Sequential, only pods from the currently processing workload will be admitted. - Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. - Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, - but provides a higher guarantee of workloads scheduling all pods together successfully. - When set to Parallel, pods from any workload will be admitted at any time. - This may lead to a deadlock where workloads are in contention for cluster capacity and - pods from another workload having successfully scheduled prevent pods from the current workload scheduling. - enum: - - Parallel - - Sequential - type: string - x-kubernetes-validations: - - message: byWorkload is only valid if policy equals ByWorkload - rule: self.policy==ByWorkload - policy: - description: |- - policy allows for changing the kinds of gang scheduling Kueue does. - This is an optional field. - The allowed values are ByWorkload and Disabled. - The default value will be Disabled. - ByWorkload allows for configuration how admission is performed - for Kueue. - enum: - - ByWorkload - - Disabled - type: string - type: object preemption: description: |- preemption is the process of evicting one or more admitted Workloads to accommodate another Workload. @@ -238,12 +246,12 @@ spec: when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. - FairSharing is a more heavy weight algorithm. - ClusterQueues with pending Workloads can preempt other Workloads + Fairsharing means that ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort. + FairSharing is a more heavy weight algorithm. enum: - Classical - FairSharing @@ -255,14 +263,17 @@ spec: The default behavior of Kueue will manage workloads that have a queue-name label. This field is optional. properties: - queueLabelPolicy: + policy: description: |- - queueLabelPolicy controls whether or not Kueue reconciles + policy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. The allowed values are QueueNameRequired and QueueNameOptional. - If set to QueueNameRequired, then those jobs will be suspended and never started unless - they are assigned a queue and eventually admitted. This also applies to - jobs created before starting the kueue controller. + QueueNameOptional means that workloads will be suspended on + creation and a label will be added via a mutating webhook. + QueueNameRequired means that workloads that are managed + by Kueue must have a the label kueue.x-k8s.io/queue-name. + If this label is not present on the workload, then Kueue will + ignore this workload. Defaults to QueueNameRequired; therefore, those jobs are not managed and if they are created unsuspended, they will start immediately. enum: diff --git a/operator/v1alpha1/zz_generated.deepcopy.go b/operator/v1alpha1/zz_generated.deepcopy.go index 2463b95f25d..8ab7d6360b8 100644 --- a/operator/v1alpha1/zz_generated.deepcopy.go +++ b/operator/v1alpha1/zz_generated.deepcopy.go @@ -272,6 +272,32 @@ func (in *ExternalFramework) DeepCopy() *ExternalFramework { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *GangSchedulingPolicy) DeepCopyInto(out *GangSchedulingPolicy) { + *out = *in + if in.Policy != nil { + in, out := &in.Policy, &out.Policy + *out = new(GangSchedulingPolicyOptions) + **out = **in + } + if in.ByWorkload != nil { + in, out := &in.ByWorkload, &out.ByWorkload + *out = new(GangSchedulingAdmissionOptions) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GangSchedulingPolicy. +func (in *GangSchedulingPolicy) DeepCopy() *GangSchedulingPolicy { + if in == nil { + return nil + } + out := new(GangSchedulingPolicy) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GenerationHistory) DeepCopyInto(out *GenerationHistory) { *out = *in @@ -463,9 +489,21 @@ func (in *Kueue) DeepCopyObject() runtime.Object { func (in *KueueConfiguration) DeepCopyInto(out *KueueConfiguration) { *out = *in in.Integrations.DeepCopyInto(&out.Integrations) - out.QueueLabelPolicy = in.QueueLabelPolicy - out.KueueGangSchedulingPolicy = in.KueueGangSchedulingPolicy - out.Preemption = in.Preemption + if in.QueueLabelPolicy != nil { + in, out := &in.QueueLabelPolicy, &out.QueueLabelPolicy + *out = new(QueueLabelPolicy) + (*in).DeepCopyInto(*out) + } + if in.GangSchedulingPolicy != nil { + in, out := &in.GangSchedulingPolicy, &out.GangSchedulingPolicy + *out = new(GangSchedulingPolicy) + (*in).DeepCopyInto(*out) + } + if in.Preemption != nil { + in, out := &in.Preemption, &out.Preemption + *out = new(Preemption) + (*in).DeepCopyInto(*out) + } return } @@ -479,22 +517,6 @@ func (in *KueueConfiguration) DeepCopy() *KueueConfiguration { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *KueueGangSchedulingPolicy) DeepCopyInto(out *KueueGangSchedulingPolicy) { - *out = *in - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KueueGangSchedulingPolicy. -func (in *KueueGangSchedulingPolicy) DeepCopy() *KueueGangSchedulingPolicy { - if in == nil { - return nil - } - out := new(KueueGangSchedulingPolicy) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KueueList) DeepCopyInto(out *KueueList) { *out = *in @@ -781,6 +803,11 @@ func (in *OperatorStatus) DeepCopy() *OperatorStatus { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Preemption) DeepCopyInto(out *Preemption) { *out = *in + if in.PreemptionStrategy != nil { + in, out := &in.PreemptionStrategy, &out.PreemptionStrategy + *out = new(PreemptionStrategy) + **out = **in + } return } @@ -797,6 +824,11 @@ func (in *Preemption) DeepCopy() *Preemption { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *QueueLabelPolicy) DeepCopyInto(out *QueueLabelPolicy) { *out = *in + if in.Policy != nil { + in, out := &in.Policy, &out.Policy + *out = new(QueueLabelNamePolicy) + **out = **in + } return } diff --git a/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml b/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml index aa57ea5e27d..5c355d070b4 100644 --- a/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml +++ b/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml @@ -17,8 +17,8 @@ spec: schema: openAPIV3Schema: description: |- - Kueue is the CRD to represent the Kueue operator - This CRD defines the configuration that the Kueue + Kueue is the CRD to represent the Kueue operator. + This CRD defines the configuration for Kueue. Compatibility level 4: No compatibility is provided, the API can change at any point for any reason. These capabilities should not be used by applications needing long term support. properties: apiVersion: @@ -46,6 +46,49 @@ spec: config is the desired configuration for the Kueue operator. properties: + gangSchedulingPolicy: + description: |- + gangSchedulingPolicy controls how Kueue admits workloads. + Gang Scheduling is the act of all or nothing scheduling, + where workloads do not become ready within a certain period, they may be evicted and later retried. + This field is optional. + properties: + byWorkload: + description: |- + byWorkload controls how admission is done. + When admission is set to Sequential, only pods from the currently processing workload will be admitted. + Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. + Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, + but provides a higher guarantee of workloads scheduling all pods together successfully. + When set to Parallel, pods from any workload will be admitted at any time. + This may lead to a deadlock where workloads are in contention for cluster capacity and + pods from another workload having successfully scheduled prevent pods from the current workload scheduling. + enum: + - Parallel + - Sequential + type: string + x-kubernetes-validations: + - message: byWorkload is only valid if policy equals ByWorkload + rule: self.policy=='ByWorkload' + policy: + description: |- + policy allows you to enable and configure gang scheduling. + This is an optional field. + The allowed values are ByWorkload and Disabled. + The default value will be Disabled. + When set to ByWorkload, this means each workload is processed and considered + for admission as a single unit. + Where workloads do not become ready over time, the entire workload may then be evicted and retried at a later time. + enum: + - ByWorkload + - Disabled + type: string + type: object + x-kubernetes-validations: + - message: byWorkload is required when policy is ByWorkload, and + forbidden otherwise + rule: 'has(self.policy) && self.policy == ''ByWorkload'' ? has(self.byWorkload) + : !has(self.byWorkload)' integrations: description: |- integrations is a required field that configures the Kueue's workload integrations. @@ -156,7 +199,7 @@ spec: properties: key: description: |- - key is the label key + key is the label key. A label key must be a valid qualified name consisting of a lower-case alphanumeric string, and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. @@ -178,6 +221,8 @@ spec: within the subdomain must start and end with an alphanumeric character. rule: '!format.qualifiedName().validate(self).hasValue()' + required: + - key type: object maxItems: 64 type: array @@ -185,43 +230,6 @@ spec: required: - frameworks type: object - kueueGangSchedulingPolicy: - description: |- - kueueGangSchedulingPolicy controls how Kueue admits workloads. - Gang Scheduling is the act of all or nothing scheduling. - Kueue provides this ability. - This field is optional. - properties: - byWorkload: - description: |- - byWorkload controls how admission is done. - When admission is set to Sequential, only pods from the currently processing workload will be admitted. - Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. - Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, - but provides a higher guarantee of workloads scheduling all pods together successfully. - When set to Parallel, pods from any workload will be admitted at any time. - This may lead to a deadlock where workloads are in contention for cluster capacity and - pods from another workload having successfully scheduled prevent pods from the current workload scheduling. - enum: - - Parallel - - Sequential - type: string - x-kubernetes-validations: - - message: byWorkload is only valid if policy equals ByWorkload - rule: self.policy==ByWorkload - policy: - description: |- - policy allows for changing the kinds of gang scheduling Kueue does. - This is an optional field. - The allowed values are ByWorkload and Disabled. - The default value will be Disabled. - ByWorkload allows for configuration how admission is performed - for Kueue. - enum: - - ByWorkload - - Disabled - type: string - type: object preemption: description: |- preemption is the process of evicting one or more admitted Workloads to accommodate another Workload. @@ -236,12 +244,12 @@ spec: when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. - FairSharing is a more heavy weight algorithm. - ClusterQueues with pending Workloads can preempt other Workloads + Fairsharing means that ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort. + FairSharing is a more heavy weight algorithm. enum: - Classical - FairSharing @@ -253,14 +261,17 @@ spec: The default behavior of Kueue will manage workloads that have a queue-name label. This field is optional. properties: - queueLabelPolicy: + policy: description: |- - queueLabelPolicy controls whether or not Kueue reconciles + policy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. The allowed values are QueueNameRequired and QueueNameOptional. - If set to QueueNameRequired, then those jobs will be suspended and never started unless - they are assigned a queue and eventually admitted. This also applies to - jobs created before starting the kueue controller. + QueueNameOptional means that workloads will be suspended on + creation and a label will be added via a mutating webhook. + QueueNameRequired means that workloads that are managed + by Kueue must have a the label kueue.x-k8s.io/queue-name. + If this label is not present on the workload, then Kueue will + ignore this workload. Defaults to QueueNameRequired; therefore, those jobs are not managed and if they are created unsuspended, they will start immediately. enum: diff --git a/operator/v1alpha1/zz_generated.swagger_doc_generated.go b/operator/v1alpha1/zz_generated.swagger_doc_generated.go index 9e09feca184..4926095f97f 100644 --- a/operator/v1alpha1/zz_generated.swagger_doc_generated.go +++ b/operator/v1alpha1/zz_generated.swagger_doc_generated.go @@ -268,6 +268,16 @@ func (ExternalFramework) SwaggerDoc() map[string]string { return map_ExternalFramework } +var map_GangSchedulingPolicy = map[string]string{ + "": "Kueue provides the ability to admit workloads all in one (gang admission) and evicts workloads if they are not ready within a specific time.", + "policy": "policy allows you to enable and configure gang scheduling. This is an optional field. The allowed values are ByWorkload and Disabled. The default value will be Disabled. When set to ByWorkload, this means each workload is processed and considered for admission as a single unit. Where workloads do not become ready over time, the entire workload may then be evicted and retried at a later time.", + "byWorkload": "byWorkload controls how admission is done. When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling.", +} + +func (GangSchedulingPolicy) SwaggerDoc() map[string]string { + return map_GangSchedulingPolicy +} + var map_Integrations = map[string]string{ "": "This is the integrations for Kueue. Kueue uses these apis to determine which jobs will be managed by Kueue.", "frameworks": "frameworks are a unique list of names to be enabled. This is required and must have at least one element. Each framework represents a type of job that Kueue will manage. Frameworks are a list of frameworks that Kueue has support for. The allowed values are BatchJob, RayJob, RayCluster, JobSet, MPIJob, PaddleJob, PytorchJob, TFJob, XGBoostJob, AppWrapper, Pod, Deployment, StatefulSet and LeaderWorkerSet.", @@ -280,7 +290,7 @@ func (Integrations) SwaggerDoc() map[string]string { } var map_Kueue = map[string]string{ - "": "Kueue is the CRD to represent the Kueue operator This CRD defines the configuration that the Kueue Compatibility level 4: No compatibility is provided, the API can change at any point for any reason. These capabilities should not be used by applications needing long term support.", + "": "Kueue is the CRD to represent the Kueue operator. This CRD defines the configuration for Kueue. Compatibility level 4: No compatibility is provided, the API can change at any point for any reason. These capabilities should not be used by applications needing long term support.", "metadata": "metadata for kueue", "spec": "spec holds user settable values for configuration", "status": "status holds observed values from the cluster. They may not be overridden.", @@ -291,26 +301,16 @@ func (Kueue) SwaggerDoc() map[string]string { } var map_KueueConfiguration = map[string]string{ - "integrations": "integrations is a required field that configures the Kueue's workload integrations. Kueue has both standard integrations, known as job frameworks, and external integrations known as external frameworks. Kueue will only manage workloads that correspond to the specified integrations.", - "queueLabelPolicy": "queueLabelPolicy controls how kueue manages workloads The default behavior of Kueue will manage workloads that have a queue-name label. This field is optional.", - "kueueGangSchedulingPolicy": "kueueGangSchedulingPolicy controls how Kueue admits workloads. Gang Scheduling is the act of all or nothing scheduling. Kueue provides this ability. This field is optional.", - "preemption": "preemption is the process of evicting one or more admitted Workloads to accommodate another Workload. Kueue has classical premption and preemption via fair sharing.", + "integrations": "integrations is a required field that configures the Kueue's workload integrations. Kueue has both standard integrations, known as job frameworks, and external integrations known as external frameworks. Kueue will only manage workloads that correspond to the specified integrations.", + "queueLabelPolicy": "queueLabelPolicy controls how kueue manages workloads The default behavior of Kueue will manage workloads that have a queue-name label. This field is optional.", + "gangSchedulingPolicy": "gangSchedulingPolicy controls how Kueue admits workloads. Gang Scheduling is the act of all or nothing scheduling, where workloads do not become ready within a certain period, they may be evicted and later retried. This field is optional.", + "preemption": "preemption is the process of evicting one or more admitted Workloads to accommodate another Workload. Kueue has classical premption and preemption via fair sharing.", } func (KueueConfiguration) SwaggerDoc() map[string]string { return map_KueueConfiguration } -var map_KueueGangSchedulingPolicy = map[string]string{ - "": "Kueue provides the ability to admit workloads all in one (gang admission) and evicts workloads if they are not ready within a specific time.", - "policy": "policy allows for changing the kinds of gang scheduling Kueue does. This is an optional field. The allowed values are ByWorkload and Disabled. The default value will be Disabled. ByWorkload allows for configuration how admission is performed for Kueue.", - "byWorkload": "byWorkload controls how admission is done. When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling.", -} - -func (KueueGangSchedulingPolicy) SwaggerDoc() map[string]string { - return map_KueueGangSchedulingPolicy -} - var map_KueueList = map[string]string{ "": "KueueList contains a list of Kueue\n\nCompatibility level 4: No compatibility is provided, the API can change at any point for any reason. These capabilities should not be used by applications needing long term support.", "metadata": "metadata for the list", @@ -338,7 +338,7 @@ func (KueueStatus) SwaggerDoc() map[string]string { } var map_LabelKeys = map[string]string{ - "key": "key is the label key A label key must be a valid qualified name consisting of a lower-case alphanumeric string, and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. The optional prefix and the name are separate by a forward slash (/).", + "key": "key is the label key. A label key must be a valid qualified name consisting of a lower-case alphanumeric string, and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. The optional prefix and the name are separate by a forward slash (/).", } func (LabelKeys) SwaggerDoc() map[string]string { @@ -346,7 +346,7 @@ func (LabelKeys) SwaggerDoc() map[string]string { } var map_Preemption = map[string]string{ - "preemptionStrategy": "preemptionStrategy are the types of preemption kueue allows. Kueue has two types of preemption: classical and fair sharing. Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. FairSharing is a more heavy weight algorithm. ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort.", + "preemptionStrategy": "preemptionStrategy are the types of preemption kueue allows. Kueue has two types of preemption: classical and fair sharing. Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. Fairsharing means that ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort. FairSharing is a more heavy weight algorithm.", } func (Preemption) SwaggerDoc() map[string]string { @@ -354,7 +354,7 @@ func (Preemption) SwaggerDoc() map[string]string { } var map_QueueLabelPolicy = map[string]string{ - "queueLabelPolicy": "queueLabelPolicy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. The allowed values are QueueNameRequired and QueueNameOptional. If set to QueueNameRequired, then those jobs will be suspended and never started unless they are assigned a queue and eventually admitted. This also applies to jobs created before starting the kueue controller. Defaults to QueueNameRequired; therefore, those jobs are not managed and if they are created unsuspended, they will start immediately.", + "policy": "policy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. The allowed values are QueueNameRequired and QueueNameOptional. QueueNameOptional means that workloads will be suspended on creation and a label will be added via a mutating webhook. QueueNameRequired means that workloads that are managed by Kueue must have a the label kueue.x-k8s.io/queue-name. If this label is not present on the workload, then Kueue will ignore this workload. Defaults to QueueNameRequired; therefore, those jobs are not managed and if they are created unsuspended, they will start immediately.", } func (QueueLabelPolicy) SwaggerDoc() map[string]string { From 49b7815bd4499886cbdd4689d402192221ca9970 Mon Sep 17 00:00:00 2001 From: Kevin Hannon Date: Wed, 2 Apr 2025 15:46:31 -0400 Subject: [PATCH 26/42] api updates --- openapi/generated_openapi/zz_generated.openapi.go | 4 ++-- openapi/openapi.json | 4 ++-- operator/v1alpha1/types_kueue.go | 7 +++---- .../v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml | 9 +++------ .../kueue.operator.openshift.io/AAA_ungated.yaml | 9 +++------ operator/v1alpha1/zz_generated.swagger_doc_generated.go | 4 ++-- 6 files changed, 15 insertions(+), 22 deletions(-) diff --git a/openapi/generated_openapi/zz_generated.openapi.go b/openapi/generated_openapi/zz_generated.openapi.go index 28f48db2da5..5775844e4c0 100644 --- a/openapi/generated_openapi/zz_generated.openapi.go +++ b/openapi/generated_openapi/zz_generated.openapi.go @@ -58039,7 +58039,7 @@ func schema_openshift_api_operator_v1alpha1_Preemption(ref common.ReferenceCallb Properties: map[string]spec.Schema{ "preemptionStrategy": { SchemaProps: spec.SchemaProps{ - Description: "preemptionStrategy are the types of preemption kueue allows. Kueue has two types of preemption: classical and fair sharing. Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. Fairsharing means that ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort. FairSharing is a more heavy weight algorithm.", + Description: "preemptionStrategy are the types of preemption kueue allows. Kueue has two types of preemption: classical and fair sharing. Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. Fairsharing means that ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort. FairSharing is a more heavy weight algorithm. The default is Classical.", Type: []string{"string"}, Format: "", }, @@ -58058,7 +58058,7 @@ func schema_openshift_api_operator_v1alpha1_QueueLabelPolicy(ref common.Referenc Properties: map[string]spec.Schema{ "policy": { SchemaProps: spec.SchemaProps{ - Description: "policy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. The allowed values are QueueNameRequired and QueueNameOptional. QueueNameOptional means that workloads will be suspended on creation and a label will be added via a mutating webhook. QueueNameRequired means that workloads that are managed by Kueue must have a the label kueue.x-k8s.io/queue-name. If this label is not present on the workload, then Kueue will ignore this workload. Defaults to QueueNameRequired; therefore, those jobs are not managed and if they are created unsuspended, they will start immediately.", + Description: "policy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. The allowed values are QueueNameRequired and QueueNameOptional. QueueNameOptional means that workloads will be suspended on creation and a label will be added via a mutating webhook. QueueNameRequired means that workloads that are managed by Kueue must have a label kueue.x-k8s.io/queue-name. If this label is not present on the workload, then Kueue will ignore this workload. Defaults to QueueNameRequired.", Type: []string{"string"}, Format: "", }, diff --git a/openapi/openapi.json b/openapi/openapi.json index 0828d7dd4bd..9472e711c3a 100644 --- a/openapi/openapi.json +++ b/openapi/openapi.json @@ -33933,7 +33933,7 @@ "type": "object", "properties": { "preemptionStrategy": { - "description": "preemptionStrategy are the types of preemption kueue allows. Kueue has two types of preemption: classical and fair sharing. Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. Fairsharing means that ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort. FairSharing is a more heavy weight algorithm.", + "description": "preemptionStrategy are the types of preemption kueue allows. Kueue has two types of preemption: classical and fair sharing. Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. Fairsharing means that ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort. FairSharing is a more heavy weight algorithm. The default is Classical.", "type": "string" } } @@ -33942,7 +33942,7 @@ "type": "object", "properties": { "policy": { - "description": "policy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. The allowed values are QueueNameRequired and QueueNameOptional. QueueNameOptional means that workloads will be suspended on creation and a label will be added via a mutating webhook. QueueNameRequired means that workloads that are managed by Kueue must have a the label kueue.x-k8s.io/queue-name. If this label is not present on the workload, then Kueue will ignore this workload. Defaults to QueueNameRequired; therefore, those jobs are not managed and if they are created unsuspended, they will start immediately.", + "description": "policy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. The allowed values are QueueNameRequired and QueueNameOptional. QueueNameOptional means that workloads will be suspended on creation and a label will be added via a mutating webhook. QueueNameRequired means that workloads that are managed by Kueue must have a label kueue.x-k8s.io/queue-name. If this label is not present on the workload, then Kueue will ignore this workload. Defaults to QueueNameRequired.", "type": "string" } } diff --git a/operator/v1alpha1/types_kueue.go b/operator/v1alpha1/types_kueue.go index 5aa25402fb0..579f34ce835 100644 --- a/operator/v1alpha1/types_kueue.go +++ b/operator/v1alpha1/types_kueue.go @@ -226,7 +226,6 @@ type GangSchedulingPolicy struct { // When set to Parallel, pods from any workload will be admitted at any time. // This may lead to a deadlock where workloads are in contention for cluster capacity and // pods from another workload having successfully scheduled prevent pods from the current workload scheduling. - // +kubebuilder:validation:XValidation:rule="self.policy=='ByWorkload'",message="byWorkload is only valid if policy equals ByWorkload" // +optional ByWorkload *GangSchedulingAdmissionOptions `json:"byWorkload,omitempty"` } @@ -246,11 +245,10 @@ type QueueLabelPolicy struct { // QueueNameOptional means that workloads will be suspended on // creation and a label will be added via a mutating webhook. // QueueNameRequired means that workloads that are managed - // by Kueue must have a the label kueue.x-k8s.io/queue-name. + // by Kueue must have a label kueue.x-k8s.io/queue-name. // If this label is not present on the workload, then Kueue will // ignore this workload. - // Defaults to QueueNameRequired; therefore, those jobs are not managed and if they are created - // unsuspended, they will start immediately. + // Defaults to QueueNameRequired. // +optional Policy *QueueLabelNamePolicy `json:"policy,omitempty"` } @@ -277,6 +275,7 @@ type Preemption struct { // The borrowable resources are the unused nominal quota // of all the ClusterQueues in the cohort. // FairSharing is a more heavy weight algorithm. + // The default is Classical. // +optional PreemptionStrategy *PreemptionStrategy `json:"preemptionStrategy,omitempty"` } diff --git a/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml b/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml index 4068048e320..070bc6a086c 100644 --- a/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml +++ b/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml @@ -69,9 +69,6 @@ spec: - Parallel - Sequential type: string - x-kubernetes-validations: - - message: byWorkload is only valid if policy equals ByWorkload - rule: self.policy=='ByWorkload' policy: description: |- policy allows you to enable and configure gang scheduling. @@ -252,6 +249,7 @@ spec: The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort. FairSharing is a more heavy weight algorithm. + The default is Classical. enum: - Classical - FairSharing @@ -271,11 +269,10 @@ spec: QueueNameOptional means that workloads will be suspended on creation and a label will be added via a mutating webhook. QueueNameRequired means that workloads that are managed - by Kueue must have a the label kueue.x-k8s.io/queue-name. + by Kueue must have a label kueue.x-k8s.io/queue-name. If this label is not present on the workload, then Kueue will ignore this workload. - Defaults to QueueNameRequired; therefore, those jobs are not managed and if they are created - unsuspended, they will start immediately. + Defaults to QueueNameRequired. enum: - QueueNameRequired - QueueNameOptional diff --git a/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml b/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml index 5c355d070b4..3740a3a2cc7 100644 --- a/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml +++ b/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml @@ -67,9 +67,6 @@ spec: - Parallel - Sequential type: string - x-kubernetes-validations: - - message: byWorkload is only valid if policy equals ByWorkload - rule: self.policy=='ByWorkload' policy: description: |- policy allows you to enable and configure gang scheduling. @@ -250,6 +247,7 @@ spec: The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort. FairSharing is a more heavy weight algorithm. + The default is Classical. enum: - Classical - FairSharing @@ -269,11 +267,10 @@ spec: QueueNameOptional means that workloads will be suspended on creation and a label will be added via a mutating webhook. QueueNameRequired means that workloads that are managed - by Kueue must have a the label kueue.x-k8s.io/queue-name. + by Kueue must have a label kueue.x-k8s.io/queue-name. If this label is not present on the workload, then Kueue will ignore this workload. - Defaults to QueueNameRequired; therefore, those jobs are not managed and if they are created - unsuspended, they will start immediately. + Defaults to QueueNameRequired. enum: - QueueNameRequired - QueueNameOptional diff --git a/operator/v1alpha1/zz_generated.swagger_doc_generated.go b/operator/v1alpha1/zz_generated.swagger_doc_generated.go index 4926095f97f..c27c2ea487f 100644 --- a/operator/v1alpha1/zz_generated.swagger_doc_generated.go +++ b/operator/v1alpha1/zz_generated.swagger_doc_generated.go @@ -346,7 +346,7 @@ func (LabelKeys) SwaggerDoc() map[string]string { } var map_Preemption = map[string]string{ - "preemptionStrategy": "preemptionStrategy are the types of preemption kueue allows. Kueue has two types of preemption: classical and fair sharing. Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. Fairsharing means that ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort. FairSharing is a more heavy weight algorithm.", + "preemptionStrategy": "preemptionStrategy are the types of preemption kueue allows. Kueue has two types of preemption: classical and fair sharing. Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. Fairsharing means that ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort. FairSharing is a more heavy weight algorithm. The default is Classical.", } func (Preemption) SwaggerDoc() map[string]string { @@ -354,7 +354,7 @@ func (Preemption) SwaggerDoc() map[string]string { } var map_QueueLabelPolicy = map[string]string{ - "policy": "policy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. The allowed values are QueueNameRequired and QueueNameOptional. QueueNameOptional means that workloads will be suspended on creation and a label will be added via a mutating webhook. QueueNameRequired means that workloads that are managed by Kueue must have a the label kueue.x-k8s.io/queue-name. If this label is not present on the workload, then Kueue will ignore this workload. Defaults to QueueNameRequired; therefore, those jobs are not managed and if they are created unsuspended, they will start immediately.", + "policy": "policy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. The allowed values are QueueNameRequired and QueueNameOptional. QueueNameOptional means that workloads will be suspended on creation and a label will be added via a mutating webhook. QueueNameRequired means that workloads that are managed by Kueue must have a label kueue.x-k8s.io/queue-name. If this label is not present on the workload, then Kueue will ignore this workload. Defaults to QueueNameRequired.", } func (QueueLabelPolicy) SwaggerDoc() map[string]string { From b17e4c5c6f066ecbca48bab3859bf4b4dcbb19d8 Mon Sep 17 00:00:00 2001 From: Kevin Hannon Date: Wed, 2 Apr 2025 15:54:16 -0400 Subject: [PATCH 27/42] updates --- openapi/generated_openapi/zz_generated.openapi.go | 2 +- openapi/openapi.json | 2 +- operator/v1alpha1/types_kueue.go | 1 + operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml | 1 + .../kueue.operator.openshift.io/AAA_ungated.yaml | 1 + operator/v1alpha1/zz_generated.swagger_doc_generated.go | 2 +- 6 files changed, 6 insertions(+), 3 deletions(-) diff --git a/openapi/generated_openapi/zz_generated.openapi.go b/openapi/generated_openapi/zz_generated.openapi.go index 5775844e4c0..265ff5cc123 100644 --- a/openapi/generated_openapi/zz_generated.openapi.go +++ b/openapi/generated_openapi/zz_generated.openapi.go @@ -57289,7 +57289,7 @@ func schema_openshift_api_operator_v1alpha1_KueueConfiguration(ref common.Refere }, "queueLabelPolicy": { SchemaProps: spec.SchemaProps{ - Description: "queueLabelPolicy controls how kueue manages workloads The default behavior of Kueue will manage workloads that have a queue-name label. This field is optional.", + Description: "queueLabelPolicy controls how kueue manages workloads The default behavior of Kueue will manage workloads that have a queue-name label. Workloads that are missing these label will be ignored by Kueue. This field is optional.", Ref: ref("github.com/openshift/api/operator/v1alpha1.QueueLabelPolicy"), }, }, diff --git a/openapi/openapi.json b/openapi/openapi.json index 9472e711c3a..cf7655eb590 100644 --- a/openapi/openapi.json +++ b/openapi/openapi.json @@ -33478,7 +33478,7 @@ "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.Preemption" }, "queueLabelPolicy": { - "description": "queueLabelPolicy controls how kueue manages workloads The default behavior of Kueue will manage workloads that have a queue-name label. This field is optional.", + "description": "queueLabelPolicy controls how kueue manages workloads The default behavior of Kueue will manage workloads that have a queue-name label. Workloads that are missing these label will be ignored by Kueue. This field is optional.", "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.QueueLabelPolicy" } } diff --git a/operator/v1alpha1/types_kueue.go b/operator/v1alpha1/types_kueue.go index 579f34ce835..2a4470af054 100644 --- a/operator/v1alpha1/types_kueue.go +++ b/operator/v1alpha1/types_kueue.go @@ -49,6 +49,7 @@ type KueueConfiguration struct { Integrations Integrations `json:"integrations"` // queueLabelPolicy controls how kueue manages workloads // The default behavior of Kueue will manage workloads that have a queue-name label. + // Workloads that are missing these label will be ignored by Kueue. // This field is optional. // +optional QueueLabelPolicy *QueueLabelPolicy `json:"queueLabelPolicy,omitempty"` diff --git a/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml b/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml index 070bc6a086c..b71e9c3459a 100644 --- a/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml +++ b/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml @@ -259,6 +259,7 @@ spec: description: |- queueLabelPolicy controls how kueue manages workloads The default behavior of Kueue will manage workloads that have a queue-name label. + Workloads that are missing these label will be ignored by Kueue. This field is optional. properties: policy: diff --git a/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml b/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml index 3740a3a2cc7..ae2b52449dc 100644 --- a/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml +++ b/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml @@ -257,6 +257,7 @@ spec: description: |- queueLabelPolicy controls how kueue manages workloads The default behavior of Kueue will manage workloads that have a queue-name label. + Workloads that are missing these label will be ignored by Kueue. This field is optional. properties: policy: diff --git a/operator/v1alpha1/zz_generated.swagger_doc_generated.go b/operator/v1alpha1/zz_generated.swagger_doc_generated.go index c27c2ea487f..5b931888648 100644 --- a/operator/v1alpha1/zz_generated.swagger_doc_generated.go +++ b/operator/v1alpha1/zz_generated.swagger_doc_generated.go @@ -302,7 +302,7 @@ func (Kueue) SwaggerDoc() map[string]string { var map_KueueConfiguration = map[string]string{ "integrations": "integrations is a required field that configures the Kueue's workload integrations. Kueue has both standard integrations, known as job frameworks, and external integrations known as external frameworks. Kueue will only manage workloads that correspond to the specified integrations.", - "queueLabelPolicy": "queueLabelPolicy controls how kueue manages workloads The default behavior of Kueue will manage workloads that have a queue-name label. This field is optional.", + "queueLabelPolicy": "queueLabelPolicy controls how kueue manages workloads The default behavior of Kueue will manage workloads that have a queue-name label. Workloads that are missing these label will be ignored by Kueue. This field is optional.", "gangSchedulingPolicy": "gangSchedulingPolicy controls how Kueue admits workloads. Gang Scheduling is the act of all or nothing scheduling, where workloads do not become ready within a certain period, they may be evicted and later retried. This field is optional.", "preemption": "preemption is the process of evicting one or more admitted Workloads to accommodate another Workload. Kueue has classical premption and preemption via fair sharing.", } From cb9cc85137f65febf0d5b27c4de519e1f5aeb457 Mon Sep 17 00:00:00 2001 From: Kevin Hannon Date: Fri, 4 Apr 2025 14:01:24 -0400 Subject: [PATCH 28/42] update api with bryce comments --- .../generated_openapi/zz_generated.openapi.go | 32 ++++++++++++++++--- openapi/openapi.json | 18 ++++++++--- operator/v1alpha1/types_kueue.go | 19 +++++++---- .../zz_generated.crd-manifests/kueue.crd.yaml | 31 ++++++++++-------- operator/v1alpha1/zz_generated.deepcopy.go | 31 +++++++++++++++--- .../AAA_ungated.yaml | 31 ++++++++++-------- .../zz_generated.swagger_doc_generated.go | 13 ++++++-- 7 files changed, 127 insertions(+), 48 deletions(-) diff --git a/openapi/generated_openapi/zz_generated.openapi.go b/openapi/generated_openapi/zz_generated.openapi.go index 265ff5cc123..6fcc561d0e5 100644 --- a/openapi/generated_openapi/zz_generated.openapi.go +++ b/openapi/generated_openapi/zz_generated.openapi.go @@ -1100,6 +1100,7 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "github.com/openshift/api/operator/v1.UpstreamResolvers": schema_openshift_api_operator_v1_UpstreamResolvers(ref), "github.com/openshift/api/operator/v1.VSphereCSIDriverConfigSpec": schema_openshift_api_operator_v1_VSphereCSIDriverConfigSpec(ref), "github.com/openshift/api/operator/v1alpha1.BackupJobReference": schema_openshift_api_operator_v1alpha1_BackupJobReference(ref), + "github.com/openshift/api/operator/v1alpha1.ByWorkload": schema_openshift_api_operator_v1alpha1_ByWorkload(ref), "github.com/openshift/api/operator/v1alpha1.ClusterVersionOperator": schema_openshift_api_operator_v1alpha1_ClusterVersionOperator(ref), "github.com/openshift/api/operator/v1alpha1.ClusterVersionOperatorList": schema_openshift_api_operator_v1alpha1_ClusterVersionOperatorList(ref), "github.com/openshift/api/operator/v1alpha1.ClusterVersionOperatorSpec": schema_openshift_api_operator_v1alpha1_ClusterVersionOperatorSpec(ref), @@ -56505,6 +56506,26 @@ func schema_openshift_api_operator_v1alpha1_BackupJobReference(ref common.Refere } } +func schema_openshift_api_operator_v1alpha1_ByWorkload(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Description: "ByWorkload controls how admission is done", + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "admission": { + SchemaProps: spec.SchemaProps{ + Description: "When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling.", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + } +} + func schema_openshift_api_operator_v1alpha1_ClusterVersionOperator(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ @@ -56904,14 +56925,15 @@ func schema_openshift_api_operator_v1alpha1_GangSchedulingPolicy(ref common.Refe }, "byWorkload": { SchemaProps: spec.SchemaProps{ - Description: "byWorkload controls how admission is done. When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling.", - Type: []string{"string"}, - Format: "", + Description: "byWorkload controls how admission is done. byWorkload is only required if policy is equal to ByWorkload.", + Ref: ref("github.com/openshift/api/operator/v1alpha1.ByWorkload"), }, }, }, }, }, + Dependencies: []string{ + "github.com/openshift/api/operator/v1alpha1.ByWorkload"}, } } @@ -58037,9 +58059,9 @@ func schema_openshift_api_operator_v1alpha1_Preemption(ref common.ReferenceCallb SchemaProps: spec.SchemaProps{ Type: []string{"object"}, Properties: map[string]spec.Schema{ - "preemptionStrategy": { + "preemptionPolicy": { SchemaProps: spec.SchemaProps{ - Description: "preemptionStrategy are the types of preemption kueue allows. Kueue has two types of preemption: classical and fair sharing. Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. Fairsharing means that ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort. FairSharing is a more heavy weight algorithm. The default is Classical.", + Description: "preemptionPolicy are the types of preemption kueue allows. Kueue has two types of preemption: classical and fair sharing. Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. Fairsharing means that ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort. FairSharing is a more heavy weight algorithm. The default is Classical.", Type: []string{"string"}, Format: "", }, diff --git a/openapi/openapi.json b/openapi/openapi.json index cf7655eb590..1e0432a49ec 100644 --- a/openapi/openapi.json +++ b/openapi/openapi.json @@ -33009,6 +33009,16 @@ } } }, + "com.github.openshift.api.operator.v1alpha1.ByWorkload": { + "description": "ByWorkload controls how admission is done", + "type": "object", + "properties": { + "admission": { + "description": "When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling.", + "type": "string" + } + } + }, "com.github.openshift.api.operator.v1alpha1.ClusterVersionOperator": { "description": "ClusterVersionOperator holds cluster-wide information about the Cluster Version Operator.\n\nCompatibility level 4: No compatibility is provided, the API can change at any point for any reason. These capabilities should not be used by applications needing long term support.", "type": "object", @@ -33237,8 +33247,8 @@ "type": "object", "properties": { "byWorkload": { - "description": "byWorkload controls how admission is done. When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling.", - "type": "string" + "description": "byWorkload controls how admission is done. byWorkload is only required if policy is equal to ByWorkload.", + "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.ByWorkload" }, "policy": { "description": "policy allows you to enable and configure gang scheduling. This is an optional field. The allowed values are ByWorkload and Disabled. The default value will be Disabled. When set to ByWorkload, this means each workload is processed and considered for admission as a single unit. Where workloads do not become ready over time, the entire workload may then be evicted and retried at a later time.", @@ -33932,8 +33942,8 @@ "com.github.openshift.api.operator.v1alpha1.Preemption": { "type": "object", "properties": { - "preemptionStrategy": { - "description": "preemptionStrategy are the types of preemption kueue allows. Kueue has two types of preemption: classical and fair sharing. Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. Fairsharing means that ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort. FairSharing is a more heavy weight algorithm. The default is Classical.", + "preemptionPolicy": { + "description": "preemptionPolicy are the types of preemption kueue allows. Kueue has two types of preemption: classical and fair sharing. Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. Fairsharing means that ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort. FairSharing is a more heavy weight algorithm. The default is Classical.", "type": "string" } } diff --git a/operator/v1alpha1/types_kueue.go b/operator/v1alpha1/types_kueue.go index 2a4470af054..3602df814b9 100644 --- a/operator/v1alpha1/types_kueue.go +++ b/operator/v1alpha1/types_kueue.go @@ -220,6 +220,13 @@ type GangSchedulingPolicy struct { // +optional Policy *GangSchedulingPolicyOptions `json:"policy,omitempty"` // byWorkload controls how admission is done. + // byWorkload is only required if policy is equal to ByWorkload. + // +optional + ByWorkload *ByWorkload `json:"byWorkload,omitempty"` +} + +// ByWorkload controls how admission is done +type ByWorkload struct { // When admission is set to Sequential, only pods from the currently processing workload will be admitted. // Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. // Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, @@ -228,7 +235,7 @@ type GangSchedulingPolicy struct { // This may lead to a deadlock where workloads are in contention for cluster capacity and // pods from another workload having successfully scheduled prevent pods from the current workload scheduling. // +optional - ByWorkload *GangSchedulingAdmissionOptions `json:"byWorkload,omitempty"` + Admission *GangSchedulingAdmissionOptions `json:"admission,omitempty"` } // +kubebuilder:validation:Enum=QueueNameRequired;QueueNameOptional @@ -255,15 +262,15 @@ type QueueLabelPolicy struct { } // +kubebuilder:validation:Enum=Classical;FairSharing -type PreemptionStrategy string +type PreemptionPolicy string const ( - PreemeptionStrategyClassical PreemptionStrategy = "Classical" - PreemeptionStrategyFairsharing PreemptionStrategy = "FairSharing" + PreemptionStrategyClassical PreemptionPolicy = "Classical" + PreemptionStrategyFairsharing PreemptionPolicy = "FairSharing" ) type Preemption struct { - // preemptionStrategy are the types of preemption kueue allows. + // preemptionPolicy are the types of preemption kueue allows. // Kueue has two types of preemption: classical and fair sharing. // Classical means that an incoming workload, which does // not fit within the unusued quota, is eligible to issue preemptions @@ -278,5 +285,5 @@ type Preemption struct { // FairSharing is a more heavy weight algorithm. // The default is Classical. // +optional - PreemptionStrategy *PreemptionStrategy `json:"preemptionStrategy,omitempty"` + PreemptionPolicy *PreemptionPolicy `json:"preemptionPolicy,omitempty"` } diff --git a/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml b/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml index b71e9c3459a..a356e6c14ce 100644 --- a/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml +++ b/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml @@ -58,17 +58,22 @@ spec: byWorkload: description: |- byWorkload controls how admission is done. - When admission is set to Sequential, only pods from the currently processing workload will be admitted. - Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. - Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, - but provides a higher guarantee of workloads scheduling all pods together successfully. - When set to Parallel, pods from any workload will be admitted at any time. - This may lead to a deadlock where workloads are in contention for cluster capacity and - pods from another workload having successfully scheduled prevent pods from the current workload scheduling. - enum: - - Parallel - - Sequential - type: string + byWorkload is only required if policy is equal to ByWorkload. + properties: + admission: + description: |- + When admission is set to Sequential, only pods from the currently processing workload will be admitted. + Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. + Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, + but provides a higher guarantee of workloads scheduling all pods together successfully. + When set to Parallel, pods from any workload will be admitted at any time. + This may lead to a deadlock where workloads are in contention for cluster capacity and + pods from another workload having successfully scheduled prevent pods from the current workload scheduling. + enum: + - Parallel + - Sequential + type: string + type: object policy: description: |- policy allows you to enable and configure gang scheduling. @@ -234,9 +239,9 @@ spec: preemption is the process of evicting one or more admitted Workloads to accommodate another Workload. Kueue has classical premption and preemption via fair sharing. properties: - preemptionStrategy: + preemptionPolicy: description: |- - preemptionStrategy are the types of preemption kueue allows. + preemptionPolicy are the types of preemption kueue allows. Kueue has two types of preemption: classical and fair sharing. Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions diff --git a/operator/v1alpha1/zz_generated.deepcopy.go b/operator/v1alpha1/zz_generated.deepcopy.go index 8ab7d6360b8..8a6bfe59f3b 100644 --- a/operator/v1alpha1/zz_generated.deepcopy.go +++ b/operator/v1alpha1/zz_generated.deepcopy.go @@ -26,6 +26,27 @@ func (in *BackupJobReference) DeepCopy() *BackupJobReference { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ByWorkload) DeepCopyInto(out *ByWorkload) { + *out = *in + if in.Admission != nil { + in, out := &in.Admission, &out.Admission + *out = new(GangSchedulingAdmissionOptions) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ByWorkload. +func (in *ByWorkload) DeepCopy() *ByWorkload { + if in == nil { + return nil + } + out := new(ByWorkload) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ClusterVersionOperator) DeepCopyInto(out *ClusterVersionOperator) { *out = *in @@ -282,8 +303,8 @@ func (in *GangSchedulingPolicy) DeepCopyInto(out *GangSchedulingPolicy) { } if in.ByWorkload != nil { in, out := &in.ByWorkload, &out.ByWorkload - *out = new(GangSchedulingAdmissionOptions) - **out = **in + *out = new(ByWorkload) + (*in).DeepCopyInto(*out) } return } @@ -803,9 +824,9 @@ func (in *OperatorStatus) DeepCopy() *OperatorStatus { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Preemption) DeepCopyInto(out *Preemption) { *out = *in - if in.PreemptionStrategy != nil { - in, out := &in.PreemptionStrategy, &out.PreemptionStrategy - *out = new(PreemptionStrategy) + if in.PreemptionPolicy != nil { + in, out := &in.PreemptionPolicy, &out.PreemptionPolicy + *out = new(PreemptionPolicy) **out = **in } return diff --git a/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml b/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml index ae2b52449dc..d2aadf87500 100644 --- a/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml +++ b/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml @@ -56,17 +56,22 @@ spec: byWorkload: description: |- byWorkload controls how admission is done. - When admission is set to Sequential, only pods from the currently processing workload will be admitted. - Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. - Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, - but provides a higher guarantee of workloads scheduling all pods together successfully. - When set to Parallel, pods from any workload will be admitted at any time. - This may lead to a deadlock where workloads are in contention for cluster capacity and - pods from another workload having successfully scheduled prevent pods from the current workload scheduling. - enum: - - Parallel - - Sequential - type: string + byWorkload is only required if policy is equal to ByWorkload. + properties: + admission: + description: |- + When admission is set to Sequential, only pods from the currently processing workload will be admitted. + Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. + Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, + but provides a higher guarantee of workloads scheduling all pods together successfully. + When set to Parallel, pods from any workload will be admitted at any time. + This may lead to a deadlock where workloads are in contention for cluster capacity and + pods from another workload having successfully scheduled prevent pods from the current workload scheduling. + enum: + - Parallel + - Sequential + type: string + type: object policy: description: |- policy allows you to enable and configure gang scheduling. @@ -232,9 +237,9 @@ spec: preemption is the process of evicting one or more admitted Workloads to accommodate another Workload. Kueue has classical premption and preemption via fair sharing. properties: - preemptionStrategy: + preemptionPolicy: description: |- - preemptionStrategy are the types of preemption kueue allows. + preemptionPolicy are the types of preemption kueue allows. Kueue has two types of preemption: classical and fair sharing. Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions diff --git a/operator/v1alpha1/zz_generated.swagger_doc_generated.go b/operator/v1alpha1/zz_generated.swagger_doc_generated.go index 5b931888648..58f61b2ee65 100644 --- a/operator/v1alpha1/zz_generated.swagger_doc_generated.go +++ b/operator/v1alpha1/zz_generated.swagger_doc_generated.go @@ -257,6 +257,15 @@ func (RepositoryDigestMirrors) SwaggerDoc() map[string]string { return map_RepositoryDigestMirrors } +var map_ByWorkload = map[string]string{ + "": "ByWorkload controls how admission is done", + "admission": "When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling.", +} + +func (ByWorkload) SwaggerDoc() map[string]string { + return map_ByWorkload +} + var map_ExternalFramework = map[string]string{ "": "This is the GVR for an external framework. Controller runtime requires this in this format for api discoverability.", "group": "group is the API group of the externalFramework. Must be a valid DNS 1123 subdomain consisting of of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character.", @@ -271,7 +280,7 @@ func (ExternalFramework) SwaggerDoc() map[string]string { var map_GangSchedulingPolicy = map[string]string{ "": "Kueue provides the ability to admit workloads all in one (gang admission) and evicts workloads if they are not ready within a specific time.", "policy": "policy allows you to enable and configure gang scheduling. This is an optional field. The allowed values are ByWorkload and Disabled. The default value will be Disabled. When set to ByWorkload, this means each workload is processed and considered for admission as a single unit. Where workloads do not become ready over time, the entire workload may then be evicted and retried at a later time.", - "byWorkload": "byWorkload controls how admission is done. When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling.", + "byWorkload": "byWorkload controls how admission is done. byWorkload is only required if policy is equal to ByWorkload.", } func (GangSchedulingPolicy) SwaggerDoc() map[string]string { @@ -346,7 +355,7 @@ func (LabelKeys) SwaggerDoc() map[string]string { } var map_Preemption = map[string]string{ - "preemptionStrategy": "preemptionStrategy are the types of preemption kueue allows. Kueue has two types of preemption: classical and fair sharing. Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. Fairsharing means that ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort. FairSharing is a more heavy weight algorithm. The default is Classical.", + "preemptionPolicy": "preemptionPolicy are the types of preemption kueue allows. Kueue has two types of preemption: classical and fair sharing. Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. Fairsharing means that ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort. FairSharing is a more heavy weight algorithm. The default is Classical.", } func (Preemption) SwaggerDoc() map[string]string { From fb3b6609d6bef226c4159b32f5a2c55cf4dbf8a8 Mon Sep 17 00:00:00 2001 From: Kevin Hannon Date: Mon, 7 Apr 2025 13:48:58 -0400 Subject: [PATCH 29/42] fix linter --- openapi/generated_openapi/zz_generated.openapi.go | 2 +- openapi/openapi.json | 2 +- operator/v1alpha1/types_kueue.go | 1 + operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml | 1 + .../kueue.operator.openshift.io/AAA_ungated.yaml | 1 + operator/v1alpha1/zz_generated.swagger_doc_generated.go | 2 +- 6 files changed, 6 insertions(+), 3 deletions(-) diff --git a/openapi/generated_openapi/zz_generated.openapi.go b/openapi/generated_openapi/zz_generated.openapi.go index 6fcc561d0e5..80e20d970a2 100644 --- a/openapi/generated_openapi/zz_generated.openapi.go +++ b/openapi/generated_openapi/zz_generated.openapi.go @@ -56515,7 +56515,7 @@ func schema_openshift_api_operator_v1alpha1_ByWorkload(ref common.ReferenceCallb Properties: map[string]spec.Schema{ "admission": { SchemaProps: spec.SchemaProps{ - Description: "When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling.", + Description: "admission controls how kueue will process workloads. When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling.", Type: []string{"string"}, Format: "", }, diff --git a/openapi/openapi.json b/openapi/openapi.json index 1e0432a49ec..a926c0b2fea 100644 --- a/openapi/openapi.json +++ b/openapi/openapi.json @@ -33014,7 +33014,7 @@ "type": "object", "properties": { "admission": { - "description": "When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling.", + "description": "admission controls how kueue will process workloads. When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling.", "type": "string" } } diff --git a/operator/v1alpha1/types_kueue.go b/operator/v1alpha1/types_kueue.go index 3602df814b9..d3d1d08efbc 100644 --- a/operator/v1alpha1/types_kueue.go +++ b/operator/v1alpha1/types_kueue.go @@ -227,6 +227,7 @@ type GangSchedulingPolicy struct { // ByWorkload controls how admission is done type ByWorkload struct { + // admission controls how kueue will process workloads. // When admission is set to Sequential, only pods from the currently processing workload will be admitted. // Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. // Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, diff --git a/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml b/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml index a356e6c14ce..3374adc6306 100644 --- a/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml +++ b/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml @@ -62,6 +62,7 @@ spec: properties: admission: description: |- + admission controls how kueue will process workloads. When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, diff --git a/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml b/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml index d2aadf87500..7c32f970106 100644 --- a/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml +++ b/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml @@ -60,6 +60,7 @@ spec: properties: admission: description: |- + admission controls how kueue will process workloads. When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, diff --git a/operator/v1alpha1/zz_generated.swagger_doc_generated.go b/operator/v1alpha1/zz_generated.swagger_doc_generated.go index 58f61b2ee65..0a8b2fc77a0 100644 --- a/operator/v1alpha1/zz_generated.swagger_doc_generated.go +++ b/operator/v1alpha1/zz_generated.swagger_doc_generated.go @@ -259,7 +259,7 @@ func (RepositoryDigestMirrors) SwaggerDoc() map[string]string { var map_ByWorkload = map[string]string{ "": "ByWorkload controls how admission is done", - "admission": "When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling.", + "admission": "admission controls how kueue will process workloads. When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling.", } func (ByWorkload) SwaggerDoc() map[string]string { From 8741f7b1b774366d82b19317fc1107404253a99b Mon Sep 17 00:00:00 2001 From: Kevin Hannon Date: Mon, 7 Apr 2025 17:03:54 -0400 Subject: [PATCH 30/42] add bryce comments --- .../generated_openapi/zz_generated.openapi.go | 60 ++++++++--------- openapi/openapi.json | 34 +++++----- operator/v1alpha1/types_kueue.go | 51 +++++++------- .../zz_generated.crd-manifests/kueue.crd.yaml | 27 ++++---- operator/v1alpha1/zz_generated.deepcopy.go | 66 +++++++++---------- .../AAA_ungated.yaml | 27 ++++---- .../zz_generated.swagger_doc_generated.go | 22 +++---- 7 files changed, 145 insertions(+), 142 deletions(-) diff --git a/openapi/generated_openapi/zz_generated.openapi.go b/openapi/generated_openapi/zz_generated.openapi.go index 80e20d970a2..b4c1d39da62 100644 --- a/openapi/generated_openapi/zz_generated.openapi.go +++ b/openapi/generated_openapi/zz_generated.openapi.go @@ -1112,7 +1112,7 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "github.com/openshift/api/operator/v1alpha1.EtcdBackupSpec": schema_openshift_api_operator_v1alpha1_EtcdBackupSpec(ref), "github.com/openshift/api/operator/v1alpha1.EtcdBackupStatus": schema_openshift_api_operator_v1alpha1_EtcdBackupStatus(ref), "github.com/openshift/api/operator/v1alpha1.ExternalFramework": schema_openshift_api_operator_v1alpha1_ExternalFramework(ref), - "github.com/openshift/api/operator/v1alpha1.GangSchedulingPolicy": schema_openshift_api_operator_v1alpha1_GangSchedulingPolicy(ref), + "github.com/openshift/api/operator/v1alpha1.GangScheduling": schema_openshift_api_operator_v1alpha1_GangScheduling(ref), "github.com/openshift/api/operator/v1alpha1.GenerationHistory": schema_openshift_api_operator_v1alpha1_GenerationHistory(ref), "github.com/openshift/api/operator/v1alpha1.GenericOperatorConfig": schema_openshift_api_operator_v1alpha1_GenericOperatorConfig(ref), "github.com/openshift/api/operator/v1alpha1.ImageContentSourcePolicy": schema_openshift_api_operator_v1alpha1_ImageContentSourcePolicy(ref), @@ -1135,10 +1135,10 @@ func GetOpenAPIDefinitions(ref common.ReferenceCallback) map[string]common.OpenA "github.com/openshift/api/operator/v1alpha1.OperatorSpec": schema_openshift_api_operator_v1alpha1_OperatorSpec(ref), "github.com/openshift/api/operator/v1alpha1.OperatorStatus": schema_openshift_api_operator_v1alpha1_OperatorStatus(ref), "github.com/openshift/api/operator/v1alpha1.Preemption": schema_openshift_api_operator_v1alpha1_Preemption(ref), - "github.com/openshift/api/operator/v1alpha1.QueueLabelPolicy": schema_openshift_api_operator_v1alpha1_QueueLabelPolicy(ref), "github.com/openshift/api/operator/v1alpha1.RepositoryDigestMirrors": schema_openshift_api_operator_v1alpha1_RepositoryDigestMirrors(ref), "github.com/openshift/api/operator/v1alpha1.StaticPodOperatorStatus": schema_openshift_api_operator_v1alpha1_StaticPodOperatorStatus(ref), "github.com/openshift/api/operator/v1alpha1.VersionAvailability": schema_openshift_api_operator_v1alpha1_VersionAvailability(ref), + "github.com/openshift/api/operator/v1alpha1.WorkloadManagement": schema_openshift_api_operator_v1alpha1_WorkloadManagement(ref), "github.com/openshift/api/operatorcontrolplane/v1alpha1.LogEntry": schema_openshift_api_operatorcontrolplane_v1alpha1_LogEntry(ref), "github.com/openshift/api/operatorcontrolplane/v1alpha1.OutageEntry": schema_openshift_api_operatorcontrolplane_v1alpha1_OutageEntry(ref), "github.com/openshift/api/operatorcontrolplane/v1alpha1.PodNetworkConnectivityCheck": schema_openshift_api_operatorcontrolplane_v1alpha1_PodNetworkConnectivityCheck(ref), @@ -56909,7 +56909,7 @@ func schema_openshift_api_operator_v1alpha1_ExternalFramework(ref common.Referen } } -func schema_openshift_api_operator_v1alpha1_GangSchedulingPolicy(ref common.ReferenceCallback) common.OpenAPIDefinition { +func schema_openshift_api_operator_v1alpha1_GangScheduling(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ SchemaProps: spec.SchemaProps{ @@ -57309,21 +57309,21 @@ func schema_openshift_api_operator_v1alpha1_KueueConfiguration(ref common.Refere Ref: ref("github.com/openshift/api/operator/v1alpha1.Integrations"), }, }, - "queueLabelPolicy": { + "workloadManagement": { SchemaProps: spec.SchemaProps{ - Description: "queueLabelPolicy controls how kueue manages workloads The default behavior of Kueue will manage workloads that have a queue-name label. Workloads that are missing these label will be ignored by Kueue. This field is optional.", - Ref: ref("github.com/openshift/api/operator/v1alpha1.QueueLabelPolicy"), + Description: "workloadManagement controls how kueue manages workloads. The default behavior of Kueue will manage workloads that have a queue-name label. Workloads that are missing these label will be ignored by Kueue. This field is optional.", + Ref: ref("github.com/openshift/api/operator/v1alpha1.WorkloadManagement"), }, }, - "gangSchedulingPolicy": { + "gangScheduling": { SchemaProps: spec.SchemaProps{ - Description: "gangSchedulingPolicy controls how Kueue admits workloads. Gang Scheduling is the act of all or nothing scheduling, where workloads do not become ready within a certain period, they may be evicted and later retried. This field is optional.", - Ref: ref("github.com/openshift/api/operator/v1alpha1.GangSchedulingPolicy"), + Description: "gangScheduling controls how Kueue admits workloads. Gang Scheduling is the act of all or nothing scheduling, where workloads do not become ready within a certain period, they may be evicted and later retried. This field is optional.", + Ref: ref("github.com/openshift/api/operator/v1alpha1.GangScheduling"), }, }, "preemption": { SchemaProps: spec.SchemaProps{ - Description: "preemption is the process of evicting one or more admitted Workloads to accommodate another Workload. Kueue has classical premption and preemption via fair sharing.", + Description: "preemption is the process of evicting one or more admitted Workloads to accommodate another Workload. Kueue has classical premption and preemption via fair sharing. This field is optional.", Ref: ref("github.com/openshift/api/operator/v1alpha1.Preemption"), }, }, @@ -57332,7 +57332,7 @@ func schema_openshift_api_operator_v1alpha1_KueueConfiguration(ref common.Refere }, }, Dependencies: []string{ - "github.com/openshift/api/operator/v1alpha1.GangSchedulingPolicy", "github.com/openshift/api/operator/v1alpha1.Integrations", "github.com/openshift/api/operator/v1alpha1.Preemption", "github.com/openshift/api/operator/v1alpha1.QueueLabelPolicy"}, + "github.com/openshift/api/operator/v1alpha1.GangScheduling", "github.com/openshift/api/operator/v1alpha1.Integrations", "github.com/openshift/api/operator/v1alpha1.Preemption", "github.com/openshift/api/operator/v1alpha1.WorkloadManagement"}, } } @@ -58072,25 +58072,6 @@ func schema_openshift_api_operator_v1alpha1_Preemption(ref common.ReferenceCallb } } -func schema_openshift_api_operator_v1alpha1_QueueLabelPolicy(ref common.ReferenceCallback) common.OpenAPIDefinition { - return common.OpenAPIDefinition{ - Schema: spec.Schema{ - SchemaProps: spec.SchemaProps{ - Type: []string{"object"}, - Properties: map[string]spec.Schema{ - "policy": { - SchemaProps: spec.SchemaProps{ - Description: "policy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. The allowed values are QueueNameRequired and QueueNameOptional. QueueNameOptional means that workloads will be suspended on creation and a label will be added via a mutating webhook. QueueNameRequired means that workloads that are managed by Kueue must have a label kueue.x-k8s.io/queue-name. If this label is not present on the workload, then Kueue will ignore this workload. Defaults to QueueNameRequired.", - Type: []string{"string"}, - Format: "", - }, - }, - }, - }, - }, - } -} - func schema_openshift_api_operator_v1alpha1_RepositoryDigestMirrors(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ @@ -58282,6 +58263,25 @@ func schema_openshift_api_operator_v1alpha1_VersionAvailability(ref common.Refer } } +func schema_openshift_api_operator_v1alpha1_WorkloadManagement(ref common.ReferenceCallback) common.OpenAPIDefinition { + return common.OpenAPIDefinition{ + Schema: spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"object"}, + Properties: map[string]spec.Schema{ + "labelPolicy": { + SchemaProps: spec.SchemaProps{ + Description: "labelPolicy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. The allowed values are QueueName and None. None means that workloads will be suspended on creation and a label will be added via a mutating webhook. QueueName means that workloads that are managed by Kueue must have a label kueue.x-k8s.io/queue-name. If this label is not present on the workload, then Kueue will ignore this workload. Defaults to QueueName.", + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, + } +} + func schema_openshift_api_operatorcontrolplane_v1alpha1_LogEntry(ref common.ReferenceCallback) common.OpenAPIDefinition { return common.OpenAPIDefinition{ Schema: spec.Schema{ diff --git a/openapi/openapi.json b/openapi/openapi.json index a926c0b2fea..8ecfdbe7eca 100644 --- a/openapi/openapi.json +++ b/openapi/openapi.json @@ -33242,7 +33242,7 @@ } } }, - "com.github.openshift.api.operator.v1alpha1.GangSchedulingPolicy": { + "com.github.openshift.api.operator.v1alpha1.GangScheduling": { "description": "Kueue provides the ability to admit workloads all in one (gang admission) and evicts workloads if they are not ready within a specific time.", "type": "object", "properties": { @@ -33474,9 +33474,9 @@ "integrations" ], "properties": { - "gangSchedulingPolicy": { - "description": "gangSchedulingPolicy controls how Kueue admits workloads. Gang Scheduling is the act of all or nothing scheduling, where workloads do not become ready within a certain period, they may be evicted and later retried. This field is optional.", - "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.GangSchedulingPolicy" + "gangScheduling": { + "description": "gangScheduling controls how Kueue admits workloads. Gang Scheduling is the act of all or nothing scheduling, where workloads do not become ready within a certain period, they may be evicted and later retried. This field is optional.", + "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.GangScheduling" }, "integrations": { "description": "integrations is a required field that configures the Kueue's workload integrations. Kueue has both standard integrations, known as job frameworks, and external integrations known as external frameworks. Kueue will only manage workloads that correspond to the specified integrations.", @@ -33484,12 +33484,12 @@ "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.Integrations" }, "preemption": { - "description": "preemption is the process of evicting one or more admitted Workloads to accommodate another Workload. Kueue has classical premption and preemption via fair sharing.", + "description": "preemption is the process of evicting one or more admitted Workloads to accommodate another Workload. Kueue has classical premption and preemption via fair sharing. This field is optional.", "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.Preemption" }, - "queueLabelPolicy": { - "description": "queueLabelPolicy controls how kueue manages workloads The default behavior of Kueue will manage workloads that have a queue-name label. Workloads that are missing these label will be ignored by Kueue. This field is optional.", - "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.QueueLabelPolicy" + "workloadManagement": { + "description": "workloadManagement controls how kueue manages workloads. The default behavior of Kueue will manage workloads that have a queue-name label. Workloads that are missing these label will be ignored by Kueue. This field is optional.", + "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.WorkloadManagement" } } }, @@ -33948,15 +33948,6 @@ } } }, - "com.github.openshift.api.operator.v1alpha1.QueueLabelPolicy": { - "type": "object", - "properties": { - "policy": { - "description": "policy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. The allowed values are QueueNameRequired and QueueNameOptional. QueueNameOptional means that workloads will be suspended on creation and a label will be added via a mutating webhook. QueueNameRequired means that workloads that are managed by Kueue must have a label kueue.x-k8s.io/queue-name. If this label is not present on the workload, then Kueue will ignore this workload. Defaults to QueueNameRequired.", - "type": "string" - } - } - }, "com.github.openshift.api.operator.v1alpha1.RepositoryDigestMirrors": { "description": "RepositoryDigestMirrors holds cluster-wide information about how to handle mirros in the registries config. Note: the mirrors only work when pulling the images that are referenced by their digests.", "type": "object", @@ -34078,6 +34069,15 @@ } } }, + "com.github.openshift.api.operator.v1alpha1.WorkloadManagement": { + "type": "object", + "properties": { + "labelPolicy": { + "description": "labelPolicy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. The allowed values are QueueName and None. None means that workloads will be suspended on creation and a label will be added via a mutating webhook. QueueName means that workloads that are managed by Kueue must have a label kueue.x-k8s.io/queue-name. If this label is not present on the workload, then Kueue will ignore this workload. Defaults to QueueName.", + "type": "string" + } + } + }, "com.github.openshift.api.operatorcontrolplane.v1alpha1.LogEntry": { "description": "LogEntry records events", "type": "object", diff --git a/operator/v1alpha1/types_kueue.go b/operator/v1alpha1/types_kueue.go index d3d1d08efbc..b2eaa184e26 100644 --- a/operator/v1alpha1/types_kueue.go +++ b/operator/v1alpha1/types_kueue.go @@ -47,20 +47,21 @@ type KueueConfiguration struct { // Kueue will only manage workloads that correspond to the specified integrations. // +required Integrations Integrations `json:"integrations"` - // queueLabelPolicy controls how kueue manages workloads + // workloadManagement controls how kueue manages workloads. // The default behavior of Kueue will manage workloads that have a queue-name label. // Workloads that are missing these label will be ignored by Kueue. // This field is optional. // +optional - QueueLabelPolicy *QueueLabelPolicy `json:"queueLabelPolicy,omitempty"` - // gangSchedulingPolicy controls how Kueue admits workloads. + WorkloadManagement *WorkloadManagement `json:"workloadManagement,omitempty"` + // gangScheduling controls how Kueue admits workloads. // Gang Scheduling is the act of all or nothing scheduling, // where workloads do not become ready within a certain period, they may be evicted and later retried. // This field is optional. // +optional - GangSchedulingPolicy *GangSchedulingPolicy `json:"gangSchedulingPolicy,omitempty"` + GangScheduling *GangScheduling `json:"gangScheduling,omitempty"` // preemption is the process of evicting one or more admitted Workloads to accommodate another Workload. // Kueue has classical premption and preemption via fair sharing. + // This field is optional. // +optional Preemption *Preemption `json:"preemption,omitempty"` } @@ -191,25 +192,25 @@ type LabelKeys struct { } // +kubebuilder:validation:Enum=ByWorkload;Disabled -type GangSchedulingPolicyOptions string +type GangSchedulingPolicy string const ( - GangSchedulingPolicyByWorkload GangSchedulingPolicyOptions = "ByWorkload" - GangSchedulingPolicyDisabled GangSchedulingPolicyOptions = "Disabled" + GangSchedulingPolicyByWorkload GangSchedulingPolicy = "ByWorkload" + GangSchedulingPolicyDisabled GangSchedulingPolicy = "Disabled" ) // +kubebuilder:validation:Enum=Parallel;Sequential -type GangSchedulingAdmissionOptions string +type GangSchedulingWorkloadAdmission string const ( - GangSchedulingAdmissionOptionsSequential GangSchedulingAdmissionOptions = "Sequential" - GangSchedulingAdmissionOptionsParallel GangSchedulingAdmissionOptions = "Parallel" + GangSchedulingWorkloadAdmissionSequential GangSchedulingWorkloadAdmission = "Sequential" + GangSchedulingWorkloadAdmissionParallel GangSchedulingWorkloadAdmission = "Parallel" ) // Kueue provides the ability to admit workloads all in one (gang admission) // and evicts workloads if they are not ready within a specific time. -// +kubebuilder:validation:XValidation:rule="has(self.policy) && self.policy == 'ByWorkload' ? has(self.byWorkload) : !has(self.byWorkload)",message="byWorkload is required when policy is ByWorkload, and forbidden otherwise" -type GangSchedulingPolicy struct { +// +kubebuilder:validation:XValidation:rule="has(self.policy) && self.policy == 'ByWorkload' ? has(self.byWorkload) : !has(self.byWorkload)",message="byWorkload is required when policy is byWorkload, and forbidden otherwise" +type GangScheduling struct { // policy allows you to enable and configure gang scheduling. // This is an optional field. // The allowed values are ByWorkload and Disabled. @@ -218,7 +219,7 @@ type GangSchedulingPolicy struct { // for admission as a single unit. // Where workloads do not become ready over time, the entire workload may then be evicted and retried at a later time. // +optional - Policy *GangSchedulingPolicyOptions `json:"policy,omitempty"` + Policy *GangSchedulingPolicy `json:"policy,omitempty"` // byWorkload controls how admission is done. // byWorkload is only required if policy is equal to ByWorkload. // +optional @@ -236,30 +237,30 @@ type ByWorkload struct { // This may lead to a deadlock where workloads are in contention for cluster capacity and // pods from another workload having successfully scheduled prevent pods from the current workload scheduling. // +optional - Admission *GangSchedulingAdmissionOptions `json:"admission,omitempty"` + Admission *GangSchedulingWorkloadAdmission `json:"admission,omitempty"` } -// +kubebuilder:validation:Enum=QueueNameRequired;QueueNameOptional -type QueueLabelNamePolicy string +// +kubebuilder:validation:Enum=QueueName;None +type LabelPolicy string const ( - QueueLabelNamePolicyRequired QueueLabelNamePolicy = "QueueNameRequired" - QueueLabelNamePolicyOptional QueueLabelNamePolicy = "QueueNameOptional" + QueueLabelNamePolicyRequired LabelPolicy = "QueueName" + QueueLabelNamePolicyOptional LabelPolicy = "None" ) -type QueueLabelPolicy struct { - // policy controls whether or not Kueue reconciles +type WorkloadManagement struct { + // labelPolicy controls whether or not Kueue reconciles // jobs that don't set the label kueue.x-k8s.io/queue-name. - // The allowed values are QueueNameRequired and QueueNameOptional. - // QueueNameOptional means that workloads will be suspended on + // The allowed values are QueueName and None. + // None means that workloads will be suspended on // creation and a label will be added via a mutating webhook. - // QueueNameRequired means that workloads that are managed + // QueueName means that workloads that are managed // by Kueue must have a label kueue.x-k8s.io/queue-name. // If this label is not present on the workload, then Kueue will // ignore this workload. - // Defaults to QueueNameRequired. + // Defaults to QueueName. // +optional - Policy *QueueLabelNamePolicy `json:"policy,omitempty"` + LabelPolicy *LabelPolicy `json:"labelPolicy,omitempty"` } // +kubebuilder:validation:Enum=Classical;FairSharing diff --git a/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml b/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml index 3374adc6306..48440baa5ee 100644 --- a/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml +++ b/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml @@ -48,9 +48,9 @@ spec: config is the desired configuration for the Kueue operator. properties: - gangSchedulingPolicy: + gangScheduling: description: |- - gangSchedulingPolicy controls how Kueue admits workloads. + gangScheduling controls how Kueue admits workloads. Gang Scheduling is the act of all or nothing scheduling, where workloads do not become ready within a certain period, they may be evicted and later retried. This field is optional. @@ -90,7 +90,7 @@ spec: type: string type: object x-kubernetes-validations: - - message: byWorkload is required when policy is ByWorkload, and + - message: byWorkload is required when policy is byWorkload, and forbidden otherwise rule: 'has(self.policy) && self.policy == ''ByWorkload'' ? has(self.byWorkload) : !has(self.byWorkload)' @@ -239,6 +239,7 @@ spec: description: |- preemption is the process of evicting one or more admitted Workloads to accommodate another Workload. Kueue has classical premption and preemption via fair sharing. + This field is optional. properties: preemptionPolicy: description: |- @@ -261,28 +262,28 @@ spec: - FairSharing type: string type: object - queueLabelPolicy: + workloadManagement: description: |- - queueLabelPolicy controls how kueue manages workloads + workloadManagement controls how kueue manages workloads. The default behavior of Kueue will manage workloads that have a queue-name label. Workloads that are missing these label will be ignored by Kueue. This field is optional. properties: - policy: + labelPolicy: description: |- - policy controls whether or not Kueue reconciles + labelPolicy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. - The allowed values are QueueNameRequired and QueueNameOptional. - QueueNameOptional means that workloads will be suspended on + The allowed values are QueueName and None. + None means that workloads will be suspended on creation and a label will be added via a mutating webhook. - QueueNameRequired means that workloads that are managed + QueueName means that workloads that are managed by Kueue must have a label kueue.x-k8s.io/queue-name. If this label is not present on the workload, then Kueue will ignore this workload. - Defaults to QueueNameRequired. + Defaults to QueueName. enum: - - QueueNameRequired - - QueueNameOptional + - QueueName + - None type: string type: object required: diff --git a/operator/v1alpha1/zz_generated.deepcopy.go b/operator/v1alpha1/zz_generated.deepcopy.go index 8a6bfe59f3b..db6bf9edaf5 100644 --- a/operator/v1alpha1/zz_generated.deepcopy.go +++ b/operator/v1alpha1/zz_generated.deepcopy.go @@ -31,7 +31,7 @@ func (in *ByWorkload) DeepCopyInto(out *ByWorkload) { *out = *in if in.Admission != nil { in, out := &in.Admission, &out.Admission - *out = new(GangSchedulingAdmissionOptions) + *out = new(GangSchedulingWorkloadAdmission) **out = **in } return @@ -294,11 +294,11 @@ func (in *ExternalFramework) DeepCopy() *ExternalFramework { } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *GangSchedulingPolicy) DeepCopyInto(out *GangSchedulingPolicy) { +func (in *GangScheduling) DeepCopyInto(out *GangScheduling) { *out = *in if in.Policy != nil { in, out := &in.Policy, &out.Policy - *out = new(GangSchedulingPolicyOptions) + *out = new(GangSchedulingPolicy) **out = **in } if in.ByWorkload != nil { @@ -309,12 +309,12 @@ func (in *GangSchedulingPolicy) DeepCopyInto(out *GangSchedulingPolicy) { return } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GangSchedulingPolicy. -func (in *GangSchedulingPolicy) DeepCopy() *GangSchedulingPolicy { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GangScheduling. +func (in *GangScheduling) DeepCopy() *GangScheduling { if in == nil { return nil } - out := new(GangSchedulingPolicy) + out := new(GangScheduling) in.DeepCopyInto(out) return out } @@ -510,14 +510,14 @@ func (in *Kueue) DeepCopyObject() runtime.Object { func (in *KueueConfiguration) DeepCopyInto(out *KueueConfiguration) { *out = *in in.Integrations.DeepCopyInto(&out.Integrations) - if in.QueueLabelPolicy != nil { - in, out := &in.QueueLabelPolicy, &out.QueueLabelPolicy - *out = new(QueueLabelPolicy) + if in.WorkloadManagement != nil { + in, out := &in.WorkloadManagement, &out.WorkloadManagement + *out = new(WorkloadManagement) (*in).DeepCopyInto(*out) } - if in.GangSchedulingPolicy != nil { - in, out := &in.GangSchedulingPolicy, &out.GangSchedulingPolicy - *out = new(GangSchedulingPolicy) + if in.GangScheduling != nil { + in, out := &in.GangScheduling, &out.GangScheduling + *out = new(GangScheduling) (*in).DeepCopyInto(*out) } if in.Preemption != nil { @@ -842,27 +842,6 @@ func (in *Preemption) DeepCopy() *Preemption { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *QueueLabelPolicy) DeepCopyInto(out *QueueLabelPolicy) { - *out = *in - if in.Policy != nil { - in, out := &in.Policy, &out.Policy - *out = new(QueueLabelNamePolicy) - **out = **in - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new QueueLabelPolicy. -func (in *QueueLabelPolicy) DeepCopy() *QueueLabelPolicy { - if in == nil { - return nil - } - out := new(QueueLabelPolicy) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RepositoryDigestMirrors) DeepCopyInto(out *RepositoryDigestMirrors) { *out = *in @@ -933,3 +912,24 @@ func (in *VersionAvailability) DeepCopy() *VersionAvailability { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *WorkloadManagement) DeepCopyInto(out *WorkloadManagement) { + *out = *in + if in.LabelPolicy != nil { + in, out := &in.LabelPolicy, &out.LabelPolicy + *out = new(LabelPolicy) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkloadManagement. +func (in *WorkloadManagement) DeepCopy() *WorkloadManagement { + if in == nil { + return nil + } + out := new(WorkloadManagement) + in.DeepCopyInto(out) + return out +} diff --git a/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml b/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml index 7c32f970106..967984a086d 100644 --- a/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml +++ b/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml @@ -46,9 +46,9 @@ spec: config is the desired configuration for the Kueue operator. properties: - gangSchedulingPolicy: + gangScheduling: description: |- - gangSchedulingPolicy controls how Kueue admits workloads. + gangScheduling controls how Kueue admits workloads. Gang Scheduling is the act of all or nothing scheduling, where workloads do not become ready within a certain period, they may be evicted and later retried. This field is optional. @@ -88,7 +88,7 @@ spec: type: string type: object x-kubernetes-validations: - - message: byWorkload is required when policy is ByWorkload, and + - message: byWorkload is required when policy is byWorkload, and forbidden otherwise rule: 'has(self.policy) && self.policy == ''ByWorkload'' ? has(self.byWorkload) : !has(self.byWorkload)' @@ -237,6 +237,7 @@ spec: description: |- preemption is the process of evicting one or more admitted Workloads to accommodate another Workload. Kueue has classical premption and preemption via fair sharing. + This field is optional. properties: preemptionPolicy: description: |- @@ -259,28 +260,28 @@ spec: - FairSharing type: string type: object - queueLabelPolicy: + workloadManagement: description: |- - queueLabelPolicy controls how kueue manages workloads + workloadManagement controls how kueue manages workloads. The default behavior of Kueue will manage workloads that have a queue-name label. Workloads that are missing these label will be ignored by Kueue. This field is optional. properties: - policy: + labelPolicy: description: |- - policy controls whether or not Kueue reconciles + labelPolicy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. - The allowed values are QueueNameRequired and QueueNameOptional. - QueueNameOptional means that workloads will be suspended on + The allowed values are QueueName and None. + None means that workloads will be suspended on creation and a label will be added via a mutating webhook. - QueueNameRequired means that workloads that are managed + QueueName means that workloads that are managed by Kueue must have a label kueue.x-k8s.io/queue-name. If this label is not present on the workload, then Kueue will ignore this workload. - Defaults to QueueNameRequired. + Defaults to QueueName. enum: - - QueueNameRequired - - QueueNameOptional + - QueueName + - None type: string type: object required: diff --git a/operator/v1alpha1/zz_generated.swagger_doc_generated.go b/operator/v1alpha1/zz_generated.swagger_doc_generated.go index 0a8b2fc77a0..3a5cd156cec 100644 --- a/operator/v1alpha1/zz_generated.swagger_doc_generated.go +++ b/operator/v1alpha1/zz_generated.swagger_doc_generated.go @@ -277,14 +277,14 @@ func (ExternalFramework) SwaggerDoc() map[string]string { return map_ExternalFramework } -var map_GangSchedulingPolicy = map[string]string{ +var map_GangScheduling = map[string]string{ "": "Kueue provides the ability to admit workloads all in one (gang admission) and evicts workloads if they are not ready within a specific time.", "policy": "policy allows you to enable and configure gang scheduling. This is an optional field. The allowed values are ByWorkload and Disabled. The default value will be Disabled. When set to ByWorkload, this means each workload is processed and considered for admission as a single unit. Where workloads do not become ready over time, the entire workload may then be evicted and retried at a later time.", "byWorkload": "byWorkload controls how admission is done. byWorkload is only required if policy is equal to ByWorkload.", } -func (GangSchedulingPolicy) SwaggerDoc() map[string]string { - return map_GangSchedulingPolicy +func (GangScheduling) SwaggerDoc() map[string]string { + return map_GangScheduling } var map_Integrations = map[string]string{ @@ -310,10 +310,10 @@ func (Kueue) SwaggerDoc() map[string]string { } var map_KueueConfiguration = map[string]string{ - "integrations": "integrations is a required field that configures the Kueue's workload integrations. Kueue has both standard integrations, known as job frameworks, and external integrations known as external frameworks. Kueue will only manage workloads that correspond to the specified integrations.", - "queueLabelPolicy": "queueLabelPolicy controls how kueue manages workloads The default behavior of Kueue will manage workloads that have a queue-name label. Workloads that are missing these label will be ignored by Kueue. This field is optional.", - "gangSchedulingPolicy": "gangSchedulingPolicy controls how Kueue admits workloads. Gang Scheduling is the act of all or nothing scheduling, where workloads do not become ready within a certain period, they may be evicted and later retried. This field is optional.", - "preemption": "preemption is the process of evicting one or more admitted Workloads to accommodate another Workload. Kueue has classical premption and preemption via fair sharing.", + "integrations": "integrations is a required field that configures the Kueue's workload integrations. Kueue has both standard integrations, known as job frameworks, and external integrations known as external frameworks. Kueue will only manage workloads that correspond to the specified integrations.", + "workloadManagement": "workloadManagement controls how kueue manages workloads. The default behavior of Kueue will manage workloads that have a queue-name label. Workloads that are missing these label will be ignored by Kueue. This field is optional.", + "gangScheduling": "gangScheduling controls how Kueue admits workloads. Gang Scheduling is the act of all or nothing scheduling, where workloads do not become ready within a certain period, they may be evicted and later retried. This field is optional.", + "preemption": "preemption is the process of evicting one or more admitted Workloads to accommodate another Workload. Kueue has classical premption and preemption via fair sharing. This field is optional.", } func (KueueConfiguration) SwaggerDoc() map[string]string { @@ -362,12 +362,12 @@ func (Preemption) SwaggerDoc() map[string]string { return map_Preemption } -var map_QueueLabelPolicy = map[string]string{ - "policy": "policy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. The allowed values are QueueNameRequired and QueueNameOptional. QueueNameOptional means that workloads will be suspended on creation and a label will be added via a mutating webhook. QueueNameRequired means that workloads that are managed by Kueue must have a label kueue.x-k8s.io/queue-name. If this label is not present on the workload, then Kueue will ignore this workload. Defaults to QueueNameRequired.", +var map_WorkloadManagement = map[string]string{ + "labelPolicy": "labelPolicy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. The allowed values are QueueName and None. None means that workloads will be suspended on creation and a label will be added via a mutating webhook. QueueName means that workloads that are managed by Kueue must have a label kueue.x-k8s.io/queue-name. If this label is not present on the workload, then Kueue will ignore this workload. Defaults to QueueName.", } -func (QueueLabelPolicy) SwaggerDoc() map[string]string { - return map_QueueLabelPolicy +func (WorkloadManagement) SwaggerDoc() map[string]string { + return map_WorkloadManagement } var map_OLM = map[string]string{ From b8d22109dda4e44aa2082f27a74cb2acb172ee79 Mon Sep 17 00:00:00 2001 From: Kevin Hannon Date: Wed, 9 Apr 2025 09:52:58 -0400 Subject: [PATCH 31/42] updates with apis --- .../generated_openapi/zz_generated.openapi.go | 38 +++++++---- openapi/openapi.json | 44 +++++++++---- operator/v1alpha1/types_kueue.go | 56 +++++++++------- .../zz_generated.crd-manifests/kueue.crd.yaml | 64 +++++++++---------- operator/v1alpha1/zz_generated.deepcopy.go | 26 +------- .../AAA_ungated.yaml | 64 +++++++++---------- .../zz_generated.swagger_doc_generated.go | 20 +++--- 7 files changed, 166 insertions(+), 146 deletions(-) diff --git a/openapi/generated_openapi/zz_generated.openapi.go b/openapi/generated_openapi/zz_generated.openapi.go index b4c1d39da62..97333e62140 100644 --- a/openapi/generated_openapi/zz_generated.openapi.go +++ b/openapi/generated_openapi/zz_generated.openapi.go @@ -56880,7 +56880,7 @@ func schema_openshift_api_operator_v1alpha1_ExternalFramework(ref common.Referen Properties: map[string]spec.Schema{ "group": { SchemaProps: spec.SchemaProps{ - Description: "group is the API group of the externalFramework. Must be a valid DNS 1123 subdomain consisting of of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character.", + Description: "group is the API group of the externalFramework. Must be a valid DNS 1123 subdomain consisting of of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. We are using matches and not cel functions to allow for use on 4.17.", Default: "", Type: []string{"string"}, Format: "", @@ -56888,7 +56888,7 @@ func schema_openshift_api_operator_v1alpha1_ExternalFramework(ref common.Referen }, "resource": { SchemaProps: spec.SchemaProps{ - Description: "resource is the Resource type of the external framework. Resource types are lowercase and plural (e.g. pods, deployments). Must be a valid DNS 1123 label consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. The value must start and end with an alphanumeric character.", + Description: "resource is the Resource type of the external framework. Resource types are lowercase and plural (e.g. pods, deployments). Must be a valid DNS 1123 label consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. The value must start and end with an alphanumeric character. We are using matches and not cel functions to allow for use on 4.17.", Default: "", Type: []string{"string"}, Format: "", @@ -56896,7 +56896,7 @@ func schema_openshift_api_operator_v1alpha1_ExternalFramework(ref common.Referen }, "version": { SchemaProps: spec.SchemaProps{ - Description: "version is the version of the api (e.g. v1alpha1, v1beta1, v1). Must be a valid DNS 1035 label consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. The value must start with an alphabetic character and end with an alphanumeric character.", + Description: "version is the version of the api (e.g. v1alpha1, v1beta1, v1). Must be a valid DNS 1035 label consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. The value must start with an alphabetic character and end with an alphanumeric character. We are using matches and not cel functions to allow for use on 4.17.", Default: "", Type: []string{"string"}, Format: "", @@ -56919,6 +56919,7 @@ func schema_openshift_api_operator_v1alpha1_GangScheduling(ref common.ReferenceC "policy": { SchemaProps: spec.SchemaProps{ Description: "policy allows you to enable and configure gang scheduling. This is an optional field. The allowed values are ByWorkload and Disabled. The default value will be Disabled. When set to ByWorkload, this means each workload is processed and considered for admission as a single unit. Where workloads do not become ready over time, the entire workload may then be evicted and retried at a later time.", + Default: "", Type: []string{"string"}, Format: "", }, @@ -56930,6 +56931,19 @@ func schema_openshift_api_operator_v1alpha1_GangScheduling(ref common.ReferenceC }, }, }, + Required: []string{"policy"}, + }, + VendorExtensible: spec.VendorExtensible{ + Extensions: spec.Extensions{ + "x-kubernetes-unions": []interface{}{ + map[string]interface{}{ + "discriminator": "policy", + "fields-to-discriminateBy": map[string]interface{}{ + "byWorkload": "ByWorkload", + }, + }, + }, + }, }, }, Dependencies: []string{ @@ -57220,7 +57234,7 @@ func schema_openshift_api_operator_v1alpha1_Integrations(ref common.ReferenceCal "labelKeysToCopy": { VendorExtensible: spec.VendorExtensible{ Extensions: spec.Extensions{ - "x-kubernetes-list-type": "atomic", + "x-kubernetes-list-type": "set", }, }, SchemaProps: spec.SchemaProps{ @@ -57249,7 +57263,7 @@ func schema_openshift_api_operator_v1alpha1_Kueue(ref common.ReferenceCallback) return common.OpenAPIDefinition{ Schema: spec.Schema{ SchemaProps: spec.SchemaProps{ - Description: "Kueue is the CRD to represent the Kueue operator. This CRD defines the configuration for Kueue. Compatibility level 4: No compatibility is provided, the API can change at any point for any reason. These capabilities should not be used by applications needing long term support.", + Description: "Kueue is the CRD to represent the Kueue operator This CRD defines the configuration that the Kueue Compatibility level 4: No compatibility is provided, the API can change at any point for any reason. These capabilities should not be used by applications needing long term support.", Type: []string{"object"}, Properties: map[string]spec.Schema{ "kind": { @@ -57311,19 +57325,19 @@ func schema_openshift_api_operator_v1alpha1_KueueConfiguration(ref common.Refere }, "workloadManagement": { SchemaProps: spec.SchemaProps{ - Description: "workloadManagement controls how kueue manages workloads. The default behavior of Kueue will manage workloads that have a queue-name label. Workloads that are missing these label will be ignored by Kueue. This field is optional.", + Description: "workloadManagement controls how kueue manages workloads. By default Kueue will manage workloads that have a queue-name label. Workloads that are missing the queue-name will be ignored by Kueue. This field is optional. If this field is not specified, Kueue will only manage workloads that have the queue-name label.", Ref: ref("github.com/openshift/api/operator/v1alpha1.WorkloadManagement"), }, }, "gangScheduling": { SchemaProps: spec.SchemaProps{ - Description: "gangScheduling controls how Kueue admits workloads. Gang Scheduling is the act of all or nothing scheduling, where workloads do not become ready within a certain period, they may be evicted and later retried. This field is optional.", + Description: "gangScheduling controls how Kueue admits workloads. Gang Scheduling is the act of all or nothing scheduling, where workloads do not become ready within a certain period, they may be evicted and later retried. This field is optional. If this field is not specified, gang scheduling will be disabled.", Ref: ref("github.com/openshift/api/operator/v1alpha1.GangScheduling"), }, }, "preemption": { SchemaProps: spec.SchemaProps{ - Description: "preemption is the process of evicting one or more admitted Workloads to accommodate another Workload. Kueue has classical premption and preemption via fair sharing. This field is optional.", + Description: "preemption is the process of evicting one or more admitted Workloads to accommodate another Workload. Kueue has classical premption and preemption via fair sharing. This field is optional. If this field is not specified, preemption will be set to Classical.", Ref: ref("github.com/openshift/api/operator/v1alpha1.Preemption"), }, }, @@ -57543,7 +57557,7 @@ func schema_openshift_api_operator_v1alpha1_LabelKeys(ref common.ReferenceCallba Properties: map[string]spec.Schema{ "key": { SchemaProps: spec.SchemaProps{ - Description: "key is the label key. A label key must be a valid qualified name consisting of a lower-case alphanumeric string, and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. The optional prefix and the name are separate by a forward slash (/).", + Description: "key is the label key. A label key must be a valid qualified name consisting of a lower-case alphanumeric string, and hyphens of at most 63 characters in length. We are using matches and not cel functions to allow for use on 4.17. The name must start and end with an alphanumeric character. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. The optional prefix and the name are separate by a forward slash (/).", Default: "", Type: []string{"string"}, Format: "", @@ -58061,7 +58075,7 @@ func schema_openshift_api_operator_v1alpha1_Preemption(ref common.ReferenceCallb Properties: map[string]spec.Schema{ "preemptionPolicy": { SchemaProps: spec.SchemaProps{ - Description: "preemptionPolicy are the types of preemption kueue allows. Kueue has two types of preemption: classical and fair sharing. Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. Fairsharing means that ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort. FairSharing is a more heavy weight algorithm. The default is Classical.", + Description: "preemptionPolicy are the types of preemption kueue allows. Kueue has two types of preemption: Classical and FairSharing. Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. FairSharing means that ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort. FairSharing is a more heavy weight algorithm. The default is Classical.", Type: []string{"string"}, Format: "", }, @@ -58271,12 +58285,14 @@ func schema_openshift_api_operator_v1alpha1_WorkloadManagement(ref common.Refere Properties: map[string]spec.Schema{ "labelPolicy": { SchemaProps: spec.SchemaProps{ - Description: "labelPolicy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. The allowed values are QueueName and None. None means that workloads will be suspended on creation and a label will be added via a mutating webhook. QueueName means that workloads that are managed by Kueue must have a label kueue.x-k8s.io/queue-name. If this label is not present on the workload, then Kueue will ignore this workload. Defaults to QueueName.", + Description: "labelPolicy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. The allowed values are QueueName and None. None means that workloads will be suspended on creation and a label will be added via a mutating webhook. This will be applied for all integrations that Kueue manages. QueueName means that workloads that are managed by Kueue must have a label kueue.x-k8s.io/queue-name. If this label is not present on the workload, then Kueue will ignore this workload. Defaults to QueueName.", + Default: "", Type: []string{"string"}, Format: "", }, }, }, + Required: []string{"labelPolicy"}, }, }, } diff --git a/openapi/openapi.json b/openapi/openapi.json index 8ecfdbe7eca..c3db9e7cfab 100644 --- a/openapi/openapi.json +++ b/openapi/openapi.json @@ -33226,17 +33226,17 @@ ], "properties": { "group": { - "description": "group is the API group of the externalFramework. Must be a valid DNS 1123 subdomain consisting of of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character.", + "description": "group is the API group of the externalFramework. Must be a valid DNS 1123 subdomain consisting of of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. We are using matches and not cel functions to allow for use on 4.17.", "type": "string", "default": "" }, "resource": { - "description": "resource is the Resource type of the external framework. Resource types are lowercase and plural (e.g. pods, deployments). Must be a valid DNS 1123 label consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. The value must start and end with an alphanumeric character.", + "description": "resource is the Resource type of the external framework. Resource types are lowercase and plural (e.g. pods, deployments). Must be a valid DNS 1123 label consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. The value must start and end with an alphanumeric character. We are using matches and not cel functions to allow for use on 4.17.", "type": "string", "default": "" }, "version": { - "description": "version is the version of the api (e.g. v1alpha1, v1beta1, v1). Must be a valid DNS 1035 label consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. The value must start with an alphabetic character and end with an alphanumeric character.", + "description": "version is the version of the api (e.g. v1alpha1, v1beta1, v1). Must be a valid DNS 1035 label consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. The value must start with an alphabetic character and end with an alphanumeric character. We are using matches and not cel functions to allow for use on 4.17.", "type": "string", "default": "" } @@ -33245,6 +33245,9 @@ "com.github.openshift.api.operator.v1alpha1.GangScheduling": { "description": "Kueue provides the ability to admit workloads all in one (gang admission) and evicts workloads if they are not ready within a specific time.", "type": "object", + "required": [ + "policy" + ], "properties": { "byWorkload": { "description": "byWorkload controls how admission is done. byWorkload is only required if policy is equal to ByWorkload.", @@ -33252,9 +33255,18 @@ }, "policy": { "description": "policy allows you to enable and configure gang scheduling. This is an optional field. The allowed values are ByWorkload and Disabled. The default value will be Disabled. When set to ByWorkload, this means each workload is processed and considered for admission as a single unit. Where workloads do not become ready over time, the entire workload may then be evicted and retried at a later time.", - "type": "string" + "type": "string", + "default": "" } - } + }, + "x-kubernetes-unions": [ + { + "discriminator": "policy", + "fields-to-discriminateBy": { + "byWorkload": "ByWorkload" + } + } + ] }, "com.github.openshift.api.operator.v1alpha1.GenerationHistory": { "description": "GenerationHistory keeps track of the generation for a given resource so that decisions about forced updated can be made. DEPRECATED: Use fields in v1.GenerationStatus instead", @@ -33432,12 +33444,12 @@ "default": {}, "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.LabelKeys" }, - "x-kubernetes-list-type": "atomic" + "x-kubernetes-list-type": "set" } } }, "com.github.openshift.api.operator.v1alpha1.Kueue": { - "description": "Kueue is the CRD to represent the Kueue operator. This CRD defines the configuration for Kueue. Compatibility level 4: No compatibility is provided, the API can change at any point for any reason. These capabilities should not be used by applications needing long term support.", + "description": "Kueue is the CRD to represent the Kueue operator This CRD defines the configuration that the Kueue Compatibility level 4: No compatibility is provided, the API can change at any point for any reason. These capabilities should not be used by applications needing long term support.", "type": "object", "required": [ "spec" @@ -33475,7 +33487,7 @@ ], "properties": { "gangScheduling": { - "description": "gangScheduling controls how Kueue admits workloads. Gang Scheduling is the act of all or nothing scheduling, where workloads do not become ready within a certain period, they may be evicted and later retried. This field is optional.", + "description": "gangScheduling controls how Kueue admits workloads. Gang Scheduling is the act of all or nothing scheduling, where workloads do not become ready within a certain period, they may be evicted and later retried. This field is optional. If this field is not specified, gang scheduling will be disabled.", "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.GangScheduling" }, "integrations": { @@ -33484,11 +33496,11 @@ "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.Integrations" }, "preemption": { - "description": "preemption is the process of evicting one or more admitted Workloads to accommodate another Workload. Kueue has classical premption and preemption via fair sharing. This field is optional.", + "description": "preemption is the process of evicting one or more admitted Workloads to accommodate another Workload. Kueue has classical premption and preemption via fair sharing. This field is optional. If this field is not specified, preemption will be set to Classical.", "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.Preemption" }, "workloadManagement": { - "description": "workloadManagement controls how kueue manages workloads. The default behavior of Kueue will manage workloads that have a queue-name label. Workloads that are missing these label will be ignored by Kueue. This field is optional.", + "description": "workloadManagement controls how kueue manages workloads. By default Kueue will manage workloads that have a queue-name label. Workloads that are missing the queue-name will be ignored by Kueue. This field is optional. If this field is not specified, Kueue will only manage workloads that have the queue-name label.", "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.WorkloadManagement" } } @@ -33621,7 +33633,7 @@ ], "properties": { "key": { - "description": "key is the label key. A label key must be a valid qualified name consisting of a lower-case alphanumeric string, and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. The optional prefix and the name are separate by a forward slash (/).", + "description": "key is the label key. A label key must be a valid qualified name consisting of a lower-case alphanumeric string, and hyphens of at most 63 characters in length. We are using matches and not cel functions to allow for use on 4.17. The name must start and end with an alphanumeric character. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. The optional prefix and the name are separate by a forward slash (/).", "type": "string", "default": "" } @@ -33943,7 +33955,7 @@ "type": "object", "properties": { "preemptionPolicy": { - "description": "preemptionPolicy are the types of preemption kueue allows. Kueue has two types of preemption: classical and fair sharing. Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. Fairsharing means that ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort. FairSharing is a more heavy weight algorithm. The default is Classical.", + "description": "preemptionPolicy are the types of preemption kueue allows. Kueue has two types of preemption: Classical and FairSharing. Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. FairSharing means that ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort. FairSharing is a more heavy weight algorithm. The default is Classical.", "type": "string" } } @@ -34071,10 +34083,14 @@ }, "com.github.openshift.api.operator.v1alpha1.WorkloadManagement": { "type": "object", + "required": [ + "labelPolicy" + ], "properties": { "labelPolicy": { - "description": "labelPolicy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. The allowed values are QueueName and None. None means that workloads will be suspended on creation and a label will be added via a mutating webhook. QueueName means that workloads that are managed by Kueue must have a label kueue.x-k8s.io/queue-name. If this label is not present on the workload, then Kueue will ignore this workload. Defaults to QueueName.", - "type": "string" + "description": "labelPolicy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. The allowed values are QueueName and None. None means that workloads will be suspended on creation and a label will be added via a mutating webhook. This will be applied for all integrations that Kueue manages. QueueName means that workloads that are managed by Kueue must have a label kueue.x-k8s.io/queue-name. If this label is not present on the workload, then Kueue will ignore this workload. Defaults to QueueName.", + "type": "string", + "default": "" } } }, diff --git a/operator/v1alpha1/types_kueue.go b/operator/v1alpha1/types_kueue.go index b2eaa184e26..df9d172163f 100644 --- a/operator/v1alpha1/types_kueue.go +++ b/operator/v1alpha1/types_kueue.go @@ -7,8 +7,8 @@ import ( // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object -// Kueue is the CRD to represent the Kueue operator. -// This CRD defines the configuration for Kueue. +// Kueue is the CRD to represent the Kueue operator +// This CRD defines the configuration that the Kueue // Compatibility level 4: No compatibility is provided, the API can change at any point for any reason. These capabilities should not be used by applications needing long term support. // +openshift:compatibility-gen:level=4 // +kubebuilder:object:root=true @@ -43,25 +43,30 @@ type KueueOperandSpec struct { type KueueConfiguration struct { // integrations is a required field that configures the Kueue's workload integrations. - // Kueue has both standard integrations, known as job frameworks, and external integrations known as external frameworks. + // Kueue has both standard integrations, known as job frameworks, and external integrations + // known as external frameworks. // Kueue will only manage workloads that correspond to the specified integrations. // +required Integrations Integrations `json:"integrations"` // workloadManagement controls how kueue manages workloads. - // The default behavior of Kueue will manage workloads that have a queue-name label. - // Workloads that are missing these label will be ignored by Kueue. + // By default Kueue will manage workloads that have a queue-name label. + // Workloads that are missing the queue-name will be ignored by Kueue. // This field is optional. + // If this field is not specified, Kueue will only manage workloads + // that have the queue-name label. // +optional WorkloadManagement *WorkloadManagement `json:"workloadManagement,omitempty"` // gangScheduling controls how Kueue admits workloads. // Gang Scheduling is the act of all or nothing scheduling, // where workloads do not become ready within a certain period, they may be evicted and later retried. // This field is optional. + // If this field is not specified, gang scheduling will be disabled. // +optional GangScheduling *GangScheduling `json:"gangScheduling,omitempty"` // preemption is the process of evicting one or more admitted Workloads to accommodate another Workload. // Kueue has classical premption and preemption via fair sharing. // This field is optional. + // If this field is not specified, preemption will be set to Classical. // +optional Preemption *Preemption `json:"preemption,omitempty"` } @@ -89,7 +94,7 @@ type KueueList struct { Items []Kueue `json:"items"` } -// +kubebuilder:validation:Enum=BatchJob;RayJob;RayCluster;JobSet;MPIJob;PaddleJob;PytorchJob;TFJob;XGBoostJob;AppWrapper;Pod;Deployment;StatefulSet;LeaderWorkerSet +// +kubebuilder:validation:Enum=BatchJob;RayJob;RayCluster;JobSet;MPIJob;PaddleJob;PyTorchJob;TFJob;XGBoostJob;AppWrapper;Pod;Deployment;StatefulSet;LeaderWorkerSet type KueueIntegration string const ( @@ -117,9 +122,10 @@ type ExternalFramework struct { // Must be a valid DNS 1123 subdomain consisting of of lower-case alphanumeric characters, // hyphens and periods, of at most 253 characters in length. // Each period separated segment within the subdomain must start and end with an alphanumeric character. + // We are using matches and not cel functions to allow for use on 4.17. // +kubebuilder:validation:MaxLength=253 // +kubebuilder:validation:MinLength=1 - // +kubebuilder:validation:XValidation:rule="self.size() == 0 || !format.dns1123Label().validate(self).hasValue()",message="a lowercase RFC 1123 subdomain must consist of lower case alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character." + // +kubebuilder:validation:XValidation:rule="self.matches(r'^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$') && self.size() <= 253" // +required Group string `json:"group"` // resource is the Resource type of the external framework. @@ -127,18 +133,20 @@ type ExternalFramework struct { // Must be a valid DNS 1123 label consisting of a lower-case alphanumeric string // and hyphens of at most 63 characters in length. // The value must start and end with an alphanumeric character. + // We are using matches and not cel functions to allow for use on 4.17. // +kubebuilder:validation:MaxLength=63 // +kubebuilder:validation:MinLength=1 - // +kubebuilder:validation:XValidation:rule="self.size() == 0 || !format.dns1123Label().validate(self).hasValue()",message="a lowercase RFC 1123 label must consist of lower case alphanumeric characters and '-', and must start and end with an alphanumeric character." + // +kubebuilder:validation:XValidation:rule="self.matches(r'^[a-z0-9]([-a-z0-9]*[a-z0-9])?$') && self.size() <= 63" // +required Resource string `json:"resource"` // version is the version of the api (e.g. v1alpha1, v1beta1, v1). // Must be a valid DNS 1035 label consisting of a lower-case alphanumeric string // and hyphens of at most 63 characters in length. // The value must start with an alphabetic character and end with an alphanumeric character. + // We are using matches and not cel functions to allow for use on 4.17. // +kubebuilder:validation:MaxLength=63 // +kubebuilder:validation:MinLength=1 - // +kubebuilder:validation:XValidation:rule="self.size() == 0 || !format.dns1035Label().validate(self).hasValue()",message="a lowercase RFC 1035 label must consist of lower case alphanumeric characters, '-' or '.', and must start with an alphabetic character and end with an alphanumeric character." + // +kubebuilder:validation:XValidation:rule="self.matches(r'^[a-z0-9]([-a-z0-9]*[a-z0-9])?$') && self.size() <= 63" // +required Version string `json:"version"` } @@ -170,7 +178,8 @@ type Integrations struct { // These keys are persisted to the internal Kueue workload object. // If not specified, only the Kueue labels will be copied. // +kubebuilder:validation:MaxItems=64 - // +listType=atomic + // +listType=set + // +listMapType=key // +optional LabelKeysToCopy []LabelKeys `json:"labelKeysToCopy,omitempty"` } @@ -179,6 +188,7 @@ type LabelKeys struct { // key is the label key. // A label key must be a valid qualified name consisting of a lower-case alphanumeric string, // and hyphens of at most 63 characters in length. + // We are using matches and not cel functions to allow for use on 4.17. // The name must start and end with an alphanumeric character. // The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, // hyphens and periods, of at most 253 characters in length. @@ -186,7 +196,7 @@ type LabelKeys struct { // The optional prefix and the name are separate by a forward slash (/). // +kubebuilder:validation:MaxLength=317 // +kubebuilder:validation:MinLength=1 - // +kubebuilder:validation:XValidation:rule="!format.qualifiedName().validate(self).hasValue()",message="a qualified name must consist of a lower-case alphanumeric and hyphenated string of at most 63 characters in length, starting and ending with alphanumeric chracters. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character." + // +kubebuilder:validation:XValidation:rule="self.matches(r'^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?([a-z0-9]([-a-z0-9]*[a-z0-9])?)$') && self.size() <= 317" // +required Key string `json:"key"` } @@ -210,6 +220,7 @@ const ( // Kueue provides the ability to admit workloads all in one (gang admission) // and evicts workloads if they are not ready within a specific time. // +kubebuilder:validation:XValidation:rule="has(self.policy) && self.policy == 'ByWorkload' ? has(self.byWorkload) : !has(self.byWorkload)",message="byWorkload is required when policy is byWorkload, and forbidden otherwise" +// +union type GangScheduling struct { // policy allows you to enable and configure gang scheduling. // This is an optional field. @@ -218,8 +229,9 @@ type GangScheduling struct { // When set to ByWorkload, this means each workload is processed and considered // for admission as a single unit. // Where workloads do not become ready over time, the entire workload may then be evicted and retried at a later time. - // +optional - Policy *GangSchedulingPolicy `json:"policy,omitempty"` + // +required + // +unionDiscriminator + Policy GangSchedulingPolicy `json:"policy"` // byWorkload controls how admission is done. // byWorkload is only required if policy is equal to ByWorkload. // +optional @@ -237,15 +249,15 @@ type ByWorkload struct { // This may lead to a deadlock where workloads are in contention for cluster capacity and // pods from another workload having successfully scheduled prevent pods from the current workload scheduling. // +optional - Admission *GangSchedulingWorkloadAdmission `json:"admission,omitempty"` + Admission GangSchedulingWorkloadAdmission `json:"admission,omitempty"` } // +kubebuilder:validation:Enum=QueueName;None type LabelPolicy string const ( - QueueLabelNamePolicyRequired LabelPolicy = "QueueName" - QueueLabelNamePolicyOptional LabelPolicy = "None" + LabelPolicyQueueName LabelPolicy = "QueueName" + LabelPolicyNone LabelPolicy = "None" ) type WorkloadManagement struct { @@ -254,13 +266,13 @@ type WorkloadManagement struct { // The allowed values are QueueName and None. // None means that workloads will be suspended on // creation and a label will be added via a mutating webhook. + // This will be applied for all integrations that Kueue manages. // QueueName means that workloads that are managed // by Kueue must have a label kueue.x-k8s.io/queue-name. // If this label is not present on the workload, then Kueue will // ignore this workload. - // Defaults to QueueName. - // +optional - LabelPolicy *LabelPolicy `json:"labelPolicy,omitempty"` + // +required + LabelPolicy LabelPolicy `json:"labelPolicy"` } // +kubebuilder:validation:Enum=Classical;FairSharing @@ -273,13 +285,13 @@ const ( type Preemption struct { // preemptionPolicy are the types of preemption kueue allows. - // Kueue has two types of preemption: classical and fair sharing. + // Kueue has two types of preemption: Classical and FairSharing. // Classical means that an incoming workload, which does // not fit within the unusued quota, is eligible to issue preemptions // when the requests of the workload are below the // resource flavor's nominal quota or borrowWithinCohort is enabled // on the Cluster Queue. - // Fairsharing means that ClusterQueues with pending Workloads can preempt other Workloads + // FairSharing means that ClusterQueues with pending Workloads can preempt other Workloads // in their cohort until the preempting ClusterQueue // obtains an equal or weighted share of the borrowable resources. // The borrowable resources are the unused nominal quota @@ -287,5 +299,5 @@ type Preemption struct { // FairSharing is a more heavy weight algorithm. // The default is Classical. // +optional - PreemptionPolicy *PreemptionPolicy `json:"preemptionPolicy,omitempty"` + PreemptionPolicy PreemptionPolicy `json:"preemptionPolicy,omitempty"` } diff --git a/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml b/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml index 48440baa5ee..e098c2cfea9 100644 --- a/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml +++ b/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml @@ -19,8 +19,8 @@ spec: schema: openAPIV3Schema: description: |- - Kueue is the CRD to represent the Kueue operator. - This CRD defines the configuration for Kueue. + Kueue is the CRD to represent the Kueue operator + This CRD defines the configuration that the Kueue Compatibility level 4: No compatibility is provided, the API can change at any point for any reason. These capabilities should not be used by applications needing long term support. properties: apiVersion: @@ -54,6 +54,7 @@ spec: Gang Scheduling is the act of all or nothing scheduling, where workloads do not become ready within a certain period, they may be evicted and later retried. This field is optional. + If this field is not specified, gang scheduling will be disabled. properties: byWorkload: description: |- @@ -88,6 +89,8 @@ spec: - ByWorkload - Disabled type: string + required: + - policy type: object x-kubernetes-validations: - message: byWorkload is required when policy is byWorkload, and @@ -97,7 +100,8 @@ spec: integrations: description: |- integrations is a required field that configures the Kueue's workload integrations. - Kueue has both standard integrations, known as job frameworks, and external integrations known as external frameworks. + Kueue has both standard integrations, known as job frameworks, and external integrations + known as external frameworks. Kueue will only manage workloads that correspond to the specified integrations. properties: externalFrameworks: @@ -118,14 +122,13 @@ spec: Must be a valid DNS 1123 subdomain consisting of of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. + We are using matches and not cel functions to allow for use on 4.17. maxLength: 253 minLength: 1 type: string x-kubernetes-validations: - - message: a lowercase RFC 1123 subdomain must consist - of lower case alphanumeric characters, '-' or '.', - and must start and end with an alphanumeric character. - rule: self.size() == 0 || !format.dns1123Label().validate(self).hasValue() + - rule: self.matches(r'^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$') + && self.size() <= 253 resource: description: |- resource is the Resource type of the external framework. @@ -133,29 +136,26 @@ spec: Must be a valid DNS 1123 label consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. The value must start and end with an alphanumeric character. + We are using matches and not cel functions to allow for use on 4.17. maxLength: 63 minLength: 1 type: string x-kubernetes-validations: - - message: a lowercase RFC 1123 label must consist of - lower case alphanumeric characters and '-', and - must start and end with an alphanumeric character. - rule: self.size() == 0 || !format.dns1123Label().validate(self).hasValue() + - rule: self.matches(r'^[a-z0-9]([-a-z0-9]*[a-z0-9])?$') + && self.size() <= 63 version: description: |- version is the version of the api (e.g. v1alpha1, v1beta1, v1). Must be a valid DNS 1035 label consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. The value must start with an alphabetic character and end with an alphanumeric character. + We are using matches and not cel functions to allow for use on 4.17. maxLength: 63 minLength: 1 type: string x-kubernetes-validations: - - message: a lowercase RFC 1035 label must consist of - lower case alphanumeric characters, '-' or '.', - and must start with an alphabetic character and - end with an alphanumeric character. - rule: self.size() == 0 || !format.dns1035Label().validate(self).hasValue() + - rule: self.matches(r'^[a-z0-9]([-a-z0-9]*[a-z0-9])?$') + && self.size() <= 63 required: - group - resource @@ -179,7 +179,7 @@ spec: - JobSet - MPIJob - PaddleJob - - PytorchJob + - PyTorchJob - TFJob - XGBoostJob - AppWrapper @@ -207,6 +207,7 @@ spec: key is the label key. A label key must be a valid qualified name consisting of a lower-case alphanumeric string, and hyphens of at most 63 characters in length. + We are using matches and not cel functions to allow for use on 4.17. The name must start and end with an alphanumeric character. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. @@ -216,22 +217,14 @@ spec: minLength: 1 type: string x-kubernetes-validations: - - message: a qualified name must consist of a lower-case - alphanumeric and hyphenated string of at most 63 - characters in length, starting and ending with alphanumeric - chracters. The name may be optionally prefixed with - a subdomain consisting of lower-case alphanumeric - characters, hyphens and periods, of at most 253 - characters in length. Each period separated segment - within the subdomain must start and end with an - alphanumeric character. - rule: '!format.qualifiedName().validate(self).hasValue()' + - rule: self.matches(r'^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?([a-z0-9]([-a-z0-9]*[a-z0-9])?)$') + && self.size() <= 317 required: - key type: object maxItems: 64 type: array - x-kubernetes-list-type: atomic + x-kubernetes-list-type: set required: - frameworks type: object @@ -240,17 +233,18 @@ spec: preemption is the process of evicting one or more admitted Workloads to accommodate another Workload. Kueue has classical premption and preemption via fair sharing. This field is optional. + If this field is not specified, preemption will be set to Classical. properties: preemptionPolicy: description: |- preemptionPolicy are the types of preemption kueue allows. - Kueue has two types of preemption: classical and fair sharing. + Kueue has two types of preemption: Classical and FairSharing. Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. - Fairsharing means that ClusterQueues with pending Workloads can preempt other Workloads + FairSharing means that ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota @@ -265,9 +259,11 @@ spec: workloadManagement: description: |- workloadManagement controls how kueue manages workloads. - The default behavior of Kueue will manage workloads that have a queue-name label. - Workloads that are missing these label will be ignored by Kueue. + By default Kueue will manage workloads that have a queue-name label. + Workloads that are missing the queue-name will be ignored by Kueue. This field is optional. + If this field is not specified, Kueue will only manage workloads + that have the queue-name label. properties: labelPolicy: description: |- @@ -276,15 +272,17 @@ spec: The allowed values are QueueName and None. None means that workloads will be suspended on creation and a label will be added via a mutating webhook. + This will be applied for all integrations that Kueue manages. QueueName means that workloads that are managed by Kueue must have a label kueue.x-k8s.io/queue-name. If this label is not present on the workload, then Kueue will ignore this workload. - Defaults to QueueName. enum: - QueueName - None type: string + required: + - labelPolicy type: object required: - integrations diff --git a/operator/v1alpha1/zz_generated.deepcopy.go b/operator/v1alpha1/zz_generated.deepcopy.go index db6bf9edaf5..fd8b139769b 100644 --- a/operator/v1alpha1/zz_generated.deepcopy.go +++ b/operator/v1alpha1/zz_generated.deepcopy.go @@ -29,11 +29,6 @@ func (in *BackupJobReference) DeepCopy() *BackupJobReference { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ByWorkload) DeepCopyInto(out *ByWorkload) { *out = *in - if in.Admission != nil { - in, out := &in.Admission, &out.Admission - *out = new(GangSchedulingWorkloadAdmission) - **out = **in - } return } @@ -296,15 +291,10 @@ func (in *ExternalFramework) DeepCopy() *ExternalFramework { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GangScheduling) DeepCopyInto(out *GangScheduling) { *out = *in - if in.Policy != nil { - in, out := &in.Policy, &out.Policy - *out = new(GangSchedulingPolicy) - **out = **in - } if in.ByWorkload != nil { in, out := &in.ByWorkload, &out.ByWorkload *out = new(ByWorkload) - (*in).DeepCopyInto(*out) + **out = **in } return } @@ -513,7 +503,7 @@ func (in *KueueConfiguration) DeepCopyInto(out *KueueConfiguration) { if in.WorkloadManagement != nil { in, out := &in.WorkloadManagement, &out.WorkloadManagement *out = new(WorkloadManagement) - (*in).DeepCopyInto(*out) + **out = **in } if in.GangScheduling != nil { in, out := &in.GangScheduling, &out.GangScheduling @@ -523,7 +513,7 @@ func (in *KueueConfiguration) DeepCopyInto(out *KueueConfiguration) { if in.Preemption != nil { in, out := &in.Preemption, &out.Preemption *out = new(Preemption) - (*in).DeepCopyInto(*out) + **out = **in } return } @@ -824,11 +814,6 @@ func (in *OperatorStatus) DeepCopy() *OperatorStatus { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Preemption) DeepCopyInto(out *Preemption) { *out = *in - if in.PreemptionPolicy != nil { - in, out := &in.PreemptionPolicy, &out.PreemptionPolicy - *out = new(PreemptionPolicy) - **out = **in - } return } @@ -916,11 +901,6 @@ func (in *VersionAvailability) DeepCopy() *VersionAvailability { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *WorkloadManagement) DeepCopyInto(out *WorkloadManagement) { *out = *in - if in.LabelPolicy != nil { - in, out := &in.LabelPolicy, &out.LabelPolicy - *out = new(LabelPolicy) - **out = **in - } return } diff --git a/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml b/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml index 967984a086d..8d5faec84aa 100644 --- a/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml +++ b/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml @@ -17,8 +17,8 @@ spec: schema: openAPIV3Schema: description: |- - Kueue is the CRD to represent the Kueue operator. - This CRD defines the configuration for Kueue. + Kueue is the CRD to represent the Kueue operator + This CRD defines the configuration that the Kueue Compatibility level 4: No compatibility is provided, the API can change at any point for any reason. These capabilities should not be used by applications needing long term support. properties: apiVersion: @@ -52,6 +52,7 @@ spec: Gang Scheduling is the act of all or nothing scheduling, where workloads do not become ready within a certain period, they may be evicted and later retried. This field is optional. + If this field is not specified, gang scheduling will be disabled. properties: byWorkload: description: |- @@ -86,6 +87,8 @@ spec: - ByWorkload - Disabled type: string + required: + - policy type: object x-kubernetes-validations: - message: byWorkload is required when policy is byWorkload, and @@ -95,7 +98,8 @@ spec: integrations: description: |- integrations is a required field that configures the Kueue's workload integrations. - Kueue has both standard integrations, known as job frameworks, and external integrations known as external frameworks. + Kueue has both standard integrations, known as job frameworks, and external integrations + known as external frameworks. Kueue will only manage workloads that correspond to the specified integrations. properties: externalFrameworks: @@ -116,14 +120,13 @@ spec: Must be a valid DNS 1123 subdomain consisting of of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. + We are using matches and not cel functions to allow for use on 4.17. maxLength: 253 minLength: 1 type: string x-kubernetes-validations: - - message: a lowercase RFC 1123 subdomain must consist - of lower case alphanumeric characters, '-' or '.', - and must start and end with an alphanumeric character. - rule: self.size() == 0 || !format.dns1123Label().validate(self).hasValue() + - rule: self.matches(r'^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$') + && self.size() <= 253 resource: description: |- resource is the Resource type of the external framework. @@ -131,29 +134,26 @@ spec: Must be a valid DNS 1123 label consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. The value must start and end with an alphanumeric character. + We are using matches and not cel functions to allow for use on 4.17. maxLength: 63 minLength: 1 type: string x-kubernetes-validations: - - message: a lowercase RFC 1123 label must consist of - lower case alphanumeric characters and '-', and - must start and end with an alphanumeric character. - rule: self.size() == 0 || !format.dns1123Label().validate(self).hasValue() + - rule: self.matches(r'^[a-z0-9]([-a-z0-9]*[a-z0-9])?$') + && self.size() <= 63 version: description: |- version is the version of the api (e.g. v1alpha1, v1beta1, v1). Must be a valid DNS 1035 label consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. The value must start with an alphabetic character and end with an alphanumeric character. + We are using matches and not cel functions to allow for use on 4.17. maxLength: 63 minLength: 1 type: string x-kubernetes-validations: - - message: a lowercase RFC 1035 label must consist of - lower case alphanumeric characters, '-' or '.', - and must start with an alphabetic character and - end with an alphanumeric character. - rule: self.size() == 0 || !format.dns1035Label().validate(self).hasValue() + - rule: self.matches(r'^[a-z0-9]([-a-z0-9]*[a-z0-9])?$') + && self.size() <= 63 required: - group - resource @@ -177,7 +177,7 @@ spec: - JobSet - MPIJob - PaddleJob - - PytorchJob + - PyTorchJob - TFJob - XGBoostJob - AppWrapper @@ -205,6 +205,7 @@ spec: key is the label key. A label key must be a valid qualified name consisting of a lower-case alphanumeric string, and hyphens of at most 63 characters in length. + We are using matches and not cel functions to allow for use on 4.17. The name must start and end with an alphanumeric character. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. @@ -214,22 +215,14 @@ spec: minLength: 1 type: string x-kubernetes-validations: - - message: a qualified name must consist of a lower-case - alphanumeric and hyphenated string of at most 63 - characters in length, starting and ending with alphanumeric - chracters. The name may be optionally prefixed with - a subdomain consisting of lower-case alphanumeric - characters, hyphens and periods, of at most 253 - characters in length. Each period separated segment - within the subdomain must start and end with an - alphanumeric character. - rule: '!format.qualifiedName().validate(self).hasValue()' + - rule: self.matches(r'^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?([a-z0-9]([-a-z0-9]*[a-z0-9])?)$') + && self.size() <= 317 required: - key type: object maxItems: 64 type: array - x-kubernetes-list-type: atomic + x-kubernetes-list-type: set required: - frameworks type: object @@ -238,17 +231,18 @@ spec: preemption is the process of evicting one or more admitted Workloads to accommodate another Workload. Kueue has classical premption and preemption via fair sharing. This field is optional. + If this field is not specified, preemption will be set to Classical. properties: preemptionPolicy: description: |- preemptionPolicy are the types of preemption kueue allows. - Kueue has two types of preemption: classical and fair sharing. + Kueue has two types of preemption: Classical and FairSharing. Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. - Fairsharing means that ClusterQueues with pending Workloads can preempt other Workloads + FairSharing means that ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota @@ -263,9 +257,11 @@ spec: workloadManagement: description: |- workloadManagement controls how kueue manages workloads. - The default behavior of Kueue will manage workloads that have a queue-name label. - Workloads that are missing these label will be ignored by Kueue. + By default Kueue will manage workloads that have a queue-name label. + Workloads that are missing the queue-name will be ignored by Kueue. This field is optional. + If this field is not specified, Kueue will only manage workloads + that have the queue-name label. properties: labelPolicy: description: |- @@ -274,15 +270,17 @@ spec: The allowed values are QueueName and None. None means that workloads will be suspended on creation and a label will be added via a mutating webhook. + This will be applied for all integrations that Kueue manages. QueueName means that workloads that are managed by Kueue must have a label kueue.x-k8s.io/queue-name. If this label is not present on the workload, then Kueue will ignore this workload. - Defaults to QueueName. enum: - QueueName - None type: string + required: + - labelPolicy type: object required: - integrations diff --git a/operator/v1alpha1/zz_generated.swagger_doc_generated.go b/operator/v1alpha1/zz_generated.swagger_doc_generated.go index 3a5cd156cec..5b811763bef 100644 --- a/operator/v1alpha1/zz_generated.swagger_doc_generated.go +++ b/operator/v1alpha1/zz_generated.swagger_doc_generated.go @@ -268,9 +268,9 @@ func (ByWorkload) SwaggerDoc() map[string]string { var map_ExternalFramework = map[string]string{ "": "This is the GVR for an external framework. Controller runtime requires this in this format for api discoverability.", - "group": "group is the API group of the externalFramework. Must be a valid DNS 1123 subdomain consisting of of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character.", - "resource": "resource is the Resource type of the external framework. Resource types are lowercase and plural (e.g. pods, deployments). Must be a valid DNS 1123 label consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. The value must start and end with an alphanumeric character.", - "version": "version is the version of the api (e.g. v1alpha1, v1beta1, v1). Must be a valid DNS 1035 label consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. The value must start with an alphabetic character and end with an alphanumeric character.", + "group": "group is the API group of the externalFramework. Must be a valid DNS 1123 subdomain consisting of of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. We are using matches and not cel functions to allow for use on 4.17.", + "resource": "resource is the Resource type of the external framework. Resource types are lowercase and plural (e.g. pods, deployments). Must be a valid DNS 1123 label consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. The value must start and end with an alphanumeric character. We are using matches and not cel functions to allow for use on 4.17.", + "version": "version is the version of the api (e.g. v1alpha1, v1beta1, v1). Must be a valid DNS 1035 label consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. The value must start with an alphabetic character and end with an alphanumeric character. We are using matches and not cel functions to allow for use on 4.17.", } func (ExternalFramework) SwaggerDoc() map[string]string { @@ -299,7 +299,7 @@ func (Integrations) SwaggerDoc() map[string]string { } var map_Kueue = map[string]string{ - "": "Kueue is the CRD to represent the Kueue operator. This CRD defines the configuration for Kueue. Compatibility level 4: No compatibility is provided, the API can change at any point for any reason. These capabilities should not be used by applications needing long term support.", + "": "Kueue is the CRD to represent the Kueue operator This CRD defines the configuration that the Kueue Compatibility level 4: No compatibility is provided, the API can change at any point for any reason. These capabilities should not be used by applications needing long term support.", "metadata": "metadata for kueue", "spec": "spec holds user settable values for configuration", "status": "status holds observed values from the cluster. They may not be overridden.", @@ -311,9 +311,9 @@ func (Kueue) SwaggerDoc() map[string]string { var map_KueueConfiguration = map[string]string{ "integrations": "integrations is a required field that configures the Kueue's workload integrations. Kueue has both standard integrations, known as job frameworks, and external integrations known as external frameworks. Kueue will only manage workloads that correspond to the specified integrations.", - "workloadManagement": "workloadManagement controls how kueue manages workloads. The default behavior of Kueue will manage workloads that have a queue-name label. Workloads that are missing these label will be ignored by Kueue. This field is optional.", - "gangScheduling": "gangScheduling controls how Kueue admits workloads. Gang Scheduling is the act of all or nothing scheduling, where workloads do not become ready within a certain period, they may be evicted and later retried. This field is optional.", - "preemption": "preemption is the process of evicting one or more admitted Workloads to accommodate another Workload. Kueue has classical premption and preemption via fair sharing. This field is optional.", + "workloadManagement": "workloadManagement controls how kueue manages workloads. By default Kueue will manage workloads that have a queue-name label. Workloads that are missing the queue-name will be ignored by Kueue. This field is optional. If this field is not specified, Kueue will only manage workloads that have the queue-name label.", + "gangScheduling": "gangScheduling controls how Kueue admits workloads. Gang Scheduling is the act of all or nothing scheduling, where workloads do not become ready within a certain period, they may be evicted and later retried. This field is optional. If this field is not specified, gang scheduling will be disabled.", + "preemption": "preemption is the process of evicting one or more admitted Workloads to accommodate another Workload. Kueue has classical premption and preemption via fair sharing. This field is optional. If this field is not specified, preemption will be set to Classical.", } func (KueueConfiguration) SwaggerDoc() map[string]string { @@ -347,7 +347,7 @@ func (KueueStatus) SwaggerDoc() map[string]string { } var map_LabelKeys = map[string]string{ - "key": "key is the label key. A label key must be a valid qualified name consisting of a lower-case alphanumeric string, and hyphens of at most 63 characters in length. The name must start and end with an alphanumeric character. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. The optional prefix and the name are separate by a forward slash (/).", + "key": "key is the label key. A label key must be a valid qualified name consisting of a lower-case alphanumeric string, and hyphens of at most 63 characters in length. We are using matches and not cel functions to allow for use on 4.17. The name must start and end with an alphanumeric character. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. The optional prefix and the name are separate by a forward slash (/).", } func (LabelKeys) SwaggerDoc() map[string]string { @@ -355,7 +355,7 @@ func (LabelKeys) SwaggerDoc() map[string]string { } var map_Preemption = map[string]string{ - "preemptionPolicy": "preemptionPolicy are the types of preemption kueue allows. Kueue has two types of preemption: classical and fair sharing. Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. Fairsharing means that ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort. FairSharing is a more heavy weight algorithm. The default is Classical.", + "preemptionPolicy": "preemptionPolicy are the types of preemption kueue allows. Kueue has two types of preemption: Classical and FairSharing. Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. FairSharing means that ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort. FairSharing is a more heavy weight algorithm. The default is Classical.", } func (Preemption) SwaggerDoc() map[string]string { @@ -363,7 +363,7 @@ func (Preemption) SwaggerDoc() map[string]string { } var map_WorkloadManagement = map[string]string{ - "labelPolicy": "labelPolicy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. The allowed values are QueueName and None. None means that workloads will be suspended on creation and a label will be added via a mutating webhook. QueueName means that workloads that are managed by Kueue must have a label kueue.x-k8s.io/queue-name. If this label is not present on the workload, then Kueue will ignore this workload. Defaults to QueueName.", + "labelPolicy": "labelPolicy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. The allowed values are QueueName and None. None means that workloads will be suspended on creation and a label will be added via a mutating webhook. This will be applied for all integrations that Kueue manages. QueueName means that workloads that are managed by Kueue must have a label kueue.x-k8s.io/queue-name. If this label is not present on the workload, then Kueue will ignore this workload.", } func (WorkloadManagement) SwaggerDoc() map[string]string { From 40b549ae7dc2ed59497a6b5e7bc881834da0b3f8 Mon Sep 17 00:00:00 2001 From: Kevin Hannon Date: Wed, 9 Apr 2025 10:47:20 -0400 Subject: [PATCH 32/42] update docs --- openapi/generated_openapi/zz_generated.openapi.go | 2 +- openapi/openapi.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/openapi/generated_openapi/zz_generated.openapi.go b/openapi/generated_openapi/zz_generated.openapi.go index 97333e62140..02c45dedc60 100644 --- a/openapi/generated_openapi/zz_generated.openapi.go +++ b/openapi/generated_openapi/zz_generated.openapi.go @@ -58285,7 +58285,7 @@ func schema_openshift_api_operator_v1alpha1_WorkloadManagement(ref common.Refere Properties: map[string]spec.Schema{ "labelPolicy": { SchemaProps: spec.SchemaProps{ - Description: "labelPolicy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. The allowed values are QueueName and None. None means that workloads will be suspended on creation and a label will be added via a mutating webhook. This will be applied for all integrations that Kueue manages. QueueName means that workloads that are managed by Kueue must have a label kueue.x-k8s.io/queue-name. If this label is not present on the workload, then Kueue will ignore this workload. Defaults to QueueName.", + Description: "labelPolicy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. The allowed values are QueueName and None. None means that workloads will be suspended on creation and a label will be added via a mutating webhook. This will be applied for all integrations that Kueue manages. QueueName means that workloads that are managed by Kueue must have a label kueue.x-k8s.io/queue-name. If this label is not present on the workload, then Kueue will ignore this workload.", Default: "", Type: []string{"string"}, Format: "", diff --git a/openapi/openapi.json b/openapi/openapi.json index c3db9e7cfab..8345a1a5d47 100644 --- a/openapi/openapi.json +++ b/openapi/openapi.json @@ -34088,7 +34088,7 @@ ], "properties": { "labelPolicy": { - "description": "labelPolicy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. The allowed values are QueueName and None. None means that workloads will be suspended on creation and a label will be added via a mutating webhook. This will be applied for all integrations that Kueue manages. QueueName means that workloads that are managed by Kueue must have a label kueue.x-k8s.io/queue-name. If this label is not present on the workload, then Kueue will ignore this workload. Defaults to QueueName.", + "description": "labelPolicy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. The allowed values are QueueName and None. None means that workloads will be suspended on creation and a label will be added via a mutating webhook. This will be applied for all integrations that Kueue manages. QueueName means that workloads that are managed by Kueue must have a label kueue.x-k8s.io/queue-name. If this label is not present on the workload, then Kueue will ignore this workload.", "type": "string", "default": "" } From e1d7706cdd6841604118f4fcc1acafa0d9cf6ace Mon Sep 17 00:00:00 2001 From: Kevin Hannon Date: Wed, 9 Apr 2025 11:25:17 -0400 Subject: [PATCH 33/42] bryce godoc comments --- openapi/generated_openapi/zz_generated.openapi.go | 4 ++-- openapi/openapi.json | 4 ++-- operator/v1alpha1/types_kueue.go | 5 +++-- operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml | 5 +++-- .../kueue.operator.openshift.io/AAA_ungated.yaml | 5 +++-- operator/v1alpha1/zz_generated.swagger_doc_generated.go | 4 ++-- 6 files changed, 15 insertions(+), 12 deletions(-) diff --git a/openapi/generated_openapi/zz_generated.openapi.go b/openapi/generated_openapi/zz_generated.openapi.go index 02c45dedc60..7a601d49118 100644 --- a/openapi/generated_openapi/zz_generated.openapi.go +++ b/openapi/generated_openapi/zz_generated.openapi.go @@ -56515,7 +56515,7 @@ func schema_openshift_api_operator_v1alpha1_ByWorkload(ref common.ReferenceCallb Properties: map[string]spec.Schema{ "admission": { SchemaProps: spec.SchemaProps{ - Description: "admission controls how kueue will process workloads. When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling.", + Description: "admission controls how kueue will process workloads. Allowed values are Sequential and Parallel. When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling.", Type: []string{"string"}, Format: "", }, @@ -56926,7 +56926,7 @@ func schema_openshift_api_operator_v1alpha1_GangScheduling(ref common.ReferenceC }, "byWorkload": { SchemaProps: spec.SchemaProps{ - Description: "byWorkload controls how admission is done. byWorkload is only required if policy is equal to ByWorkload.", + Description: "byWorkload configures how Kueue will process workloads for admission. byWorkload is required when policy is ByWorkload, and forbidden otherwise.", Ref: ref("github.com/openshift/api/operator/v1alpha1.ByWorkload"), }, }, diff --git a/openapi/openapi.json b/openapi/openapi.json index 8345a1a5d47..8a750e06a8c 100644 --- a/openapi/openapi.json +++ b/openapi/openapi.json @@ -33014,7 +33014,7 @@ "type": "object", "properties": { "admission": { - "description": "admission controls how kueue will process workloads. When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling.", + "description": "admission controls how kueue will process workloads. Allowed values are Sequential and Parallel. When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling.", "type": "string" } } @@ -33250,7 +33250,7 @@ ], "properties": { "byWorkload": { - "description": "byWorkload controls how admission is done. byWorkload is only required if policy is equal to ByWorkload.", + "description": "byWorkload configures how Kueue will process workloads for admission. byWorkload is required when policy is ByWorkload, and forbidden otherwise.", "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.ByWorkload" }, "policy": { diff --git a/operator/v1alpha1/types_kueue.go b/operator/v1alpha1/types_kueue.go index df9d172163f..4ecc13518b4 100644 --- a/operator/v1alpha1/types_kueue.go +++ b/operator/v1alpha1/types_kueue.go @@ -232,8 +232,8 @@ type GangScheduling struct { // +required // +unionDiscriminator Policy GangSchedulingPolicy `json:"policy"` - // byWorkload controls how admission is done. - // byWorkload is only required if policy is equal to ByWorkload. + // byWorkload configures how Kueue will process workloads for admission. + // byWorkload is required when policy is ByWorkload, and forbidden otherwise. // +optional ByWorkload *ByWorkload `json:"byWorkload,omitempty"` } @@ -241,6 +241,7 @@ type GangScheduling struct { // ByWorkload controls how admission is done type ByWorkload struct { // admission controls how kueue will process workloads. + // Allowed values are Sequential and Parallel. // When admission is set to Sequential, only pods from the currently processing workload will be admitted. // Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. // Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, diff --git a/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml b/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml index e098c2cfea9..1d2c956656c 100644 --- a/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml +++ b/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml @@ -58,12 +58,13 @@ spec: properties: byWorkload: description: |- - byWorkload controls how admission is done. - byWorkload is only required if policy is equal to ByWorkload. + byWorkload configures how Kueue will process workloads for admission. + byWorkload is required when policy is ByWorkload, and forbidden otherwise. properties: admission: description: |- admission controls how kueue will process workloads. + Allowed values are Sequential and Parallel. When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, diff --git a/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml b/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml index 8d5faec84aa..7cb27fbd691 100644 --- a/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml +++ b/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml @@ -56,12 +56,13 @@ spec: properties: byWorkload: description: |- - byWorkload controls how admission is done. - byWorkload is only required if policy is equal to ByWorkload. + byWorkload configures how Kueue will process workloads for admission. + byWorkload is required when policy is ByWorkload, and forbidden otherwise. properties: admission: description: |- admission controls how kueue will process workloads. + Allowed values are Sequential and Parallel. When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, diff --git a/operator/v1alpha1/zz_generated.swagger_doc_generated.go b/operator/v1alpha1/zz_generated.swagger_doc_generated.go index 5b811763bef..187aeff2b89 100644 --- a/operator/v1alpha1/zz_generated.swagger_doc_generated.go +++ b/operator/v1alpha1/zz_generated.swagger_doc_generated.go @@ -259,7 +259,7 @@ func (RepositoryDigestMirrors) SwaggerDoc() map[string]string { var map_ByWorkload = map[string]string{ "": "ByWorkload controls how admission is done", - "admission": "admission controls how kueue will process workloads. When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling.", + "admission": "admission controls how kueue will process workloads. Allowed values are Sequential and Parallel. When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling.", } func (ByWorkload) SwaggerDoc() map[string]string { @@ -280,7 +280,7 @@ func (ExternalFramework) SwaggerDoc() map[string]string { var map_GangScheduling = map[string]string{ "": "Kueue provides the ability to admit workloads all in one (gang admission) and evicts workloads if they are not ready within a specific time.", "policy": "policy allows you to enable and configure gang scheduling. This is an optional field. The allowed values are ByWorkload and Disabled. The default value will be Disabled. When set to ByWorkload, this means each workload is processed and considered for admission as a single unit. Where workloads do not become ready over time, the entire workload may then be evicted and retried at a later time.", - "byWorkload": "byWorkload controls how admission is done. byWorkload is only required if policy is equal to ByWorkload.", + "byWorkload": "byWorkload configures how Kueue will process workloads for admission. byWorkload is required when policy is ByWorkload, and forbidden otherwise.", } func (GangScheduling) SwaggerDoc() map[string]string { From 12d66d55895967cd4a754e93a2bc6da39037d4ff Mon Sep 17 00:00:00 2001 From: Kevin Hannon Date: Wed, 9 Apr 2025 13:00:25 -0400 Subject: [PATCH 34/42] bryce minor comments --- openapi/generated_openapi/zz_generated.openapi.go | 8 ++++---- openapi/openapi.json | 8 ++++---- operator/v1alpha1/types_kueue.go | 11 ++++++----- .../zz_generated.crd-manifests/kueue.crd.yaml | 11 ++++++----- .../kueue.operator.openshift.io/AAA_ungated.yaml | 11 ++++++----- .../v1alpha1/zz_generated.swagger_doc_generated.go | 8 ++++---- 6 files changed, 30 insertions(+), 27 deletions(-) diff --git a/openapi/generated_openapi/zz_generated.openapi.go b/openapi/generated_openapi/zz_generated.openapi.go index 7a601d49118..39c06951094 100644 --- a/openapi/generated_openapi/zz_generated.openapi.go +++ b/openapi/generated_openapi/zz_generated.openapi.go @@ -56515,7 +56515,7 @@ func schema_openshift_api_operator_v1alpha1_ByWorkload(ref common.ReferenceCallb Properties: map[string]spec.Schema{ "admission": { SchemaProps: spec.SchemaProps{ - Description: "admission controls how kueue will process workloads. Allowed values are Sequential and Parallel. When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling.", + Description: "admission controls how Kueue will process workloads. Allowed values are Sequential and Parallel. When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling.", Type: []string{"string"}, Format: "", }, @@ -57282,7 +57282,7 @@ func schema_openshift_api_operator_v1alpha1_Kueue(ref common.ReferenceCallback) }, "metadata": { SchemaProps: spec.SchemaProps{ - Description: "metadata for kueue", + Description: "metadata for Kueue.", Default: map[string]interface{}{}, Ref: ref("k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta"), }, @@ -57325,7 +57325,7 @@ func schema_openshift_api_operator_v1alpha1_KueueConfiguration(ref common.Refere }, "workloadManagement": { SchemaProps: spec.SchemaProps{ - Description: "workloadManagement controls how kueue manages workloads. By default Kueue will manage workloads that have a queue-name label. Workloads that are missing the queue-name will be ignored by Kueue. This field is optional. If this field is not specified, Kueue will only manage workloads that have the queue-name label.", + Description: "workloadManagement controls how Kueue manages workloads. By default Kueue will manage workloads that have a queue-name label. Workloads that are missing the queue-name will be ignored by Kueue. This field is optional. If this field is not specified, Kueue will only manage workloads that have the queue-name label.", Ref: ref("github.com/openshift/api/operator/v1alpha1.WorkloadManagement"), }, }, @@ -58075,7 +58075,7 @@ func schema_openshift_api_operator_v1alpha1_Preemption(ref common.ReferenceCallb Properties: map[string]spec.Schema{ "preemptionPolicy": { SchemaProps: spec.SchemaProps{ - Description: "preemptionPolicy are the types of preemption kueue allows. Kueue has two types of preemption: Classical and FairSharing. Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. FairSharing means that ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort. FairSharing is a more heavy weight algorithm. The default is Classical.", + Description: "preemptionPolicy are the types of preemption Kueue allows. Kueue has two types of preemption: Classical and FairSharing. Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. FairSharing means that ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort. FairSharing is a more heavy weight algorithm. The default is Classical.", Type: []string{"string"}, Format: "", }, diff --git a/openapi/openapi.json b/openapi/openapi.json index 8a750e06a8c..7b779b8f1eb 100644 --- a/openapi/openapi.json +++ b/openapi/openapi.json @@ -33014,7 +33014,7 @@ "type": "object", "properties": { "admission": { - "description": "admission controls how kueue will process workloads. Allowed values are Sequential and Parallel. When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling.", + "description": "admission controls how Kueue will process workloads. Allowed values are Sequential and Parallel. When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling.", "type": "string" } } @@ -33464,7 +33464,7 @@ "type": "string" }, "metadata": { - "description": "metadata for kueue", + "description": "metadata for Kueue.", "default": {}, "$ref": "#/definitions/io.k8s.apimachinery.pkg.apis.meta.v1.ObjectMeta" }, @@ -33500,7 +33500,7 @@ "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.Preemption" }, "workloadManagement": { - "description": "workloadManagement controls how kueue manages workloads. By default Kueue will manage workloads that have a queue-name label. Workloads that are missing the queue-name will be ignored by Kueue. This field is optional. If this field is not specified, Kueue will only manage workloads that have the queue-name label.", + "description": "workloadManagement controls how Kueue manages workloads. By default Kueue will manage workloads that have a queue-name label. Workloads that are missing the queue-name will be ignored by Kueue. This field is optional. If this field is not specified, Kueue will only manage workloads that have the queue-name label.", "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.WorkloadManagement" } } @@ -33955,7 +33955,7 @@ "type": "object", "properties": { "preemptionPolicy": { - "description": "preemptionPolicy are the types of preemption kueue allows. Kueue has two types of preemption: Classical and FairSharing. Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. FairSharing means that ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort. FairSharing is a more heavy weight algorithm. The default is Classical.", + "description": "preemptionPolicy are the types of preemption Kueue allows. Kueue has two types of preemption: Classical and FairSharing. Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. FairSharing means that ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort. FairSharing is a more heavy weight algorithm. The default is Classical.", "type": "string" } } diff --git a/operator/v1alpha1/types_kueue.go b/operator/v1alpha1/types_kueue.go index 4ecc13518b4..23509858916 100644 --- a/operator/v1alpha1/types_kueue.go +++ b/operator/v1alpha1/types_kueue.go @@ -21,14 +21,15 @@ import ( // +kubebuilder:validation:XValidation:rule="self.metadata.name == 'cluster'",message="Kueue is a singleton, .metadata.name must be 'cluster'" type Kueue struct { metav1.TypeMeta `json:",inline"` - // metadata for kueue + // metadata for Kueue. // +optional metav1.ObjectMeta `json:"metadata,omitempty"` // spec holds user settable values for configuration // +required Spec KueueOperandSpec `json:"spec"` - // status holds observed values from the cluster. They may not be overridden. + // status holds observed values from the cluster. + // They may not be overridden. // +optional Status KueueStatus `json:"status,omitempty"` } @@ -48,7 +49,7 @@ type KueueConfiguration struct { // Kueue will only manage workloads that correspond to the specified integrations. // +required Integrations Integrations `json:"integrations"` - // workloadManagement controls how kueue manages workloads. + // workloadManagement controls how Kueue manages workloads. // By default Kueue will manage workloads that have a queue-name label. // Workloads that are missing the queue-name will be ignored by Kueue. // This field is optional. @@ -240,7 +241,7 @@ type GangScheduling struct { // ByWorkload controls how admission is done type ByWorkload struct { - // admission controls how kueue will process workloads. + // admission controls how Kueue will process workloads. // Allowed values are Sequential and Parallel. // When admission is set to Sequential, only pods from the currently processing workload will be admitted. // Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. @@ -285,7 +286,7 @@ const ( ) type Preemption struct { - // preemptionPolicy are the types of preemption kueue allows. + // preemptionPolicy are the types of preemption Kueue allows. // Kueue has two types of preemption: Classical and FairSharing. // Classical means that an incoming workload, which does // not fit within the unusued quota, is eligible to issue preemptions diff --git a/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml b/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml index 1d2c956656c..bee4020b0aa 100644 --- a/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml +++ b/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml @@ -63,7 +63,7 @@ spec: properties: admission: description: |- - admission controls how kueue will process workloads. + admission controls how Kueue will process workloads. Allowed values are Sequential and Parallel. When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. @@ -238,7 +238,7 @@ spec: properties: preemptionPolicy: description: |- - preemptionPolicy are the types of preemption kueue allows. + preemptionPolicy are the types of preemption Kueue allows. Kueue has two types of preemption: Classical and FairSharing. Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions @@ -259,7 +259,7 @@ spec: type: object workloadManagement: description: |- - workloadManagement controls how kueue manages workloads. + workloadManagement controls how Kueue manages workloads. By default Kueue will manage workloads that have a queue-name label. Workloads that are missing the queue-name will be ignored by Kueue. This field is optional. @@ -344,8 +344,9 @@ spec: - config type: object status: - description: status holds observed values from the cluster. They may not - be overridden. + description: |- + status holds observed values from the cluster. + They may not be overridden. properties: conditions: description: conditions is a list of conditions and their status diff --git a/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml b/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml index 7cb27fbd691..5842e681830 100644 --- a/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml +++ b/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml @@ -61,7 +61,7 @@ spec: properties: admission: description: |- - admission controls how kueue will process workloads. + admission controls how Kueue will process workloads. Allowed values are Sequential and Parallel. When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. @@ -236,7 +236,7 @@ spec: properties: preemptionPolicy: description: |- - preemptionPolicy are the types of preemption kueue allows. + preemptionPolicy are the types of preemption Kueue allows. Kueue has two types of preemption: Classical and FairSharing. Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions @@ -257,7 +257,7 @@ spec: type: object workloadManagement: description: |- - workloadManagement controls how kueue manages workloads. + workloadManagement controls how Kueue manages workloads. By default Kueue will manage workloads that have a queue-name label. Workloads that are missing the queue-name will be ignored by Kueue. This field is optional. @@ -342,8 +342,9 @@ spec: - config type: object status: - description: status holds observed values from the cluster. They may not - be overridden. + description: |- + status holds observed values from the cluster. + They may not be overridden. properties: conditions: description: conditions is a list of conditions and their status diff --git a/operator/v1alpha1/zz_generated.swagger_doc_generated.go b/operator/v1alpha1/zz_generated.swagger_doc_generated.go index 187aeff2b89..50069be317e 100644 --- a/operator/v1alpha1/zz_generated.swagger_doc_generated.go +++ b/operator/v1alpha1/zz_generated.swagger_doc_generated.go @@ -259,7 +259,7 @@ func (RepositoryDigestMirrors) SwaggerDoc() map[string]string { var map_ByWorkload = map[string]string{ "": "ByWorkload controls how admission is done", - "admission": "admission controls how kueue will process workloads. Allowed values are Sequential and Parallel. When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling.", + "admission": "admission controls how Kueue will process workloads. Allowed values are Sequential and Parallel. When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling.", } func (ByWorkload) SwaggerDoc() map[string]string { @@ -300,7 +300,7 @@ func (Integrations) SwaggerDoc() map[string]string { var map_Kueue = map[string]string{ "": "Kueue is the CRD to represent the Kueue operator This CRD defines the configuration that the Kueue Compatibility level 4: No compatibility is provided, the API can change at any point for any reason. These capabilities should not be used by applications needing long term support.", - "metadata": "metadata for kueue", + "metadata": "metadata for Kueue.", "spec": "spec holds user settable values for configuration", "status": "status holds observed values from the cluster. They may not be overridden.", } @@ -311,7 +311,7 @@ func (Kueue) SwaggerDoc() map[string]string { var map_KueueConfiguration = map[string]string{ "integrations": "integrations is a required field that configures the Kueue's workload integrations. Kueue has both standard integrations, known as job frameworks, and external integrations known as external frameworks. Kueue will only manage workloads that correspond to the specified integrations.", - "workloadManagement": "workloadManagement controls how kueue manages workloads. By default Kueue will manage workloads that have a queue-name label. Workloads that are missing the queue-name will be ignored by Kueue. This field is optional. If this field is not specified, Kueue will only manage workloads that have the queue-name label.", + "workloadManagement": "workloadManagement controls how Kueue manages workloads. By default Kueue will manage workloads that have a queue-name label. Workloads that are missing the queue-name will be ignored by Kueue. This field is optional. If this field is not specified, Kueue will only manage workloads that have the queue-name label.", "gangScheduling": "gangScheduling controls how Kueue admits workloads. Gang Scheduling is the act of all or nothing scheduling, where workloads do not become ready within a certain period, they may be evicted and later retried. This field is optional. If this field is not specified, gang scheduling will be disabled.", "preemption": "preemption is the process of evicting one or more admitted Workloads to accommodate another Workload. Kueue has classical premption and preemption via fair sharing. This field is optional. If this field is not specified, preemption will be set to Classical.", } @@ -355,7 +355,7 @@ func (LabelKeys) SwaggerDoc() map[string]string { } var map_Preemption = map[string]string{ - "preemptionPolicy": "preemptionPolicy are the types of preemption kueue allows. Kueue has two types of preemption: Classical and FairSharing. Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. FairSharing means that ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort. FairSharing is a more heavy weight algorithm. The default is Classical.", + "preemptionPolicy": "preemptionPolicy are the types of preemption Kueue allows. Kueue has two types of preemption: Classical and FairSharing. Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. FairSharing means that ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort. FairSharing is a more heavy weight algorithm. The default is Classical.", } func (Preemption) SwaggerDoc() map[string]string { From 1b672eea42d079673c4f3e0a40642804d7e55992 Mon Sep 17 00:00:00 2001 From: Kevin Hannon Date: Thu, 10 Apr 2025 12:25:39 -0400 Subject: [PATCH 35/42] godocs --- .../generated_openapi/zz_generated.openapi.go | 26 +++++----- openapi/openapi.json | 26 +++++----- operator/v1alpha1/types_kueue.go | 47 +++++++++++-------- .../zz_generated.crd-manifests/kueue.crd.yaml | 45 ++++++++++-------- .../AAA_ungated.yaml | 45 ++++++++++-------- .../zz_generated.swagger_doc_generated.go | 24 +++++----- 6 files changed, 117 insertions(+), 96 deletions(-) diff --git a/openapi/generated_openapi/zz_generated.openapi.go b/openapi/generated_openapi/zz_generated.openapi.go index 39c06951094..80ba2a30771 100644 --- a/openapi/generated_openapi/zz_generated.openapi.go +++ b/openapi/generated_openapi/zz_generated.openapi.go @@ -56515,7 +56515,7 @@ func schema_openshift_api_operator_v1alpha1_ByWorkload(ref common.ReferenceCallb Properties: map[string]spec.Schema{ "admission": { SchemaProps: spec.SchemaProps{ - Description: "admission controls how Kueue will process workloads. Allowed values are Sequential and Parallel. When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling.", + Description: "admission controls how Kueue will process workloads. admission is an optional field. Allowed values are Sequential and Parallel. When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling.", Type: []string{"string"}, Format: "", }, @@ -56880,7 +56880,7 @@ func schema_openshift_api_operator_v1alpha1_ExternalFramework(ref common.Referen Properties: map[string]spec.Schema{ "group": { SchemaProps: spec.SchemaProps{ - Description: "group is the API group of the externalFramework. Must be a valid DNS 1123 subdomain consisting of of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. We are using matches and not cel functions to allow for use on 4.17.", + Description: "group is the API group of the externalFramework. Must be a valid DNS 1123 subdomain consisting of of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. group uses matches and not cel functions to allow for use on 4.17.", Default: "", Type: []string{"string"}, Format: "", @@ -56888,7 +56888,7 @@ func schema_openshift_api_operator_v1alpha1_ExternalFramework(ref common.Referen }, "resource": { SchemaProps: spec.SchemaProps{ - Description: "resource is the Resource type of the external framework. Resource types are lowercase and plural (e.g. pods, deployments). Must be a valid DNS 1123 label consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. The value must start and end with an alphanumeric character. We are using matches and not cel functions to allow for use on 4.17.", + Description: "resource is the Resource type of the external framework. Resource types are lowercase and plural (e.g. pods, deployments). Must be a valid DNS 1123 label consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. The value must start and end with an alphanumeric character. resource uses matches and not cel functions to allow for use on 4.17.", Default: "", Type: []string{"string"}, Format: "", @@ -56896,7 +56896,7 @@ func schema_openshift_api_operator_v1alpha1_ExternalFramework(ref common.Referen }, "version": { SchemaProps: spec.SchemaProps{ - Description: "version is the version of the api (e.g. v1alpha1, v1beta1, v1). Must be a valid DNS 1035 label consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. The value must start with an alphabetic character and end with an alphanumeric character. We are using matches and not cel functions to allow for use on 4.17.", + Description: "version is the version of the api (e.g. v1alpha1, v1beta1, v1). Must be a valid DNS 1035 label consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. The value must start with an alphabetic character and end with an alphanumeric character. version uses matches and not cel functions to allow for use on 4.17.", Default: "", Type: []string{"string"}, Format: "", @@ -56918,7 +56918,7 @@ func schema_openshift_api_operator_v1alpha1_GangScheduling(ref common.ReferenceC Properties: map[string]spec.Schema{ "policy": { SchemaProps: spec.SchemaProps{ - Description: "policy allows you to enable and configure gang scheduling. This is an optional field. The allowed values are ByWorkload and Disabled. The default value will be Disabled. When set to ByWorkload, this means each workload is processed and considered for admission as a single unit. Where workloads do not become ready over time, the entire workload may then be evicted and retried at a later time.", + Description: "policy allows you to enable and configure gang scheduling. The allowed values are ByWorkload and None. The default value will be None. When set to ByWorkload, this means each workload is processed and considered for admission as a single unit. Where workloads do not become ready over time, the entire workload may then be evicted and retried at a later time. policy is a required field.", Default: "", Type: []string{"string"}, Format: "", @@ -57199,7 +57199,7 @@ func schema_openshift_api_operator_v1alpha1_Integrations(ref common.ReferenceCal }, }, SchemaProps: spec.SchemaProps{ - Description: "frameworks are a unique list of names to be enabled. This is required and must have at least one element. Each framework represents a type of job that Kueue will manage. Frameworks are a list of frameworks that Kueue has support for. The allowed values are BatchJob, RayJob, RayCluster, JobSet, MPIJob, PaddleJob, PytorchJob, TFJob, XGBoostJob, AppWrapper, Pod, Deployment, StatefulSet and LeaderWorkerSet.", + Description: "frameworks are a list of frameworks that Kueue has support for. The allowed values are BatchJob, RayJob, RayCluster, JobSet, MPIJob, PaddleJob, PyTorchJob, TFJob, XGBoostJob, AppWrapper, Pod, Deployment, StatefulSet and LeaderWorkerSet. frameworks are required and must have at least one element. frameworks can not have more than 14 elements. Each framework represents a type of job that Kueue will manage.", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -57219,7 +57219,7 @@ func schema_openshift_api_operator_v1alpha1_Integrations(ref common.ReferenceCal }, }, SchemaProps: spec.SchemaProps{ - Description: "externalFrameworks are a list of GroupVersionResources that are managed for Kueue by external controllers. These are optional and should only be used if you have an external controller that integrates with Kueue.", + Description: "externalFrameworks are a list of GroupVersionResources that are managed for Kueue by external controllers. externalFrameworks are optional and should only be used if you have an external controller that integrates with Kueue. externalFrameworks, if specified, must have at least 1 item. externalFrameworks, if specified, can not have more than 32 items.", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -57234,11 +57234,11 @@ func schema_openshift_api_operator_v1alpha1_Integrations(ref common.ReferenceCal "labelKeysToCopy": { VendorExtensible: spec.VendorExtensible{ Extensions: spec.Extensions{ - "x-kubernetes-list-type": "set", + "x-kubernetes-list-type": "map", }, }, SchemaProps: spec.SchemaProps{ - Description: "labelKeysToCopy are a list of label keys that are copied once a workload is created. These keys are persisted to the internal Kueue workload object. If not specified, only the Kueue labels will be copied.", + Description: "labelKeysToCopy are a list of label keys that are copied once a workload is created. These keys are persisted to the internal Kueue workload object. If not specified, only the Kueue labels will be copied. labelKeysToCopy, if specified, is limited to a maximum of 64 items. labelKeysToCopy, if specified, must have at least one item.", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -57325,19 +57325,19 @@ func schema_openshift_api_operator_v1alpha1_KueueConfiguration(ref common.Refere }, "workloadManagement": { SchemaProps: spec.SchemaProps{ - Description: "workloadManagement controls how Kueue manages workloads. By default Kueue will manage workloads that have a queue-name label. Workloads that are missing the queue-name will be ignored by Kueue. This field is optional. If this field is not specified, Kueue will only manage workloads that have the queue-name label.", + Description: "workloadManagement controls how Kueue manages workloads. By default Kueue will manage workloads that have a queue-name label. Workloads that are missing the queue-name will be ignored by Kueue. If tworkloadManagement is not specified, Kueue will only manage workloads that have the queue-name label. This field is optional.", Ref: ref("github.com/openshift/api/operator/v1alpha1.WorkloadManagement"), }, }, "gangScheduling": { SchemaProps: spec.SchemaProps{ - Description: "gangScheduling controls how Kueue admits workloads. Gang Scheduling is the act of all or nothing scheduling, where workloads do not become ready within a certain period, they may be evicted and later retried. This field is optional. If this field is not specified, gang scheduling will be disabled.", + Description: "gangScheduling controls how Kueue admits workloads. Gang Scheduling is the act of all or nothing scheduling, where workloads do not become ready within a certain period, they may be evicted and later retried. This field is optional. If gangScheduling is not specified, gang scheduling will be disabled.", Ref: ref("github.com/openshift/api/operator/v1alpha1.GangScheduling"), }, }, "preemption": { SchemaProps: spec.SchemaProps{ - Description: "preemption is the process of evicting one or more admitted Workloads to accommodate another Workload. Kueue has classical premption and preemption via fair sharing. This field is optional. If this field is not specified, preemption will be set to Classical.", + Description: "preemption is the process of evicting one or more admitted Workloads to accommodate another Workload. Kueue has classical premption and preemption via fair sharing. preemption is optional. If preemption is not specified, preemption will be set to Classical.", Ref: ref("github.com/openshift/api/operator/v1alpha1.Preemption"), }, }, @@ -57557,7 +57557,7 @@ func schema_openshift_api_operator_v1alpha1_LabelKeys(ref common.ReferenceCallba Properties: map[string]spec.Schema{ "key": { SchemaProps: spec.SchemaProps{ - Description: "key is the label key. A label key must be a valid qualified name consisting of a lower-case alphanumeric string, and hyphens of at most 63 characters in length. We are using matches and not cel functions to allow for use on 4.17. The name must start and end with an alphanumeric character. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. The optional prefix and the name are separate by a forward slash (/).", + Description: "key is the label key. A label key must be a valid qualified name consisting of a lower-case alphanumeric string, and hyphens of at most 63 characters in length. To support older openshift versions, matches is used instead of CEL validation. The name must start and end with an alphanumeric character. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. The optional prefix and the name are separate by a forward slash (/).", Default: "", Type: []string{"string"}, Format: "", diff --git a/openapi/openapi.json b/openapi/openapi.json index 7b779b8f1eb..05a850b66d1 100644 --- a/openapi/openapi.json +++ b/openapi/openapi.json @@ -33014,7 +33014,7 @@ "type": "object", "properties": { "admission": { - "description": "admission controls how Kueue will process workloads. Allowed values are Sequential and Parallel. When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling.", + "description": "admission controls how Kueue will process workloads. admission is an optional field. Allowed values are Sequential and Parallel. When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling.", "type": "string" } } @@ -33226,17 +33226,17 @@ ], "properties": { "group": { - "description": "group is the API group of the externalFramework. Must be a valid DNS 1123 subdomain consisting of of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. We are using matches and not cel functions to allow for use on 4.17.", + "description": "group is the API group of the externalFramework. Must be a valid DNS 1123 subdomain consisting of of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. group uses matches and not cel functions to allow for use on 4.17.", "type": "string", "default": "" }, "resource": { - "description": "resource is the Resource type of the external framework. Resource types are lowercase and plural (e.g. pods, deployments). Must be a valid DNS 1123 label consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. The value must start and end with an alphanumeric character. We are using matches and not cel functions to allow for use on 4.17.", + "description": "resource is the Resource type of the external framework. Resource types are lowercase and plural (e.g. pods, deployments). Must be a valid DNS 1123 label consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. The value must start and end with an alphanumeric character. resource uses matches and not cel functions to allow for use on 4.17.", "type": "string", "default": "" }, "version": { - "description": "version is the version of the api (e.g. v1alpha1, v1beta1, v1). Must be a valid DNS 1035 label consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. The value must start with an alphabetic character and end with an alphanumeric character. We are using matches and not cel functions to allow for use on 4.17.", + "description": "version is the version of the api (e.g. v1alpha1, v1beta1, v1). Must be a valid DNS 1035 label consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. The value must start with an alphabetic character and end with an alphanumeric character. version uses matches and not cel functions to allow for use on 4.17.", "type": "string", "default": "" } @@ -33254,7 +33254,7 @@ "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.ByWorkload" }, "policy": { - "description": "policy allows you to enable and configure gang scheduling. This is an optional field. The allowed values are ByWorkload and Disabled. The default value will be Disabled. When set to ByWorkload, this means each workload is processed and considered for admission as a single unit. Where workloads do not become ready over time, the entire workload may then be evicted and retried at a later time.", + "description": "policy allows you to enable and configure gang scheduling. The allowed values are ByWorkload and None. The default value will be None. When set to ByWorkload, this means each workload is processed and considered for admission as a single unit. Where workloads do not become ready over time, the entire workload may then be evicted and retried at a later time. policy is a required field.", "type": "string", "default": "" } @@ -33420,7 +33420,7 @@ ], "properties": { "externalFrameworks": { - "description": "externalFrameworks are a list of GroupVersionResources that are managed for Kueue by external controllers. These are optional and should only be used if you have an external controller that integrates with Kueue.", + "description": "externalFrameworks are a list of GroupVersionResources that are managed for Kueue by external controllers. externalFrameworks are optional and should only be used if you have an external controller that integrates with Kueue. externalFrameworks, if specified, must have at least 1 item. externalFrameworks, if specified, can not have more than 32 items.", "type": "array", "items": { "default": {}, @@ -33429,7 +33429,7 @@ "x-kubernetes-list-type": "atomic" }, "frameworks": { - "description": "frameworks are a unique list of names to be enabled. This is required and must have at least one element. Each framework represents a type of job that Kueue will manage. Frameworks are a list of frameworks that Kueue has support for. The allowed values are BatchJob, RayJob, RayCluster, JobSet, MPIJob, PaddleJob, PytorchJob, TFJob, XGBoostJob, AppWrapper, Pod, Deployment, StatefulSet and LeaderWorkerSet.", + "description": "frameworks are a list of frameworks that Kueue has support for. The allowed values are BatchJob, RayJob, RayCluster, JobSet, MPIJob, PaddleJob, PyTorchJob, TFJob, XGBoostJob, AppWrapper, Pod, Deployment, StatefulSet and LeaderWorkerSet. frameworks are required and must have at least one element. frameworks can not have more than 14 elements. Each framework represents a type of job that Kueue will manage.", "type": "array", "items": { "type": "string", @@ -33438,13 +33438,13 @@ "x-kubernetes-list-type": "atomic" }, "labelKeysToCopy": { - "description": "labelKeysToCopy are a list of label keys that are copied once a workload is created. These keys are persisted to the internal Kueue workload object. If not specified, only the Kueue labels will be copied.", + "description": "labelKeysToCopy are a list of label keys that are copied once a workload is created. These keys are persisted to the internal Kueue workload object. If not specified, only the Kueue labels will be copied. labelKeysToCopy, if specified, is limited to a maximum of 64 items. labelKeysToCopy, if specified, must have at least one item.", "type": "array", "items": { "default": {}, "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.LabelKeys" }, - "x-kubernetes-list-type": "set" + "x-kubernetes-list-type": "map" } } }, @@ -33487,7 +33487,7 @@ ], "properties": { "gangScheduling": { - "description": "gangScheduling controls how Kueue admits workloads. Gang Scheduling is the act of all or nothing scheduling, where workloads do not become ready within a certain period, they may be evicted and later retried. This field is optional. If this field is not specified, gang scheduling will be disabled.", + "description": "gangScheduling controls how Kueue admits workloads. Gang Scheduling is the act of all or nothing scheduling, where workloads do not become ready within a certain period, they may be evicted and later retried. This field is optional. If gangScheduling is not specified, gang scheduling will be disabled.", "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.GangScheduling" }, "integrations": { @@ -33496,11 +33496,11 @@ "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.Integrations" }, "preemption": { - "description": "preemption is the process of evicting one or more admitted Workloads to accommodate another Workload. Kueue has classical premption and preemption via fair sharing. This field is optional. If this field is not specified, preemption will be set to Classical.", + "description": "preemption is the process of evicting one or more admitted Workloads to accommodate another Workload. Kueue has classical premption and preemption via fair sharing. preemption is optional. If preemption is not specified, preemption will be set to Classical.", "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.Preemption" }, "workloadManagement": { - "description": "workloadManagement controls how Kueue manages workloads. By default Kueue will manage workloads that have a queue-name label. Workloads that are missing the queue-name will be ignored by Kueue. This field is optional. If this field is not specified, Kueue will only manage workloads that have the queue-name label.", + "description": "workloadManagement controls how Kueue manages workloads. By default Kueue will manage workloads that have a queue-name label. Workloads that are missing the queue-name will be ignored by Kueue. If tworkloadManagement is not specified, Kueue will only manage workloads that have the queue-name label. This field is optional.", "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.WorkloadManagement" } } @@ -33633,7 +33633,7 @@ ], "properties": { "key": { - "description": "key is the label key. A label key must be a valid qualified name consisting of a lower-case alphanumeric string, and hyphens of at most 63 characters in length. We are using matches and not cel functions to allow for use on 4.17. The name must start and end with an alphanumeric character. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. The optional prefix and the name are separate by a forward slash (/).", + "description": "key is the label key. A label key must be a valid qualified name consisting of a lower-case alphanumeric string, and hyphens of at most 63 characters in length. To support older openshift versions, matches is used instead of CEL validation. The name must start and end with an alphanumeric character. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. The optional prefix and the name are separate by a forward slash (/).", "type": "string", "default": "" } diff --git a/operator/v1alpha1/types_kueue.go b/operator/v1alpha1/types_kueue.go index 23509858916..28bf998fc8e 100644 --- a/operator/v1alpha1/types_kueue.go +++ b/operator/v1alpha1/types_kueue.go @@ -52,22 +52,22 @@ type KueueConfiguration struct { // workloadManagement controls how Kueue manages workloads. // By default Kueue will manage workloads that have a queue-name label. // Workloads that are missing the queue-name will be ignored by Kueue. - // This field is optional. - // If this field is not specified, Kueue will only manage workloads + // If tworkloadManagement is not specified, Kueue will only manage workloads // that have the queue-name label. + // This field is optional. // +optional WorkloadManagement *WorkloadManagement `json:"workloadManagement,omitempty"` // gangScheduling controls how Kueue admits workloads. // Gang Scheduling is the act of all or nothing scheduling, // where workloads do not become ready within a certain period, they may be evicted and later retried. // This field is optional. - // If this field is not specified, gang scheduling will be disabled. + // If gangScheduling is not specified, gang scheduling will be disabled. // +optional GangScheduling *GangScheduling `json:"gangScheduling,omitempty"` // preemption is the process of evicting one or more admitted Workloads to accommodate another Workload. // Kueue has classical premption and preemption via fair sharing. - // This field is optional. - // If this field is not specified, preemption will be set to Classical. + // preemption is optional. + // If preemption is not specified, preemption will be set to Classical. // +optional Preemption *Preemption `json:"preemption,omitempty"` } @@ -123,7 +123,7 @@ type ExternalFramework struct { // Must be a valid DNS 1123 subdomain consisting of of lower-case alphanumeric characters, // hyphens and periods, of at most 253 characters in length. // Each period separated segment within the subdomain must start and end with an alphanumeric character. - // We are using matches and not cel functions to allow for use on 4.17. + // group uses matches and not cel functions to allow for use on 4.17. // +kubebuilder:validation:MaxLength=253 // +kubebuilder:validation:MinLength=1 // +kubebuilder:validation:XValidation:rule="self.matches(r'^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$') && self.size() <= 253" @@ -134,7 +134,7 @@ type ExternalFramework struct { // Must be a valid DNS 1123 label consisting of a lower-case alphanumeric string // and hyphens of at most 63 characters in length. // The value must start and end with an alphanumeric character. - // We are using matches and not cel functions to allow for use on 4.17. + // resource uses matches and not cel functions to allow for use on 4.17. // +kubebuilder:validation:MaxLength=63 // +kubebuilder:validation:MinLength=1 // +kubebuilder:validation:XValidation:rule="self.matches(r'^[a-z0-9]([-a-z0-9]*[a-z0-9])?$') && self.size() <= 63" @@ -144,7 +144,7 @@ type ExternalFramework struct { // Must be a valid DNS 1035 label consisting of a lower-case alphanumeric string // and hyphens of at most 63 characters in length. // The value must start with an alphabetic character and end with an alphanumeric character. - // We are using matches and not cel functions to allow for use on 4.17. + // version uses matches and not cel functions to allow for use on 4.17. // +kubebuilder:validation:MaxLength=63 // +kubebuilder:validation:MinLength=1 // +kubebuilder:validation:XValidation:rule="self.matches(r'^[a-z0-9]([-a-z0-9]*[a-z0-9])?$') && self.size() <= 63" @@ -156,11 +156,11 @@ type ExternalFramework struct { // Kueue uses these apis to determine // which jobs will be managed by Kueue. type Integrations struct { - // frameworks are a unique list of names to be enabled. - // This is required and must have at least one element. + // frameworks are a list of frameworks that Kueue has support for. + // The allowed values are BatchJob, RayJob, RayCluster, JobSet, MPIJob, PaddleJob, PyTorchJob, TFJob, XGBoostJob, AppWrapper, Pod, Deployment, StatefulSet and LeaderWorkerSet. + // frameworks are required and must have at least one element. + // frameworks can not have more than 14 elements. // Each framework represents a type of job that Kueue will manage. - // Frameworks are a list of frameworks that Kueue has support for. - // The allowed values are BatchJob, RayJob, RayCluster, JobSet, MPIJob, PaddleJob, PytorchJob, TFJob, XGBoostJob, AppWrapper, Pod, Deployment, StatefulSet and LeaderWorkerSet. // +kubebuilder:validation:MaxItems=14 // +kubebuilder:validation:MinItems=1 // +kubebuilder:validation:XValidation:rule="self.all(x, self.exists_one(y, x == y))",message="each item in frameworks must be unique" @@ -169,17 +169,23 @@ type Integrations struct { Frameworks []KueueIntegration `json:"frameworks"` // externalFrameworks are a list of GroupVersionResources // that are managed for Kueue by external controllers. - // These are optional and should only be used if you have an external controller + // externalFrameworks are optional and should only be used if you have an external controller // that integrates with Kueue. + // externalFrameworks, if specified, must have at least 1 item. + // externalFrameworks, if specified, can not have more than 32 items. // +listType=atomic + // +kubebuilder:validation:MinItems=1 // +kubebuilder:validation:MaxItems=32 // +optional ExternalFrameworks []ExternalFramework `json:"externalFrameworks,omitempty"` // labelKeysToCopy are a list of label keys that are copied once a workload is created. // These keys are persisted to the internal Kueue workload object. // If not specified, only the Kueue labels will be copied. + // labelKeysToCopy, if specified, is limited to a maximum of 64 items. + // labelKeysToCopy, if specified, must have at least one item. // +kubebuilder:validation:MaxItems=64 - // +listType=set + // +kubebuilder:validation:MinItems=1 + // +listType=map // +listMapType=key // +optional LabelKeysToCopy []LabelKeys `json:"labelKeysToCopy,omitempty"` @@ -189,7 +195,7 @@ type LabelKeys struct { // key is the label key. // A label key must be a valid qualified name consisting of a lower-case alphanumeric string, // and hyphens of at most 63 characters in length. - // We are using matches and not cel functions to allow for use on 4.17. + // To support older openshift versions, matches is used instead of CEL validation. // The name must start and end with an alphanumeric character. // The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, // hyphens and periods, of at most 253 characters in length. @@ -202,12 +208,12 @@ type LabelKeys struct { Key string `json:"key"` } -// +kubebuilder:validation:Enum=ByWorkload;Disabled +// +kubebuilder:validation:Enum=ByWorkload;None type GangSchedulingPolicy string const ( GangSchedulingPolicyByWorkload GangSchedulingPolicy = "ByWorkload" - GangSchedulingPolicyDisabled GangSchedulingPolicy = "Disabled" + GangSchedulingPolicyNone GangSchedulingPolicy = "None" ) // +kubebuilder:validation:Enum=Parallel;Sequential @@ -224,12 +230,12 @@ const ( // +union type GangScheduling struct { // policy allows you to enable and configure gang scheduling. - // This is an optional field. - // The allowed values are ByWorkload and Disabled. - // The default value will be Disabled. + // The allowed values are ByWorkload and None. + // The default value will be None. // When set to ByWorkload, this means each workload is processed and considered // for admission as a single unit. // Where workloads do not become ready over time, the entire workload may then be evicted and retried at a later time. + // policy is a required field. // +required // +unionDiscriminator Policy GangSchedulingPolicy `json:"policy"` @@ -242,6 +248,7 @@ type GangScheduling struct { // ByWorkload controls how admission is done type ByWorkload struct { // admission controls how Kueue will process workloads. + // admission is an optional field. // Allowed values are Sequential and Parallel. // When admission is set to Sequential, only pods from the currently processing workload will be admitted. // Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. diff --git a/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml b/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml index bee4020b0aa..81b0188008e 100644 --- a/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml +++ b/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml @@ -54,7 +54,7 @@ spec: Gang Scheduling is the act of all or nothing scheduling, where workloads do not become ready within a certain period, they may be evicted and later retried. This field is optional. - If this field is not specified, gang scheduling will be disabled. + If gangScheduling is not specified, gang scheduling will be disabled. properties: byWorkload: description: |- @@ -64,6 +64,7 @@ spec: admission: description: |- admission controls how Kueue will process workloads. + admission is an optional field. Allowed values are Sequential and Parallel. When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. @@ -80,15 +81,15 @@ spec: policy: description: |- policy allows you to enable and configure gang scheduling. - This is an optional field. - The allowed values are ByWorkload and Disabled. - The default value will be Disabled. + The allowed values are ByWorkload and None. + The default value will be None. When set to ByWorkload, this means each workload is processed and considered for admission as a single unit. Where workloads do not become ready over time, the entire workload may then be evicted and retried at a later time. + policy is a required field. enum: - ByWorkload - - Disabled + - None type: string required: - policy @@ -109,8 +110,10 @@ spec: description: |- externalFrameworks are a list of GroupVersionResources that are managed for Kueue by external controllers. - These are optional and should only be used if you have an external controller + externalFrameworks are optional and should only be used if you have an external controller that integrates with Kueue. + externalFrameworks, if specified, must have at least 1 item. + externalFrameworks, if specified, can not have more than 32 items. items: description: |- This is the GVR for an external framework. @@ -123,7 +126,7 @@ spec: Must be a valid DNS 1123 subdomain consisting of of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. - We are using matches and not cel functions to allow for use on 4.17. + group uses matches and not cel functions to allow for use on 4.17. maxLength: 253 minLength: 1 type: string @@ -137,7 +140,7 @@ spec: Must be a valid DNS 1123 label consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. The value must start and end with an alphanumeric character. - We are using matches and not cel functions to allow for use on 4.17. + resource uses matches and not cel functions to allow for use on 4.17. maxLength: 63 minLength: 1 type: string @@ -150,7 +153,7 @@ spec: Must be a valid DNS 1035 label consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. The value must start with an alphabetic character and end with an alphanumeric character. - We are using matches and not cel functions to allow for use on 4.17. + version uses matches and not cel functions to allow for use on 4.17. maxLength: 63 minLength: 1 type: string @@ -163,15 +166,16 @@ spec: - version type: object maxItems: 32 + minItems: 1 type: array x-kubernetes-list-type: atomic frameworks: description: |- - frameworks are a unique list of names to be enabled. - This is required and must have at least one element. + frameworks are a list of frameworks that Kueue has support for. + The allowed values are BatchJob, RayJob, RayCluster, JobSet, MPIJob, PaddleJob, PyTorchJob, TFJob, XGBoostJob, AppWrapper, Pod, Deployment, StatefulSet and LeaderWorkerSet. + frameworks are required and must have at least one element. + frameworks can not have more than 14 elements. Each framework represents a type of job that Kueue will manage. - Frameworks are a list of frameworks that Kueue has support for. - The allowed values are BatchJob, RayJob, RayCluster, JobSet, MPIJob, PaddleJob, PytorchJob, TFJob, XGBoostJob, AppWrapper, Pod, Deployment, StatefulSet and LeaderWorkerSet. items: enum: - BatchJob @@ -201,6 +205,8 @@ spec: labelKeysToCopy are a list of label keys that are copied once a workload is created. These keys are persisted to the internal Kueue workload object. If not specified, only the Kueue labels will be copied. + labelKeysToCopy, if specified, is limited to a maximum of 64 items. + labelKeysToCopy, if specified, must have at least one item. items: properties: key: @@ -208,7 +214,7 @@ spec: key is the label key. A label key must be a valid qualified name consisting of a lower-case alphanumeric string, and hyphens of at most 63 characters in length. - We are using matches and not cel functions to allow for use on 4.17. + To support older openshift versions, matches is used instead of CEL validation. The name must start and end with an alphanumeric character. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. @@ -224,8 +230,9 @@ spec: - key type: object maxItems: 64 + minItems: 1 type: array - x-kubernetes-list-type: set + x-kubernetes-list-type: map required: - frameworks type: object @@ -233,8 +240,8 @@ spec: description: |- preemption is the process of evicting one or more admitted Workloads to accommodate another Workload. Kueue has classical premption and preemption via fair sharing. - This field is optional. - If this field is not specified, preemption will be set to Classical. + preemption is optional. + If preemption is not specified, preemption will be set to Classical. properties: preemptionPolicy: description: |- @@ -262,9 +269,9 @@ spec: workloadManagement controls how Kueue manages workloads. By default Kueue will manage workloads that have a queue-name label. Workloads that are missing the queue-name will be ignored by Kueue. - This field is optional. - If this field is not specified, Kueue will only manage workloads + If tworkloadManagement is not specified, Kueue will only manage workloads that have the queue-name label. + This field is optional. properties: labelPolicy: description: |- diff --git a/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml b/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml index 5842e681830..ec38ae89198 100644 --- a/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml +++ b/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml @@ -52,7 +52,7 @@ spec: Gang Scheduling is the act of all or nothing scheduling, where workloads do not become ready within a certain period, they may be evicted and later retried. This field is optional. - If this field is not specified, gang scheduling will be disabled. + If gangScheduling is not specified, gang scheduling will be disabled. properties: byWorkload: description: |- @@ -62,6 +62,7 @@ spec: admission: description: |- admission controls how Kueue will process workloads. + admission is an optional field. Allowed values are Sequential and Parallel. When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. @@ -78,15 +79,15 @@ spec: policy: description: |- policy allows you to enable and configure gang scheduling. - This is an optional field. - The allowed values are ByWorkload and Disabled. - The default value will be Disabled. + The allowed values are ByWorkload and None. + The default value will be None. When set to ByWorkload, this means each workload is processed and considered for admission as a single unit. Where workloads do not become ready over time, the entire workload may then be evicted and retried at a later time. + policy is a required field. enum: - ByWorkload - - Disabled + - None type: string required: - policy @@ -107,8 +108,10 @@ spec: description: |- externalFrameworks are a list of GroupVersionResources that are managed for Kueue by external controllers. - These are optional and should only be used if you have an external controller + externalFrameworks are optional and should only be used if you have an external controller that integrates with Kueue. + externalFrameworks, if specified, must have at least 1 item. + externalFrameworks, if specified, can not have more than 32 items. items: description: |- This is the GVR for an external framework. @@ -121,7 +124,7 @@ spec: Must be a valid DNS 1123 subdomain consisting of of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. - We are using matches and not cel functions to allow for use on 4.17. + group uses matches and not cel functions to allow for use on 4.17. maxLength: 253 minLength: 1 type: string @@ -135,7 +138,7 @@ spec: Must be a valid DNS 1123 label consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. The value must start and end with an alphanumeric character. - We are using matches and not cel functions to allow for use on 4.17. + resource uses matches and not cel functions to allow for use on 4.17. maxLength: 63 minLength: 1 type: string @@ -148,7 +151,7 @@ spec: Must be a valid DNS 1035 label consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. The value must start with an alphabetic character and end with an alphanumeric character. - We are using matches and not cel functions to allow for use on 4.17. + version uses matches and not cel functions to allow for use on 4.17. maxLength: 63 minLength: 1 type: string @@ -161,15 +164,16 @@ spec: - version type: object maxItems: 32 + minItems: 1 type: array x-kubernetes-list-type: atomic frameworks: description: |- - frameworks are a unique list of names to be enabled. - This is required and must have at least one element. + frameworks are a list of frameworks that Kueue has support for. + The allowed values are BatchJob, RayJob, RayCluster, JobSet, MPIJob, PaddleJob, PyTorchJob, TFJob, XGBoostJob, AppWrapper, Pod, Deployment, StatefulSet and LeaderWorkerSet. + frameworks are required and must have at least one element. + frameworks can not have more than 14 elements. Each framework represents a type of job that Kueue will manage. - Frameworks are a list of frameworks that Kueue has support for. - The allowed values are BatchJob, RayJob, RayCluster, JobSet, MPIJob, PaddleJob, PytorchJob, TFJob, XGBoostJob, AppWrapper, Pod, Deployment, StatefulSet and LeaderWorkerSet. items: enum: - BatchJob @@ -199,6 +203,8 @@ spec: labelKeysToCopy are a list of label keys that are copied once a workload is created. These keys are persisted to the internal Kueue workload object. If not specified, only the Kueue labels will be copied. + labelKeysToCopy, if specified, is limited to a maximum of 64 items. + labelKeysToCopy, if specified, must have at least one item. items: properties: key: @@ -206,7 +212,7 @@ spec: key is the label key. A label key must be a valid qualified name consisting of a lower-case alphanumeric string, and hyphens of at most 63 characters in length. - We are using matches and not cel functions to allow for use on 4.17. + To support older openshift versions, matches is used instead of CEL validation. The name must start and end with an alphanumeric character. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. @@ -222,8 +228,9 @@ spec: - key type: object maxItems: 64 + minItems: 1 type: array - x-kubernetes-list-type: set + x-kubernetes-list-type: map required: - frameworks type: object @@ -231,8 +238,8 @@ spec: description: |- preemption is the process of evicting one or more admitted Workloads to accommodate another Workload. Kueue has classical premption and preemption via fair sharing. - This field is optional. - If this field is not specified, preemption will be set to Classical. + preemption is optional. + If preemption is not specified, preemption will be set to Classical. properties: preemptionPolicy: description: |- @@ -260,9 +267,9 @@ spec: workloadManagement controls how Kueue manages workloads. By default Kueue will manage workloads that have a queue-name label. Workloads that are missing the queue-name will be ignored by Kueue. - This field is optional. - If this field is not specified, Kueue will only manage workloads + If tworkloadManagement is not specified, Kueue will only manage workloads that have the queue-name label. + This field is optional. properties: labelPolicy: description: |- diff --git a/operator/v1alpha1/zz_generated.swagger_doc_generated.go b/operator/v1alpha1/zz_generated.swagger_doc_generated.go index 50069be317e..fced28c46a4 100644 --- a/operator/v1alpha1/zz_generated.swagger_doc_generated.go +++ b/operator/v1alpha1/zz_generated.swagger_doc_generated.go @@ -259,7 +259,7 @@ func (RepositoryDigestMirrors) SwaggerDoc() map[string]string { var map_ByWorkload = map[string]string{ "": "ByWorkload controls how admission is done", - "admission": "admission controls how Kueue will process workloads. Allowed values are Sequential and Parallel. When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling.", + "admission": "admission controls how Kueue will process workloads. admission is an optional field. Allowed values are Sequential and Parallel. When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling.", } func (ByWorkload) SwaggerDoc() map[string]string { @@ -268,9 +268,9 @@ func (ByWorkload) SwaggerDoc() map[string]string { var map_ExternalFramework = map[string]string{ "": "This is the GVR for an external framework. Controller runtime requires this in this format for api discoverability.", - "group": "group is the API group of the externalFramework. Must be a valid DNS 1123 subdomain consisting of of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. We are using matches and not cel functions to allow for use on 4.17.", - "resource": "resource is the Resource type of the external framework. Resource types are lowercase and plural (e.g. pods, deployments). Must be a valid DNS 1123 label consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. The value must start and end with an alphanumeric character. We are using matches and not cel functions to allow for use on 4.17.", - "version": "version is the version of the api (e.g. v1alpha1, v1beta1, v1). Must be a valid DNS 1035 label consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. The value must start with an alphabetic character and end with an alphanumeric character. We are using matches and not cel functions to allow for use on 4.17.", + "group": "group is the API group of the externalFramework. Must be a valid DNS 1123 subdomain consisting of of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. group uses matches and not cel functions to allow for use on 4.17.", + "resource": "resource is the Resource type of the external framework. Resource types are lowercase and plural (e.g. pods, deployments). Must be a valid DNS 1123 label consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. The value must start and end with an alphanumeric character. resource uses matches and not cel functions to allow for use on 4.17.", + "version": "version is the version of the api (e.g. v1alpha1, v1beta1, v1). Must be a valid DNS 1035 label consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. The value must start with an alphabetic character and end with an alphanumeric character. version uses matches and not cel functions to allow for use on 4.17.", } func (ExternalFramework) SwaggerDoc() map[string]string { @@ -279,7 +279,7 @@ func (ExternalFramework) SwaggerDoc() map[string]string { var map_GangScheduling = map[string]string{ "": "Kueue provides the ability to admit workloads all in one (gang admission) and evicts workloads if they are not ready within a specific time.", - "policy": "policy allows you to enable and configure gang scheduling. This is an optional field. The allowed values are ByWorkload and Disabled. The default value will be Disabled. When set to ByWorkload, this means each workload is processed and considered for admission as a single unit. Where workloads do not become ready over time, the entire workload may then be evicted and retried at a later time.", + "policy": "policy allows you to enable and configure gang scheduling. The allowed values are ByWorkload and None. The default value will be None. When set to ByWorkload, this means each workload is processed and considered for admission as a single unit. Where workloads do not become ready over time, the entire workload may then be evicted and retried at a later time. policy is a required field.", "byWorkload": "byWorkload configures how Kueue will process workloads for admission. byWorkload is required when policy is ByWorkload, and forbidden otherwise.", } @@ -289,9 +289,9 @@ func (GangScheduling) SwaggerDoc() map[string]string { var map_Integrations = map[string]string{ "": "This is the integrations for Kueue. Kueue uses these apis to determine which jobs will be managed by Kueue.", - "frameworks": "frameworks are a unique list of names to be enabled. This is required and must have at least one element. Each framework represents a type of job that Kueue will manage. Frameworks are a list of frameworks that Kueue has support for. The allowed values are BatchJob, RayJob, RayCluster, JobSet, MPIJob, PaddleJob, PytorchJob, TFJob, XGBoostJob, AppWrapper, Pod, Deployment, StatefulSet and LeaderWorkerSet.", - "externalFrameworks": "externalFrameworks are a list of GroupVersionResources that are managed for Kueue by external controllers. These are optional and should only be used if you have an external controller that integrates with Kueue.", - "labelKeysToCopy": "labelKeysToCopy are a list of label keys that are copied once a workload is created. These keys are persisted to the internal Kueue workload object. If not specified, only the Kueue labels will be copied.", + "frameworks": "frameworks are a list of frameworks that Kueue has support for. The allowed values are BatchJob, RayJob, RayCluster, JobSet, MPIJob, PaddleJob, PyTorchJob, TFJob, XGBoostJob, AppWrapper, Pod, Deployment, StatefulSet and LeaderWorkerSet. frameworks are required and must have at least one element. frameworks can not have more than 14 elements. Each framework represents a type of job that Kueue will manage.", + "externalFrameworks": "externalFrameworks are a list of GroupVersionResources that are managed for Kueue by external controllers. externalFrameworks are optional and should only be used if you have an external controller that integrates with Kueue. externalFrameworks, if specified, must have at least 1 item. externalFrameworks, if specified, can not have more than 32 items.", + "labelKeysToCopy": "labelKeysToCopy are a list of label keys that are copied once a workload is created. These keys are persisted to the internal Kueue workload object. If not specified, only the Kueue labels will be copied. labelKeysToCopy, if specified, is limited to a maximum of 64 items. labelKeysToCopy, if specified, must have at least one item.", } func (Integrations) SwaggerDoc() map[string]string { @@ -311,9 +311,9 @@ func (Kueue) SwaggerDoc() map[string]string { var map_KueueConfiguration = map[string]string{ "integrations": "integrations is a required field that configures the Kueue's workload integrations. Kueue has both standard integrations, known as job frameworks, and external integrations known as external frameworks. Kueue will only manage workloads that correspond to the specified integrations.", - "workloadManagement": "workloadManagement controls how Kueue manages workloads. By default Kueue will manage workloads that have a queue-name label. Workloads that are missing the queue-name will be ignored by Kueue. This field is optional. If this field is not specified, Kueue will only manage workloads that have the queue-name label.", - "gangScheduling": "gangScheduling controls how Kueue admits workloads. Gang Scheduling is the act of all or nothing scheduling, where workloads do not become ready within a certain period, they may be evicted and later retried. This field is optional. If this field is not specified, gang scheduling will be disabled.", - "preemption": "preemption is the process of evicting one or more admitted Workloads to accommodate another Workload. Kueue has classical premption and preemption via fair sharing. This field is optional. If this field is not specified, preemption will be set to Classical.", + "workloadManagement": "workloadManagement controls how Kueue manages workloads. By default Kueue will manage workloads that have a queue-name label. Workloads that are missing the queue-name will be ignored by Kueue. If tworkloadManagement is not specified, Kueue will only manage workloads that have the queue-name label. This field is optional.", + "gangScheduling": "gangScheduling controls how Kueue admits workloads. Gang Scheduling is the act of all or nothing scheduling, where workloads do not become ready within a certain period, they may be evicted and later retried. This field is optional. If gangScheduling is not specified, gang scheduling will be disabled.", + "preemption": "preemption is the process of evicting one or more admitted Workloads to accommodate another Workload. Kueue has classical premption and preemption via fair sharing. preemption is optional. If preemption is not specified, preemption will be set to Classical.", } func (KueueConfiguration) SwaggerDoc() map[string]string { @@ -347,7 +347,7 @@ func (KueueStatus) SwaggerDoc() map[string]string { } var map_LabelKeys = map[string]string{ - "key": "key is the label key. A label key must be a valid qualified name consisting of a lower-case alphanumeric string, and hyphens of at most 63 characters in length. We are using matches and not cel functions to allow for use on 4.17. The name must start and end with an alphanumeric character. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. The optional prefix and the name are separate by a forward slash (/).", + "key": "key is the label key. A label key must be a valid qualified name consisting of a lower-case alphanumeric string, and hyphens of at most 63 characters in length. To support older openshift versions, matches is used instead of CEL validation. The name must start and end with an alphanumeric character. The name may be optionally prefixed with a subdomain consisting of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. The optional prefix and the name are separate by a forward slash (/).", } func (LabelKeys) SwaggerDoc() map[string]string { From bedbe63033f82a2ffd189d038fe39809834f6d11 Mon Sep 17 00:00:00 2001 From: Kevin Hannon Date: Thu, 10 Apr 2025 13:24:06 -0400 Subject: [PATCH 36/42] consistent api --- openapi/generated_openapi/zz_generated.openapi.go | 8 +++++--- openapi/openapi.json | 12 ++++++++---- operator/v1alpha1/types_kueue.go | 11 +++++++---- .../zz_generated.crd-manifests/kueue.crd.yaml | 9 +++++++-- .../kueue.operator.openshift.io/AAA_ungated.yaml | 9 +++++++-- .../v1alpha1/zz_generated.swagger_doc_generated.go | 6 +++--- 6 files changed, 37 insertions(+), 18 deletions(-) diff --git a/openapi/generated_openapi/zz_generated.openapi.go b/openapi/generated_openapi/zz_generated.openapi.go index 80ba2a30771..23d4bcbab5f 100644 --- a/openapi/generated_openapi/zz_generated.openapi.go +++ b/openapi/generated_openapi/zz_generated.openapi.go @@ -56515,12 +56515,14 @@ func schema_openshift_api_operator_v1alpha1_ByWorkload(ref common.ReferenceCallb Properties: map[string]spec.Schema{ "admission": { SchemaProps: spec.SchemaProps{ - Description: "admission controls how Kueue will process workloads. admission is an optional field. Allowed values are Sequential and Parallel. When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling.", + Description: "admission controls how Kueue will process workloads. admission is a required field with policy is set to ByWorkload. admission is only required if policy is specified to ByWorkload. Allowed values are Sequential and Parallel. When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling.", + Default: "", Type: []string{"string"}, Format: "", }, }, }, + Required: []string{"admission"}, }, }, } @@ -58075,7 +58077,7 @@ func schema_openshift_api_operator_v1alpha1_Preemption(ref common.ReferenceCallb Properties: map[string]spec.Schema{ "preemptionPolicy": { SchemaProps: spec.SchemaProps{ - Description: "preemptionPolicy are the types of preemption Kueue allows. Kueue has two types of preemption: Classical and FairSharing. Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. FairSharing means that ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort. FairSharing is a more heavy weight algorithm. The default is Classical.", + Description: "preemptionPolicy are the types of preemption Kueue allows. preemptionPolicy is an optional field. The allowed values are Classical and FairSharing. Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. FairSharing means that ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort. FairSharing is a more heavy weight algorithm. The default is Classical.", Type: []string{"string"}, Format: "", }, @@ -58285,7 +58287,7 @@ func schema_openshift_api_operator_v1alpha1_WorkloadManagement(ref common.Refere Properties: map[string]spec.Schema{ "labelPolicy": { SchemaProps: spec.SchemaProps{ - Description: "labelPolicy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. The allowed values are QueueName and None. None means that workloads will be suspended on creation and a label will be added via a mutating webhook. This will be applied for all integrations that Kueue manages. QueueName means that workloads that are managed by Kueue must have a label kueue.x-k8s.io/queue-name. If this label is not present on the workload, then Kueue will ignore this workload.", + Description: "labelPolicy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. labelPolicy is a required field. The allowed values are QueueName and None. None means that workloads will be suspended on creation and a label will be added via a mutating webhook. This will be applied for all integrations that Kueue manages. QueueName means that workloads that are managed by Kueue must have a label kueue.x-k8s.io/queue-name. If this label is not present on the workload, then Kueue will ignore this workload.", Default: "", Type: []string{"string"}, Format: "", diff --git a/openapi/openapi.json b/openapi/openapi.json index 05a850b66d1..265a27e6e9a 100644 --- a/openapi/openapi.json +++ b/openapi/openapi.json @@ -33012,10 +33012,14 @@ "com.github.openshift.api.operator.v1alpha1.ByWorkload": { "description": "ByWorkload controls how admission is done", "type": "object", + "required": [ + "admission" + ], "properties": { "admission": { - "description": "admission controls how Kueue will process workloads. admission is an optional field. Allowed values are Sequential and Parallel. When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling.", - "type": "string" + "description": "admission controls how Kueue will process workloads. admission is a required field with policy is set to ByWorkload. admission is only required if policy is specified to ByWorkload. Allowed values are Sequential and Parallel. When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling.", + "type": "string", + "default": "" } } }, @@ -33955,7 +33959,7 @@ "type": "object", "properties": { "preemptionPolicy": { - "description": "preemptionPolicy are the types of preemption Kueue allows. Kueue has two types of preemption: Classical and FairSharing. Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. FairSharing means that ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort. FairSharing is a more heavy weight algorithm. The default is Classical.", + "description": "preemptionPolicy are the types of preemption Kueue allows. preemptionPolicy is an optional field. The allowed values are Classical and FairSharing. Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. FairSharing means that ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort. FairSharing is a more heavy weight algorithm. The default is Classical.", "type": "string" } } @@ -34088,7 +34092,7 @@ ], "properties": { "labelPolicy": { - "description": "labelPolicy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. The allowed values are QueueName and None. None means that workloads will be suspended on creation and a label will be added via a mutating webhook. This will be applied for all integrations that Kueue manages. QueueName means that workloads that are managed by Kueue must have a label kueue.x-k8s.io/queue-name. If this label is not present on the workload, then Kueue will ignore this workload.", + "description": "labelPolicy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. labelPolicy is a required field. The allowed values are QueueName and None. None means that workloads will be suspended on creation and a label will be added via a mutating webhook. This will be applied for all integrations that Kueue manages. QueueName means that workloads that are managed by Kueue must have a label kueue.x-k8s.io/queue-name. If this label is not present on the workload, then Kueue will ignore this workload.", "type": "string", "default": "" } diff --git a/operator/v1alpha1/types_kueue.go b/operator/v1alpha1/types_kueue.go index 28bf998fc8e..1e9926db314 100644 --- a/operator/v1alpha1/types_kueue.go +++ b/operator/v1alpha1/types_kueue.go @@ -248,7 +248,8 @@ type GangScheduling struct { // ByWorkload controls how admission is done type ByWorkload struct { // admission controls how Kueue will process workloads. - // admission is an optional field. + // admission is a required field with policy is set to ByWorkload. + // admission is only required if policy is specified to ByWorkload. // Allowed values are Sequential and Parallel. // When admission is set to Sequential, only pods from the currently processing workload will be admitted. // Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. @@ -257,8 +258,8 @@ type ByWorkload struct { // When set to Parallel, pods from any workload will be admitted at any time. // This may lead to a deadlock where workloads are in contention for cluster capacity and // pods from another workload having successfully scheduled prevent pods from the current workload scheduling. - // +optional - Admission GangSchedulingWorkloadAdmission `json:"admission,omitempty"` + // +required + Admission GangSchedulingWorkloadAdmission `json:"admission"` } // +kubebuilder:validation:Enum=QueueName;None @@ -272,6 +273,7 @@ const ( type WorkloadManagement struct { // labelPolicy controls whether or not Kueue reconciles // jobs that don't set the label kueue.x-k8s.io/queue-name. + // labelPolicy is a required field. // The allowed values are QueueName and None. // None means that workloads will be suspended on // creation and a label will be added via a mutating webhook. @@ -294,7 +296,8 @@ const ( type Preemption struct { // preemptionPolicy are the types of preemption Kueue allows. - // Kueue has two types of preemption: Classical and FairSharing. + // preemptionPolicy is an optional field. + // The allowed values are Classical and FairSharing. // Classical means that an incoming workload, which does // not fit within the unusued quota, is eligible to issue preemptions // when the requests of the workload are below the diff --git a/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml b/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml index 81b0188008e..d42c490ba23 100644 --- a/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml +++ b/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml @@ -64,7 +64,8 @@ spec: admission: description: |- admission controls how Kueue will process workloads. - admission is an optional field. + admission is a required field with policy is set to ByWorkload. + admission is only required if policy is specified to ByWorkload. Allowed values are Sequential and Parallel. When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. @@ -77,6 +78,8 @@ spec: - Parallel - Sequential type: string + required: + - admission type: object policy: description: |- @@ -246,7 +249,8 @@ spec: preemptionPolicy: description: |- preemptionPolicy are the types of preemption Kueue allows. - Kueue has two types of preemption: Classical and FairSharing. + preemptionPolicy is an optional field. + The allowed values are Classical and FairSharing. Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the @@ -277,6 +281,7 @@ spec: description: |- labelPolicy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. + labelPolicy is a required field. The allowed values are QueueName and None. None means that workloads will be suspended on creation and a label will be added via a mutating webhook. diff --git a/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml b/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml index ec38ae89198..92e5b1fc22a 100644 --- a/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml +++ b/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml @@ -62,7 +62,8 @@ spec: admission: description: |- admission controls how Kueue will process workloads. - admission is an optional field. + admission is a required field with policy is set to ByWorkload. + admission is only required if policy is specified to ByWorkload. Allowed values are Sequential and Parallel. When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. @@ -75,6 +76,8 @@ spec: - Parallel - Sequential type: string + required: + - admission type: object policy: description: |- @@ -244,7 +247,8 @@ spec: preemptionPolicy: description: |- preemptionPolicy are the types of preemption Kueue allows. - Kueue has two types of preemption: Classical and FairSharing. + preemptionPolicy is an optional field. + The allowed values are Classical and FairSharing. Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the @@ -275,6 +279,7 @@ spec: description: |- labelPolicy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. + labelPolicy is a required field. The allowed values are QueueName and None. None means that workloads will be suspended on creation and a label will be added via a mutating webhook. diff --git a/operator/v1alpha1/zz_generated.swagger_doc_generated.go b/operator/v1alpha1/zz_generated.swagger_doc_generated.go index fced28c46a4..8ec170f9a7d 100644 --- a/operator/v1alpha1/zz_generated.swagger_doc_generated.go +++ b/operator/v1alpha1/zz_generated.swagger_doc_generated.go @@ -259,7 +259,7 @@ func (RepositoryDigestMirrors) SwaggerDoc() map[string]string { var map_ByWorkload = map[string]string{ "": "ByWorkload controls how admission is done", - "admission": "admission controls how Kueue will process workloads. admission is an optional field. Allowed values are Sequential and Parallel. When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling.", + "admission": "admission controls how Kueue will process workloads. admission is a required field with policy is set to ByWorkload. admission is only required if policy is specified to ByWorkload. Allowed values are Sequential and Parallel. When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling.", } func (ByWorkload) SwaggerDoc() map[string]string { @@ -355,7 +355,7 @@ func (LabelKeys) SwaggerDoc() map[string]string { } var map_Preemption = map[string]string{ - "preemptionPolicy": "preemptionPolicy are the types of preemption Kueue allows. Kueue has two types of preemption: Classical and FairSharing. Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. FairSharing means that ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort. FairSharing is a more heavy weight algorithm. The default is Classical.", + "preemptionPolicy": "preemptionPolicy are the types of preemption Kueue allows. preemptionPolicy is an optional field. The allowed values are Classical and FairSharing. Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. FairSharing means that ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort. FairSharing is a more heavy weight algorithm. The default is Classical.", } func (Preemption) SwaggerDoc() map[string]string { @@ -363,7 +363,7 @@ func (Preemption) SwaggerDoc() map[string]string { } var map_WorkloadManagement = map[string]string{ - "labelPolicy": "labelPolicy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. The allowed values are QueueName and None. None means that workloads will be suspended on creation and a label will be added via a mutating webhook. This will be applied for all integrations that Kueue manages. QueueName means that workloads that are managed by Kueue must have a label kueue.x-k8s.io/queue-name. If this label is not present on the workload, then Kueue will ignore this workload.", + "labelPolicy": "labelPolicy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. labelPolicy is a required field. The allowed values are QueueName and None. None means that workloads will be suspended on creation and a label will be added via a mutating webhook. This will be applied for all integrations that Kueue manages. QueueName means that workloads that are managed by Kueue must have a label kueue.x-k8s.io/queue-name. If this label is not present on the workload, then Kueue will ignore this workload.", } func (WorkloadManagement) SwaggerDoc() map[string]string { From 2bf301aa71bdc34e2e86872f8d3f525ea5667890 Mon Sep 17 00:00:00 2001 From: Kevin Hannon Date: Thu, 10 Apr 2025 16:18:04 -0400 Subject: [PATCH 37/42] more comments --- openapi/generated_openapi/zz_generated.openapi.go | 10 ++++++++-- openapi/openapi.json | 10 ++++++++-- operator/v1alpha1/types_kueue.go | 6 +++--- .../v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml | 7 +++++-- .../kueue.operator.openshift.io/AAA_ungated.yaml | 7 +++++-- .../v1alpha1/zz_generated.swagger_doc_generated.go | 2 +- 6 files changed, 30 insertions(+), 12 deletions(-) diff --git a/openapi/generated_openapi/zz_generated.openapi.go b/openapi/generated_openapi/zz_generated.openapi.go index 23d4bcbab5f..2a804a2d461 100644 --- a/openapi/generated_openapi/zz_generated.openapi.go +++ b/openapi/generated_openapi/zz_generated.openapi.go @@ -56920,7 +56920,7 @@ func schema_openshift_api_operator_v1alpha1_GangScheduling(ref common.ReferenceC Properties: map[string]spec.Schema{ "policy": { SchemaProps: spec.SchemaProps{ - Description: "policy allows you to enable and configure gang scheduling. The allowed values are ByWorkload and None. The default value will be None. When set to ByWorkload, this means each workload is processed and considered for admission as a single unit. Where workloads do not become ready over time, the entire workload may then be evicted and retried at a later time. policy is a required field.", + Description: "policy allows you to enable and configure gang scheduling. The allowed values are ByWorkload and None. When set to ByWorkload, this means each workload is processed and considered for admission as a single unit. Where workloads do not become ready over time, the entire workload may then be evicted and retried at a later time. policy is a required field.", Default: "", Type: []string{"string"}, Format: "", @@ -57217,7 +57217,10 @@ func schema_openshift_api_operator_v1alpha1_Integrations(ref common.ReferenceCal "externalFrameworks": { VendorExtensible: spec.VendorExtensible{ Extensions: spec.Extensions{ - "x-kubernetes-list-type": "atomic", + "x-kubernetes-list-map-keys": []interface{}{ + "group", + }, + "x-kubernetes-list-type": "map", }, }, SchemaProps: spec.SchemaProps{ @@ -57236,6 +57239,9 @@ func schema_openshift_api_operator_v1alpha1_Integrations(ref common.ReferenceCal "labelKeysToCopy": { VendorExtensible: spec.VendorExtensible{ Extensions: spec.Extensions{ + "x-kubernetes-list-map-keys": []interface{}{ + "key", + }, "x-kubernetes-list-type": "map", }, }, diff --git a/openapi/openapi.json b/openapi/openapi.json index 265a27e6e9a..499629f2864 100644 --- a/openapi/openapi.json +++ b/openapi/openapi.json @@ -33258,7 +33258,7 @@ "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.ByWorkload" }, "policy": { - "description": "policy allows you to enable and configure gang scheduling. The allowed values are ByWorkload and None. The default value will be None. When set to ByWorkload, this means each workload is processed and considered for admission as a single unit. Where workloads do not become ready over time, the entire workload may then be evicted and retried at a later time. policy is a required field.", + "description": "policy allows you to enable and configure gang scheduling. The allowed values are ByWorkload and None. When set to ByWorkload, this means each workload is processed and considered for admission as a single unit. Where workloads do not become ready over time, the entire workload may then be evicted and retried at a later time. policy is a required field.", "type": "string", "default": "" } @@ -33430,7 +33430,10 @@ "default": {}, "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.ExternalFramework" }, - "x-kubernetes-list-type": "atomic" + "x-kubernetes-list-map-keys": [ + "group" + ], + "x-kubernetes-list-type": "map" }, "frameworks": { "description": "frameworks are a list of frameworks that Kueue has support for. The allowed values are BatchJob, RayJob, RayCluster, JobSet, MPIJob, PaddleJob, PyTorchJob, TFJob, XGBoostJob, AppWrapper, Pod, Deployment, StatefulSet and LeaderWorkerSet. frameworks are required and must have at least one element. frameworks can not have more than 14 elements. Each framework represents a type of job that Kueue will manage.", @@ -33448,6 +33451,9 @@ "default": {}, "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.LabelKeys" }, + "x-kubernetes-list-map-keys": [ + "key" + ], "x-kubernetes-list-type": "map" } } diff --git a/operator/v1alpha1/types_kueue.go b/operator/v1alpha1/types_kueue.go index 1e9926db314..a0008e0fec7 100644 --- a/operator/v1alpha1/types_kueue.go +++ b/operator/v1alpha1/types_kueue.go @@ -173,7 +173,8 @@ type Integrations struct { // that integrates with Kueue. // externalFrameworks, if specified, must have at least 1 item. // externalFrameworks, if specified, can not have more than 32 items. - // +listType=atomic + // +listType=map + // +listMapKey=group // +kubebuilder:validation:MinItems=1 // +kubebuilder:validation:MaxItems=32 // +optional @@ -186,7 +187,7 @@ type Integrations struct { // +kubebuilder:validation:MaxItems=64 // +kubebuilder:validation:MinItems=1 // +listType=map - // +listMapType=key + // +listMapKey=key // +optional LabelKeysToCopy []LabelKeys `json:"labelKeysToCopy,omitempty"` } @@ -231,7 +232,6 @@ const ( type GangScheduling struct { // policy allows you to enable and configure gang scheduling. // The allowed values are ByWorkload and None. - // The default value will be None. // When set to ByWorkload, this means each workload is processed and considered // for admission as a single unit. // Where workloads do not become ready over time, the entire workload may then be evicted and retried at a later time. diff --git a/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml b/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml index d42c490ba23..78c943d9d99 100644 --- a/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml +++ b/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml @@ -85,7 +85,6 @@ spec: description: |- policy allows you to enable and configure gang scheduling. The allowed values are ByWorkload and None. - The default value will be None. When set to ByWorkload, this means each workload is processed and considered for admission as a single unit. Where workloads do not become ready over time, the entire workload may then be evicted and retried at a later time. @@ -171,7 +170,9 @@ spec: maxItems: 32 minItems: 1 type: array - x-kubernetes-list-type: atomic + x-kubernetes-list-map-keys: + - group + x-kubernetes-list-type: map frameworks: description: |- frameworks are a list of frameworks that Kueue has support for. @@ -235,6 +236,8 @@ spec: maxItems: 64 minItems: 1 type: array + x-kubernetes-list-map-keys: + - key x-kubernetes-list-type: map required: - frameworks diff --git a/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml b/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml index 92e5b1fc22a..ef61498aee8 100644 --- a/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml +++ b/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml @@ -83,7 +83,6 @@ spec: description: |- policy allows you to enable and configure gang scheduling. The allowed values are ByWorkload and None. - The default value will be None. When set to ByWorkload, this means each workload is processed and considered for admission as a single unit. Where workloads do not become ready over time, the entire workload may then be evicted and retried at a later time. @@ -169,7 +168,9 @@ spec: maxItems: 32 minItems: 1 type: array - x-kubernetes-list-type: atomic + x-kubernetes-list-map-keys: + - group + x-kubernetes-list-type: map frameworks: description: |- frameworks are a list of frameworks that Kueue has support for. @@ -233,6 +234,8 @@ spec: maxItems: 64 minItems: 1 type: array + x-kubernetes-list-map-keys: + - key x-kubernetes-list-type: map required: - frameworks diff --git a/operator/v1alpha1/zz_generated.swagger_doc_generated.go b/operator/v1alpha1/zz_generated.swagger_doc_generated.go index 8ec170f9a7d..3d2f0ba5c53 100644 --- a/operator/v1alpha1/zz_generated.swagger_doc_generated.go +++ b/operator/v1alpha1/zz_generated.swagger_doc_generated.go @@ -279,7 +279,7 @@ func (ExternalFramework) SwaggerDoc() map[string]string { var map_GangScheduling = map[string]string{ "": "Kueue provides the ability to admit workloads all in one (gang admission) and evicts workloads if they are not ready within a specific time.", - "policy": "policy allows you to enable and configure gang scheduling. The allowed values are ByWorkload and None. The default value will be None. When set to ByWorkload, this means each workload is processed and considered for admission as a single unit. Where workloads do not become ready over time, the entire workload may then be evicted and retried at a later time. policy is a required field.", + "policy": "policy allows you to enable and configure gang scheduling. The allowed values are ByWorkload and None. When set to ByWorkload, this means each workload is processed and considered for admission as a single unit. Where workloads do not become ready over time, the entire workload may then be evicted and retried at a later time. policy is a required field.", "byWorkload": "byWorkload configures how Kueue will process workloads for admission. byWorkload is required when policy is ByWorkload, and forbidden otherwise.", } From bde4a926f3b8108285a453020d16d0a06fb2305c Mon Sep 17 00:00:00 2001 From: Kevin Hannon Date: Sun, 13 Apr 2025 13:58:44 -0400 Subject: [PATCH 38/42] remove omit empty --- openapi/generated_openapi/zz_generated.openapi.go | 1 + openapi/openapi.json | 3 ++- operator/v1alpha1/types_kueue.go | 14 +++++++------- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/openapi/generated_openapi/zz_generated.openapi.go b/openapi/generated_openapi/zz_generated.openapi.go index 2a804a2d461..734b9e958ba 100644 --- a/openapi/generated_openapi/zz_generated.openapi.go +++ b/openapi/generated_openapi/zz_generated.openapi.go @@ -58084,6 +58084,7 @@ func schema_openshift_api_operator_v1alpha1_Preemption(ref common.ReferenceCallb "preemptionPolicy": { SchemaProps: spec.SchemaProps{ Description: "preemptionPolicy are the types of preemption Kueue allows. preemptionPolicy is an optional field. The allowed values are Classical and FairSharing. Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. FairSharing means that ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort. FairSharing is a more heavy weight algorithm. The default is Classical.", + Default: "", Type: []string{"string"}, Format: "", }, diff --git a/openapi/openapi.json b/openapi/openapi.json index 499629f2864..2f029106bec 100644 --- a/openapi/openapi.json +++ b/openapi/openapi.json @@ -33966,7 +33966,8 @@ "properties": { "preemptionPolicy": { "description": "preemptionPolicy are the types of preemption Kueue allows. preemptionPolicy is an optional field. The allowed values are Classical and FairSharing. Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. FairSharing means that ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort. FairSharing is a more heavy weight algorithm. The default is Classical.", - "type": "string" + "type": "string", + "default": "" } } }, diff --git a/operator/v1alpha1/types_kueue.go b/operator/v1alpha1/types_kueue.go index a0008e0fec7..e077bf349b1 100644 --- a/operator/v1alpha1/types_kueue.go +++ b/operator/v1alpha1/types_kueue.go @@ -56,20 +56,20 @@ type KueueConfiguration struct { // that have the queue-name label. // This field is optional. // +optional - WorkloadManagement *WorkloadManagement `json:"workloadManagement,omitempty"` + WorkloadManagement *WorkloadManagement `json:"workloadManagement"` // gangScheduling controls how Kueue admits workloads. // Gang Scheduling is the act of all or nothing scheduling, // where workloads do not become ready within a certain period, they may be evicted and later retried. // This field is optional. // If gangScheduling is not specified, gang scheduling will be disabled. // +optional - GangScheduling *GangScheduling `json:"gangScheduling,omitempty"` + GangScheduling *GangScheduling `json:"gangScheduling"` // preemption is the process of evicting one or more admitted Workloads to accommodate another Workload. // Kueue has classical premption and preemption via fair sharing. // preemption is optional. // If preemption is not specified, preemption will be set to Classical. // +optional - Preemption *Preemption `json:"preemption,omitempty"` + Preemption *Preemption `json:"preemption"` } // KueueStatus defines the observed state of Kueue @@ -178,7 +178,7 @@ type Integrations struct { // +kubebuilder:validation:MinItems=1 // +kubebuilder:validation:MaxItems=32 // +optional - ExternalFrameworks []ExternalFramework `json:"externalFrameworks,omitempty"` + ExternalFrameworks []ExternalFramework `json:"externalFrameworks"` // labelKeysToCopy are a list of label keys that are copied once a workload is created. // These keys are persisted to the internal Kueue workload object. // If not specified, only the Kueue labels will be copied. @@ -189,7 +189,7 @@ type Integrations struct { // +listType=map // +listMapKey=key // +optional - LabelKeysToCopy []LabelKeys `json:"labelKeysToCopy,omitempty"` + LabelKeysToCopy []LabelKeys `json:"labelKeysToCopy"` } type LabelKeys struct { @@ -242,7 +242,7 @@ type GangScheduling struct { // byWorkload configures how Kueue will process workloads for admission. // byWorkload is required when policy is ByWorkload, and forbidden otherwise. // +optional - ByWorkload *ByWorkload `json:"byWorkload,omitempty"` + ByWorkload *ByWorkload `json:"byWorkload"` } // ByWorkload controls how admission is done @@ -311,5 +311,5 @@ type Preemption struct { // FairSharing is a more heavy weight algorithm. // The default is Classical. // +optional - PreemptionPolicy PreemptionPolicy `json:"preemptionPolicy,omitempty"` + PreemptionPolicy PreemptionPolicy `json:"preemptionPolicy"` } From 185dd841480f68d76c0bcc028a74bd6ef6b6d654 Mon Sep 17 00:00:00 2001 From: Kevin Hannon Date: Mon, 14 Apr 2025 10:41:33 -0400 Subject: [PATCH 39/42] fix issues with discoverability --- .../generated_openapi/zz_generated.openapi.go | 28 +++++----- openapi/openapi.json | 28 +++++----- operator/v1alpha1/types_kueue.go | 54 +++++++++---------- .../zz_generated.crd-manifests/kueue.crd.yaml | 34 ++++++------ operator/v1alpha1/zz_generated.deepcopy.go | 24 ++------- .../AAA_ungated.yaml | 34 ++++++------ .../zz_generated.swagger_doc_generated.go | 24 ++++----- 7 files changed, 105 insertions(+), 121 deletions(-) diff --git a/openapi/generated_openapi/zz_generated.openapi.go b/openapi/generated_openapi/zz_generated.openapi.go index 734b9e958ba..e1573a044b9 100644 --- a/openapi/generated_openapi/zz_generated.openapi.go +++ b/openapi/generated_openapi/zz_generated.openapi.go @@ -56515,7 +56515,7 @@ func schema_openshift_api_operator_v1alpha1_ByWorkload(ref common.ReferenceCallb Properties: map[string]spec.Schema{ "admission": { SchemaProps: spec.SchemaProps{ - Description: "admission controls how Kueue will process workloads. admission is a required field with policy is set to ByWorkload. admission is only required if policy is specified to ByWorkload. Allowed values are Sequential and Parallel. When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling.", + Description: "admission controls how Kueue will process workloads. admission is a required field with policy is set to ByWorkload. admission is only required if policy is specified to ByWorkload. Allowed values are Sequential, Parallel and \"\". When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling.", Default: "", Type: []string{"string"}, Format: "", @@ -56882,7 +56882,7 @@ func schema_openshift_api_operator_v1alpha1_ExternalFramework(ref common.Referen Properties: map[string]spec.Schema{ "group": { SchemaProps: spec.SchemaProps{ - Description: "group is the API group of the externalFramework. Must be a valid DNS 1123 subdomain consisting of of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. group uses matches and not cel functions to allow for use on 4.17.", + Description: "group is the API group of the externalFramework. Must be a valid DNS 1123 subdomain consisting of of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character.", Default: "", Type: []string{"string"}, Format: "", @@ -56890,7 +56890,7 @@ func schema_openshift_api_operator_v1alpha1_ExternalFramework(ref common.Referen }, "resource": { SchemaProps: spec.SchemaProps{ - Description: "resource is the Resource type of the external framework. Resource types are lowercase and plural (e.g. pods, deployments). Must be a valid DNS 1123 label consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. The value must start and end with an alphanumeric character. resource uses matches and not cel functions to allow for use on 4.17.", + Description: "resource is the Resource type of the external framework. Resource types are lowercase and plural (e.g. pods, deployments). Must be a valid DNS 1123 label consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. The value must start and end with an alphanumeric character.", Default: "", Type: []string{"string"}, Format: "", @@ -56898,7 +56898,7 @@ func schema_openshift_api_operator_v1alpha1_ExternalFramework(ref common.Referen }, "version": { SchemaProps: spec.SchemaProps{ - Description: "version is the version of the api (e.g. v1alpha1, v1beta1, v1). Must be a valid DNS 1035 label consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. The value must start with an alphabetic character and end with an alphanumeric character. version uses matches and not cel functions to allow for use on 4.17.", + Description: "version is the version of the api (e.g. v1alpha1, v1beta1, v1). Must be a valid DNS 1035 label consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. The value must start with an alphabetic character and end with an alphanumeric character.", Default: "", Type: []string{"string"}, Format: "", @@ -56920,7 +56920,7 @@ func schema_openshift_api_operator_v1alpha1_GangScheduling(ref common.ReferenceC Properties: map[string]spec.Schema{ "policy": { SchemaProps: spec.SchemaProps{ - Description: "policy allows you to enable and configure gang scheduling. The allowed values are ByWorkload and None. When set to ByWorkload, this means each workload is processed and considered for admission as a single unit. Where workloads do not become ready over time, the entire workload may then be evicted and retried at a later time. policy is a required field.", + Description: "policy allows you to enable and configure gang scheduling. The allowed values are ByWorkload, None and \"\". When set to ByWorkload, this means each workload is processed and considered for admission as a single unit. Where workloads do not become ready over time, the entire workload may then be evicted and retried at a later time. policy is a required field.", Default: "", Type: []string{"string"}, Format: "", @@ -56929,6 +56929,7 @@ func schema_openshift_api_operator_v1alpha1_GangScheduling(ref common.ReferenceC "byWorkload": { SchemaProps: spec.SchemaProps{ Description: "byWorkload configures how Kueue will process workloads for admission. byWorkload is required when policy is ByWorkload, and forbidden otherwise.", + Default: map[string]interface{}{}, Ref: ref("github.com/openshift/api/operator/v1alpha1.ByWorkload"), }, }, @@ -57224,7 +57225,7 @@ func schema_openshift_api_operator_v1alpha1_Integrations(ref common.ReferenceCal }, }, SchemaProps: spec.SchemaProps{ - Description: "externalFrameworks are a list of GroupVersionResources that are managed for Kueue by external controllers. externalFrameworks are optional and should only be used if you have an external controller that integrates with Kueue. externalFrameworks, if specified, must have at least 1 item. externalFrameworks, if specified, can not have more than 32 items.", + Description: "externalFrameworks are a list of GroupVersionResources that are managed for Kueue by external controllers. externalFrameworks are optional and should only be used if you have an external controller that integrates with Kueue. externalFrameworks, if specified, can not have more than 32 items.", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -57246,7 +57247,7 @@ func schema_openshift_api_operator_v1alpha1_Integrations(ref common.ReferenceCal }, }, SchemaProps: spec.SchemaProps{ - Description: "labelKeysToCopy are a list of label keys that are copied once a workload is created. These keys are persisted to the internal Kueue workload object. If not specified, only the Kueue labels will be copied. labelKeysToCopy, if specified, is limited to a maximum of 64 items. labelKeysToCopy, if specified, must have at least one item.", + Description: "labelKeysToCopy are a list of label keys that are copied once a workload is created. These keys are persisted to the internal Kueue workload object. If not specified, only the Kueue labels will be copied. labelKeysToCopy, if specified, is limited to a maximum of 64 items.", Type: []string{"array"}, Items: &spec.SchemaOrArray{ Schema: &spec.Schema{ @@ -57333,19 +57334,22 @@ func schema_openshift_api_operator_v1alpha1_KueueConfiguration(ref common.Refere }, "workloadManagement": { SchemaProps: spec.SchemaProps{ - Description: "workloadManagement controls how Kueue manages workloads. By default Kueue will manage workloads that have a queue-name label. Workloads that are missing the queue-name will be ignored by Kueue. If tworkloadManagement is not specified, Kueue will only manage workloads that have the queue-name label. This field is optional.", + Description: "workloadManagement controls how Kueue manages workloads. By default Kueue will manage workloads that have a queue-name label. Workloads that are missing the queue-name will be ignored by Kueue. If workloadManagement is not specified, the operator will decide the default. This default could change over time. This field is optional.", + Default: map[string]interface{}{}, Ref: ref("github.com/openshift/api/operator/v1alpha1.WorkloadManagement"), }, }, "gangScheduling": { SchemaProps: spec.SchemaProps{ - Description: "gangScheduling controls how Kueue admits workloads. Gang Scheduling is the act of all or nothing scheduling, where workloads do not become ready within a certain period, they may be evicted and later retried. This field is optional. If gangScheduling is not specified, gang scheduling will be disabled.", + Description: "gangScheduling controls how Kueue admits workloads. Gang Scheduling is the act of all or nothing scheduling, where workloads do not become ready within a certain period, they may be evicted and later retried. This field is optional. If gangScheduling is not specified, the operator will decide the default. This default could change over time.", + Default: map[string]interface{}{}, Ref: ref("github.com/openshift/api/operator/v1alpha1.GangScheduling"), }, }, "preemption": { SchemaProps: spec.SchemaProps{ - Description: "preemption is the process of evicting one or more admitted Workloads to accommodate another Workload. Kueue has classical premption and preemption via fair sharing. preemption is optional. If preemption is not specified, preemption will be set to Classical.", + Description: "preemption is the process of evicting one or more admitted Workloads to accommodate another Workload. Kueue has classical premption and preemption via fair sharing. preemption is optional. If preemption is not specified, the operator will decide the default. This default could change over time.", + Default: map[string]interface{}{}, Ref: ref("github.com/openshift/api/operator/v1alpha1.Preemption"), }, }, @@ -58083,7 +58087,7 @@ func schema_openshift_api_operator_v1alpha1_Preemption(ref common.ReferenceCallb Properties: map[string]spec.Schema{ "preemptionPolicy": { SchemaProps: spec.SchemaProps{ - Description: "preemptionPolicy are the types of preemption Kueue allows. preemptionPolicy is an optional field. The allowed values are Classical and FairSharing. Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. FairSharing means that ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort. FairSharing is a more heavy weight algorithm. The default is Classical.", + Description: "preemptionPolicy are the types of preemption Kueue allows. preemptionPolicy is an optional field. The allowed values are Classical, FairSharing and \"\". Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. FairSharing means that ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort. FairSharing is a more heavy weight algorithm. The default is Classical.", Default: "", Type: []string{"string"}, Format: "", @@ -58294,7 +58298,7 @@ func schema_openshift_api_operator_v1alpha1_WorkloadManagement(ref common.Refere Properties: map[string]spec.Schema{ "labelPolicy": { SchemaProps: spec.SchemaProps{ - Description: "labelPolicy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. labelPolicy is a required field. The allowed values are QueueName and None. None means that workloads will be suspended on creation and a label will be added via a mutating webhook. This will be applied for all integrations that Kueue manages. QueueName means that workloads that are managed by Kueue must have a label kueue.x-k8s.io/queue-name. If this label is not present on the workload, then Kueue will ignore this workload.", + Description: "labelPolicy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. labelPolicy is a required field. The allowed values are QueueName, None and \"\". None means that workloads will be suspended on creation and a label will be added via a mutating webhook. This will be applied for all integrations that Kueue manages. QueueName means that workloads that are managed by Kueue must have a label kueue.x-k8s.io/queue-name. If this label is not present on the workload, then Kueue will ignore this workload.", Default: "", Type: []string{"string"}, Format: "", diff --git a/openapi/openapi.json b/openapi/openapi.json index 2f029106bec..27b9d8c4073 100644 --- a/openapi/openapi.json +++ b/openapi/openapi.json @@ -33017,7 +33017,7 @@ ], "properties": { "admission": { - "description": "admission controls how Kueue will process workloads. admission is a required field with policy is set to ByWorkload. admission is only required if policy is specified to ByWorkload. Allowed values are Sequential and Parallel. When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling.", + "description": "admission controls how Kueue will process workloads. admission is a required field with policy is set to ByWorkload. admission is only required if policy is specified to ByWorkload. Allowed values are Sequential, Parallel and \"\". When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling.", "type": "string", "default": "" } @@ -33230,17 +33230,17 @@ ], "properties": { "group": { - "description": "group is the API group of the externalFramework. Must be a valid DNS 1123 subdomain consisting of of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. group uses matches and not cel functions to allow for use on 4.17.", + "description": "group is the API group of the externalFramework. Must be a valid DNS 1123 subdomain consisting of of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character.", "type": "string", "default": "" }, "resource": { - "description": "resource is the Resource type of the external framework. Resource types are lowercase and plural (e.g. pods, deployments). Must be a valid DNS 1123 label consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. The value must start and end with an alphanumeric character. resource uses matches and not cel functions to allow for use on 4.17.", + "description": "resource is the Resource type of the external framework. Resource types are lowercase and plural (e.g. pods, deployments). Must be a valid DNS 1123 label consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. The value must start and end with an alphanumeric character.", "type": "string", "default": "" }, "version": { - "description": "version is the version of the api (e.g. v1alpha1, v1beta1, v1). Must be a valid DNS 1035 label consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. The value must start with an alphabetic character and end with an alphanumeric character. version uses matches and not cel functions to allow for use on 4.17.", + "description": "version is the version of the api (e.g. v1alpha1, v1beta1, v1). Must be a valid DNS 1035 label consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. The value must start with an alphabetic character and end with an alphanumeric character.", "type": "string", "default": "" } @@ -33255,10 +33255,11 @@ "properties": { "byWorkload": { "description": "byWorkload configures how Kueue will process workloads for admission. byWorkload is required when policy is ByWorkload, and forbidden otherwise.", + "default": {}, "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.ByWorkload" }, "policy": { - "description": "policy allows you to enable and configure gang scheduling. The allowed values are ByWorkload and None. When set to ByWorkload, this means each workload is processed and considered for admission as a single unit. Where workloads do not become ready over time, the entire workload may then be evicted and retried at a later time. policy is a required field.", + "description": "policy allows you to enable and configure gang scheduling. The allowed values are ByWorkload, None and \"\". When set to ByWorkload, this means each workload is processed and considered for admission as a single unit. Where workloads do not become ready over time, the entire workload may then be evicted and retried at a later time. policy is a required field.", "type": "string", "default": "" } @@ -33424,7 +33425,7 @@ ], "properties": { "externalFrameworks": { - "description": "externalFrameworks are a list of GroupVersionResources that are managed for Kueue by external controllers. externalFrameworks are optional and should only be used if you have an external controller that integrates with Kueue. externalFrameworks, if specified, must have at least 1 item. externalFrameworks, if specified, can not have more than 32 items.", + "description": "externalFrameworks are a list of GroupVersionResources that are managed for Kueue by external controllers. externalFrameworks are optional and should only be used if you have an external controller that integrates with Kueue. externalFrameworks, if specified, can not have more than 32 items.", "type": "array", "items": { "default": {}, @@ -33445,7 +33446,7 @@ "x-kubernetes-list-type": "atomic" }, "labelKeysToCopy": { - "description": "labelKeysToCopy are a list of label keys that are copied once a workload is created. These keys are persisted to the internal Kueue workload object. If not specified, only the Kueue labels will be copied. labelKeysToCopy, if specified, is limited to a maximum of 64 items. labelKeysToCopy, if specified, must have at least one item.", + "description": "labelKeysToCopy are a list of label keys that are copied once a workload is created. These keys are persisted to the internal Kueue workload object. If not specified, only the Kueue labels will be copied. labelKeysToCopy, if specified, is limited to a maximum of 64 items.", "type": "array", "items": { "default": {}, @@ -33497,7 +33498,8 @@ ], "properties": { "gangScheduling": { - "description": "gangScheduling controls how Kueue admits workloads. Gang Scheduling is the act of all or nothing scheduling, where workloads do not become ready within a certain period, they may be evicted and later retried. This field is optional. If gangScheduling is not specified, gang scheduling will be disabled.", + "description": "gangScheduling controls how Kueue admits workloads. Gang Scheduling is the act of all or nothing scheduling, where workloads do not become ready within a certain period, they may be evicted and later retried. This field is optional. If gangScheduling is not specified, the operator will decide the default. This default could change over time.", + "default": {}, "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.GangScheduling" }, "integrations": { @@ -33506,11 +33508,13 @@ "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.Integrations" }, "preemption": { - "description": "preemption is the process of evicting one or more admitted Workloads to accommodate another Workload. Kueue has classical premption and preemption via fair sharing. preemption is optional. If preemption is not specified, preemption will be set to Classical.", + "description": "preemption is the process of evicting one or more admitted Workloads to accommodate another Workload. Kueue has classical premption and preemption via fair sharing. preemption is optional. If preemption is not specified, the operator will decide the default. This default could change over time.", + "default": {}, "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.Preemption" }, "workloadManagement": { - "description": "workloadManagement controls how Kueue manages workloads. By default Kueue will manage workloads that have a queue-name label. Workloads that are missing the queue-name will be ignored by Kueue. If tworkloadManagement is not specified, Kueue will only manage workloads that have the queue-name label. This field is optional.", + "description": "workloadManagement controls how Kueue manages workloads. By default Kueue will manage workloads that have a queue-name label. Workloads that are missing the queue-name will be ignored by Kueue. If workloadManagement is not specified, the operator will decide the default. This default could change over time. This field is optional.", + "default": {}, "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.WorkloadManagement" } } @@ -33965,7 +33969,7 @@ "type": "object", "properties": { "preemptionPolicy": { - "description": "preemptionPolicy are the types of preemption Kueue allows. preemptionPolicy is an optional field. The allowed values are Classical and FairSharing. Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. FairSharing means that ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort. FairSharing is a more heavy weight algorithm. The default is Classical.", + "description": "preemptionPolicy are the types of preemption Kueue allows. preemptionPolicy is an optional field. The allowed values are Classical, FairSharing and \"\". Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. FairSharing means that ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort. FairSharing is a more heavy weight algorithm. The default is Classical.", "type": "string", "default": "" } @@ -34099,7 +34103,7 @@ ], "properties": { "labelPolicy": { - "description": "labelPolicy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. labelPolicy is a required field. The allowed values are QueueName and None. None means that workloads will be suspended on creation and a label will be added via a mutating webhook. This will be applied for all integrations that Kueue manages. QueueName means that workloads that are managed by Kueue must have a label kueue.x-k8s.io/queue-name. If this label is not present on the workload, then Kueue will ignore this workload.", + "description": "labelPolicy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. labelPolicy is a required field. The allowed values are QueueName, None and \"\". None means that workloads will be suspended on creation and a label will be added via a mutating webhook. This will be applied for all integrations that Kueue manages. QueueName means that workloads that are managed by Kueue must have a label kueue.x-k8s.io/queue-name. If this label is not present on the workload, then Kueue will ignore this workload.", "type": "string", "default": "" } diff --git a/operator/v1alpha1/types_kueue.go b/operator/v1alpha1/types_kueue.go index e077bf349b1..02f49221563 100644 --- a/operator/v1alpha1/types_kueue.go +++ b/operator/v1alpha1/types_kueue.go @@ -52,24 +52,27 @@ type KueueConfiguration struct { // workloadManagement controls how Kueue manages workloads. // By default Kueue will manage workloads that have a queue-name label. // Workloads that are missing the queue-name will be ignored by Kueue. - // If tworkloadManagement is not specified, Kueue will only manage workloads - // that have the queue-name label. + // If workloadManagement is not specified, the operator will decide the + // default. + // This default could change over time. // This field is optional. // +optional - WorkloadManagement *WorkloadManagement `json:"workloadManagement"` + WorkloadManagement WorkloadManagement `json:"workloadManagement"` // gangScheduling controls how Kueue admits workloads. // Gang Scheduling is the act of all or nothing scheduling, // where workloads do not become ready within a certain period, they may be evicted and later retried. // This field is optional. - // If gangScheduling is not specified, gang scheduling will be disabled. + // If gangScheduling is not specified, the operator will decide the default. + // This default could change over time. // +optional - GangScheduling *GangScheduling `json:"gangScheduling"` + GangScheduling GangScheduling `json:"gangScheduling"` // preemption is the process of evicting one or more admitted Workloads to accommodate another Workload. // Kueue has classical premption and preemption via fair sharing. // preemption is optional. - // If preemption is not specified, preemption will be set to Classical. + // If preemption is not specified, the operator will decide the default. + // This default could change over time. // +optional - Preemption *Preemption `json:"preemption"` + Preemption Preemption `json:"preemption"` } // KueueStatus defines the observed state of Kueue @@ -123,10 +126,10 @@ type ExternalFramework struct { // Must be a valid DNS 1123 subdomain consisting of of lower-case alphanumeric characters, // hyphens and periods, of at most 253 characters in length. // Each period separated segment within the subdomain must start and end with an alphanumeric character. - // group uses matches and not cel functions to allow for use on 4.17. + // +group uses matches and not cel functions to allow for use on 4.17. // +kubebuilder:validation:MaxLength=253 // +kubebuilder:validation:MinLength=1 - // +kubebuilder:validation:XValidation:rule="self.matches(r'^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$') && self.size() <= 253" + // +kubebuilder:validation:XValidation:rule="self.matches(r'^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$')" // +required Group string `json:"group"` // resource is the Resource type of the external framework. @@ -134,20 +137,20 @@ type ExternalFramework struct { // Must be a valid DNS 1123 label consisting of a lower-case alphanumeric string // and hyphens of at most 63 characters in length. // The value must start and end with an alphanumeric character. - // resource uses matches and not cel functions to allow for use on 4.17. + // +resource uses matches and not cel functions to allow for use on 4.17. // +kubebuilder:validation:MaxLength=63 // +kubebuilder:validation:MinLength=1 - // +kubebuilder:validation:XValidation:rule="self.matches(r'^[a-z0-9]([-a-z0-9]*[a-z0-9])?$') && self.size() <= 63" + // +kubebuilder:validation:XValidation:rule="self.matches(r'^[a-z0-9]([-a-z0-9]*[a-z0-9])?$')" // +required Resource string `json:"resource"` // version is the version of the api (e.g. v1alpha1, v1beta1, v1). // Must be a valid DNS 1035 label consisting of a lower-case alphanumeric string // and hyphens of at most 63 characters in length. // The value must start with an alphabetic character and end with an alphanumeric character. - // version uses matches and not cel functions to allow for use on 4.17. + // +version uses matches and not cel functions to allow for use on 4.17. // +kubebuilder:validation:MaxLength=63 // +kubebuilder:validation:MinLength=1 - // +kubebuilder:validation:XValidation:rule="self.matches(r'^[a-z0-9]([-a-z0-9]*[a-z0-9])?$') && self.size() <= 63" + // +kubebuilder:validation:XValidation:rule="self.matches(r'^[a-z0-9]([-a-z0-9]*[a-z0-9])?$')" // +required Version string `json:"version"` } @@ -171,11 +174,9 @@ type Integrations struct { // that are managed for Kueue by external controllers. // externalFrameworks are optional and should only be used if you have an external controller // that integrates with Kueue. - // externalFrameworks, if specified, must have at least 1 item. // externalFrameworks, if specified, can not have more than 32 items. // +listType=map // +listMapKey=group - // +kubebuilder:validation:MinItems=1 // +kubebuilder:validation:MaxItems=32 // +optional ExternalFrameworks []ExternalFramework `json:"externalFrameworks"` @@ -183,9 +184,7 @@ type Integrations struct { // These keys are persisted to the internal Kueue workload object. // If not specified, only the Kueue labels will be copied. // labelKeysToCopy, if specified, is limited to a maximum of 64 items. - // labelKeysToCopy, if specified, must have at least one item. // +kubebuilder:validation:MaxItems=64 - // +kubebuilder:validation:MinItems=1 // +listType=map // +listMapKey=key // +optional @@ -202,14 +201,15 @@ type LabelKeys struct { // hyphens and periods, of at most 253 characters in length. // Each period separated segment within the subdomain must start and end with an alphanumeric character. // The optional prefix and the name are separate by a forward slash (/). + // +key uses matches and not cel functions to allow for use on 4.17. // +kubebuilder:validation:MaxLength=317 // +kubebuilder:validation:MinLength=1 - // +kubebuilder:validation:XValidation:rule="self.matches(r'^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?([a-z0-9]([-a-z0-9]*[a-z0-9])?)$') && self.size() <= 317" + // +kubebuilder:validation:XValidation:rule="self.matches(r'^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?([a-z0-9]([-a-z0-9]*[a-z0-9])?)$')" // +required Key string `json:"key"` } -// +kubebuilder:validation:Enum=ByWorkload;None +// +kubebuilder:validation:Enum=ByWorkload;None;"" type GangSchedulingPolicy string const ( @@ -217,7 +217,7 @@ const ( GangSchedulingPolicyNone GangSchedulingPolicy = "None" ) -// +kubebuilder:validation:Enum=Parallel;Sequential +// +kubebuilder:validation:Enum="";Parallel;Sequential type GangSchedulingWorkloadAdmission string const ( @@ -231,7 +231,7 @@ const ( // +union type GangScheduling struct { // policy allows you to enable and configure gang scheduling. - // The allowed values are ByWorkload and None. + // The allowed values are ByWorkload, None and "". // When set to ByWorkload, this means each workload is processed and considered // for admission as a single unit. // Where workloads do not become ready over time, the entire workload may then be evicted and retried at a later time. @@ -242,7 +242,7 @@ type GangScheduling struct { // byWorkload configures how Kueue will process workloads for admission. // byWorkload is required when policy is ByWorkload, and forbidden otherwise. // +optional - ByWorkload *ByWorkload `json:"byWorkload"` + ByWorkload ByWorkload `json:"byWorkload"` } // ByWorkload controls how admission is done @@ -250,7 +250,7 @@ type ByWorkload struct { // admission controls how Kueue will process workloads. // admission is a required field with policy is set to ByWorkload. // admission is only required if policy is specified to ByWorkload. - // Allowed values are Sequential and Parallel. + // Allowed values are Sequential, Parallel and "". // When admission is set to Sequential, only pods from the currently processing workload will be admitted. // Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. // Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, @@ -262,7 +262,7 @@ type ByWorkload struct { Admission GangSchedulingWorkloadAdmission `json:"admission"` } -// +kubebuilder:validation:Enum=QueueName;None +// +kubebuilder:validation:Enum="";QueueName;None type LabelPolicy string const ( @@ -274,7 +274,7 @@ type WorkloadManagement struct { // labelPolicy controls whether or not Kueue reconciles // jobs that don't set the label kueue.x-k8s.io/queue-name. // labelPolicy is a required field. - // The allowed values are QueueName and None. + // The allowed values are QueueName, None and "". // None means that workloads will be suspended on // creation and a label will be added via a mutating webhook. // This will be applied for all integrations that Kueue manages. @@ -286,7 +286,7 @@ type WorkloadManagement struct { LabelPolicy LabelPolicy `json:"labelPolicy"` } -// +kubebuilder:validation:Enum=Classical;FairSharing +// +kubebuilder:validation:Enum="";Classical;FairSharing type PreemptionPolicy string const ( @@ -297,7 +297,7 @@ const ( type Preemption struct { // preemptionPolicy are the types of preemption Kueue allows. // preemptionPolicy is an optional field. - // The allowed values are Classical and FairSharing. + // The allowed values are Classical, FairSharing and "". // Classical means that an incoming workload, which does // not fit within the unusued quota, is eligible to issue preemptions // when the requests of the workload are below the diff --git a/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml b/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml index 78c943d9d99..cea17b66639 100644 --- a/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml +++ b/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml @@ -54,7 +54,8 @@ spec: Gang Scheduling is the act of all or nothing scheduling, where workloads do not become ready within a certain period, they may be evicted and later retried. This field is optional. - If gangScheduling is not specified, gang scheduling will be disabled. + If gangScheduling is not specified, the operator will decide the default. + This default could change over time. properties: byWorkload: description: |- @@ -66,7 +67,7 @@ spec: admission controls how Kueue will process workloads. admission is a required field with policy is set to ByWorkload. admission is only required if policy is specified to ByWorkload. - Allowed values are Sequential and Parallel. + Allowed values are Sequential, Parallel and "". When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, @@ -75,6 +76,7 @@ spec: This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling. enum: + - "" - Parallel - Sequential type: string @@ -84,7 +86,7 @@ spec: policy: description: |- policy allows you to enable and configure gang scheduling. - The allowed values are ByWorkload and None. + The allowed values are ByWorkload, None and "". When set to ByWorkload, this means each workload is processed and considered for admission as a single unit. Where workloads do not become ready over time, the entire workload may then be evicted and retried at a later time. @@ -92,6 +94,7 @@ spec: enum: - ByWorkload - None + - "" type: string required: - policy @@ -114,7 +117,6 @@ spec: that are managed for Kueue by external controllers. externalFrameworks are optional and should only be used if you have an external controller that integrates with Kueue. - externalFrameworks, if specified, must have at least 1 item. externalFrameworks, if specified, can not have more than 32 items. items: description: |- @@ -128,13 +130,11 @@ spec: Must be a valid DNS 1123 subdomain consisting of of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. - group uses matches and not cel functions to allow for use on 4.17. maxLength: 253 minLength: 1 type: string x-kubernetes-validations: - rule: self.matches(r'^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$') - && self.size() <= 253 resource: description: |- resource is the Resource type of the external framework. @@ -142,33 +142,28 @@ spec: Must be a valid DNS 1123 label consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. The value must start and end with an alphanumeric character. - resource uses matches and not cel functions to allow for use on 4.17. maxLength: 63 minLength: 1 type: string x-kubernetes-validations: - rule: self.matches(r'^[a-z0-9]([-a-z0-9]*[a-z0-9])?$') - && self.size() <= 63 version: description: |- version is the version of the api (e.g. v1alpha1, v1beta1, v1). Must be a valid DNS 1035 label consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. The value must start with an alphabetic character and end with an alphanumeric character. - version uses matches and not cel functions to allow for use on 4.17. maxLength: 63 minLength: 1 type: string x-kubernetes-validations: - rule: self.matches(r'^[a-z0-9]([-a-z0-9]*[a-z0-9])?$') - && self.size() <= 63 required: - group - resource - version type: object maxItems: 32 - minItems: 1 type: array x-kubernetes-list-map-keys: - group @@ -210,7 +205,6 @@ spec: These keys are persisted to the internal Kueue workload object. If not specified, only the Kueue labels will be copied. labelKeysToCopy, if specified, is limited to a maximum of 64 items. - labelKeysToCopy, if specified, must have at least one item. items: properties: key: @@ -229,12 +223,10 @@ spec: type: string x-kubernetes-validations: - rule: self.matches(r'^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?([a-z0-9]([-a-z0-9]*[a-z0-9])?)$') - && self.size() <= 317 required: - key type: object maxItems: 64 - minItems: 1 type: array x-kubernetes-list-map-keys: - key @@ -247,13 +239,14 @@ spec: preemption is the process of evicting one or more admitted Workloads to accommodate another Workload. Kueue has classical premption and preemption via fair sharing. preemption is optional. - If preemption is not specified, preemption will be set to Classical. + If preemption is not specified, the operator will decide the default. + This default could change over time. properties: preemptionPolicy: description: |- preemptionPolicy are the types of preemption Kueue allows. preemptionPolicy is an optional field. - The allowed values are Classical and FairSharing. + The allowed values are Classical, FairSharing and "". Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the @@ -267,6 +260,7 @@ spec: FairSharing is a more heavy weight algorithm. The default is Classical. enum: + - "" - Classical - FairSharing type: string @@ -276,8 +270,9 @@ spec: workloadManagement controls how Kueue manages workloads. By default Kueue will manage workloads that have a queue-name label. Workloads that are missing the queue-name will be ignored by Kueue. - If tworkloadManagement is not specified, Kueue will only manage workloads - that have the queue-name label. + If workloadManagement is not specified, the operator will decide the + default. + This default could change over time. This field is optional. properties: labelPolicy: @@ -285,7 +280,7 @@ spec: labelPolicy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. labelPolicy is a required field. - The allowed values are QueueName and None. + The allowed values are QueueName, None and "". None means that workloads will be suspended on creation and a label will be added via a mutating webhook. This will be applied for all integrations that Kueue manages. @@ -294,6 +289,7 @@ spec: If this label is not present on the workload, then Kueue will ignore this workload. enum: + - "" - QueueName - None type: string diff --git a/operator/v1alpha1/zz_generated.deepcopy.go b/operator/v1alpha1/zz_generated.deepcopy.go index fd8b139769b..7037959ab0d 100644 --- a/operator/v1alpha1/zz_generated.deepcopy.go +++ b/operator/v1alpha1/zz_generated.deepcopy.go @@ -291,11 +291,7 @@ func (in *ExternalFramework) DeepCopy() *ExternalFramework { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GangScheduling) DeepCopyInto(out *GangScheduling) { *out = *in - if in.ByWorkload != nil { - in, out := &in.ByWorkload, &out.ByWorkload - *out = new(ByWorkload) - **out = **in - } + out.ByWorkload = in.ByWorkload return } @@ -500,21 +496,9 @@ func (in *Kueue) DeepCopyObject() runtime.Object { func (in *KueueConfiguration) DeepCopyInto(out *KueueConfiguration) { *out = *in in.Integrations.DeepCopyInto(&out.Integrations) - if in.WorkloadManagement != nil { - in, out := &in.WorkloadManagement, &out.WorkloadManagement - *out = new(WorkloadManagement) - **out = **in - } - if in.GangScheduling != nil { - in, out := &in.GangScheduling, &out.GangScheduling - *out = new(GangScheduling) - (*in).DeepCopyInto(*out) - } - if in.Preemption != nil { - in, out := &in.Preemption, &out.Preemption - *out = new(Preemption) - **out = **in - } + out.WorkloadManagement = in.WorkloadManagement + out.GangScheduling = in.GangScheduling + out.Preemption = in.Preemption return } diff --git a/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml b/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml index ef61498aee8..10f6e47d958 100644 --- a/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml +++ b/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml @@ -52,7 +52,8 @@ spec: Gang Scheduling is the act of all or nothing scheduling, where workloads do not become ready within a certain period, they may be evicted and later retried. This field is optional. - If gangScheduling is not specified, gang scheduling will be disabled. + If gangScheduling is not specified, the operator will decide the default. + This default could change over time. properties: byWorkload: description: |- @@ -64,7 +65,7 @@ spec: admission controls how Kueue will process workloads. admission is a required field with policy is set to ByWorkload. admission is only required if policy is specified to ByWorkload. - Allowed values are Sequential and Parallel. + Allowed values are Sequential, Parallel and "". When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, @@ -73,6 +74,7 @@ spec: This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling. enum: + - "" - Parallel - Sequential type: string @@ -82,7 +84,7 @@ spec: policy: description: |- policy allows you to enable and configure gang scheduling. - The allowed values are ByWorkload and None. + The allowed values are ByWorkload, None and "". When set to ByWorkload, this means each workload is processed and considered for admission as a single unit. Where workloads do not become ready over time, the entire workload may then be evicted and retried at a later time. @@ -90,6 +92,7 @@ spec: enum: - ByWorkload - None + - "" type: string required: - policy @@ -112,7 +115,6 @@ spec: that are managed for Kueue by external controllers. externalFrameworks are optional and should only be used if you have an external controller that integrates with Kueue. - externalFrameworks, if specified, must have at least 1 item. externalFrameworks, if specified, can not have more than 32 items. items: description: |- @@ -126,13 +128,11 @@ spec: Must be a valid DNS 1123 subdomain consisting of of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. - group uses matches and not cel functions to allow for use on 4.17. maxLength: 253 minLength: 1 type: string x-kubernetes-validations: - rule: self.matches(r'^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$') - && self.size() <= 253 resource: description: |- resource is the Resource type of the external framework. @@ -140,33 +140,28 @@ spec: Must be a valid DNS 1123 label consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. The value must start and end with an alphanumeric character. - resource uses matches and not cel functions to allow for use on 4.17. maxLength: 63 minLength: 1 type: string x-kubernetes-validations: - rule: self.matches(r'^[a-z0-9]([-a-z0-9]*[a-z0-9])?$') - && self.size() <= 63 version: description: |- version is the version of the api (e.g. v1alpha1, v1beta1, v1). Must be a valid DNS 1035 label consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. The value must start with an alphabetic character and end with an alphanumeric character. - version uses matches and not cel functions to allow for use on 4.17. maxLength: 63 minLength: 1 type: string x-kubernetes-validations: - rule: self.matches(r'^[a-z0-9]([-a-z0-9]*[a-z0-9])?$') - && self.size() <= 63 required: - group - resource - version type: object maxItems: 32 - minItems: 1 type: array x-kubernetes-list-map-keys: - group @@ -208,7 +203,6 @@ spec: These keys are persisted to the internal Kueue workload object. If not specified, only the Kueue labels will be copied. labelKeysToCopy, if specified, is limited to a maximum of 64 items. - labelKeysToCopy, if specified, must have at least one item. items: properties: key: @@ -227,12 +221,10 @@ spec: type: string x-kubernetes-validations: - rule: self.matches(r'^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?([a-z0-9]([-a-z0-9]*[a-z0-9])?)$') - && self.size() <= 317 required: - key type: object maxItems: 64 - minItems: 1 type: array x-kubernetes-list-map-keys: - key @@ -245,13 +237,14 @@ spec: preemption is the process of evicting one or more admitted Workloads to accommodate another Workload. Kueue has classical premption and preemption via fair sharing. preemption is optional. - If preemption is not specified, preemption will be set to Classical. + If preemption is not specified, the operator will decide the default. + This default could change over time. properties: preemptionPolicy: description: |- preemptionPolicy are the types of preemption Kueue allows. preemptionPolicy is an optional field. - The allowed values are Classical and FairSharing. + The allowed values are Classical, FairSharing and "". Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the @@ -265,6 +258,7 @@ spec: FairSharing is a more heavy weight algorithm. The default is Classical. enum: + - "" - Classical - FairSharing type: string @@ -274,8 +268,9 @@ spec: workloadManagement controls how Kueue manages workloads. By default Kueue will manage workloads that have a queue-name label. Workloads that are missing the queue-name will be ignored by Kueue. - If tworkloadManagement is not specified, Kueue will only manage workloads - that have the queue-name label. + If workloadManagement is not specified, the operator will decide the + default. + This default could change over time. This field is optional. properties: labelPolicy: @@ -283,7 +278,7 @@ spec: labelPolicy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. labelPolicy is a required field. - The allowed values are QueueName and None. + The allowed values are QueueName, None and "". None means that workloads will be suspended on creation and a label will be added via a mutating webhook. This will be applied for all integrations that Kueue manages. @@ -292,6 +287,7 @@ spec: If this label is not present on the workload, then Kueue will ignore this workload. enum: + - "" - QueueName - None type: string diff --git a/operator/v1alpha1/zz_generated.swagger_doc_generated.go b/operator/v1alpha1/zz_generated.swagger_doc_generated.go index 3d2f0ba5c53..f2b8a42c3f0 100644 --- a/operator/v1alpha1/zz_generated.swagger_doc_generated.go +++ b/operator/v1alpha1/zz_generated.swagger_doc_generated.go @@ -259,7 +259,7 @@ func (RepositoryDigestMirrors) SwaggerDoc() map[string]string { var map_ByWorkload = map[string]string{ "": "ByWorkload controls how admission is done", - "admission": "admission controls how Kueue will process workloads. admission is a required field with policy is set to ByWorkload. admission is only required if policy is specified to ByWorkload. Allowed values are Sequential and Parallel. When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling.", + "admission": "admission controls how Kueue will process workloads. admission is a required field with policy is set to ByWorkload. admission is only required if policy is specified to ByWorkload. Allowed values are Sequential, Parallel and \"\". When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling.", } func (ByWorkload) SwaggerDoc() map[string]string { @@ -268,9 +268,9 @@ func (ByWorkload) SwaggerDoc() map[string]string { var map_ExternalFramework = map[string]string{ "": "This is the GVR for an external framework. Controller runtime requires this in this format for api discoverability.", - "group": "group is the API group of the externalFramework. Must be a valid DNS 1123 subdomain consisting of of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character. group uses matches and not cel functions to allow for use on 4.17.", - "resource": "resource is the Resource type of the external framework. Resource types are lowercase and plural (e.g. pods, deployments). Must be a valid DNS 1123 label consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. The value must start and end with an alphanumeric character. resource uses matches and not cel functions to allow for use on 4.17.", - "version": "version is the version of the api (e.g. v1alpha1, v1beta1, v1). Must be a valid DNS 1035 label consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. The value must start with an alphabetic character and end with an alphanumeric character. version uses matches and not cel functions to allow for use on 4.17.", + "group": "group is the API group of the externalFramework. Must be a valid DNS 1123 subdomain consisting of of lower-case alphanumeric characters, hyphens and periods, of at most 253 characters in length. Each period separated segment within the subdomain must start and end with an alphanumeric character.", + "resource": "resource is the Resource type of the external framework. Resource types are lowercase and plural (e.g. pods, deployments). Must be a valid DNS 1123 label consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. The value must start and end with an alphanumeric character.", + "version": "version is the version of the api (e.g. v1alpha1, v1beta1, v1). Must be a valid DNS 1035 label consisting of a lower-case alphanumeric string and hyphens of at most 63 characters in length. The value must start with an alphabetic character and end with an alphanumeric character.", } func (ExternalFramework) SwaggerDoc() map[string]string { @@ -279,7 +279,7 @@ func (ExternalFramework) SwaggerDoc() map[string]string { var map_GangScheduling = map[string]string{ "": "Kueue provides the ability to admit workloads all in one (gang admission) and evicts workloads if they are not ready within a specific time.", - "policy": "policy allows you to enable and configure gang scheduling. The allowed values are ByWorkload and None. When set to ByWorkload, this means each workload is processed and considered for admission as a single unit. Where workloads do not become ready over time, the entire workload may then be evicted and retried at a later time. policy is a required field.", + "policy": "policy allows you to enable and configure gang scheduling. The allowed values are ByWorkload, None and \"\". When set to ByWorkload, this means each workload is processed and considered for admission as a single unit. Where workloads do not become ready over time, the entire workload may then be evicted and retried at a later time. policy is a required field.", "byWorkload": "byWorkload configures how Kueue will process workloads for admission. byWorkload is required when policy is ByWorkload, and forbidden otherwise.", } @@ -290,8 +290,8 @@ func (GangScheduling) SwaggerDoc() map[string]string { var map_Integrations = map[string]string{ "": "This is the integrations for Kueue. Kueue uses these apis to determine which jobs will be managed by Kueue.", "frameworks": "frameworks are a list of frameworks that Kueue has support for. The allowed values are BatchJob, RayJob, RayCluster, JobSet, MPIJob, PaddleJob, PyTorchJob, TFJob, XGBoostJob, AppWrapper, Pod, Deployment, StatefulSet and LeaderWorkerSet. frameworks are required and must have at least one element. frameworks can not have more than 14 elements. Each framework represents a type of job that Kueue will manage.", - "externalFrameworks": "externalFrameworks are a list of GroupVersionResources that are managed for Kueue by external controllers. externalFrameworks are optional and should only be used if you have an external controller that integrates with Kueue. externalFrameworks, if specified, must have at least 1 item. externalFrameworks, if specified, can not have more than 32 items.", - "labelKeysToCopy": "labelKeysToCopy are a list of label keys that are copied once a workload is created. These keys are persisted to the internal Kueue workload object. If not specified, only the Kueue labels will be copied. labelKeysToCopy, if specified, is limited to a maximum of 64 items. labelKeysToCopy, if specified, must have at least one item.", + "externalFrameworks": "externalFrameworks are a list of GroupVersionResources that are managed for Kueue by external controllers. externalFrameworks are optional and should only be used if you have an external controller that integrates with Kueue. externalFrameworks, if specified, can not have more than 32 items.", + "labelKeysToCopy": "labelKeysToCopy are a list of label keys that are copied once a workload is created. These keys are persisted to the internal Kueue workload object. If not specified, only the Kueue labels will be copied. labelKeysToCopy, if specified, is limited to a maximum of 64 items.", } func (Integrations) SwaggerDoc() map[string]string { @@ -311,9 +311,9 @@ func (Kueue) SwaggerDoc() map[string]string { var map_KueueConfiguration = map[string]string{ "integrations": "integrations is a required field that configures the Kueue's workload integrations. Kueue has both standard integrations, known as job frameworks, and external integrations known as external frameworks. Kueue will only manage workloads that correspond to the specified integrations.", - "workloadManagement": "workloadManagement controls how Kueue manages workloads. By default Kueue will manage workloads that have a queue-name label. Workloads that are missing the queue-name will be ignored by Kueue. If tworkloadManagement is not specified, Kueue will only manage workloads that have the queue-name label. This field is optional.", - "gangScheduling": "gangScheduling controls how Kueue admits workloads. Gang Scheduling is the act of all or nothing scheduling, where workloads do not become ready within a certain period, they may be evicted and later retried. This field is optional. If gangScheduling is not specified, gang scheduling will be disabled.", - "preemption": "preemption is the process of evicting one or more admitted Workloads to accommodate another Workload. Kueue has classical premption and preemption via fair sharing. preemption is optional. If preemption is not specified, preemption will be set to Classical.", + "workloadManagement": "workloadManagement controls how Kueue manages workloads. By default Kueue will manage workloads that have a queue-name label. Workloads that are missing the queue-name will be ignored by Kueue. If workloadManagement is not specified, the operator will decide the default. This default could change over time. This field is optional.", + "gangScheduling": "gangScheduling controls how Kueue admits workloads. Gang Scheduling is the act of all or nothing scheduling, where workloads do not become ready within a certain period, they may be evicted and later retried. This field is optional. If gangScheduling is not specified, the operator will decide the default. This default could change over time.", + "preemption": "preemption is the process of evicting one or more admitted Workloads to accommodate another Workload. Kueue has classical premption and preemption via fair sharing. preemption is optional. If preemption is not specified, the operator will decide the default. This default could change over time.", } func (KueueConfiguration) SwaggerDoc() map[string]string { @@ -355,7 +355,7 @@ func (LabelKeys) SwaggerDoc() map[string]string { } var map_Preemption = map[string]string{ - "preemptionPolicy": "preemptionPolicy are the types of preemption Kueue allows. preemptionPolicy is an optional field. The allowed values are Classical and FairSharing. Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. FairSharing means that ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort. FairSharing is a more heavy weight algorithm. The default is Classical.", + "preemptionPolicy": "preemptionPolicy are the types of preemption Kueue allows. preemptionPolicy is an optional field. The allowed values are Classical, FairSharing and \"\". Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. FairSharing means that ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort. FairSharing is a more heavy weight algorithm. The default is Classical.", } func (Preemption) SwaggerDoc() map[string]string { @@ -363,7 +363,7 @@ func (Preemption) SwaggerDoc() map[string]string { } var map_WorkloadManagement = map[string]string{ - "labelPolicy": "labelPolicy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. labelPolicy is a required field. The allowed values are QueueName and None. None means that workloads will be suspended on creation and a label will be added via a mutating webhook. This will be applied for all integrations that Kueue manages. QueueName means that workloads that are managed by Kueue must have a label kueue.x-k8s.io/queue-name. If this label is not present on the workload, then Kueue will ignore this workload.", + "labelPolicy": "labelPolicy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. labelPolicy is a required field. The allowed values are QueueName, None and \"\". None means that workloads will be suspended on creation and a label will be added via a mutating webhook. This will be applied for all integrations that Kueue manages. QueueName means that workloads that are managed by Kueue must have a label kueue.x-k8s.io/queue-name. If this label is not present on the workload, then Kueue will ignore this workload.", } func (WorkloadManagement) SwaggerDoc() map[string]string { From 9ca52b825ec18672d5aee38ba86c53f4ab1bac15 Mon Sep 17 00:00:00 2001 From: Kevin Hannon Date: Mon, 14 Apr 2025 13:39:40 -0400 Subject: [PATCH 40/42] address feedback --- openapi/generated_openapi/zz_generated.openapi.go | 10 +++++----- openapi/openapi.json | 12 +++++++----- operator/v1alpha1/types_kueue.go | 15 ++++++++++----- .../zz_generated.crd-manifests/kueue.crd.yaml | 10 +++++++--- operator/v1alpha1/zz_generated.deepcopy.go | 8 ++++++-- .../kueue.operator.openshift.io/AAA_ungated.yaml | 10 +++++++--- .../zz_generated.swagger_doc_generated.go | 6 +++--- 7 files changed, 45 insertions(+), 26 deletions(-) diff --git a/openapi/generated_openapi/zz_generated.openapi.go b/openapi/generated_openapi/zz_generated.openapi.go index e1573a044b9..9607e4c9de3 100644 --- a/openapi/generated_openapi/zz_generated.openapi.go +++ b/openapi/generated_openapi/zz_generated.openapi.go @@ -56515,7 +56515,7 @@ func schema_openshift_api_operator_v1alpha1_ByWorkload(ref common.ReferenceCallb Properties: map[string]spec.Schema{ "admission": { SchemaProps: spec.SchemaProps{ - Description: "admission controls how Kueue will process workloads. admission is a required field with policy is set to ByWorkload. admission is only required if policy is specified to ByWorkload. Allowed values are Sequential, Parallel and \"\". When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling.", + Description: "admission controls how Kueue will process workloads. admission is only required if policy is specified to ByWorkload. Allowed values are Sequential, Parallel and \"\". When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling.", Default: "", Type: []string{"string"}, Format: "", @@ -56929,7 +56929,6 @@ func schema_openshift_api_operator_v1alpha1_GangScheduling(ref common.ReferenceC "byWorkload": { SchemaProps: spec.SchemaProps{ Description: "byWorkload configures how Kueue will process workloads for admission. byWorkload is required when policy is ByWorkload, and forbidden otherwise.", - Default: map[string]interface{}{}, Ref: ref("github.com/openshift/api/operator/v1alpha1.ByWorkload"), }, }, @@ -57198,7 +57197,7 @@ func schema_openshift_api_operator_v1alpha1_Integrations(ref common.ReferenceCal "frameworks": { VendorExtensible: spec.VendorExtensible{ Extensions: spec.Extensions{ - "x-kubernetes-list-type": "atomic", + "x-kubernetes-list-type": "set", }, }, SchemaProps: spec.SchemaProps{ @@ -58087,13 +58086,14 @@ func schema_openshift_api_operator_v1alpha1_Preemption(ref common.ReferenceCallb Properties: map[string]spec.Schema{ "preemptionPolicy": { SchemaProps: spec.SchemaProps{ - Description: "preemptionPolicy are the types of preemption Kueue allows. preemptionPolicy is an optional field. The allowed values are Classical, FairSharing and \"\". Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. FairSharing means that ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort. FairSharing is a more heavy weight algorithm. The default is Classical.", + Description: "preemptionPolicy are the types of preemption Kueue allows. preemptionPolicy is an optional field. The allowed values are Classical, FairSharing and \"\". Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. FairSharing means that ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort. FairSharing is a more heavy weight algorithm. \"\" is our default setting which means that the operator will decide the default.", Default: "", Type: []string{"string"}, Format: "", }, }, }, + Required: []string{"preemptionPolicy"}, }, }, } @@ -58298,7 +58298,7 @@ func schema_openshift_api_operator_v1alpha1_WorkloadManagement(ref common.Refere Properties: map[string]spec.Schema{ "labelPolicy": { SchemaProps: spec.SchemaProps{ - Description: "labelPolicy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. labelPolicy is a required field. The allowed values are QueueName, None and \"\". None means that workloads will be suspended on creation and a label will be added via a mutating webhook. This will be applied for all integrations that Kueue manages. QueueName means that workloads that are managed by Kueue must have a label kueue.x-k8s.io/queue-name. If this label is not present on the workload, then Kueue will ignore this workload.", + Description: "labelPolicy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. labelPolicy is a required field. The allowed values are QueueName, None and \"\". None means that workloads will be suspended on creation and a label will be added via a mutating webhook. This will be applied for all integrations that Kueue manages. QueueName means that workloads that are managed by Kueue must have a label kueue.x-k8s.io/queue-name. \"\" is our default setting which the operator will decide the default. If this label is not present on the workload, then Kueue will ignore this workload.", Default: "", Type: []string{"string"}, Format: "", diff --git a/openapi/openapi.json b/openapi/openapi.json index 27b9d8c4073..21657a719c6 100644 --- a/openapi/openapi.json +++ b/openapi/openapi.json @@ -33017,7 +33017,7 @@ ], "properties": { "admission": { - "description": "admission controls how Kueue will process workloads. admission is a required field with policy is set to ByWorkload. admission is only required if policy is specified to ByWorkload. Allowed values are Sequential, Parallel and \"\". When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling.", + "description": "admission controls how Kueue will process workloads. admission is only required if policy is specified to ByWorkload. Allowed values are Sequential, Parallel and \"\". When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling.", "type": "string", "default": "" } @@ -33255,7 +33255,6 @@ "properties": { "byWorkload": { "description": "byWorkload configures how Kueue will process workloads for admission. byWorkload is required when policy is ByWorkload, and forbidden otherwise.", - "default": {}, "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.ByWorkload" }, "policy": { @@ -33443,7 +33442,7 @@ "type": "string", "default": "" }, - "x-kubernetes-list-type": "atomic" + "x-kubernetes-list-type": "set" }, "labelKeysToCopy": { "description": "labelKeysToCopy are a list of label keys that are copied once a workload is created. These keys are persisted to the internal Kueue workload object. If not specified, only the Kueue labels will be copied. labelKeysToCopy, if specified, is limited to a maximum of 64 items.", @@ -33967,9 +33966,12 @@ }, "com.github.openshift.api.operator.v1alpha1.Preemption": { "type": "object", + "required": [ + "preemptionPolicy" + ], "properties": { "preemptionPolicy": { - "description": "preemptionPolicy are the types of preemption Kueue allows. preemptionPolicy is an optional field. The allowed values are Classical, FairSharing and \"\". Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. FairSharing means that ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort. FairSharing is a more heavy weight algorithm. The default is Classical.", + "description": "preemptionPolicy are the types of preemption Kueue allows. preemptionPolicy is an optional field. The allowed values are Classical, FairSharing and \"\". Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. FairSharing means that ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort. FairSharing is a more heavy weight algorithm. \"\" is our default setting which means that the operator will decide the default.", "type": "string", "default": "" } @@ -34103,7 +34105,7 @@ ], "properties": { "labelPolicy": { - "description": "labelPolicy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. labelPolicy is a required field. The allowed values are QueueName, None and \"\". None means that workloads will be suspended on creation and a label will be added via a mutating webhook. This will be applied for all integrations that Kueue manages. QueueName means that workloads that are managed by Kueue must have a label kueue.x-k8s.io/queue-name. If this label is not present on the workload, then Kueue will ignore this workload.", + "description": "labelPolicy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. labelPolicy is a required field. The allowed values are QueueName, None and \"\". None means that workloads will be suspended on creation and a label will be added via a mutating webhook. This will be applied for all integrations that Kueue manages. QueueName means that workloads that are managed by Kueue must have a label kueue.x-k8s.io/queue-name. \"\" is our default setting which the operator will decide the default. If this label is not present on the workload, then Kueue will ignore this workload.", "type": "string", "default": "" } diff --git a/operator/v1alpha1/types_kueue.go b/operator/v1alpha1/types_kueue.go index 02f49221563..ab66526ece3 100644 --- a/operator/v1alpha1/types_kueue.go +++ b/operator/v1alpha1/types_kueue.go @@ -167,7 +167,7 @@ type Integrations struct { // +kubebuilder:validation:MaxItems=14 // +kubebuilder:validation:MinItems=1 // +kubebuilder:validation:XValidation:rule="self.all(x, self.exists_one(y, x == y))",message="each item in frameworks must be unique" - // +listType=atomic + // +listType=set // +required Frameworks []KueueIntegration `json:"frameworks"` // externalFrameworks are a list of GroupVersionResources @@ -235,6 +235,8 @@ type GangScheduling struct { // When set to ByWorkload, this means each workload is processed and considered // for admission as a single unit. // Where workloads do not become ready over time, the entire workload may then be evicted and retried at a later time. + // None means gang scheduling is disabled. + // "" means the the operator will decide the default. // policy is a required field. // +required // +unionDiscriminator @@ -242,13 +244,12 @@ type GangScheduling struct { // byWorkload configures how Kueue will process workloads for admission. // byWorkload is required when policy is ByWorkload, and forbidden otherwise. // +optional - ByWorkload ByWorkload `json:"byWorkload"` + ByWorkload *ByWorkload `json:"byWorkload,omitempty"` } // ByWorkload controls how admission is done type ByWorkload struct { // admission controls how Kueue will process workloads. - // admission is a required field with policy is set to ByWorkload. // admission is only required if policy is specified to ByWorkload. // Allowed values are Sequential, Parallel and "". // When admission is set to Sequential, only pods from the currently processing workload will be admitted. @@ -258,6 +259,7 @@ type ByWorkload struct { // When set to Parallel, pods from any workload will be admitted at any time. // This may lead to a deadlock where workloads are in contention for cluster capacity and // pods from another workload having successfully scheduled prevent pods from the current workload scheduling. + // When set to "", the operator will decide // +required Admission GangSchedulingWorkloadAdmission `json:"admission"` } @@ -280,6 +282,8 @@ type WorkloadManagement struct { // This will be applied for all integrations that Kueue manages. // QueueName means that workloads that are managed // by Kueue must have a label kueue.x-k8s.io/queue-name. + // "" is our default setting which the operator will + // decide the default. // If this label is not present on the workload, then Kueue will // ignore this workload. // +required @@ -309,7 +313,8 @@ type Preemption struct { // The borrowable resources are the unused nominal quota // of all the ClusterQueues in the cohort. // FairSharing is a more heavy weight algorithm. - // The default is Classical. - // +optional + // "" is our default setting which means that the operator will + // decide the default. + // +required PreemptionPolicy PreemptionPolicy `json:"preemptionPolicy"` } diff --git a/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml b/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml index cea17b66639..827e115eeb6 100644 --- a/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml +++ b/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml @@ -65,7 +65,6 @@ spec: admission: description: |- admission controls how Kueue will process workloads. - admission is a required field with policy is set to ByWorkload. admission is only required if policy is specified to ByWorkload. Allowed values are Sequential, Parallel and "". When admission is set to Sequential, only pods from the currently processing workload will be admitted. @@ -195,7 +194,7 @@ spec: maxItems: 14 minItems: 1 type: array - x-kubernetes-list-type: atomic + x-kubernetes-list-type: set x-kubernetes-validations: - message: each item in frameworks must be unique rule: self.all(x, self.exists_one(y, x == y)) @@ -258,12 +257,15 @@ spec: The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort. FairSharing is a more heavy weight algorithm. - The default is Classical. + "" is our default setting which means that the operator will + decide the default. enum: - "" - Classical - FairSharing type: string + required: + - preemptionPolicy type: object workloadManagement: description: |- @@ -286,6 +288,8 @@ spec: This will be applied for all integrations that Kueue manages. QueueName means that workloads that are managed by Kueue must have a label kueue.x-k8s.io/queue-name. + "" is our default setting which the operator will + decide the default. If this label is not present on the workload, then Kueue will ignore this workload. enum: diff --git a/operator/v1alpha1/zz_generated.deepcopy.go b/operator/v1alpha1/zz_generated.deepcopy.go index 7037959ab0d..6681c8217d0 100644 --- a/operator/v1alpha1/zz_generated.deepcopy.go +++ b/operator/v1alpha1/zz_generated.deepcopy.go @@ -291,7 +291,11 @@ func (in *ExternalFramework) DeepCopy() *ExternalFramework { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GangScheduling) DeepCopyInto(out *GangScheduling) { *out = *in - out.ByWorkload = in.ByWorkload + if in.ByWorkload != nil { + in, out := &in.ByWorkload, &out.ByWorkload + *out = new(ByWorkload) + **out = **in + } return } @@ -497,7 +501,7 @@ func (in *KueueConfiguration) DeepCopyInto(out *KueueConfiguration) { *out = *in in.Integrations.DeepCopyInto(&out.Integrations) out.WorkloadManagement = in.WorkloadManagement - out.GangScheduling = in.GangScheduling + in.GangScheduling.DeepCopyInto(&out.GangScheduling) out.Preemption = in.Preemption return } diff --git a/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml b/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml index 10f6e47d958..034a6fbd167 100644 --- a/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml +++ b/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml @@ -63,7 +63,6 @@ spec: admission: description: |- admission controls how Kueue will process workloads. - admission is a required field with policy is set to ByWorkload. admission is only required if policy is specified to ByWorkload. Allowed values are Sequential, Parallel and "". When admission is set to Sequential, only pods from the currently processing workload will be admitted. @@ -193,7 +192,7 @@ spec: maxItems: 14 minItems: 1 type: array - x-kubernetes-list-type: atomic + x-kubernetes-list-type: set x-kubernetes-validations: - message: each item in frameworks must be unique rule: self.all(x, self.exists_one(y, x == y)) @@ -256,12 +255,15 @@ spec: The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort. FairSharing is a more heavy weight algorithm. - The default is Classical. + "" is our default setting which means that the operator will + decide the default. enum: - "" - Classical - FairSharing type: string + required: + - preemptionPolicy type: object workloadManagement: description: |- @@ -284,6 +286,8 @@ spec: This will be applied for all integrations that Kueue manages. QueueName means that workloads that are managed by Kueue must have a label kueue.x-k8s.io/queue-name. + "" is our default setting which the operator will + decide the default. If this label is not present on the workload, then Kueue will ignore this workload. enum: diff --git a/operator/v1alpha1/zz_generated.swagger_doc_generated.go b/operator/v1alpha1/zz_generated.swagger_doc_generated.go index f2b8a42c3f0..efcd13df61d 100644 --- a/operator/v1alpha1/zz_generated.swagger_doc_generated.go +++ b/operator/v1alpha1/zz_generated.swagger_doc_generated.go @@ -259,7 +259,7 @@ func (RepositoryDigestMirrors) SwaggerDoc() map[string]string { var map_ByWorkload = map[string]string{ "": "ByWorkload controls how admission is done", - "admission": "admission controls how Kueue will process workloads. admission is a required field with policy is set to ByWorkload. admission is only required if policy is specified to ByWorkload. Allowed values are Sequential, Parallel and \"\". When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling.", + "admission": "admission controls how Kueue will process workloads. admission is only required if policy is specified to ByWorkload. Allowed values are Sequential, Parallel and \"\". When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling.", } func (ByWorkload) SwaggerDoc() map[string]string { @@ -355,7 +355,7 @@ func (LabelKeys) SwaggerDoc() map[string]string { } var map_Preemption = map[string]string{ - "preemptionPolicy": "preemptionPolicy are the types of preemption Kueue allows. preemptionPolicy is an optional field. The allowed values are Classical, FairSharing and \"\". Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. FairSharing means that ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort. FairSharing is a more heavy weight algorithm. The default is Classical.", + "preemptionPolicy": "preemptionPolicy are the types of preemption Kueue allows. preemptionPolicy is an optional field. The allowed values are Classical, FairSharing and \"\". Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. FairSharing means that ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort. FairSharing is a more heavy weight algorithm. \"\" is our default setting which means that the operator will decide the default.", } func (Preemption) SwaggerDoc() map[string]string { @@ -363,7 +363,7 @@ func (Preemption) SwaggerDoc() map[string]string { } var map_WorkloadManagement = map[string]string{ - "labelPolicy": "labelPolicy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. labelPolicy is a required field. The allowed values are QueueName, None and \"\". None means that workloads will be suspended on creation and a label will be added via a mutating webhook. This will be applied for all integrations that Kueue manages. QueueName means that workloads that are managed by Kueue must have a label kueue.x-k8s.io/queue-name. If this label is not present on the workload, then Kueue will ignore this workload.", + "labelPolicy": "labelPolicy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. labelPolicy is a required field. The allowed values are QueueName, None and \"\". None means that workloads will be suspended on creation and a label will be added via a mutating webhook. This will be applied for all integrations that Kueue manages. QueueName means that workloads that are managed by Kueue must have a label kueue.x-k8s.io/queue-name. \"\" is our default setting which the operator will decide the default. If this label is not present on the workload, then Kueue will ignore this workload.", } func (WorkloadManagement) SwaggerDoc() map[string]string { From 405ba5e18ff2f4bdbf54d36b1055b899b9a00670 Mon Sep 17 00:00:00 2001 From: Kevin Hannon Date: Tue, 15 Apr 2025 11:05:52 -0400 Subject: [PATCH 41/42] update with latest godocs --- openapi/generated_openapi/zz_generated.openapi.go | 4 ++-- openapi/openapi.json | 4 ++-- operator/v1alpha1/types_kueue.go | 2 +- operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml | 5 ++++- .../kueue.operator.openshift.io/AAA_ungated.yaml | 5 ++++- operator/v1alpha1/zz_generated.swagger_doc_generated.go | 4 ++-- 6 files changed, 15 insertions(+), 9 deletions(-) diff --git a/openapi/generated_openapi/zz_generated.openapi.go b/openapi/generated_openapi/zz_generated.openapi.go index 9607e4c9de3..9c559701241 100644 --- a/openapi/generated_openapi/zz_generated.openapi.go +++ b/openapi/generated_openapi/zz_generated.openapi.go @@ -56515,7 +56515,7 @@ func schema_openshift_api_operator_v1alpha1_ByWorkload(ref common.ReferenceCallb Properties: map[string]spec.Schema{ "admission": { SchemaProps: spec.SchemaProps{ - Description: "admission controls how Kueue will process workloads. admission is only required if policy is specified to ByWorkload. Allowed values are Sequential, Parallel and \"\". When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling.", + Description: "admission controls how Kueue will process workloads. admission is required. Allowed values are Sequential, Parallel and \"\". When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling. When set to \"\", the operator will decide", Default: "", Type: []string{"string"}, Format: "", @@ -56920,7 +56920,7 @@ func schema_openshift_api_operator_v1alpha1_GangScheduling(ref common.ReferenceC Properties: map[string]spec.Schema{ "policy": { SchemaProps: spec.SchemaProps{ - Description: "policy allows you to enable and configure gang scheduling. The allowed values are ByWorkload, None and \"\". When set to ByWorkload, this means each workload is processed and considered for admission as a single unit. Where workloads do not become ready over time, the entire workload may then be evicted and retried at a later time. policy is a required field.", + Description: "policy allows you to enable and configure gang scheduling. The allowed values are ByWorkload, None and \"\". When set to ByWorkload, this means each workload is processed and considered for admission as a single unit. Where workloads do not become ready over time, the entire workload may then be evicted and retried at a later time. None means gang scheduling is disabled. \"\" means the the operator will decide the default. policy is a required field.", Default: "", Type: []string{"string"}, Format: "", diff --git a/openapi/openapi.json b/openapi/openapi.json index 21657a719c6..7c632c186e1 100644 --- a/openapi/openapi.json +++ b/openapi/openapi.json @@ -33017,7 +33017,7 @@ ], "properties": { "admission": { - "description": "admission controls how Kueue will process workloads. admission is only required if policy is specified to ByWorkload. Allowed values are Sequential, Parallel and \"\". When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling.", + "description": "admission controls how Kueue will process workloads. admission is required. Allowed values are Sequential, Parallel and \"\". When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling. When set to \"\", the operator will decide", "type": "string", "default": "" } @@ -33258,7 +33258,7 @@ "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.ByWorkload" }, "policy": { - "description": "policy allows you to enable and configure gang scheduling. The allowed values are ByWorkload, None and \"\". When set to ByWorkload, this means each workload is processed and considered for admission as a single unit. Where workloads do not become ready over time, the entire workload may then be evicted and retried at a later time. policy is a required field.", + "description": "policy allows you to enable and configure gang scheduling. The allowed values are ByWorkload, None and \"\". When set to ByWorkload, this means each workload is processed and considered for admission as a single unit. Where workloads do not become ready over time, the entire workload may then be evicted and retried at a later time. None means gang scheduling is disabled. \"\" means the the operator will decide the default. policy is a required field.", "type": "string", "default": "" } diff --git a/operator/v1alpha1/types_kueue.go b/operator/v1alpha1/types_kueue.go index ab66526ece3..8c2d921eba5 100644 --- a/operator/v1alpha1/types_kueue.go +++ b/operator/v1alpha1/types_kueue.go @@ -250,7 +250,7 @@ type GangScheduling struct { // ByWorkload controls how admission is done type ByWorkload struct { // admission controls how Kueue will process workloads. - // admission is only required if policy is specified to ByWorkload. + // admission is required. // Allowed values are Sequential, Parallel and "". // When admission is set to Sequential, only pods from the currently processing workload will be admitted. // Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. diff --git a/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml b/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml index 827e115eeb6..21b8c8676f7 100644 --- a/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml +++ b/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml @@ -65,7 +65,7 @@ spec: admission: description: |- admission controls how Kueue will process workloads. - admission is only required if policy is specified to ByWorkload. + admission is required. Allowed values are Sequential, Parallel and "". When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. @@ -74,6 +74,7 @@ spec: When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling. + When set to "", the operator will decide enum: - "" - Parallel @@ -89,6 +90,8 @@ spec: When set to ByWorkload, this means each workload is processed and considered for admission as a single unit. Where workloads do not become ready over time, the entire workload may then be evicted and retried at a later time. + None means gang scheduling is disabled. + "" means the the operator will decide the default. policy is a required field. enum: - ByWorkload diff --git a/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml b/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml index 034a6fbd167..edcfe51bee8 100644 --- a/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml +++ b/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml @@ -63,7 +63,7 @@ spec: admission: description: |- admission controls how Kueue will process workloads. - admission is only required if policy is specified to ByWorkload. + admission is required. Allowed values are Sequential, Parallel and "". When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. @@ -72,6 +72,7 @@ spec: When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling. + When set to "", the operator will decide enum: - "" - Parallel @@ -87,6 +88,8 @@ spec: When set to ByWorkload, this means each workload is processed and considered for admission as a single unit. Where workloads do not become ready over time, the entire workload may then be evicted and retried at a later time. + None means gang scheduling is disabled. + "" means the the operator will decide the default. policy is a required field. enum: - ByWorkload diff --git a/operator/v1alpha1/zz_generated.swagger_doc_generated.go b/operator/v1alpha1/zz_generated.swagger_doc_generated.go index efcd13df61d..210059bf934 100644 --- a/operator/v1alpha1/zz_generated.swagger_doc_generated.go +++ b/operator/v1alpha1/zz_generated.swagger_doc_generated.go @@ -259,7 +259,7 @@ func (RepositoryDigestMirrors) SwaggerDoc() map[string]string { var map_ByWorkload = map[string]string{ "": "ByWorkload controls how admission is done", - "admission": "admission controls how Kueue will process workloads. admission is only required if policy is specified to ByWorkload. Allowed values are Sequential, Parallel and \"\". When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling.", + "admission": "admission controls how Kueue will process workloads. admission is required. Allowed values are Sequential, Parallel and \"\". When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling. When set to \"\", the operator will decide", } func (ByWorkload) SwaggerDoc() map[string]string { @@ -279,7 +279,7 @@ func (ExternalFramework) SwaggerDoc() map[string]string { var map_GangScheduling = map[string]string{ "": "Kueue provides the ability to admit workloads all in one (gang admission) and evicts workloads if they are not ready within a specific time.", - "policy": "policy allows you to enable and configure gang scheduling. The allowed values are ByWorkload, None and \"\". When set to ByWorkload, this means each workload is processed and considered for admission as a single unit. Where workloads do not become ready over time, the entire workload may then be evicted and retried at a later time. policy is a required field.", + "policy": "policy allows you to enable and configure gang scheduling. The allowed values are ByWorkload, None and \"\". When set to ByWorkload, this means each workload is processed and considered for admission as a single unit. Where workloads do not become ready over time, the entire workload may then be evicted and retried at a later time. None means gang scheduling is disabled. \"\" means the the operator will decide the default. policy is a required field.", "byWorkload": "byWorkload configures how Kueue will process workloads for admission. byWorkload is required when policy is ByWorkload, and forbidden otherwise.", } From e92caa99d7def7c38a5b84df70490cae3207d4bc Mon Sep 17 00:00:00 2001 From: Kevin Hannon Date: Tue, 15 Apr 2025 12:01:51 -0400 Subject: [PATCH 42/42] default godocs --- .../generated_openapi/zz_generated.openapi.go | 8 ++++---- openapi/openapi.json | 8 ++++---- operator/v1alpha1/types_kueue.go | 18 ++++++++++++------ .../zz_generated.crd-manifests/kueue.crd.yaml | 18 ++++++++++++------ .../AAA_ungated.yaml | 18 ++++++++++++------ .../zz_generated.swagger_doc_generated.go | 8 ++++---- 6 files changed, 48 insertions(+), 30 deletions(-) diff --git a/openapi/generated_openapi/zz_generated.openapi.go b/openapi/generated_openapi/zz_generated.openapi.go index 9c559701241..cc7a7a41362 100644 --- a/openapi/generated_openapi/zz_generated.openapi.go +++ b/openapi/generated_openapi/zz_generated.openapi.go @@ -56515,7 +56515,7 @@ func schema_openshift_api_operator_v1alpha1_ByWorkload(ref common.ReferenceCallb Properties: map[string]spec.Schema{ "admission": { SchemaProps: spec.SchemaProps{ - Description: "admission controls how Kueue will process workloads. admission is required. Allowed values are Sequential, Parallel and \"\". When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling. When set to \"\", the operator will decide", + Description: "admission controls how Kueue will process workloads. admission is required. Allowed values are Sequential, Parallel and \"\". When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling. When set to \"\", this means no opinion and the operator is left to choose a reasonable default, which is subject to change over time. The current default is Parallel.", Default: "", Type: []string{"string"}, Format: "", @@ -56920,7 +56920,7 @@ func schema_openshift_api_operator_v1alpha1_GangScheduling(ref common.ReferenceC Properties: map[string]spec.Schema{ "policy": { SchemaProps: spec.SchemaProps{ - Description: "policy allows you to enable and configure gang scheduling. The allowed values are ByWorkload, None and \"\". When set to ByWorkload, this means each workload is processed and considered for admission as a single unit. Where workloads do not become ready over time, the entire workload may then be evicted and retried at a later time. None means gang scheduling is disabled. \"\" means the the operator will decide the default. policy is a required field.", + Description: "policy allows you to enable and configure gang scheduling. The allowed values are ByWorkload, None and \"\". When set to ByWorkload, this means each workload is processed and considered for admission as a single unit. Where workloads do not become ready over time, the entire workload may then be evicted and retried at a later time. None means gang scheduling is disabled. When set to \"\", this means no opinion and the operator is left to choose a reasonable default, which is subject to change over time. The current default is None. policy is a required field.", Default: "", Type: []string{"string"}, Format: "", @@ -58086,7 +58086,7 @@ func schema_openshift_api_operator_v1alpha1_Preemption(ref common.ReferenceCallb Properties: map[string]spec.Schema{ "preemptionPolicy": { SchemaProps: spec.SchemaProps{ - Description: "preemptionPolicy are the types of preemption Kueue allows. preemptionPolicy is an optional field. The allowed values are Classical, FairSharing and \"\". Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. FairSharing means that ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort. FairSharing is a more heavy weight algorithm. \"\" is our default setting which means that the operator will decide the default.", + Description: "preemptionPolicy are the types of preemption Kueue allows. preemptionPolicy is an optional field. The allowed values are Classical, FairSharing and \"\". Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. FairSharing means that ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort. FairSharing is a more heavy weight algorithm. When set to \"\", this means no opinion and the operator is left to choose a reasonable default, which is subject to change over time. The current default is Classical.", Default: "", Type: []string{"string"}, Format: "", @@ -58298,7 +58298,7 @@ func schema_openshift_api_operator_v1alpha1_WorkloadManagement(ref common.Refere Properties: map[string]spec.Schema{ "labelPolicy": { SchemaProps: spec.SchemaProps{ - Description: "labelPolicy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. labelPolicy is a required field. The allowed values are QueueName, None and \"\". None means that workloads will be suspended on creation and a label will be added via a mutating webhook. This will be applied for all integrations that Kueue manages. QueueName means that workloads that are managed by Kueue must have a label kueue.x-k8s.io/queue-name. \"\" is our default setting which the operator will decide the default. If this label is not present on the workload, then Kueue will ignore this workload.", + Description: "labelPolicy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. labelPolicy is a required field. The allowed values are QueueName, None and \"\". None means that workloads will be suspended on creation and a label will be added via a mutating webhook. This will be applied for all integrations that Kueue manages. QueueName means that workloads that are managed by Kueue must have a label kueue.x-k8s.io/queue-name. If this label is not present on the workload, then Kueue will ignore this workload. When set to \"\", this means no opinion and the operator is left to choose a reasonable default, which is subject to change over time. The current default is QueueName.", Default: "", Type: []string{"string"}, Format: "", diff --git a/openapi/openapi.json b/openapi/openapi.json index 7c632c186e1..5a834c7885c 100644 --- a/openapi/openapi.json +++ b/openapi/openapi.json @@ -33017,7 +33017,7 @@ ], "properties": { "admission": { - "description": "admission controls how Kueue will process workloads. admission is required. Allowed values are Sequential, Parallel and \"\". When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling. When set to \"\", the operator will decide", + "description": "admission controls how Kueue will process workloads. admission is required. Allowed values are Sequential, Parallel and \"\". When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling. When set to \"\", this means no opinion and the operator is left to choose a reasonable default, which is subject to change over time. The current default is Parallel.", "type": "string", "default": "" } @@ -33258,7 +33258,7 @@ "$ref": "#/definitions/com.github.openshift.api.operator.v1alpha1.ByWorkload" }, "policy": { - "description": "policy allows you to enable and configure gang scheduling. The allowed values are ByWorkload, None and \"\". When set to ByWorkload, this means each workload is processed and considered for admission as a single unit. Where workloads do not become ready over time, the entire workload may then be evicted and retried at a later time. None means gang scheduling is disabled. \"\" means the the operator will decide the default. policy is a required field.", + "description": "policy allows you to enable and configure gang scheduling. The allowed values are ByWorkload, None and \"\". When set to ByWorkload, this means each workload is processed and considered for admission as a single unit. Where workloads do not become ready over time, the entire workload may then be evicted and retried at a later time. None means gang scheduling is disabled. When set to \"\", this means no opinion and the operator is left to choose a reasonable default, which is subject to change over time. The current default is None. policy is a required field.", "type": "string", "default": "" } @@ -33971,7 +33971,7 @@ ], "properties": { "preemptionPolicy": { - "description": "preemptionPolicy are the types of preemption Kueue allows. preemptionPolicy is an optional field. The allowed values are Classical, FairSharing and \"\". Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. FairSharing means that ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort. FairSharing is a more heavy weight algorithm. \"\" is our default setting which means that the operator will decide the default.", + "description": "preemptionPolicy are the types of preemption Kueue allows. preemptionPolicy is an optional field. The allowed values are Classical, FairSharing and \"\". Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. FairSharing means that ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort. FairSharing is a more heavy weight algorithm. When set to \"\", this means no opinion and the operator is left to choose a reasonable default, which is subject to change over time. The current default is Classical.", "type": "string", "default": "" } @@ -34105,7 +34105,7 @@ ], "properties": { "labelPolicy": { - "description": "labelPolicy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. labelPolicy is a required field. The allowed values are QueueName, None and \"\". None means that workloads will be suspended on creation and a label will be added via a mutating webhook. This will be applied for all integrations that Kueue manages. QueueName means that workloads that are managed by Kueue must have a label kueue.x-k8s.io/queue-name. \"\" is our default setting which the operator will decide the default. If this label is not present on the workload, then Kueue will ignore this workload.", + "description": "labelPolicy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. labelPolicy is a required field. The allowed values are QueueName, None and \"\". None means that workloads will be suspended on creation and a label will be added via a mutating webhook. This will be applied for all integrations that Kueue manages. QueueName means that workloads that are managed by Kueue must have a label kueue.x-k8s.io/queue-name. If this label is not present on the workload, then Kueue will ignore this workload. When set to \"\", this means no opinion and the operator is left to choose a reasonable default, which is subject to change over time. The current default is QueueName.", "type": "string", "default": "" } diff --git a/operator/v1alpha1/types_kueue.go b/operator/v1alpha1/types_kueue.go index 8c2d921eba5..dd4a55f1cdb 100644 --- a/operator/v1alpha1/types_kueue.go +++ b/operator/v1alpha1/types_kueue.go @@ -236,7 +236,9 @@ type GangScheduling struct { // for admission as a single unit. // Where workloads do not become ready over time, the entire workload may then be evicted and retried at a later time. // None means gang scheduling is disabled. - // "" means the the operator will decide the default. + // When set to "", this means no opinion and the operator is left + // to choose a reasonable default, which is subject to change over time. + // The current default is None. // policy is a required field. // +required // +unionDiscriminator @@ -259,7 +261,9 @@ type ByWorkload struct { // When set to Parallel, pods from any workload will be admitted at any time. // This may lead to a deadlock where workloads are in contention for cluster capacity and // pods from another workload having successfully scheduled prevent pods from the current workload scheduling. - // When set to "", the operator will decide + // When set to "", this means no opinion and the operator is left + // to choose a reasonable default, which is subject to change over time. + // The current default is Parallel. // +required Admission GangSchedulingWorkloadAdmission `json:"admission"` } @@ -282,10 +286,11 @@ type WorkloadManagement struct { // This will be applied for all integrations that Kueue manages. // QueueName means that workloads that are managed // by Kueue must have a label kueue.x-k8s.io/queue-name. - // "" is our default setting which the operator will - // decide the default. // If this label is not present on the workload, then Kueue will // ignore this workload. + // When set to "", this means no opinion and the operator is left + // to choose a reasonable default, which is subject to change over time. + // The current default is QueueName. // +required LabelPolicy LabelPolicy `json:"labelPolicy"` } @@ -313,8 +318,9 @@ type Preemption struct { // The borrowable resources are the unused nominal quota // of all the ClusterQueues in the cohort. // FairSharing is a more heavy weight algorithm. - // "" is our default setting which means that the operator will - // decide the default. + // When set to "", this means no opinion and the operator is left + // to choose a reasonable default, which is subject to change over time. + // The current default is Classical. // +required PreemptionPolicy PreemptionPolicy `json:"preemptionPolicy"` } diff --git a/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml b/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml index 21b8c8676f7..f3e3be5396d 100644 --- a/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml +++ b/operator/v1alpha1/zz_generated.crd-manifests/kueue.crd.yaml @@ -74,7 +74,9 @@ spec: When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling. - When set to "", the operator will decide + When set to "", this means no opinion and the operator is left + to choose a reasonable default, which is subject to change over time. + The current default is Parallel. enum: - "" - Parallel @@ -91,7 +93,9 @@ spec: for admission as a single unit. Where workloads do not become ready over time, the entire workload may then be evicted and retried at a later time. None means gang scheduling is disabled. - "" means the the operator will decide the default. + When set to "", this means no opinion and the operator is left + to choose a reasonable default, which is subject to change over time. + The current default is None. policy is a required field. enum: - ByWorkload @@ -260,8 +264,9 @@ spec: The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort. FairSharing is a more heavy weight algorithm. - "" is our default setting which means that the operator will - decide the default. + When set to "", this means no opinion and the operator is left + to choose a reasonable default, which is subject to change over time. + The current default is Classical. enum: - "" - Classical @@ -291,10 +296,11 @@ spec: This will be applied for all integrations that Kueue manages. QueueName means that workloads that are managed by Kueue must have a label kueue.x-k8s.io/queue-name. - "" is our default setting which the operator will - decide the default. If this label is not present on the workload, then Kueue will ignore this workload. + When set to "", this means no opinion and the operator is left + to choose a reasonable default, which is subject to change over time. + The current default is QueueName. enum: - "" - QueueName diff --git a/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml b/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml index edcfe51bee8..a6a6ca67ab9 100644 --- a/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml +++ b/operator/v1alpha1/zz_generated.featuregated-crd-manifests/kueue.operator.openshift.io/AAA_ungated.yaml @@ -72,7 +72,9 @@ spec: When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling. - When set to "", the operator will decide + When set to "", this means no opinion and the operator is left + to choose a reasonable default, which is subject to change over time. + The current default is Parallel. enum: - "" - Parallel @@ -89,7 +91,9 @@ spec: for admission as a single unit. Where workloads do not become ready over time, the entire workload may then be evicted and retried at a later time. None means gang scheduling is disabled. - "" means the the operator will decide the default. + When set to "", this means no opinion and the operator is left + to choose a reasonable default, which is subject to change over time. + The current default is None. policy is a required field. enum: - ByWorkload @@ -258,8 +262,9 @@ spec: The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort. FairSharing is a more heavy weight algorithm. - "" is our default setting which means that the operator will - decide the default. + When set to "", this means no opinion and the operator is left + to choose a reasonable default, which is subject to change over time. + The current default is Classical. enum: - "" - Classical @@ -289,10 +294,11 @@ spec: This will be applied for all integrations that Kueue manages. QueueName means that workloads that are managed by Kueue must have a label kueue.x-k8s.io/queue-name. - "" is our default setting which the operator will - decide the default. If this label is not present on the workload, then Kueue will ignore this workload. + When set to "", this means no opinion and the operator is left + to choose a reasonable default, which is subject to change over time. + The current default is QueueName. enum: - "" - QueueName diff --git a/operator/v1alpha1/zz_generated.swagger_doc_generated.go b/operator/v1alpha1/zz_generated.swagger_doc_generated.go index 210059bf934..19bcbee462e 100644 --- a/operator/v1alpha1/zz_generated.swagger_doc_generated.go +++ b/operator/v1alpha1/zz_generated.swagger_doc_generated.go @@ -259,7 +259,7 @@ func (RepositoryDigestMirrors) SwaggerDoc() map[string]string { var map_ByWorkload = map[string]string{ "": "ByWorkload controls how admission is done", - "admission": "admission controls how Kueue will process workloads. admission is required. Allowed values are Sequential, Parallel and \"\". When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling. When set to \"\", the operator will decide", + "admission": "admission controls how Kueue will process workloads. admission is required. Allowed values are Sequential, Parallel and \"\". When admission is set to Sequential, only pods from the currently processing workload will be admitted. Once all pods from the current workload are admitted, and ready, Kueue will process the next workload. Sequential processing may slow down admission when the cluster has sufficient capacity for multiple workloads, but provides a higher guarantee of workloads scheduling all pods together successfully. When set to Parallel, pods from any workload will be admitted at any time. This may lead to a deadlock where workloads are in contention for cluster capacity and pods from another workload having successfully scheduled prevent pods from the current workload scheduling. When set to \"\", this means no opinion and the operator is left to choose a reasonable default, which is subject to change over time. The current default is Parallel.", } func (ByWorkload) SwaggerDoc() map[string]string { @@ -279,7 +279,7 @@ func (ExternalFramework) SwaggerDoc() map[string]string { var map_GangScheduling = map[string]string{ "": "Kueue provides the ability to admit workloads all in one (gang admission) and evicts workloads if they are not ready within a specific time.", - "policy": "policy allows you to enable and configure gang scheduling. The allowed values are ByWorkload, None and \"\". When set to ByWorkload, this means each workload is processed and considered for admission as a single unit. Where workloads do not become ready over time, the entire workload may then be evicted and retried at a later time. None means gang scheduling is disabled. \"\" means the the operator will decide the default. policy is a required field.", + "policy": "policy allows you to enable and configure gang scheduling. The allowed values are ByWorkload, None and \"\". When set to ByWorkload, this means each workload is processed and considered for admission as a single unit. Where workloads do not become ready over time, the entire workload may then be evicted and retried at a later time. None means gang scheduling is disabled. When set to \"\", this means no opinion and the operator is left to choose a reasonable default, which is subject to change over time. The current default is None. policy is a required field.", "byWorkload": "byWorkload configures how Kueue will process workloads for admission. byWorkload is required when policy is ByWorkload, and forbidden otherwise.", } @@ -355,7 +355,7 @@ func (LabelKeys) SwaggerDoc() map[string]string { } var map_Preemption = map[string]string{ - "preemptionPolicy": "preemptionPolicy are the types of preemption Kueue allows. preemptionPolicy is an optional field. The allowed values are Classical, FairSharing and \"\". Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. FairSharing means that ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort. FairSharing is a more heavy weight algorithm. \"\" is our default setting which means that the operator will decide the default.", + "preemptionPolicy": "preemptionPolicy are the types of preemption Kueue allows. preemptionPolicy is an optional field. The allowed values are Classical, FairSharing and \"\". Classical means that an incoming workload, which does not fit within the unusued quota, is eligible to issue preemptions when the requests of the workload are below the resource flavor's nominal quota or borrowWithinCohort is enabled on the Cluster Queue. FairSharing means that ClusterQueues with pending Workloads can preempt other Workloads in their cohort until the preempting ClusterQueue obtains an equal or weighted share of the borrowable resources. The borrowable resources are the unused nominal quota of all the ClusterQueues in the cohort. FairSharing is a more heavy weight algorithm. When set to \"\", this means no opinion and the operator is left to choose a reasonable default, which is subject to change over time. The current default is Classical.", } func (Preemption) SwaggerDoc() map[string]string { @@ -363,7 +363,7 @@ func (Preemption) SwaggerDoc() map[string]string { } var map_WorkloadManagement = map[string]string{ - "labelPolicy": "labelPolicy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. labelPolicy is a required field. The allowed values are QueueName, None and \"\". None means that workloads will be suspended on creation and a label will be added via a mutating webhook. This will be applied for all integrations that Kueue manages. QueueName means that workloads that are managed by Kueue must have a label kueue.x-k8s.io/queue-name. \"\" is our default setting which the operator will decide the default. If this label is not present on the workload, then Kueue will ignore this workload.", + "labelPolicy": "labelPolicy controls whether or not Kueue reconciles jobs that don't set the label kueue.x-k8s.io/queue-name. labelPolicy is a required field. The allowed values are QueueName, None and \"\". None means that workloads will be suspended on creation and a label will be added via a mutating webhook. This will be applied for all integrations that Kueue manages. QueueName means that workloads that are managed by Kueue must have a label kueue.x-k8s.io/queue-name. If this label is not present on the workload, then Kueue will ignore this workload. When set to \"\", this means no opinion and the operator is left to choose a reasonable default, which is subject to change over time. The current default is QueueName.", } func (WorkloadManagement) SwaggerDoc() map[string]string {