Upgrade spark-operator to v2.1.1

charts/flyte-deps/Chart.yaml

@@ -10,7 +10,7 @@ dependencies:
     condition: contour.enabled
   - name: spark-operator
     alias: sparkoperator
-    version: 1.1.15
+    version: 2.1.1
     repository: https://kubeflow.github.io/spark-operator
     condition: sparkoperator.enabled
   - name: dask-kubernetes-operator

charts/flyte-deps/README.md

@@ -10,7 +10,7 @@ A Helm chart for Flyte dependency
 |------------|------|---------|
 | https://charts.bitnami.com/bitnami | contour | 7.10.1 |
 | https://helm.dask.org | daskoperator(dask-kubernetes-operator) | 2022.12.0 |
-| https://kubeflow.github.io/spark-operator | sparkoperator(spark-operator) | 1.1.15 |
+| https://kubeflow.github.io/spark-operator | sparkoperator(spark-operator) | 2.1.1 |
 | https://kubernetes-retired.github.io/dashboard/ | kubernetes-dashboard | 4.0.2 |
 
 ### SANDBOX INSTALLATION:

charts/flyte/Chart.yaml

@@ -14,7 +14,7 @@ dependencies:
     condition: contour.enabled
   - name: spark-operator
     alias: sparkoperator
-    version: 1.1.15
+    version: 2.1.1
     repository: https://kubeflow.github.io/spark-operator
     condition: sparkoperator.enabled
   - name: dask-kubernetes-operator

charts/flyte/README.md

@@ -11,7 +11,7 @@ A Helm chart for Flyte Sandbox
 | file://../flyte-core | flyte(flyte-core) | v0.1.10 |
 | https://charts.bitnami.com/bitnami | contour | 7.10.1 |
 | https://helm.dask.org | daskoperator(dask-kubernetes-operator) | 2022.12.0 |
-| https://kubeflow.github.io/spark-operator | sparkoperator(spark-operator) | 1.1.15 |
+| https://kubeflow.github.io/spark-operator | sparkoperator(spark-operator) | 2.1.1 |
 | https://kubernetes-retired.github.io/dashboard/ | kubernetes-dashboard | 4.0.2 |
 
 **NOTE:** Flyte sandbox helm chart is deprecated, From now follow the sandbox [docs](https://docs.flyte.org/en/latest/deployment/sandbox.html) for installing it on cloud

flyteadmin/go.mod

@@ -3,11 +3,11 @@ module github.com/flyteorg/flyte/flyteadmin
 go 1.26.0
 
 require (
-	cloud.google.com/go/iam v1.1.5
-	cloud.google.com/go/storage v1.36.0
+	cloud.google.com/go/iam v1.2.1
+	cloud.google.com/go/storage v1.45.0
 	github.com/NYTimes/gizmo v1.3.6
 	github.com/Selvatico/go-mocket v1.0.7
-	github.com/aws/aws-sdk-go v1.47.11
+	github.com/aws/aws-sdk-go v1.55.5
 	github.com/benbjohnson/clock v1.3.0
 	github.com/cloudevents/sdk-go/binding/format/protobuf/v2 v2.8.0
 	github.com/cloudevents/sdk-go/v2 v2.15.2
@@ -25,7 +25,7 @@ require (
 	github.com/golang/glog v1.2.5
 	github.com/golang/protobuf v1.5.4
 	github.com/google/uuid v1.6.0
-	github.com/googleapis/gax-go/v2 v2.12.0
+	github.com/googleapis/gax-go/v2 v2.13.0
 	github.com/gorilla/handlers v1.5.1
 	github.com/gorilla/securecookie v1.1.1
 	github.com/grpc-ecosystem/go-grpc-middleware v1.4.0
@@ -35,7 +35,7 @@ require (
 	github.com/jackc/pgconn v1.14.3
 	github.com/jackc/pgx/v5 v5.9.2
 	github.com/lestrrat-go/jwx v1.2.29
-	github.com/magiconair/properties v1.8.6
+	github.com/magiconair/properties v1.8.7
 	github.com/nats-io/nats.go v1.31.0
 	github.com/ory/fosite v0.42.2
 	github.com/ory/x v0.0.214
@@ -56,8 +56,8 @@ require (
 	golang.org/x/oauth2 v0.35.0
 	golang.org/x/sync v0.20.0
 	golang.org/x/time v0.12.0
-	google.golang.org/api v0.155.0
-	google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80
+	google.golang.org/api v0.197.0
+	google.golang.org/genproto v0.0.0-20240903143218-8af14fe29dc1
 	google.golang.org/grpc v1.80.0
 	google.golang.org/protobuf v1.36.11
 	gorm.io/driver/mysql v1.4.4
@@ -78,11 +78,18 @@ require (
 )
 
 require (
-	cloud.google.com/go v0.112.0 // indirect
+	cel.dev/expr v0.25.1 // indirect
+	cloud.google.com/go v0.115.1 // indirect
+	cloud.google.com/go/auth v0.9.3 // indirect
+	cloud.google.com/go/auth/oauth2adapt v0.2.4 // indirect
 	cloud.google.com/go/compute/metadata v0.9.0 // indirect
-	cloud.google.com/go/pubsub v1.34.0 // indirect
+	cloud.google.com/go/monitoring v1.21.0 // indirect
+	cloud.google.com/go/pubsub v1.42.0 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 // indirect
 	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
+	github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.31.0 // indirect
+	github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.48.1 // indirect
+	github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.48.1 // indirect
 	github.com/Masterminds/semver v1.5.0 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535 // indirect
 	github.com/benlaurie/objecthash v0.0.0-20180202135721-d1e3d6079fc1 // indirect
@@ -91,6 +98,7 @@ require (
 	github.com/cenkalti/backoff/v5 v5.0.3 // indirect
 	github.com/cespare/xxhash v1.1.0 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5 // indirect
 	github.com/coocood/freecache v1.1.1 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
@@ -99,12 +107,15 @@ require (
 	github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect
 	github.com/eapache/queue v1.1.0 // indirect
 	github.com/emicklei/go-restful/v3 v3.13.0 // indirect
+	github.com/envoyproxy/go-control-plane/envoy v1.36.0 // indirect
+	github.com/envoyproxy/protoc-gen-validate v1.3.0 // indirect
 	github.com/evanphx/json-patch/v5 v5.9.11 // indirect
 	github.com/fatih/color v1.18.0 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/fsnotify/fsnotify v1.9.0 // indirect
 	github.com/fxamacker/cbor/v2 v2.9.0 // indirect
 	github.com/go-jose/go-jose/v3 v3.0.5 // indirect
+	github.com/go-jose/go-jose/v4 v4.1.4 // indirect
 	github.com/go-logr/logr v1.4.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-openapi/jsonpointer v0.21.2 // indirect
@@ -121,8 +132,8 @@ require (
 	github.com/google/btree v1.1.3 // indirect
 	github.com/google/gnostic-models v0.7.0 // indirect
 	github.com/google/go-cmp v0.7.0 // indirect
-	github.com/google/s2a-go v0.1.7 // indirect
-	github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
+	github.com/google/s2a-go v0.1.8 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect
 	github.com/gorilla/websocket v1.5.4-0.20250319132907-e064f32e3674 // indirect
 	github.com/hashicorp/go-uuid v1.0.2 // indirect
 	github.com/hashicorp/golang-lru v0.5.4 // indirect
@@ -170,6 +181,7 @@ require (
 	github.com/pelletier/go-toml/v2 v2.2.4 // indirect
 	github.com/pierrec/lz4 v2.5.2+incompatible // indirect
 	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
+	github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/prometheus/common v0.65.0 // indirect
 	github.com/prometheus/procfs v0.17.0 // indirect
@@ -183,6 +195,7 @@ require (
 	github.com/spf13/cast v1.10.0 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/viper v1.21.0 // indirect
+	github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect
 	github.com/stretchr/objx v0.5.2 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
 	github.com/tidwall/gjson v1.17.1 // indirect
@@ -194,6 +207,7 @@ require (
 	go.etcd.io/etcd/api/v3 v3.6.4 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
+	go.opentelemetry.io/contrib/detectors/gcp v1.39.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 // indirect
 	go.opentelemetry.io/otel/exporters/jaeger v1.17.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 // indirect
@@ -202,6 +216,7 @@ require (
 	go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.22.0 // indirect
 	go.opentelemetry.io/otel/metric v1.43.0 // indirect
 	go.opentelemetry.io/otel/sdk v1.43.0 // indirect
+	go.opentelemetry.io/otel/sdk/metric v1.43.0 // indirect
 	go.opentelemetry.io/otel/trace v1.43.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.10.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect