From 7406264468ccb8dd6e3600637580493d5a1e07f6 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Sun, 26 Jan 2025 15:11:02 -0800 Subject: [PATCH] [SPARK-50996][K8S] Increase `spark.kubernetes.allocation.batch.size` to 10 --- docs/core-migration-guide.md | 2 ++ docs/running-on-kubernetes.md | 2 +- .../src/main/scala/org/apache/spark/deploy/k8s/Config.scala | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/core-migration-guide.md b/docs/core-migration-guide.md index 9dcf4ad8a2984..7f92fa14f170a 100644 --- a/docs/core-migration-guide.md +++ b/docs/core-migration-guide.md @@ -36,6 +36,8 @@ license: | - In Spark 4.0, support for Apache Mesos as a resource manager was removed. +- Since Spark 4.0, Spark will allocate executor pods with a batch size of `10`. To restore the legacy behavior, you can set `spark.kubernetes.allocation.batch.size` to `5`. + - Since Spark 4.0, Spark uses `ReadWriteOncePod` instead of `ReadWriteOnce` access mode in persistence volume claims. To restore the legacy behavior, you can set `spark.kubernetes.legacy.useReadWriteOnceAccessMode` to `true`. - Since Spark 4.0, Spark reports its executor pod status by checking all containers of that pod. To restore the legacy behavior, you can set `spark.kubernetes.executor.checkAllContainers` to `false`. diff --git a/docs/running-on-kubernetes.md b/docs/running-on-kubernetes.md index c7f5d67a6cd85..bd5e1956a6279 100644 --- a/docs/running-on-kubernetes.md +++ b/docs/running-on-kubernetes.md @@ -682,7 +682,7 @@ See the [configuration page](configuration.html) for information on Spark config spark.kubernetes.allocation.batch.size - 5 + 10 Number of pods to launch at once in each round of executor pod allocation. diff --git a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala index db7fc85976c2a..4467f73e70568 100644 --- a/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala +++ b/resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/k8s/Config.scala @@ -450,7 +450,7 @@ private[spark] object Config extends Logging { .version("2.3.0") .intConf .checkValue(value => value > 0, "Allocation batch size should be a positive integer") - .createWithDefault(5) + .createWithDefault(10) val KUBERNETES_ALLOCATION_BATCH_DELAY = ConfigBuilder("spark.kubernetes.allocation.batch.delay")