Suspend/Resume feature on SparkApplication

api/v1beta2/sparkapplication_types.go

@@ -35,6 +35,13 @@ type SparkApplicationSpec struct {
 	// INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
 	// Important: Run "make generate" to regenerate code after modifying this file
 
+	// Suspend indicates whether the SparkApplication should be suspended.
+	// When true, the controller skips submitting the Spark job.
+	// If a SparkApplication is suspended after creation
+	// (i.e. the flag goes from false to true), the Spark operator will delete
+	// all active Pods associated with this SparkApplication.
+	// Users must design their Spark application to gracefully handle this.
+	Suspend *bool `json:"suspend,omitempty"`
 	// Type tells the type of the Spark application.
 	// +kubebuilder:validation:Enum={Java,Python,Scala,R}
 	Type SparkApplicationType `json:"type"`
@@ -177,6 +184,7 @@ type SparkApplicationStatus struct {
 // +kubebuilder:metadata:annotations="api-approved.kubernetes.io=https://github.com/kubeflow/spark-operator/pull/1298"
 // +kubebuilder:resource:scope=Namespaced,shortName=sparkapp,singular=sparkapplication
 // +kubebuilder:subresource:status
+// +kubebuilder:printcolumn:JSONPath=.spec.suspend,name=Suspend,type=boolean
 // +kubebuilder:printcolumn:JSONPath=.status.applicationState.state,name=Status,type=string
 // +kubebuilder:printcolumn:JSONPath=.status.executionAttempts,name=Attempts,type=string
 // +kubebuilder:printcolumn:JSONPath=.status.lastSubmissionAttemptTime,name=Start,type=string
@@ -346,6 +354,9 @@ const (
 	ApplicationStateInvalidating     ApplicationStateType = "INVALIDATING"
 	ApplicationStateSucceeding       ApplicationStateType = "SUCCEEDING"
 	ApplicationStateFailing          ApplicationStateType = "FAILING"
+	ApplicationStateSuspending       ApplicationStateType = "SUSPENDING"
+	ApplicationStateSuspended        ApplicationStateType = "SUSPENDED"
+	ApplicationStateResuming         ApplicationStateType = "RESUMING"
 	ApplicationStateUnknown          ApplicationStateType = "UNKNOWN"
 )
 

charts/spark-operator-chart/crds/sparkoperator.k8s.io_scheduledsparkapplications.yaml

@@ -10570,6 +10570,15 @@ spec:
                     description: SparkVersion is the version of Spark the application
                       uses.
                     type: string
+                  suspend:
+                    description: |-
+                      Suspend indicates whether the SparkApplication should be suspended.
+                      When true, the controller skips submitting the Spark job.
+                      If a SparkApplication is suspended after creation
+                      (i.e. the flag goes from false to true), the Spark operator will delete
+                      all active Pods associated with this SparkApplication.
+                      Users must design their Spark application to gracefully handle this.
+                    type: boolean
                   timeToLiveSeconds:
                     description: |-
                       TimeToLiveSeconds defines the Time-To-Live (TTL) duration in seconds for this SparkApplication

charts/spark-operator-chart/crds/sparkoperator.k8s.io_sparkapplications.yaml

@@ -18,6 +18,9 @@ spec:
   scope: Namespaced
   versions:
   - additionalPrinterColumns:
+    - jsonPath: .spec.suspend
+      name: Suspend
+      type: boolean
     - jsonPath: .status.applicationState.state
       name: Status
       type: string
@@ -10488,6 +10491,15 @@ spec:
                 description: SparkVersion is the version of Spark the application
                   uses.
                 type: string
+              suspend:
+                description: |-
+                  Suspend indicates whether the SparkApplication should be suspended.
+                  When true, the controller skips submitting the Spark job.
+                  If a SparkApplication is suspended after creation
+                  (i.e. the flag goes from false to true), the Spark operator will delete
+                  all active Pods associated with this SparkApplication.
+                  Users must design their Spark application to gracefully handle this.
+                type: boolean
               timeToLiveSeconds:
                 description: |-
                   TimeToLiveSeconds defines the Time-To-Live (TTL) duration in seconds for this SparkApplication

config/crd/bases/sparkoperator.k8s.io_scheduledsparkapplications.yaml

@@ -10570,6 +10570,15 @@ spec:
                     description: SparkVersion is the version of Spark the application
                       uses.
                     type: string
+                  suspend:
+                    description: |-
+                      Suspend indicates whether the SparkApplication should be suspended.
+                      When true, the controller skips submitting the Spark job.
+                      If a SparkApplication is suspended after creation
+                      (i.e. the flag goes from false to true), the Spark operator will delete
+                      all active Pods associated with this SparkApplication.
+                      Users must design their Spark application to gracefully handle this.
+                    type: boolean
                   timeToLiveSeconds:
                     description: |-
                       TimeToLiveSeconds defines the Time-To-Live (TTL) duration in seconds for this SparkApplication

config/crd/bases/sparkoperator.k8s.io_sparkapplications.yaml

@@ -18,6 +18,9 @@ spec:
   scope: Namespaced
   versions:
   - additionalPrinterColumns:
+    - jsonPath: .spec.suspend
+      name: Suspend
+      type: boolean
     - jsonPath: .status.applicationState.state
       name: Status
       type: string
@@ -10488,6 +10491,15 @@ spec:
                 description: SparkVersion is the version of Spark the application
                   uses.
                 type: string
+              suspend:
+                description: |-
+                  Suspend indicates whether the SparkApplication should be suspended.
+                  When true, the controller skips submitting the Spark job.
+                  If a SparkApplication is suspended after creation
+                  (i.e. the flag goes from false to true), the Spark operator will delete
+                  all active Pods associated with this SparkApplication.
+                  Users must design their Spark application to gracefully handle this.
+                type: boolean
               timeToLiveSeconds:
                 description: |-
                   TimeToLiveSeconds defines the Time-To-Live (TTL) duration in seconds for this SparkApplication

docs/api-docs.md

@@ -238,6 +238,22 @@ SparkApplicationSpec
 <table>
 <tr>
 <td>
+<code>suspend</code><br/>
+<em>
+bool
+</em>
+</td>
+<td>
+<p>Suspend indicates whether the SparkApplication should be suspended.
+When true, the controller skips submitting the Spark job.
+If a SparkApplication is suspended after creation
+(i.e. the flag goes from false to true), the Spark operator will delete
+all active Pods associated with this SparkApplication.
+Users must design their Spark application to gracefully handle this.</p>
+</td>
+</tr>
+<tr>
+<td>
 <code>type</code><br/>
 <em>
 <a href="#sparkoperator.k8s.io/v1beta2.SparkApplicationType">
@@ -729,12 +745,18 @@ string
 <td></td>
 </tr><tr><td><p>&#34;PENDING_RERUN&#34;</p></td>
 <td></td>
+</tr><tr><td><p>&#34;RESUMING&#34;</p></td>
+<td></td>
 </tr><tr><td><p>&#34;RUNNING&#34;</p></td>
 <td></td>
 </tr><tr><td><p>&#34;SUBMITTED&#34;</p></td>
 <td></td>
 </tr><tr><td><p>&#34;SUCCEEDING&#34;</p></td>
 <td></td>
+</tr><tr><td><p>&#34;SUSPENDED&#34;</p></td>
+<td></td>
+</tr><tr><td><p>&#34;SUSPENDING&#34;</p></td>
+<td></td>
 </tr><tr><td><p>&#34;UNKNOWN&#34;</p></td>
 <td></td>
 </tr></tbody>
@@ -2331,6 +2353,22 @@ It carries every pieces of information a spark-submit command takes and recogniz
 <tbody>
 <tr>
 <td>
+<code>suspend</code><br/>
+<em>
+bool
+</em>
+</td>
+<td>
+<p>Suspend indicates whether the SparkApplication should be suspended.
+When true, the controller skips submitting the Spark job.
+If a SparkApplication is suspended after creation
+(i.e. the flag goes from false to true), the Spark operator will delete
+all active Pods associated with this SparkApplication.
+Users must design their Spark application to gracefully handle this.</p>
+</td>
+</tr>
+<tr>
+<td>
 <code>type</code><br/>
 <em>
 <a href="#sparkoperator.k8s.io/v1beta2.SparkApplicationType">