CSPL-3354: REBASED Add Lifecycle Hooks and Configurable Termination Grace Period to Splunk Operator

api/v4/common_types.go

@@ -112,6 +112,8 @@ type Spec struct {
 
 	// TopologySpreadConstraint https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/
 	TopologySpreadConstraints []corev1.TopologySpreadConstraint `json:"topologySpreadConstraints,omitempty"`
+
+	TerminationGracePeriodSeconds int64 `json:"terminationGracePeriodSeconds"`
 }
 
 // Phase is used to represent the current phase of a custom resource

config/crd/bases/enterprise.splunk.com_clustermanagers.yaml

@@ -2132,6 +2132,9 @@ spec:
                     format: int32
                     type: integer
                 type: object
+              terminationGracePeriodSeconds:
+                format: int64
+                type: integer
               tolerations:
                 description: Pod's tolerations for Kubernetes node's taint
                 items:

config/crd/bases/enterprise.splunk.com_clustermasters.yaml

@@ -2128,6 +2128,9 @@ spec:
                     format: int32
                     type: integer
                 type: object
+              terminationGracePeriodSeconds:
+                format: int64
+                type: integer
               tolerations:
                 description: Pod's tolerations for Kubernetes node's taint
                 items:

config/crd/bases/enterprise.splunk.com_indexerclusters.yaml

@@ -1868,6 +1868,9 @@ spec:
                     format: int32
                     type: integer
                 type: object
+              terminationGracePeriodSeconds:
+                format: int64
+                type: integer
               tolerations:
                 description: Pod's tolerations for Kubernetes node's taint
                 items:
@@ -5643,6 +5646,9 @@ spec:
                     format: int32
                     type: integer
                 type: object
+              terminationGracePeriodSeconds:
+                format: int64
+                type: integer
               tolerations:
                 description: Pod's tolerations for Kubernetes node's taint
                 items:

config/crd/bases/enterprise.splunk.com_licensemanagers.yaml

@@ -2005,6 +2005,9 @@ spec:
                     format: int32
                     type: integer
                 type: object
+              terminationGracePeriodSeconds:
+                format: int64
+                type: integer
               tolerations:
                 description: Pod's tolerations for Kubernetes node's taint
                 items:

config/crd/bases/enterprise.splunk.com_licensemasters.yaml

@@ -2000,6 +2000,9 @@ spec:
                     format: int32
                     type: integer
                 type: object
+              terminationGracePeriodSeconds:
+                format: int64
+                type: integer
               tolerations:
                 description: Pod's tolerations for Kubernetes node's taint
                 items:

config/crd/bases/enterprise.splunk.com_monitoringconsoles.yaml

@@ -2007,6 +2007,9 @@ spec:
                     format: int32
                     type: integer
                 type: object
+              terminationGracePeriodSeconds:
+                format: int64
+                type: integer
               tolerations:
                 description: Pod's tolerations for Kubernetes node's taint
                 items:
@@ -6127,6 +6130,9 @@ spec:
                     format: int32
                     type: integer
                 type: object
+              terminationGracePeriodSeconds:
+                format: int64
+                type: integer
               tolerations:
                 description: Pod's tolerations for Kubernetes node's taint
                 items:

config/crd/bases/enterprise.splunk.com_searchheadclusters.yaml

@@ -2018,6 +2018,9 @@ spec:
                     format: int32
                     type: integer
                 type: object
+              terminationGracePeriodSeconds:
+                format: int64
+                type: integer
               tolerations:
                 description: Pod's tolerations for Kubernetes node's taint
                 items:
@@ -6466,6 +6469,9 @@ spec:
                     format: int32
                     type: integer
                 type: object
+              terminationGracePeriodSeconds:
+                format: int64
+                type: integer
               tolerations:
                 description: Pod's tolerations for Kubernetes node's taint
                 items:

config/crd/bases/enterprise.splunk.com_standalones.yaml

@@ -2129,6 +2129,9 @@ spec:
                     format: int32
                     type: integer
                 type: object
+              terminationGracePeriodSeconds:
+                format: int64
+                type: integer
               tolerations:
                 description: Pod's tolerations for Kubernetes node's taint
                 items:
@@ -6493,6 +6496,9 @@ spec:
                     format: int32
                     type: integer
                 type: object
+              terminationGracePeriodSeconds:
+                format: int64
+                type: integer
               tolerations:
                 description: Pod's tolerations for Kubernetes node's taint
                 items:

pkg/splunk/controller/util.go

@@ -166,6 +166,18 @@ func MergePodSpecUpdates(ctx context.Context, current *corev1.PodSpec, revised *
 		}
 	}
 
+	if (current.TerminationGracePeriodSeconds == nil && revised.TerminationGracePeriodSeconds != nil) ||
+		(current.TerminationGracePeriodSeconds != nil && revised.TerminationGracePeriodSeconds == nil) ||
+		(current.TerminationGracePeriodSeconds != nil && revised.TerminationGracePeriodSeconds != nil &&
+			*current.TerminationGracePeriodSeconds != *revised.TerminationGracePeriodSeconds) {
+
+		scopedLog.Info("Pod TerminationGracePeriodSeconds differs",
+			"current", current.TerminationGracePeriodSeconds,
+			"revised", revised.TerminationGracePeriodSeconds)
+		current.TerminationGracePeriodSeconds = revised.TerminationGracePeriodSeconds
+		result = true
+	}
+
 	// check for changes in container images; assume that the ordering is same for pods with > 1 container
 	if len(current.Containers) != len(revised.Containers) {
 		scopedLog.Info("Pod Container counts differ",
@@ -244,12 +256,54 @@ func MergePodSpecUpdates(ctx context.Context, current *corev1.PodSpec, revised *
 				current.Containers[idx].StartupProbe = revised.Containers[idx].StartupProbe
 				result = true
 			}
+
+			// check PreStop Lifecycle
+			if hasPreStopChanged(current.Containers[idx].Lifecycle, revised.Containers[idx].Lifecycle) {
+				scopedLog.Info("Pod Container PreStop Lifecycle differ",
+					"current", current.Containers[idx].Lifecycle,
+					"revised", revised.Containers[idx].Lifecycle)
+				setPreStopLifecycleHandler(current, idx)
+				result = true
+			}
 		}
 	}
 
 	return result
 }
 
+// Function to check if PreStop Lifecycle has changed
+func hasPreStopChanged(current, revised *corev1.Lifecycle) bool {
+	// If both are nil, there's no change
+	if current == nil && revised == nil {
+		return false
+	}
+
+	// If one is nil and the other isn't, there's a change
+	if (current == nil || current.PreStop == nil || current.PreStop.Exec == nil) &&
+		(revised != nil && revised.PreStop != nil && revised.PreStop.Exec != nil) {
+		return true
+	}
+
+	if (revised == nil || revised.PreStop == nil || revised.PreStop.Exec == nil) &&
+		(current != nil && current.PreStop != nil && current.PreStop.Exec != nil) {
+		return true
+	}
+
+	// If both are non-nil, compare the command
+	return !reflect.DeepEqual(current.PreStop.Exec.Command, revised.PreStop.Exec.Command)
+}
+
+// Set the PreStop lifecycle handler for the specified container index
+func setPreStopLifecycleHandler(podSpec *corev1.PodSpec, idx int) {
+	podSpec.Containers[idx].Lifecycle = &corev1.Lifecycle{
+		PreStop: &corev1.LifecycleHandler{
+			Exec: &corev1.ExecAction{
+				Command: []string{"/bin/sh", "-c", "/opt/splunk/bin/splunk offline && /opt/splunk/bin/splunk stop"},
+			},
+		},
+	}
+}
+
 // SortStatefulSetSlices sorts required slices in a statefulSet
 func SortStatefulSetSlices(ctx context.Context, current *corev1.PodSpec, name string) error {
 	reqLogger := log.FromContext(ctx)

pkg/splunk/controller/util_test.go

@@ -100,6 +100,20 @@ func TestMergePodUpdates(t *testing.T) {
 	matcher = func() bool { return reflect.DeepEqual(current.Spec.InitContainers, revised.Spec.InitContainers) }
 	podUpdateTester("InitContainer image changed")
 
+	// check Termination Grace Period updated
+	terminationGracePeriodSeconds := int64(60)
+	revised.Spec.TerminationGracePeriodSeconds = &terminationGracePeriodSeconds
+	matcher = func() bool {
+		if current.Spec.TerminationGracePeriodSeconds == nil {
+			return true
+		}
+		if current.Spec.TerminationGracePeriodSeconds == revised.Spec.TerminationGracePeriodSeconds {
+			return true
+		}
+		return false
+	}
+	podUpdateTester("TerminationGracePeriod updated")
+
 	// check new container added
 	revised.Spec.Containers = []corev1.Container{{Image: "splunk/splunk"}}
 	matcher = func() bool { return reflect.DeepEqual(current.Spec.Containers, revised.Spec.Containers) }
@@ -108,12 +122,16 @@ func TestMergePodUpdates(t *testing.T) {
 	// check container different Ports
 	revised.Spec.Containers = []corev1.Container{{Image: "splunk/splunk"}}
 	revised.Spec.Containers[0].Ports = []corev1.ContainerPort{{ContainerPort: 8000}}
-	matcher = func() bool { return reflect.DeepEqual(current.Spec.Containers, revised.Spec.Containers) }
+	matcher = func() bool {
+		return current.Spec.Containers[0].Ports[0].ContainerPort == revised.Spec.Containers[0].Ports[0].ContainerPort
+	}
 	podUpdateTester("Container Ports")
 
 	// check container different VolumeMounts
 	revised.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{{Name: "mnt-splunk"}}
-	matcher = func() bool { return reflect.DeepEqual(current.Spec.Containers, revised.Spec.Containers) }
+	matcher = func() bool {
+		return revised.Spec.Containers[0].VolumeMounts[0].Name == current.Spec.Containers[0].VolumeMounts[0].Name
+	}
 	podUpdateTester("Container VolumeMounts")
 
 	// check container different Resources
@@ -127,7 +145,20 @@ func TestMergePodUpdates(t *testing.T) {
 			corev1.ResourceMemory: resource.MustParse("512Mi"),
 		},
 	}
-	matcher = func() bool { return reflect.DeepEqual(current.Spec.Containers, revised.Spec.Containers) }
+	matcher = func() bool {
+		if len(current.Spec.Containers) != len(revised.Spec.Containers) {
+			return false
+		}
+
+		for i := range current.Spec.Containers {
+			if !reflect.DeepEqual(current.Spec.Containers[i].Resources.Requests, revised.Spec.Containers[i].Resources.Requests) ||
+				!reflect.DeepEqual(current.Spec.Containers[i].Resources.Limits, revised.Spec.Containers[i].Resources.Limits) {
+				return false
+			}
+		}
+
+		return true
+	}
 	podUpdateTester("Container Resources")
 
 	// check pod env update
@@ -206,6 +237,12 @@ func TestMergePodUpdates(t *testing.T) {
 	}
 	podUpdateTester("Pod TopologySpreadConstraints changed")
 
+	// check Pre Stop Lifecycle Handler updated
+	idx := 0
+	setPreStopLifecycleHandler(&revised.Spec, idx)
+	matcher = func() bool { return reflect.DeepEqual(current.Spec.Containers, revised.Spec.Containers) }
+	podUpdateTester("PreStopLifecycleHandler updated")
+
 	// check container removed
 	revised.Spec.Containers = []corev1.Container{}
 	matcher = func() bool { return reflect.DeepEqual(current.Spec.Containers, revised.Spec.Containers) }