dnorman3/priority system

pkg/apis/atlassian/v1/nodegroup_types.go

@@ -39,6 +39,10 @@ type NodeGroupSpec struct {
 
 	// SkipPreTerminationChecks is an optional flag to skip pre-termination checks during cycling
 	SkipPreTerminationChecks bool `json:"skipPreTerminationChecks,omitempty"`
+
+    // Priority controls the ordering of CNR creation for this NodeGroup.
+    // Lower values are higher priority. 0 runs first, then 1, 2, ...
+    Priority int32 `json:"priority,omitempty"`
 }
 
 // NodeGroupStatus defines the observed state of NodeGroup

pkg/generation/cnr.go

@@ -97,7 +97,10 @@ func GenerateCNR(nodeGroup atlassianv1.NodeGroup, nodes []string, name, namespac
 		}
 	}
 
-	return atlassianv1.CycleNodeRequest{
+    // Default/normalize priority for backward compatibility (not stored on CNR anymore)
+    _ = max(nodeGroup.Spec.Priority, 0)
+
+    return atlassianv1.CycleNodeRequest{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      finalName,
 			Namespace: namespace,
@@ -111,7 +114,7 @@ func GenerateCNR(nodeGroup atlassianv1.NodeGroup, nodes []string, name, namespac
 			HealthChecks:             nodeGroup.Spec.HealthChecks,
 			PreTerminationChecks:     nodeGroup.Spec.PreTerminationChecks,
 			SkipInitialHealthChecks:  nodeGroup.Spec.SkipInitialHealthChecks,
-			SkipPreTerminationChecks: nodeGroup.Spec.SkipPreTerminationChecks,
+            SkipPreTerminationChecks: nodeGroup.Spec.SkipPreTerminationChecks,
 			ValidationOptions:        nodeGroup.Spec.ValidationOptions,
 		},
 	}

pkg/observer/controller.go

@@ -273,31 +273,77 @@ func (c *controller) dropInProgressNodeGroups(nodeGroups v1.NodeGroupList, cnrs
 
 // createCNRs generates and applies CNRs from the changedNodeGroups
 func (c *controller) createCNRs(changedNodeGroups []*ListedNodeGroups) {
-	klog.V(3).Infoln("applying")
-	for _, nodeGroup := range changedNodeGroups {
-		nodeNames := make([]string, 0, len(nodeGroup.List))
-		for _, node := range nodeGroup.List {
-			nodeNames = append(nodeNames, node.Name)
-		}
-		// generate cnr with prefix and use generate name method
-		cnr := generation.GenerateCNR(*nodeGroup.NodeGroup, nodeNames, c.CNRPrefix, c.Namespace)
-		generation.UseGenerateNameCNR(&cnr)
-		generation.GiveReason(&cnr, nodeGroup.Reason)
-		generation.SetAPIVersion(&cnr, apiVersion)
+    klog.V(3).Infoln("applying")
+    for _, nodeGroup := range changedNodeGroups {
+        nodeNames := make([]string, 0, len(nodeGroup.List))
+        for _, node := range nodeGroup.List {
+            nodeNames = append(nodeNames, node.Name)
+        }
+        // generate cnr with prefix and use generate name method
+        cnr := generation.GenerateCNR(*nodeGroup.NodeGroup, nodeNames, c.CNRPrefix, c.Namespace)
+        generation.UseGenerateNameCNR(&cnr)
+        generation.GiveReason(&cnr, nodeGroup.Reason)
+        generation.SetAPIVersion(&cnr, apiVersion)
+
+        name := generation.GetName(cnr.ObjectMeta)
+
+        if err := generation.ApplyCNR(c.client, c.DryMode, cnr); err != nil {
+            klog.Errorf("failed to apply cnr %q for nodegroup %q: %s", name, nodeGroup.NodeGroup.Name, err)
+        } else {
+            var drymodeStr string
+            if c.DryMode {
+                drymodeStr = "[drymode] "
+            }
+            klog.V(2).Infof("%ssuccessfully applied cnr %q for nodegroup %q", drymodeStr, name, nodeGroup.NodeGroup.Name)
+            c.CNRsCreated.WithLabelValues(nodeGroup.NodeGroup.Name).Inc()
+        }
+    }
+}
 
-		name := generation.GetName(cnr.ObjectMeta)
+// filterNodeGroupsByPriority returns only the node groups at the lowest priority value
+// Backward compatibility: negative or missing priorities are treated as 0
+func (c *controller) filterNodeGroupsByPriority(changedNodeGroups []*ListedNodeGroups) []*ListedNodeGroups {
+    if len(changedNodeGroups) == 0 {
+        return nil
+    }
+    getPriority := func(ng *v1.NodeGroup) int32 { return max(ng.Spec.Priority, 0) }
+    minPriority := getPriority(changedNodeGroups[0].NodeGroup)
+    for i := 1; i < len(changedNodeGroups); i++ {
+        p := getPriority(changedNodeGroups[i].NodeGroup)
+        if p < minPriority {
+            minPriority = p
+        }
+    }
+    var filtered []*ListedNodeGroups
+    for _, g := range changedNodeGroups {
+        if getPriority(g.NodeGroup) == minPriority {
+            filtered = append(filtered, g)
+        }
+    }
+    return filtered
+}
 
-		if err := generation.ApplyCNR(c.client, c.DryMode, cnr); err != nil {
-			klog.Errorf("failed to apply cnr %q for nodegroup %q: %s", name, nodeGroup.NodeGroup.Name, err)
-		} else {
-			var drymodeStr string
-			if c.DryMode {
-				drymodeStr = "[drymode] "
-			}
-			klog.V(2).Infof("%ssuccessfully applied cnr %q for nodegroup %q", drymodeStr, name, nodeGroup.NodeGroup.Name)
-			c.CNRsCreated.WithLabelValues(nodeGroup.NodeGroup.Name).Inc()
-		}
-	}
+// isLowerPriorityInProgress returns true if any in-progress CNR belongs to a NodeGroup with
+// a priority lower than the provided minPriority (i.e., must finish before creating higher priorities)
+func (c *controller) isLowerPriorityInProgress(minPriority int32, inProg v1.CycleNodeRequestList) bool {
+    if len(inProg.Items) == 0 {
+        return false
+    }
+    // Build a list of valid nodegroups to map CNRs to priorities
+    allNG := c.validNodeGroups()
+    for _, cnr := range inProg.Items {
+        for i := range allNG.Items {
+            ng := allNG.Items[i]
+            if cnr.IsFromNodeGroup(ng) {
+                p := max(ng.Spec.Priority, 0)
+                if p < minPriority {
+                    return true
+                }
+                break
+            }
+        }
+    }
+    return false
 }
 
 // nextRunTime returns the next time the controller loop will run from now in UTC
@@ -319,8 +365,9 @@ func (c *controller) Run() {
 		nodeGroups = c.dropInProgressNodeGroups(nodeGroups, inProgressCNRs)
 	}
 
-	// observer the changes using the remaining nodegroups. This is stateless and will pickup changes again if restarted
-	changedNodeGroups := c.observeChanges(nodeGroups)
+    // observe the changes using the remaining nodegroups. This is stateless and will pickup changes again if restarted
+    changedNodeGroups := c.observeChanges(nodeGroups)
+    c.NodeGroupsDetectedChanges.WithLabelValues().Set(float64(len(changedNodeGroups)))
 	if len(changedNodeGroups) == 0 {
 		klog.V(2).Infoln("all nodegroups up to date. next check in", c.CheckInterval)
 		return
@@ -334,12 +381,31 @@ func (c *controller) Run() {
 		}
 	}
 
-	// wait for the desired amount to allow any in progress changes to batch up
+    // Filter to only the lowest priority nodegroups
+    filtered := c.filterNodeGroupsByPriority(changedNodeGroups)
+    if len(filtered) == 0 {
+        klog.V(2).Infoln("no nodegroups to apply after priority filter")
+        return
+    }
+
+    // If any lower priority CNRs are still in progress, skip this run
+    minP := max(filtered[0].NodeGroup.Spec.Priority, 0)
+    if c.isLowerPriorityInProgress(minP, inProgressCNRs) {
+        c.NodeGroupsBlocked.WithLabelValues().Set(float64(len(filtered)))
+        klog.V(2).Infof("lower priority CNRs still in progress for priority < %d; skipping creation", minP)
+        return
+    }
+    // Not blocked
+    c.NodeGroupsBlocked.WithLabelValues().Set(0)
+
+    // wait for the desired amount to allow any in progress changes to batch up
 	klog.V(3).Infof("waiting for %v to allow changes to settle", c.WaitInterval)
 	select {
 	case <-time.After(c.WaitInterval):
-		klog.V(3).Infof("applying %d CNRs", len(changedNodeGroups))
-		c.createCNRs(changedNodeGroups)
+        klog.V(3).Infof("applying %d CNRs (lowest priority batch)", len(filtered))
+        c.NodeGroupsApplying.WithLabelValues().Set(float64(len(filtered)))
+        c.createCNRs(filtered)
+        c.NodeGroupsApplying.WithLabelValues().Set(0)
 		if c.RunOnce {
 			klog.V(3).Infoln("done creating CNRs after runOnce. exiting")
 		} else {