make reapply interval configurable (#289)

Author: cbarbian-sap

pkg/component/component.go

@@ -30,6 +30,20 @@ func getSpec[T Component](component T) any {
 	return spec.Interface()
 }
 
+/*
+// TODO: should we use this instead of the typed assert functions?
+// Check if given component or its spec implements the specified Configuration type (and return it).
+func assertConfiguration[T Component, C any](component T) (C, bool) {
+	if configuration, ok := Component(component).(C); ok {
+		return configuration, true
+	}
+	if configuration, ok := getSpec(component).(C); ok {
+		return configuration, true
+	}
+	return *new(C), false
+}
+*/
+
 // Check if given component or its spec implements PlacementConfiguration (and return it).
 func assertPlacementConfiguration[T Component](component T) (PlacementConfiguration, bool) {
 	if placementConfiguration, ok := Component(component).(PlacementConfiguration); ok {
@@ -118,6 +132,17 @@ func assertTypeConfiguration[T Component](component T) (TypeConfiguration, bool)
 	return nil, false
 }
 
+// Check if given component or its spec implements ReapplyConfiguration (and return it).
+func assertReapplyConfiguration[T Component](component T) (ReapplyConfiguration, bool) {
+	if reapplyConfiguration, ok := Component(component).(ReapplyConfiguration); ok {
+		return reapplyConfiguration, true
+	}
+	if reapplyConfiguration, ok := getSpec(component).(ReapplyConfiguration); ok {
+		return reapplyConfiguration, true
+	}
+	return nil, false
+}
+
 // Implement the PlacementConfiguration interface.
 func (s *PlacementSpec) GetDeploymentNamespace() string {
 	return s.Namespace
@@ -197,6 +222,14 @@ func (s *TypeSpec) GetAdditionalManagedTypes() []reconciler.TypeInfo {
 	return s.AdditionalManagedTypes
 }
 
+// Implement the ReapplyConfiguration interface.
+func (s *ReapplySpec) GetReapplyInterval() time.Duration {
+	if s.ReapplyInterval != nil {
+		return s.ReapplyInterval.Duration
+	}
+	return time.Duration(0)
+}
+
 // Check if state is Ready.
 func (s *Status) IsReady() bool {
 	// caveat: this operates only on the status, so it does not check that observedGeneration == generation

pkg/component/reconciler.go

@@ -78,6 +78,8 @@ const (
 	ReadyConditionReasonDeletionProcessing = "DeletionProcessing"
 
 	triggerBufferSize = 1024
+
+	defaultReapplyInterval = 60 * time.Minute
 )
 
 // TODO: should we pass cluster.Client to hooks instead of just client.Client?
@@ -124,6 +126,8 @@ type ReconcilerOptions struct {
 	// Whether namespaces are auto-created if missing.
 	// If unspecified, MissingNamespacesPolicyCreate is assumed.
 	MissingNamespacesPolicy *reconciler.MissingNamespacesPolicy
+	// Interval after which an object will be force-reapplied, even if it seems to be synced.
+	ReapplyInterval *time.Duration
 	// SchemeBuilder allows to define additional schemes to be made available in the
 	// target client.
 	SchemeBuilder types.SchemeBuilder
@@ -182,6 +186,9 @@ func NewReconciler[T Component](name string, resourceGenerator manifests.Generat
 	if options.MissingNamespacesPolicy == nil {
 		options.MissingNamespacesPolicy = ref(reconciler.MissingNamespacesPolicyCreate)
 	}
+	if options.ReapplyInterval == nil {
+		options.ReapplyInterval = ref(defaultReapplyInterval)
+	}
 
 	return &Reconciler[T]{
 		name:              name,
@@ -796,6 +803,7 @@ func (r *Reconciler[T]) getOptionsForComponent(component T) reconciler.Reconcile
 		UpdatePolicy:            r.options.UpdatePolicy,
 		DeletePolicy:            r.options.DeletePolicy,
 		MissingNamespacesPolicy: r.options.MissingNamespacesPolicy,
+		ReapplyInterval:         r.options.ReapplyInterval,
 		StatusAnalyzer:          r.statusAnalyzer,
 		Metrics: reconciler.ReconcilerMetrics{
 			ReadCounter:   metrics.Operations.WithLabelValues(r.controllerName, "read"),
@@ -822,5 +830,10 @@ func (r *Reconciler[T]) getOptionsForComponent(component T) reconciler.Reconcile
 	if typeConfiguration, ok := assertTypeConfiguration(component); ok {
 		options.AdditionalManagedTypes = typeConfiguration.GetAdditionalManagedTypes()
 	}
+	if reapplyConfiguration, ok := assertReapplyConfiguration(component); ok {
+		if reapplyInterval := reapplyConfiguration.GetReapplyInterval(); reapplyInterval > 0 {
+			options.ReapplyInterval = &reapplyInterval
+		}
+	}
 	return options
 }

pkg/component/types.go

@@ -67,20 +67,23 @@ type ImpersonationConfiguration interface {
 // tweaking the requeue interval (by default, it would be 10 minutes).
 type RequeueConfiguration interface {
 	// Get requeue interval. Should be greater than 1 minute.
+	// A return value of zero means to use the framework default.
 	GetRequeueInterval() time.Duration
 }
 
 // The RetryConfiguration interface is meant to be implemented by components (or their spec) which offer
 // tweaking the retry interval (by default, it would be the value of the requeue interval).
 type RetryConfiguration interface {
 	// Get retry interval. Should be greater than 1 minute.
+	// A return value of zero means to use the framework default.
 	GetRetryInterval() time.Duration
 }
 
 // The TimeoutConfiguration interface is meant to be implemented by components (or their spec) which offer
 // tweaking the processing timeout (by default, it would be the value of the requeue interval).
 type TimeoutConfiguration interface {
 	// Get timeout. Should be greater than 1 minute.
+	// A return value of zero means to use the framework default.
 	GetTimeout() time.Duration
 }
 
@@ -112,6 +115,15 @@ type TypeConfiguration interface {
 	GetAdditionalManagedTypes() []reconciler.TypeInfo
 }
 
+// The ReapplyConfiguration interface is meant to be implemented by components (or their spec) which allow
+// to tune the force-reapply interval.
+type ReapplyConfiguration interface {
+	// Get force-reapply interval. Should be greater than the effective requeue interval. If a value smaller than the
+	// effective requeue interval is specified, the force-reapply might be delayed until the requeue happens.
+	// A return value of zero means to use the framework default.
+	GetReapplyInterval() time.Duration
+}
+
 // +kubebuilder:object:generate=true
 
 // Legacy placement spec. Components may include this into their spec.
@@ -222,6 +234,16 @@ var _ TypeConfiguration = &TypeSpec{}
 
 // +kubebuilder:object:generate=true
 
+// ReapplySpec allows to specify the force-reapply interval.
+// Components providing ReapplyConfiguration may include this into their spec.
+type ReapplySpec struct {
+	ReapplyInterval *metav1.Duration `json:"reapplyInterval,omitempty"`
+}
+
+var _ ReapplyConfiguration = &ReapplySpec{}
+
+// +kubebuilder:object:generate=true
+
 // Component Status. Components must include this into their status.
 type Status struct {
 	ObservedGeneration int64        `json:"observedGeneration"`

pkg/component/zz_generated.deepcopy.go

@@ -59,7 +59,7 @@ const (
 )
 
 const (
-	forceReapplyPeriod = 60 * time.Minute
+	defaultReapplyInterval = 60 * time.Minute
 )
 
 var adoptionPolicyByAnnotation = map[string]AdoptionPolicy{
@@ -116,6 +116,8 @@ type ReconcilerOptions struct {
 	// a typical example of such additional managed types are CRDs which are implicitly created
 	// by the workloads of the component, but not part of the manifests.
 	AdditionalManagedTypes []TypeInfo
+	// Interval after which an object will be force-reapplied, even if it seems to be synced.
+	ReapplyInterval *time.Duration
 	// How to analyze the state of the dependent objects.
 	// If unspecified, an optimized kstatus based implementation is used.
 	StatusAnalyzer status.StatusAnalyzer
@@ -145,13 +147,15 @@ type Reconciler struct {
 	deletePolicy                 DeletePolicy
 	missingNamespacesPolicy      MissingNamespacesPolicy
 	additionalManagedTypes       []TypeInfo
+	reapplyInterval              time.Duration
 	labelKeyOwnerId              string
 	annotationKeyOwnerId         string
 	annotationKeyDigest          string
 	annotationKeyAdoptionPolicy  string
 	annotationKeyReconcilePolicy string
 	annotationKeyUpdatePolicy    string
 	annotationKeyDeletePolicy    string
+	annotationKeyReapplyInterval string
 	annotationKeyApplyOrder      string
 	annotationKeyPurgeOrder      string
 	annotationKeyDeleteOrder     string
@@ -180,6 +184,9 @@ func NewReconciler(name string, clnt cluster.Client, options ReconcilerOptions)
 	if options.MissingNamespacesPolicy == nil {
 		options.MissingNamespacesPolicy = ref(MissingNamespacesPolicyCreate)
 	}
+	if options.ReapplyInterval == nil {
+		options.ReapplyInterval = ref(defaultReapplyInterval)
+	}
 	if options.StatusAnalyzer == nil {
 		options.StatusAnalyzer = status.NewStatusAnalyzer(name)
 	}
@@ -196,13 +203,15 @@ func NewReconciler(name string, clnt cluster.Client, options ReconcilerOptions)
 		deletePolicy:                 *options.DeletePolicy,
 		missingNamespacesPolicy:      *options.MissingNamespacesPolicy,
 		additionalManagedTypes:       options.AdditionalManagedTypes,
+		reapplyInterval:              *options.ReapplyInterval,
 		labelKeyOwnerId:              name + "/" + types.LabelKeySuffixOwnerId,
 		annotationKeyOwnerId:         name + "/" + types.AnnotationKeySuffixOwnerId,
 		annotationKeyDigest:          name + "/" + types.AnnotationKeySuffixDigest,
 		annotationKeyAdoptionPolicy:  name + "/" + types.AnnotationKeySuffixAdoptionPolicy,
 		annotationKeyReconcilePolicy: name + "/" + types.AnnotationKeySuffixReconcilePolicy,
 		annotationKeyUpdatePolicy:    name + "/" + types.AnnotationKeySuffixUpdatePolicy,
 		annotationKeyDeletePolicy:    name + "/" + types.AnnotationKeySuffixDeletePolicy,
+		annotationKeyReapplyInterval: name + "/" + types.AnnotationKeySuffixReapplyInterval,
 		annotationKeyApplyOrder:      name + "/" + types.AnnotationKeySuffixApplyOrder,
 		annotationKeyPurgeOrder:      name + "/" + types.AnnotationKeySuffixPurgeOrder,
 		annotationKeyDeleteOrder:     name + "/" + types.AnnotationKeySuffixDeleteOrder,
@@ -223,7 +232,7 @@ func NewReconciler(name string, clnt cluster.Client, options ReconcilerOptions)
 // An update of an existing object will be performed if it is considered to be out of sync; that means:
 //   - the object's manifest has changed, and the effective reconcile policy is ReconcilePolicyOnObjectChange or ReconcilePolicyOnObjectOrComponentChange or
 //   - the specified component has changed and the effective reconcile policy is ReconcilePolicyOnObjectOrComponentChange or
-//   - periodically after forceReapplyPeriod.
+//   - periodically after the specified force-reapply interval.
 //
 // The update itself will be done as follows:
 //   - if the effective update policy is UpdatePolicyReplace, a http PUT request will be sent to the Kubernetes API
@@ -348,6 +357,9 @@ func (r *Reconciler) Apply(ctx context.Context, inventory *[]*InventoryItem, obj
 		if _, err := r.getDeletePolicy(object); err != nil {
 			return false, errors.Wrapf(err, "error validating object %s", types.ObjectKeyToString(object))
 		}
+		if _, err := r.getReapplyInterval(object); err != nil {
+			return false, errors.Wrapf(err, "error validating object %s", types.ObjectKeyToString(object))
+		}
 		if _, err := r.getApplyOrder(object); err != nil {
 			return false, errors.Wrapf(err, "error validating object %s", types.ObjectKeyToString(object))
 		}
@@ -377,6 +389,10 @@ func (r *Reconciler) Apply(ctx context.Context, inventory *[]*InventoryItem, obj
 		// note: this must() is ok because we checked the generated objects above, and this function will be called for these objects only
 		return must(r.getDeletePolicy(object))
 	}
+	getReapplyInterval := func(object client.Object) time.Duration {
+		// note: this must() is ok because we checked the generated objects above, and this function will be called for these objects only
+		return must(r.getReapplyInterval(object))
+	}
 	getApplyOrder := func(object client.Object) int {
 		// note: this must() is ok because we checked the generated objects above, and this function will be called for these objects only
 		return must(r.getApplyOrder(object))
@@ -692,6 +708,7 @@ func (r *Reconciler) Apply(ctx context.Context, inventory *[]*InventoryItem, obj
 				setAnnotation(object, r.annotationKeyDigest, item.Digest)
 
 				updatePolicy := getUpdatePolicy(object)
+				reapplyInterval := getReapplyInterval(object)
 				now := time.Now()
 				if existingObject == nil {
 					if err := r.createObject(ctx, object, nil, updatePolicy); err != nil {
@@ -702,8 +719,7 @@ func (r *Reconciler) Apply(ctx context.Context, inventory *[]*InventoryItem, obj
 					item.LastAppliedAt = &metav1.Time{Time: now}
 					numUnready++
 				} else if existingObject.GetDeletionTimestamp().IsZero() &&
-					// TODO: make force-reconcile period (60 minutes as of now) configurable
-					(existingObject.GetAnnotations()[r.annotationKeyDigest] != item.Digest || item.LastAppliedAt == nil || item.LastAppliedAt.Time.Before(now.Add(-forceReapplyPeriod))) {
+					(existingObject.GetAnnotations()[r.annotationKeyDigest] != item.Digest || item.LastAppliedAt == nil || item.LastAppliedAt.Time.Before(now.Add(-reapplyInterval))) {
 					switch updatePolicy {
 					case UpdatePolicyRecreate:
 						if err := r.deleteObject(ctx, object, existingObject, hashedOwnerId); err != nil {
@@ -1343,6 +1359,18 @@ func (r *Reconciler) getDeletePolicy(object client.Object) (DeletePolicy, error)
 	}
 }
 
+func (r *Reconciler) getReapplyInterval(object client.Object) (time.Duration, error) {
+	value, ok := object.GetAnnotations()[r.annotationKeyReapplyInterval]
+	if !ok {
+		return r.reapplyInterval, nil
+	}
+	reapplyInterval, err := time.ParseDuration(value)
+	if err != nil {
+		return 0, errors.Wrapf(err, "invalid value for annotation %s: %s", r.annotationKeyReapplyInterval, value)
+	}
+	return reapplyInterval, nil
+}
+
 func (r *Reconciler) getApplyOrder(object client.Object) (int, error) {
 	value, ok := object.GetAnnotations()[r.annotationKeyApplyOrder]
 	if !ok {

pkg/reconciler/reconciler.go

@@ -13,6 +13,7 @@ const (
 	AnnotationKeySuffixReconcilePolicy = "reconcile-policy"
 	AnnotationKeySuffixUpdatePolicy    = "update-policy"
 	AnnotationKeySuffixDeletePolicy    = "delete-policy"
+	AnnotationKeySuffixReapplyInterval = "reapply-interval"
 	AnnotationKeySuffixApplyOrder      = "apply-order"
 	AnnotationKeySuffixPurgeOrder      = "purge-order"
 	AnnotationKeySuffixDeleteOrder     = "delete-order"

pkg/types/constants.go

@@ -44,6 +44,7 @@ To support such cases, the `Generator` implementation can set the following anno
 - `mycomponent-operator.mydomain.io/apply-order`: the wave in which this object will be reconciled; dependents will be reconciled wave by wave; that is, objects of the same wave will be deployed in a canonical order, and the reconciler will only proceed to the next wave if all objects of previous waves are ready; specified orders can be negative or positive numbers between -32768 and 32767, objects with no explicit order set are treated as if they would specify order 0
 - `mycomponent-operator.mydomain.io/purge-order` (optional): the wave by which this object will be purged; here, purged means that, while applying the dependents, the object will be deleted from the cluster at the end of the specified wave; the according record in `status.Inventory` will be set to phase `Completed`; setting purge orders is useful to spawn ad-hoc objects during the reconcilation, which are not permanently needed; so it's comparable to Helm hooks, in a certain sense
 - `mycomponent-operator.mydomain.io/delete-order` (optional): the wave by which this object will be deleted; that is, if the dependent is no longer part of the component, or if the whole component is being deleted; dependents will be deleted wave by wave; that is, objects of the same wave will be deleted in a canonical order, and the reconciler will only proceed to the next wave if all objects of previous saves are gone; specified orders can be negative or positive numbers between -32768 and 32767, objects with no explicit order set are treated as if they would specify order 0; note that the delete order is completely independent of the apply order
+- `mycomponent-operator.mydomain.io/reapply-interval` (optional): the interval after which a force-reapply of the object will be performed (even it is in sync otherwise); if not specified, the reconciler default is used; note that, even if the specified force-reapply interval has passed, the next reconcile may happen only after the current requeue interval is over; because of that, it makes sense to set the reapply interval to a value (significantly) larger than the effective requeue interval.
 - `mycomponent-operator.mydomain.io/status-hint` (optional): a comma-separated list of hints that may help the framework to properly identify the state of the annotated dependent object; currently, the following hints are possible:
   - `has-observed-generation`: tells the framework that the dependent object has a `status.observedGeneration` field, even if it is not (yet) set by the responsible controller (some controllers are known to set the observed generation lazily, with the consequence that there is a period right after creation of the dependent object, where the field is missing in the dependent's status)
   - `has-ready-condition`: tells the framework to count with a ready condition; if it is absent, the condition status will be considered as `Unknown`