OCPBUGS-14346: Fix when DNS operator reports Degraded #373
+837
−254
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -181,7 +181,7 @@ func (r *reconciler) Reconcile(ctx context.Context, request reconcile.Request) ( | |
| co.Status.Conditions = mergeConditions(co.Status.Conditions, | ||
| computeOperatorAvailableCondition(state.haveDNS, &state.dns), | ||
| operatorProgressingCondition, | ||
| computeOperatorDegradedCondition(state.haveDNS, &state.dns, oldVersions, newVersions, curVersions), | ||
| ) | ||
| co.Status.Versions = computeOperatorStatusVersions(curVersions) | ||
| co.Status.Conditions = mergeConditions(co.Status.Conditions, computeOperatorUpgradeableCondition(&state.dns)) | ||
|
|
@@ -401,7 +401,9 @@ func computeOperatorUpgradeableCondition(dns *operatorv1.DNS) configv1.ClusterOp | |
| } | ||
|
|
||
| // computeOperatorDegradedCondition computes the operator's current Degraded status state. | ||
| func computeOperatorDegradedCondition(haveDNS bool, dns *operatorv1.DNS, oldVersions, newVersions, curVersions map[string]string) configv1.ClusterOperatorStatusCondition { | ||
| var messages []string | ||
|
|
||
| if !haveDNS { | ||
| return configv1.ClusterOperatorStatusCondition{ | ||
| Type: configv1.OperatorDegraded, | ||
|
|
@@ -411,18 +413,23 @@ func computeOperatorDegradedCondition(haveDNS bool, dns *operatorv1.DNS) configv | |
| } | ||
| } | ||
|
|
||
| // See OCPBUGS-14346. If the operator is upgrading, we can't consider it as degraded. | ||
| upgrading, _ := isUpgrading(curVersions, oldVersions, newVersions) | ||
| if !upgrading { | ||
| var degraded bool | ||
| for _, cond := range dns.Status.Conditions { | ||
| if cond.Type == operatorv1.OperatorStatusTypeDegraded && cond.Status == operatorv1.ConditionTrue { | ||
| degraded = true | ||
| messages = append(messages, cond.Message) | ||
| } | ||
| } | ||
| if degraded { | ||
| return configv1.ClusterOperatorStatusCondition{ | ||
| Type: configv1.OperatorDegraded, | ||
| Status: configv1.ConditionTrue, | ||
| Reason: "DNSDegraded", | ||
| Message: fmt.Sprintf("DNS %s is degraded: %s", dns.Name, strings.Join(messages, "\n")), | ||
| } | ||
| } | ||
| } | ||
| return configv1.ClusterOperatorStatusCondition{ | ||
|
|
@@ -468,24 +475,17 @@ func computeOperatorProgressingCondition(haveDNS bool, dns *operatorv1.DNS, oldV | |
| } | ||
| } | ||
|
|
||
| // If the operator is upgrading, note it as a Progressing reason and add the upgrading messages | ||
| upgrading, upgradingMessages := isUpgrading(curVersions, oldVersions, newVersions) | ||
| if upgrading { | ||
| status = configv1.ConditionTrue | ||
| messages = append(messages, strings.Join(upgradingMessages, "And ")) | ||
| progressingReasons = append(progressingReasons, "Upgrading") | ||
| } | ||
|
|
||
| if len(progressingReasons) != 0 { | ||
| progressingCondition.Status = status | ||
| progressingCondition.Reason = strings.Join(progressingReasons, "And ") | ||
|
Contributor
Author
There was a problem hiding this comment. @bentito found this can produce a reason like |
||
| progressingCondition.Message = strings.Join(messages, "\n") | ||
| } else { | ||
| progressingCondition.Status = configv1.ConditionFalse | ||
|
|
@@ -496,6 +496,22 @@ func computeOperatorProgressingCondition(haveDNS bool, dns *operatorv1.DNS, oldV | |
| return progressingCondition | ||
| } | ||
|
|
||
| func isUpgrading(curVersions, oldVersions, newVersions map[string]string) (bool, []string) { | ||
| var messages []string | ||
| upgrading := false | ||
|
|
||
| for name, curVersion := range curVersions { | ||
| if oldVersion, ok := oldVersions[name]; ok && oldVersion != curVersion { | ||
| messages = append(messages, fmt.Sprintf("Upgraded %s to %q.", name, curVersion)) | ||
| } | ||
| if newVersion, ok := newVersions[name]; ok && curVersion != newVersion { | ||
| upgrading = true | ||
| messages = append(messages, fmt.Sprintf("Upgrading %s to %q.", name, newVersion)) | ||
| } | ||
| } | ||
| return upgrading, messages | ||
| } | ||
|
|
||
| // computeOldVersions returns a map of operand name to version computed from the | ||
| // given clusteroperator status. | ||
| func computeOldVersions(oldVersions []configv1.OperandVersion) map[string]string { | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,32 @@ | ||
| // Package conditions is copied in from openshift/cluster-ingress-operator and should be moved to library-go to be shared. | ||
| package conditions | ||
|
|
||
| import ( | ||
| "fmt" | ||
|
|
||
| operatorv1 "github.com/openshift/api/operator/v1" | ||
|
|
||
| "github.com/google/go-cmp/cmp" | ||
| "github.com/google/go-cmp/cmp/cmpopts" | ||
| ) | ||
|
|
||
| func FormatConditions(conditions []*operatorv1.OperatorCondition) string { | ||
| var formatted string | ||
| if len(conditions) == 0 { | ||
| return "" | ||
| } | ||
| for _, cond := range conditions { | ||
| formatted = formatted + fmt.Sprintf(", %s=%s (%s: %s)", cond.Type, cond.Status, cond.Reason, cond.Message) | ||
| } | ||
| formatted = formatted[2:] | ||
| return formatted | ||
| } | ||
|
|
||
| func ConditionsEqual(a, b []operatorv1.OperatorCondition) bool { | ||
| conditionCmpOpts := []cmp.Option{ | ||
| cmpopts.EquateEmpty(), | ||
| cmpopts.SortSlices(func(a, b operatorv1.OperatorCondition) bool { return a.Type < b.Type }), | ||
| } | ||
|
|
||
| return cmp.Equal(a, b, conditionCmpOpts...) | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,60 @@ | ||
| // Package retryableerror is copied in from openshift/cluster-ingress-operator and should be moved to library-go to be shared. | ||
| package retryableerror | ||
|
|
||
| import ( | ||
| "time" | ||
|
|
||
| utilerrors "k8s.io/apimachinery/pkg/util/errors" | ||
| ) | ||
|
|
||
| // Error represents an error for an operation that should be retried after the | ||
| // specified duration. | ||
| type Error interface { | ||
| error | ||
| // After is the time period after which the operation that caused the | ||
| // error should be retried. | ||
| After() time.Duration | ||
| } | ||
|
|
||
| // New returns a new RetryableError with the given error and time period. | ||
| func New(err error, after time.Duration) Error { | ||
| return retryableError{err, after} | ||
| } | ||
|
|
||
| type retryableError struct { | ||
| error | ||
| after time.Duration | ||
| } | ||
|
|
||
| // After returns the time period after which the operation that caused the error | ||
| // should be retried. | ||
| func (r retryableError) After() time.Duration { | ||
| return r.after | ||
| } | ||
|
|
||
| // NewMaybeRetryableAggregate converts a slice of errors into a single error | ||
| // value. Nil values will be filtered from the slice. If the filtered slice is | ||
| // empty, the return value will be nil. Else, if any values are non-retryable | ||
| // errors, the result will be an Aggregate interface. Else, if all errors are | ||
| // retryable, the result will be a retryable Error interface, with After() equal | ||
| // to the minimum of all the errors' After() values. | ||
| func NewMaybeRetryableAggregate(errs []error) error { | ||
| aggregate := utilerrors.NewAggregate(errs) | ||
| if aggregate == nil { | ||
| return nil | ||
| } | ||
| afterHasInitialValue := false | ||
| var after time.Duration | ||
| for _, err := range aggregate.Errors() { | ||
| switch e := err.(type) { | ||
| case Error: | ||
| if !afterHasInitialValue || e.After() < after { | ||
| after = e.After() | ||
| } | ||
| afterHasInitialValue = true | ||
| default: | ||
| return aggregate | ||
| } | ||
| } | ||
| return New(aggregate, after) | ||
| } |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.