Skip to content

Commit 405a276

Browse files
committed
fix: fail Deployment tracking fast on ReplicaSet pod-create errors
Problem: during a Deployment rollout, when the child ReplicaSet cannot create pods (for example `FailedCreate ... exceeded quota`), kubedog did not surface the failure and tracking hung until timeout. Root cause: the ReplicaSet controller emits these Warning events with InvolvedObject set to the ReplicaSet, not the Deployment. The deployment tracker only ran an event informer for the Deployment object itself, so ReplicaSet-scoped failures were never observed. StatefulSet and DaemonSet are unaffected because their controllers emit FailedCreate on the controller object kubedog already watches. The fix: start an additional event informer for each new ReplicaSet as soon as it is discovered (tracked per RS name, so a new ReplicaSet appearing mid-rollout gets its own informer too), feeding the same resourceFailed channel the Deployment tracker already reacts to. Any "Failed*" reason (including FailedCreate: exceeded quota) now interrupts tracking, consistent with StatefulSet/DaemonSet behavior. Honors KUBEDOG_DISABLE_EVENTS=1. It does NOT change the event package, the failure-counting semantics of dyntracker, or add a new immediate-abort bypass; it only widens what the Deployment tracker observes. Verified with go build ./... and go vet ./... on this branch. The behavior fix was reproduced against the main branch (PR #399), whose deployment tracker differs from this one only in import paths, using a kind cluster with a ResourceQuota blocking pod creation: before, Deployment tracking hung for the full timeout; after, it failed within seconds with the FailedCreate reason. Closes #398 Refs #216, #363 Signed-off-by: Alexey Gorovenko <sharvashinho@gmail.com>
1 parent 5d57834 commit 405a276

1 file changed

Lines changed: 20 additions & 9 deletions

File tree

pkg/tracker/deployment/tracker.go

Lines changed: 20 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -47,11 +47,12 @@ type Tracker struct {
4747
Conditions []string
4848
NewReplicaSetName string
4949

50-
knownReplicaSets map[string]*appsv1.ReplicaSet
51-
lastObject *appsv1.Deployment
52-
failedReason string
53-
podStatuses map[string]pod.PodStatus
54-
rsNameByPod map[string]string
50+
knownReplicaSets map[string]*appsv1.ReplicaSet
51+
lastObject *appsv1.Deployment
52+
failedReason string
53+
rsEventsInformerStartedForRSs map[string]bool
54+
podStatuses map[string]pod.PodStatus
55+
rsNameByPod map[string]string
5556

5657
ignoreLogs bool
5758
ignoreReadinessProbeFailsByContainerName map[string]time.Duration
@@ -109,9 +110,10 @@ func NewTracker(name, namespace string, kube kubernetes.Interface, informerFacto
109110
PodLogChunk: make(chan *replicaset.ReplicaSetPodLogChunk, 1000),
110111
PodError: make(chan PodErrorReport),
111112

112-
knownReplicaSets: make(map[string]*appsv1.ReplicaSet),
113-
podStatuses: make(map[string]pod.PodStatus),
114-
rsNameByPod: make(map[string]string),
113+
knownReplicaSets: make(map[string]*appsv1.ReplicaSet),
114+
rsEventsInformerStartedForRSs: make(map[string]bool),
115+
podStatuses: make(map[string]pod.PodStatus),
116+
rsNameByPod: make(map[string]string),
115117

116118
ignoreLogs: opts.IgnoreLogs,
117119
ignoreReadinessProbeFailsByContainerName: opts.IgnoreReadinessProbeFailsByContainerName,
@@ -165,6 +167,7 @@ func (d *Tracker) Track(ctx context.Context) (err error) {
165167
d.State = tracker.ResourceDeleted
166168
d.lastObject = nil
167169
d.knownReplicaSets = make(map[string]*appsv1.ReplicaSet)
170+
d.rsEventsInformerStartedForRSs = make(map[string]bool)
168171
d.podStatuses = make(map[string]pod.PodStatus)
169172
d.rsNameByPod = make(map[string]string)
170173
d.TrackedPodsNames = nil
@@ -203,6 +206,14 @@ func (d *Tracker) Track(ctx context.Context) (err error) {
203206
if len(d.knownReplicaSets) == 0 {
204207
rsNew = true
205208
}
209+
if rsNew && !d.rsEventsInformerStartedForRSs[rs.Name] && os.Getenv("KUBEDOG_DISABLE_EVENTS") != "1" {
210+
rsEventsInformerCleanupFn, err := d.runEventsInformer(ctx, rs)
211+
if err != nil {
212+
return err
213+
}
214+
defer rsEventsInformerCleanupFn()
215+
d.rsEventsInformerStartedForRSs[rs.Name] = true
216+
}
206217

207218
d.StatusGeneration++
208219
newPodsNames, err := d.getNewPodsNames()
@@ -655,7 +666,7 @@ func (d *Tracker) handleDeploymentState(ctx context.Context, object *appsv1.Depl
655666
return cleanupFn, nil
656667
}
657668

658-
// runEventsInformer watch for Deployment events
669+
// runEventsInformer watch for Deployment and newest ReplicaSet events
659670
func (d *Tracker) runEventsInformer(ctx context.Context, resource interface{}) (cleanupFn func(), err error) {
660671
eventInformer := event.NewEventInformer(&d.Tracker, resource)
661672
eventInformer.WithChannels(d.EventMsg, d.resourceFailed, d.errors)

0 commit comments

Comments
 (0)