diff --git a/pkg/emitter/exporter.go b/pkg/emitter/exporter.go index f5e25777..2dcd737e 100644 --- a/pkg/emitter/exporter.go +++ b/pkg/emitter/exporter.go @@ -68,9 +68,11 @@ func (de *defaultExporter) Start(interval time.Duration) bool { // take a snapshot of the current cluster state snapshot, err := de.snapshotProvider.SnapshotOf(de.ds) - if err != nil { + if err != nil && snapshot == nil { log.Errorf("failed to take snapshot for initialization phase: %v", err) return + } else if err != nil { + log.Warnf("initialization snapshot completed with partial data: %v", err) } for _, emitter := range de.emitters { @@ -92,11 +94,13 @@ func (de *defaultExporter) Start(interval time.Duration) bool { case <-time.After(interval): } - // take a snapshot of the current cluster state -- on failure, continue to next iteration + // take a snapshot of the current cluster state -- on total failure, skip to next iteration snapshot, err := de.snapshotProvider.SnapshotOf(de.ds) - if err != nil { + if err != nil && snapshot == nil { log.Errorf("failed to take snapshot: %v", err) continue + } else if err != nil { + log.Warnf("snapshot completed with partial data: %v", err) } var emitTasks sync.WaitGroup diff --git a/pkg/emitter/exporter_integration_test.go b/pkg/emitter/exporter_integration_test.go new file mode 100644 index 00000000..591ad10f --- /dev/null +++ b/pkg/emitter/exporter_integration_test.go @@ -0,0 +1,402 @@ +package emitter + +import ( + "context" + "fmt" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/ibm/finops-agent/internal/mocks" + "github.com/ibm/finops-agent/pkg/core" +) + +// waitForUint32 polls counter until it reaches at least target, or the timeout expires. +func waitForUint32(counter *atomic.Uint32, target uint32, timeout time.Duration) bool { + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + if counter.Load() >= target { + return true + } + time.Sleep(10 * time.Millisecond) + } + return counter.Load() >= target +} + +// waitForInt32 polls counter until it reaches at least target, or the timeout expires. +func waitForInt32(counter *int32, target int32, timeout time.Duration) bool { + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + if atomic.LoadInt32(counter) >= target { + return true + } + time.Sleep(10 * time.Millisecond) + } + return atomic.LoadInt32(counter) >= target +} + +// waitForBool polls flag until it becomes true, or the timeout expires. +func waitForBool(flag *atomic.Bool, timeout time.Duration) bool { + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + if flag.Load() { + return true + } + time.Sleep(10 * time.Millisecond) + } + return flag.Load() +} + +// --------------------------------------------------------------------------- +// Integration: Full exporter lifecycle with real snapshot provider +// --------------------------------------------------------------------------- + +func TestExporterWithRealSnapshotProvider(t *testing.T) { + ds := mocks.NewMockDataSource() + config := DefaultSnapshotConfig() + snapshotProvider := NewConcurrentSnapshotProvider(config) + + emitter := newCountingEmitter("real-snapshot-emitter") + exporter := NewExporter(ds, snapshotProvider, emitter) + + if !exporter.Start(200 * time.Millisecond) { + t.Fatal("Failed to start exporter") + } + + if !waitForUint32(&emitter.count, 3, 5*time.Second) { + exporter.Stop() + t.Fatalf("Expected at least 3 emissions, got %d", emitter.count.Load()) + } + exporter.Stop() +} + +// --------------------------------------------------------------------------- +// Integration: Exporter with failing snapshot provider +// --------------------------------------------------------------------------- + +func TestExporterHandlesSnapshotFailure(t *testing.T) { + ds := mocks.NewMockDataSource() + + callCount := int32(0) + sp := &intermittentSnapshotProvider{ + callCount: &callCount, + failEvery: 2, + } + + emitter := newCountingEmitter("snapshot-failure-emitter") + exporter := NewExporter(ds, sp, emitter) + + if !exporter.Start(100 * time.Millisecond) { + t.Fatal("Failed to start exporter") + } + + // Wait until we've had enough snapshot calls to observe both successes and failures. + // With failEvery=2, we need at least 3 calls to guarantee at least 1 failure and 2 successes. + if !waitForInt32(&callCount, 3, 5*time.Second) { + exporter.Stop() + t.Fatalf("Expected at least 3 snapshot calls, got %d", atomic.LoadInt32(&callCount)) + } + exporter.Stop() + + // Not all snapshots succeed, so emission count should be less than total cycles + emitCount := emitter.count.Load() + totalCalls := atomic.LoadInt32(&callCount) + + if emitCount == 0 { + t.Error("Expected at least some emissions despite snapshot failures") + } + if emitCount >= uint32(totalCalls) { + t.Error("Expected fewer emissions than snapshot attempts due to failures") + } +} + +// --------------------------------------------------------------------------- +// Integration: Exporter initialization phase +// --------------------------------------------------------------------------- + +func TestExporterInitializesEmitters(t *testing.T) { + ds := mocks.NewMockDataSource() + snapshotProvider := newEmptySnapshotProvider() + + emitter := &initTrackingEmitter{name: "init-tracker"} + exporter := NewExporter(ds, snapshotProvider, emitter) + + if !exporter.Start(time.Hour) { // Long interval so only Init runs + t.Fatal("Failed to start exporter") + } + + if !waitForBool(&emitter.initialized, 5*time.Second) { + t.Error("Expected emitter.Init() to be called during start") + } + + exporter.Stop() +} + +func TestExporterInitFailureDoesNotPreventLoop(t *testing.T) { + ds := mocks.NewMockDataSource() + snapshotProvider := newEmptySnapshotProvider() + + failInit := &failingInitEmitter{name: "fail-init"} + goodEmitter := newCountingEmitter("good-emitter") + + exporter := NewExporter(ds, snapshotProvider, failInit, goodEmitter) + + if !exporter.Start(100 * time.Millisecond) { + t.Fatal("Failed to start exporter") + } + + if !waitForUint32(&goodEmitter.count, 2, 5*time.Second) { + exporter.Stop() + t.Fatalf("Good emitter should have received at least 2 emissions, got %d", goodEmitter.count.Load()) + } + exporter.Stop() +} + +// --------------------------------------------------------------------------- +// Integration: Emitter panic recovery +// --------------------------------------------------------------------------- + +func TestExporterRecoverFromEmitterPanic(t *testing.T) { + ds := mocks.NewMockDataSource() + snapshotProvider := newEmptySnapshotProvider() + + panicEmitter := &panickingEmitter{name: "panic-emitter"} + goodEmitter := newCountingEmitter("survivor-emitter") + + exporter := NewExporter(ds, snapshotProvider, panicEmitter, goodEmitter) + + if !exporter.Start(100 * time.Millisecond) { + t.Fatal("Failed to start exporter") + } + + if !waitForUint32(&goodEmitter.count, 2, 5*time.Second) { + exporter.Stop() + t.Fatalf("Good emitter should still receive emissions, got %d", goodEmitter.count.Load()) + } + exporter.Stop() +} + +// --------------------------------------------------------------------------- +// Integration: Emitter ID tracking +// --------------------------------------------------------------------------- + +func TestExporterEmitterIDs(t *testing.T) { + ds := mocks.NewMockDataSource() + snapshotProvider := newEmptySnapshotProvider() + + a := newCountingEmitter("alpha") + b := newCountingEmitter("beta") + c := newCountingEmitter("gamma") + + exporter := NewExporter(ds, snapshotProvider, a, b, c) + ids := exporter.Emitters() + + if len(ids) != 3 { + t.Fatalf("Expected 3 emitter IDs, got %d", len(ids)) + } + + expected := map[EmitterID]bool{"alpha": true, "beta": true, "gamma": true} + for _, id := range ids { + if !expected[id] { + t.Errorf("Unexpected emitter ID: %s", id) + } + } +} + +// --------------------------------------------------------------------------- +// Integration: Start/Stop/Restart cycle +// --------------------------------------------------------------------------- + +func TestExporterRestartAfterStop(t *testing.T) { + ds := mocks.NewMockDataSource() + snapshotProvider := newEmptySnapshotProvider() + emitter := newCountingEmitter("restart-emitter") + + exporter := NewExporter(ds, snapshotProvider, emitter) + + // First run + if !exporter.Start(100 * time.Millisecond) { + t.Fatal("Failed to start exporter (first time)") + } + if !waitForUint32(&emitter.count, 2, 5*time.Second) { + exporter.Stop() + t.Fatalf("Expected at least 2 emissions in first run, got %d", emitter.count.Load()) + } + exporter.Stop() + firstRunCount := emitter.count.Load() + + // Second run + if !exporter.Start(100 * time.Millisecond) { + t.Fatal("Failed to start exporter (second time)") + } + if !waitForUint32(&emitter.count, firstRunCount+2, 5*time.Second) { + exporter.Stop() + t.Fatalf("Expected at least 2 new emissions in second run, got %d", emitter.count.Load()-firstRunCount) + } + exporter.Stop() +} + +// --------------------------------------------------------------------------- +// Integration: Multiple emitter coordination under load +// --------------------------------------------------------------------------- + +func TestMultipleEmittersReceiveSameSnapshot(t *testing.T) { + ds := mocks.NewMockDataSource() + snapshotProvider := newEmptySnapshotProvider() + + const numEmitters = 5 + emitters := make([]Emitter, numEmitters) + trackers := make([]*snapshotTrackingEmitter, numEmitters) + + for i := 0; i < numEmitters; i++ { + tracker := &snapshotTrackingEmitter{ + name: fmt.Sprintf("tracker-%d", i), + } + trackers[i] = tracker + emitters[i] = tracker + } + + exporter := NewExporter(ds, snapshotProvider, emitters...) + + if !exporter.Start(100 * time.Millisecond) { + t.Fatal("Failed to start exporter") + } + + // Wait until all emitters have received at least 1 emission + for i, tracker := range trackers { + if !waitForUint32(&tracker.count, 1, 5*time.Second) { + exporter.Stop() + t.Fatalf("Emitter %d received 0 emissions within timeout", i) + } + } + exporter.Stop() + + // All emitters should have received at least 1 emission + for i, tracker := range trackers { + count := tracker.count.Load() + if count < 1 { + t.Errorf("Emitter %d received %d emissions, expected at least 1", i, count) + } + } + + // All emitters should have the same emission count + firstCount := trackers[0].count.Load() + for i, tracker := range trackers[1:] { + if tracker.count.Load() != firstCount { + t.Errorf("Emitter %d has %d emissions, but emitter 0 has %d", i+1, tracker.count.Load(), firstCount) + } + } +} + +// --------------------------------------------------------------------------- +// Integration: Slow emitter doesn't block fast emitter +// --------------------------------------------------------------------------- + +func TestSlowEmitterDoesNotBlockFastEmitter(t *testing.T) { + ds := mocks.NewMockDataSource() + snapshotProvider := newEmptySnapshotProvider() + + slow := &delayEmitter{ + name: "slow", + delay: 50 * time.Millisecond, + } + fast := newCountingEmitter("fast") + + exporter := NewExporter(ds, snapshotProvider, slow, fast) + + if !exporter.Start(100 * time.Millisecond) { + t.Fatal("Failed to start exporter") + } + + if !waitForUint32(&slow.count, 3, 5*time.Second) { + exporter.Stop() + t.Fatalf("Slow emitter should have at least 3 emissions, got %d", slow.count.Load()) + } + exporter.Stop() + + fastCount := fast.count.Load() + if fastCount < 3 { + t.Errorf("Fast emitter should have at least 3 emissions, got %d", fastCount) + } +} + +// --------------------------------------------------------------------------- +// Test helpers +// --------------------------------------------------------------------------- + +type intermittentSnapshotProvider struct { + callCount *int32 + failEvery int32 +} + +func (i *intermittentSnapshotProvider) SnapshotOf(ds core.DataSource) (*ClusterSnapshot, error) { + count := atomic.AddInt32(i.callCount, 1) + if count%i.failEvery == 0 { + return nil, fmt.Errorf("snapshot failure at call %d", count) + } + return &ClusterSnapshot{}, nil +} + +type initTrackingEmitter struct { + name string + initialized atomic.Bool +} + +func (e *initTrackingEmitter) ID() EmitterID { return EmitterID(e.name) } +func (e *initTrackingEmitter) Init(_ *ClusterSnapshot) error { + e.initialized.Store(true) + return nil +} +func (e *initTrackingEmitter) Emit(_ context.Context, _ *ClusterSnapshot) error { return nil } + +type failingInitEmitter struct { + name string +} + +func (e *failingInitEmitter) ID() EmitterID { return EmitterID(e.name) } +func (e *failingInitEmitter) Init(_ *ClusterSnapshot) error { + return fmt.Errorf("init failed for %s", e.name) +} +func (e *failingInitEmitter) Emit(_ context.Context, _ *ClusterSnapshot) error { return nil } + +type panickingEmitter struct { + name string +} + +func (e *panickingEmitter) ID() EmitterID { return EmitterID(e.name) } +func (e *panickingEmitter) Init(_ *ClusterSnapshot) error { return nil } +func (e *panickingEmitter) Emit(_ context.Context, _ *ClusterSnapshot) error { + panic("emitter went boom!") +} + +type snapshotTrackingEmitter struct { + name string + count atomic.Uint32 + mu sync.Mutex + snaps []*ClusterSnapshot +} + +func (e *snapshotTrackingEmitter) ID() EmitterID { return EmitterID(e.name) } +func (e *snapshotTrackingEmitter) Init(_ *ClusterSnapshot) error { return nil } +func (e *snapshotTrackingEmitter) Emit(_ context.Context, snapshot *ClusterSnapshot) error { + e.count.Add(1) + e.mu.Lock() + e.snaps = append(e.snaps, snapshot) + e.mu.Unlock() + return nil +} + +type delayEmitter struct { + name string + delay time.Duration + count atomic.Uint32 +} + +func (e *delayEmitter) ID() EmitterID { return EmitterID(e.name) } +func (e *delayEmitter) Init(_ *ClusterSnapshot) error { return nil } +func (e *delayEmitter) Emit(_ context.Context, _ *ClusterSnapshot) error { + time.Sleep(e.delay) + e.count.Add(1) + return nil +} diff --git a/pkg/emitter/snapshot.go b/pkg/emitter/snapshot.go index e7e43bd8..82af6ddc 100644 --- a/pkg/emitter/snapshot.go +++ b/pkg/emitter/snapshot.go @@ -3,6 +3,7 @@ package emitter import ( "errors" "fmt" + "sync" "time" "github.com/hashicorp/go-multierror" @@ -28,8 +29,9 @@ type SnapshotProvider interface { var metricsSummaryCacheDuration time.Duration = 5 * time.Minute // ConcurrentSnapshotProvider is a struct that implements the `SnapshotProvider` interface and executes the -// snapshot generation process concurrently. +// snapshot generation process concurrently. It is safe for use from multiple goroutines. type ConcurrentSnapshotProvider struct { + mu sync.Mutex config *SnapshotConfig metricsSummary *MetricsSummary lastSnapshot time.Time @@ -61,7 +63,9 @@ func (csp *ConcurrentSnapshotProvider) SnapshotOf(ds core.DataSource) (*ClusterS // we _always_ want to set the last snapshot time upon completion, success or failure defer func() { + csp.mu.Lock() csp.lastSnapshot = now + csp.mu.Unlock() }() // Cluster Info Snapshot @@ -96,9 +100,16 @@ func (csp *ConcurrentSnapshotProvider) SnapshotOf(ds core.DataSource) (*ClusterS return err }) - err := group.Wait() - if err != nil { - return nil, fmt.Errorf("failed to generate cluster snapshot: %w", err) + merr := group.Wait() + if err := merr.ErrorOrNil(); err != nil { + log.Warnf("cluster snapshot completed with errors: %v", err) + + return &ClusterSnapshot{ + ClusterInfo: clusterInfo, + Kubernetes: k8sSnapshot, + NodeStats: nodeStats, + Metrics: metricsSnapshot, + }, err } return &ClusterSnapshot{ @@ -112,27 +123,37 @@ func (csp *ConcurrentSnapshotProvider) SnapshotOf(ds core.DataSource) (*ClusterS // temporary caching of metrics summary every 5 minutes to avoid overloading the prometheus data source until // prometheus can be replaced. func (csp *ConcurrentSnapshotProvider) cachedMetricsSummary(querier source.MetricsQuerier, now time.Time, config *SnapshotConfig) (*MetricsSummary, error) { + csp.mu.Lock() + lastSnapshot := csp.lastSnapshot + if !config.UseMetricsCache { - return snapshotMetricsSummary(querier, now, csp.lastSnapshot, config) + csp.mu.Unlock() + return snapshotMetricsSummary(querier, now, lastSnapshot, config) } // FIXME: (bolt) use a metrics summary cache duration of 5 minutes while we're using a prometheus data source. // FIXME: (bolt) this should be fine to run on a much faster frequency with a non-promethues metrics querier. if !csp.lastMetricsSummary.IsZero() && time.Since(csp.lastMetricsSummary) < metricsSummaryCacheDuration { - return csp.metricsSummary, nil + cached := csp.metricsSummary + csp.mu.Unlock() + return cached, nil } + csp.mu.Unlock() - metricsSummary, err := snapshotMetricsSummary(querier, now, csp.lastSnapshot, config) + metricsSummary, err := snapshotMetricsSummary(querier, now, lastSnapshot, config) if err != nil { - return nil, err + log.Warnf("metrics snapshot completed with errors: %v", err) } - // Note: (bolt) assuming we're not calling the SnapshotOf() in multiple goroutines [which there shouldn't be], - // Note: (bolt) there's no need to lock on cache updates. Temporary solution until we can drop prom fully. - csp.lastMetricsSummary = now - csp.metricsSummary = metricsSummary + // Cache partial results so we don't re-query within the cache window. + csp.mu.Lock() + if metricsSummary != nil { + csp.lastMetricsSummary = now + csp.metricsSummary = metricsSummary + } + csp.mu.Unlock() - return metricsSummary, nil + return metricsSummary, err } func snapshotClusterInfo(infoProvider clusters.ClusterInfoProvider) (*clusters.ClusterInfo, error) { @@ -235,10 +256,11 @@ func snapshotMetricsSummary(querier source.MetricsQuerier, now time.Time, lastSn snapshotErrors = append(snapshotErrors, errs...) } - // collect errors and return all errors joined + // return partial data with any errors joined, so callers have visibility into failures + // while still receiving whatever data was successfully collected var err error if len(snapshotErrors) > 0 { - return nil, errors.Join(snapshotErrors...) + err = errors.Join(snapshotErrors...) } return &MetricsSummary{ @@ -269,10 +291,11 @@ func snapshotWindowedMetrics( snapshot, err := snapshotMetrics(querier, *window.Start(), *window.End()) if err != nil { errors = append(errors, fmt.Errorf("failed to generate metrics snapshot for resolution: %d minutes: %w", int(resolution.Minutes()), err)) - continue } - snapshots = append(snapshots, snapshot) + if snapshot != nil { + snapshots = append(snapshots, snapshot) + } } return snapshots, errors @@ -510,8 +533,13 @@ func snapshotMetrics(mq source.MetricsQuerier, start, end time.Time) (*MetricsSn resourceQuotaStatusUsedRamLimitAvg, _ := resourceQuotaStatusUsedRamLimitAvgFuture.Await() resourceQuotaStatusUsedRamLimitMax, _ := resourceQuotaStatusUsedRamLimitMaxFuture.Await() + var metricsErr error if grp.HasErrors() { - return nil, grp.Error() + queryErrors := grp.Errors() + for _, qErr := range queryErrors { + log.Warnf("metrics query failure: %v", qErr) + } + metricsErr = fmt.Errorf("%d metrics query failures: %w", len(queryErrors), grp.Error()) } return &MetricsSnapshot{ @@ -615,5 +643,5 @@ func snapshotMetrics(mq source.MetricsQuerier, start, end time.Time) (*MetricsSn ResourceQuotaStatusUsedCPULimitMax: resourceQuotaStatusUsedCpuLimitMax, ResourceQuotaStatusUsedRAMLimitAvg: resourceQuotaStatusUsedRamLimitAvg, ResourceQuotaStatusUsedRAMLimitMax: resourceQuotaStatusUsedRamLimitMax, - }, nil + }, metricsErr } diff --git a/pkg/emitter/snapshot_provider_test.go b/pkg/emitter/snapshot_provider_test.go new file mode 100644 index 00000000..92d1f69e --- /dev/null +++ b/pkg/emitter/snapshot_provider_test.go @@ -0,0 +1,282 @@ +package emitter + +import ( + "fmt" + "sync" + "testing" + "time" + + "github.com/ibm/finops-agent/internal/mocks" + "github.com/ibm/finops-agent/pkg/core" + "github.com/ibm/finops-agent/pkg/nodes" + stats "k8s.io/kubelet/pkg/apis/stats/v1alpha1" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestSnapshotWithPopulatedClusterCache(t *testing.T) { + ds := mocks.NewMockDataSource() + + ds.ClusterCache.Nodes = []*v1.Node{ + {ObjectMeta: metav1.ObjectMeta{Name: "node-1"}}, + {ObjectMeta: metav1.ObjectMeta{Name: "node-2"}}, + } + ds.ClusterCache.Pods = []*v1.Pod{ + {ObjectMeta: metav1.ObjectMeta{Name: "pod-1", Namespace: "default"}}, + {ObjectMeta: metav1.ObjectMeta{Name: "pod-2", Namespace: "kube-system"}}, + {ObjectMeta: metav1.ObjectMeta{Name: "pod-3", Namespace: "default"}}, + } + ds.ClusterCache.Namespaces = []*v1.Namespace{ + {ObjectMeta: metav1.ObjectMeta{Name: "default"}}, + {ObjectMeta: metav1.ObjectMeta{Name: "kube-system"}}, + } + ds.ClusterCache.Services = []*v1.Service{ + {ObjectMeta: metav1.ObjectMeta{Name: "svc-1", Namespace: "default"}}, + } + + config := DefaultSnapshotConfig() + provider := NewConcurrentSnapshotProvider(config) + + snapshot, err := provider.SnapshotOf(ds) + if err != nil { + t.Fatalf("Failed to create snapshot: %v", err) + } + + if len(snapshot.Kubernetes.Nodes) != 2 { + t.Errorf("Expected 2 nodes, got %d", len(snapshot.Kubernetes.Nodes)) + } + if len(snapshot.Kubernetes.Pods) != 3 { + t.Errorf("Expected 3 pods, got %d", len(snapshot.Kubernetes.Pods)) + } + if len(snapshot.Kubernetes.Namespaces) != 2 { + t.Errorf("Expected 2 namespaces, got %d", len(snapshot.Kubernetes.Namespaces)) + } + if len(snapshot.Kubernetes.Services) != 1 { + t.Errorf("Expected 1 service, got %d", len(snapshot.Kubernetes.Services)) + } + if snapshot.ClusterInfo == nil { + t.Fatal("Expected ClusterInfo to be populated") + } + if snapshot.ClusterInfo.ID != "mock-cluster-id" { + t.Errorf("Expected cluster ID 'mock-cluster-id', got '%s'", snapshot.ClusterInfo.ID) + } + if snapshot.Metrics == nil { + t.Fatal("Expected Metrics to be non-nil") + } +} + +func TestSnapshotWithSelectiveKubernetesResources(t *testing.T) { + ds := mocks.NewMockDataSource() + ds.ClusterCache.Nodes = []*v1.Node{ + {ObjectMeta: metav1.ObjectMeta{Name: "node-1"}}, + } + ds.ClusterCache.Pods = []*v1.Pod{ + {ObjectMeta: metav1.ObjectMeta{Name: "pod-1", Namespace: "default"}}, + } + + config := DefaultSnapshotConfig() + config.KubernetesSnapshot = NewKubernetesSnapshotConfig() + config.KubernetesSnapshot.Nodes = true + config.KubernetesSnapshot.Pods = false + + provider := NewConcurrentSnapshotProvider(config) + snapshot, err := provider.SnapshotOf(ds) + if err != nil { + t.Fatalf("Failed to create snapshot: %v", err) + } + + if len(snapshot.Kubernetes.Nodes) != 1 { + t.Errorf("Expected 1 node, got %d", len(snapshot.Kubernetes.Nodes)) + } + if len(snapshot.Kubernetes.Pods) != 0 { + t.Errorf("Expected 0 pods (disabled), got %d", len(snapshot.Kubernetes.Pods)) + } + if ds.ClusterCache.Calls["GetAllPods"] != 0 { + t.Errorf("Expected GetAllPods to not be called, but it was called %d times", ds.ClusterCache.Calls["GetAllPods"]) + } + if ds.ClusterCache.Calls["GetAllNodes"] != 1 { + t.Errorf("Expected GetAllNodes to be called once, got %d", ds.ClusterCache.Calls["GetAllNodes"]) + } +} + +func TestSnapshotNodeStatsPartialFailure(t *testing.T) { + ds := &partialFailureDataSource{ + MockDataSource: mocks.NewMockDataSource(), + statsClient: &partialFailureStatsClient{ + successNodes: []*stats.Summary{ + {Node: stats.NodeStats{NodeName: "healthy-node"}}, + }, + }, + } + + config := DefaultSnapshotConfig() + provider := NewConcurrentSnapshotProvider(config) + + snapshot, err := provider.SnapshotOf(ds) + if err != nil { + t.Fatalf("Snapshot should succeed with partial data: %v", err) + } + + if snapshot.NodeStats == nil { + t.Fatal("Expected NodeStats to be non-nil") + } + if len(snapshot.NodeStats.Stats) != 1 { + t.Errorf("Expected 1 node stat, got %d", len(snapshot.NodeStats.Stats)) + } + if snapshot.NodeStats.Stats[0].Node.NodeName != "healthy-node" { + t.Errorf("Expected 'healthy-node', got '%s'", snapshot.NodeStats.Stats[0].Node.NodeName) + } +} + +func TestSnapshotNodeStatsCompleteFailure(t *testing.T) { + ds := &partialFailureDataSource{ + MockDataSource: mocks.NewMockDataSource(), + statsClient: &completeFailureStatsClient{}, + } + + config := DefaultSnapshotConfig() + provider := NewConcurrentSnapshotProvider(config) + + _, err := provider.SnapshotOf(ds) + if err == nil { + t.Fatal("Snapshot should fail when node stats completely fail") + } +} + +func TestMetricsCachePopulatesAndExpires(t *testing.T) { + previousCacheDuration := metricsSummaryCacheDuration + metricsSummaryCacheDuration = 200 * time.Millisecond + defer func() { + metricsSummaryCacheDuration = previousCacheDuration + }() + + ds := mocks.NewMockDataSource() + config := DefaultSnapshotConfig() + config.UseMetricsCache = true + + provider := NewConcurrentSnapshotProvider(config) + + snap1, err := provider.SnapshotOf(ds) + if err != nil { + t.Fatalf("First snapshot failed: %v", err) + } + + snap2, err := provider.SnapshotOf(ds) + if err != nil { + t.Fatalf("Second snapshot failed: %v", err) + } + + if snap1.Metrics != snap2.Metrics { + t.Fatal("Expected cached metrics (same pointer) for second snapshot") + } + + time.Sleep(300 * time.Millisecond) + + snap3, err := provider.SnapshotOf(ds) + if err != nil { + t.Fatalf("Third snapshot failed: %v", err) + } + + if snap1.Metrics == snap3.Metrics { + t.Fatal("Expected new metrics (different pointer) after cache expiry") + } +} + +func TestMetricsCacheDisabled(t *testing.T) { + ds := mocks.NewMockDataSource() + config := DefaultSnapshotConfig() + config.UseMetricsCache = false + + provider := NewConcurrentSnapshotProvider(config) + + snap1, err := provider.SnapshotOf(ds) + if err != nil { + t.Fatalf("First snapshot failed: %v", err) + } + + snap2, err := provider.SnapshotOf(ds) + if err != nil { + t.Fatalf("Second snapshot failed: %v", err) + } + + if snap1.Metrics == snap2.Metrics { + t.Fatal("Expected different metrics pointers when cache is disabled") + } +} + +func TestConcurrentSnapshotProviderUnderLoad(t *testing.T) { + config := DefaultSnapshotConfig() + provider := NewConcurrentSnapshotProvider(config) + + const concurrency = 10 + var wg sync.WaitGroup + errors := make(chan error, concurrency) + snapshots := make(chan *ClusterSnapshot, concurrency) + + for i := 0; i < concurrency; i++ { + wg.Add(1) + go func() { + defer wg.Done() + ds := mocks.NewMockDataSource() + ds.ClusterCache.Nodes = []*v1.Node{ + {ObjectMeta: metav1.ObjectMeta{Name: "node-1"}}, + } + snap, err := provider.SnapshotOf(ds) + if err != nil { + errors <- err + return + } + snapshots <- snap + }() + } + + wg.Wait() + close(errors) + close(snapshots) + + for err := range errors { + t.Errorf("Concurrent snapshot failed: %v", err) + } + + count := 0 + for snap := range snapshots { + count++ + if snap.Kubernetes == nil { + t.Error("Expected Kubernetes snapshot to be populated") + } + } + + if count != concurrency { + t.Errorf("Expected %d successful snapshots, got %d", concurrency, count) + } +} + +// --------------------------------------------------------------------------- +// Test helpers +// --------------------------------------------------------------------------- + +type partialFailureDataSource struct { + *mocks.MockDataSource + statsClient nodes.StatSummaryClient +} + +func (d *partialFailureDataSource) StatsSummary() nodes.StatSummaryClient { + return d.statsClient +} + +type partialFailureStatsClient struct { + successNodes []*stats.Summary +} + +func (p *partialFailureStatsClient) GetNodeData() ([]*stats.Summary, error) { + return p.successNodes, fmt.Errorf("some nodes failed: node-bad-1, node-bad-2") +} + +type completeFailureStatsClient struct{} + +func (c *completeFailureStatsClient) GetNodeData() ([]*stats.Summary, error) { + return nil, fmt.Errorf("all nodes unreachable") +} + +var _ core.DataSource = (*mocks.MockDataSource)(nil) diff --git a/pkg/emitter/snapshot_windows_test.go b/pkg/emitter/snapshot_windows_test.go new file mode 100644 index 00000000..e4da5d0e --- /dev/null +++ b/pkg/emitter/snapshot_windows_test.go @@ -0,0 +1,148 @@ +package emitter + +import ( + "testing" + "time" + + "github.com/ibm/finops-agent/internal/mocks" +) + +func TestSnapshotWindowsForZeroLastSnapshot(t *testing.T) { + now := time.Date(2025, 1, 15, 9, 5, 30, 0, time.UTC) + windows := snapshotWindowsFor(now, time.Time{}, 10*time.Minute) + + if len(windows) != 1 { + t.Fatalf("Expected 1 window for zero lastSnapshot, got %d", len(windows)) + } + + start := windows[0].Start() + end := windows[0].End() + if !start.Equal(time.Date(2025, 1, 15, 9, 0, 0, 0, time.UTC)) { + t.Errorf("Unexpected window start: %v", *start) + } + if !end.Equal(time.Date(2025, 1, 15, 9, 10, 0, 0, time.UTC)) { + t.Errorf("Unexpected window end: %v", *end) + } +} + +func TestSnapshotWindowsForSameWindow(t *testing.T) { + now := time.Date(2025, 1, 15, 9, 7, 30, 0, time.UTC) + last := time.Date(2025, 1, 15, 9, 3, 0, 0, time.UTC) + windows := snapshotWindowsFor(now, last, 10*time.Minute) + + if len(windows) != 1 { + t.Fatalf("Expected 1 window (same boundary), got %d", len(windows)) + } +} + +func TestSnapshotWindowsForBoundaryCrossing(t *testing.T) { + now := time.Date(2025, 1, 15, 9, 11, 0, 0, time.UTC) + last := time.Date(2025, 1, 15, 9, 9, 0, 0, time.UTC) + windows := snapshotWindowsFor(now, last, 10*time.Minute) + + if len(windows) != 2 { + t.Fatalf("Expected 2 windows (boundary crossing), got %d", len(windows)) + } + + s0 := windows[0].Start() + e0 := windows[0].End() + if !s0.Equal(time.Date(2025, 1, 15, 9, 0, 0, 0, time.UTC)) { + t.Errorf("Window[0] unexpected start: %v", *s0) + } + if !e0.Equal(time.Date(2025, 1, 15, 9, 10, 0, 0, time.UTC)) { + t.Errorf("Window[0] unexpected end: %v", *e0) + } + + s1 := windows[1].Start() + e1 := windows[1].End() + if !s1.Equal(time.Date(2025, 1, 15, 9, 10, 0, 0, time.UTC)) { + t.Errorf("Window[1] unexpected start: %v", *s1) + } + if !e1.Equal(time.Date(2025, 1, 15, 9, 20, 0, 0, time.UTC)) { + t.Errorf("Window[1] unexpected end: %v", *e1) + } +} + +func TestSnapshotWindowsHourlyBoundaryCrossing(t *testing.T) { + now := time.Date(2025, 1, 15, 16, 0, 30, 0, time.UTC) + last := time.Date(2025, 1, 15, 15, 59, 0, 0, time.UTC) + windows := snapshotWindowsFor(now, last, time.Hour) + + if len(windows) != 2 { + t.Fatalf("Expected 2 windows (hourly boundary crossing), got %d", len(windows)) + } +} + +func TestSnapshotWindowsDailyBoundaryCrossing(t *testing.T) { + now := time.Date(2025, 1, 16, 0, 0, 30, 0, time.UTC) + last := time.Date(2025, 1, 15, 23, 59, 0, 0, time.UTC) + windows := snapshotWindowsFor(now, last, 24*time.Hour) + + if len(windows) != 2 { + t.Fatalf("Expected 2 windows (daily boundary crossing), got %d", len(windows)) + } +} + +func TestSnapshotMinutelyMetricsEnabled(t *testing.T) { + t.Setenv("MINUTE_METRICS_ENABLED", "true") + + ds := mocks.NewMockDataSource() + config := NewSnapshotConfigFromEnv() + + provider := NewConcurrentSnapshotProvider(config) + snapshot, err := provider.SnapshotOf(ds) + if err != nil { + t.Fatalf("Snapshot failed: %v", err) + } + + if snapshot.Metrics.Minutely == nil { + t.Error("Expected minutely metrics to be populated when enabled") + } +} + +func TestSnapshotMinutelyMetricsDisabled(t *testing.T) { + ds := mocks.NewMockDataSource() + config := DefaultSnapshotConfig() + config.MinutelyMetricsEnabled = false + + provider := NewConcurrentSnapshotProvider(config) + snapshot, err := provider.SnapshotOf(ds) + if err != nil { + t.Fatalf("Snapshot failed: %v", err) + } + + if len(snapshot.Metrics.Minutely) != 0 { + t.Error("Expected minutely metrics to be empty when disabled") + } +} + +func TestSequentialSnapshotsTrackWindows(t *testing.T) { + ds := mocks.NewMockDataSource() + bender := newTimeBender() + + config := DefaultSnapshotConfig() + config.MinutelyMetricsEnabled = true + config.Now = bender.now + + provider := NewConcurrentSnapshotProvider(config) + + bender.current = time.Date(2025, 6, 1, 15, 5, 0, 0, time.UTC) + snap1, err := provider.SnapshotOf(ds) + if err != nil { + t.Fatalf("First snapshot failed: %v", err) + } + + if len(snap1.Metrics.Hourly) != 1 { + t.Errorf("First snapshot: expected 1 hourly window, got %d", len(snap1.Metrics.Hourly)) + } + + bender.current = time.Date(2025, 6, 1, 16, 1, 0, 0, time.UTC) + snap2, err := provider.SnapshotOf(ds) + if err != nil { + t.Fatalf("Second snapshot failed: %v", err) + } + + if len(snap2.Metrics.Hourly) != 2 { + t.Errorf("Second snapshot: expected 2 hourly windows, got %d", len(snap2.Metrics.Hourly)) + } +} diff --git a/pkg/nodes/collection_integration_test.go b/pkg/nodes/collection_integration_test.go new file mode 100644 index 00000000..2e89e3cf --- /dev/null +++ b/pkg/nodes/collection_integration_test.go @@ -0,0 +1,368 @@ +package nodes + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "sync/atomic" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" + stats "k8s.io/kubelet/pkg/apis/stats/v1alpha1" +) + +var _ = Describe("Node data collection with HTTP servers", func() { + var ( + tempBearerFile string + summaryData []byte + ) + + BeforeEach(func() { + file, err := os.CreateTemp("", "bearer-token-*") + Expect(err).ToNot(HaveOccurred()) + _, err = file.WriteString("test-token") + Expect(err).ToNot(HaveOccurred()) + file.Close() + tempBearerFile = file.Name() + + summary := stats.Summary{ + Node: stats.NodeStats{ + NodeName: "test-node", + }, + } + summaryData, err = json.Marshal(summary) + Expect(err).ToNot(HaveOccurred()) + }) + + AfterEach(func() { + os.Remove(tempBearerFile) + }) + + Context("with an IPv4 HTTP test server", func() { + It("collects node stats from a direct node endpoint", func() { + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + Expect(r.URL.Path).To(Equal("/stats/summary")) + w.Header().Set("Content-Type", "application/json") + w.Write(summaryData) + })) + defer server.Close() + + host, port := hostPort(server.URL) + node := newTestNode("ipv4-node", host, port) + cache := &singleNodeCache{nodes: []*v1.Node{node}} + + client := NewClient(server.Client(), 0) + config := NodeClientConfig{ + ClusterName: "test-cluster", + ConcurrentPollers: 2, + DirectNodeClient: client, + InClusterClient: client, + } + + nssc := &NodeStatsSummaryClient{ + config: config, + cache: cache, + endpoint: "stats/summary", + clusterHostUrl: server.URL, + bearerTokenFile: tempBearerFile, + } + + data, err := nssc.GetNodeData() + Expect(err).ToNot(HaveOccurred()) + Expect(data).To(HaveLen(1)) + Expect(data[0].Node.NodeName).To(Equal("test-node")) + }) + + It("collects node stats via proxy endpoint", func() { + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + Expect(r.URL.Path).To(Equal("/api/v1/nodes/proxy-node/proxy/stats/summary")) + w.Header().Set("Content-Type", "application/json") + w.Write(summaryData) + })) + defer server.Close() + + node := newTestNode("proxy-node", "192.168.1.100", 10250) + cache := &singleNodeCache{nodes: []*v1.Node{node}} + + client := NewClient(server.Client(), 0) + config := NodeClientConfig{ + ClusterName: "test-cluster", + ConcurrentPollers: 2, + DirectNodeClient: Client{client: &failingHTTPClient{}, retries: 0}, + InClusterClient: client, + ProxyConfig: NodeClientProxyConfig{ForceKubeProxy: true}, + } + + nssc := &NodeStatsSummaryClient{ + config: config, + cache: cache, + endpoint: "stats/summary", + clusterHostUrl: server.URL, + bearerTokenFile: tempBearerFile, + } + + data, err := nssc.GetNodeData() + Expect(err).ToNot(HaveOccurred()) + Expect(data).To(HaveLen(1)) + Expect(data[0].Node.NodeName).To(Equal("test-node")) + }) + }) + + Context("with a local proxy", func() { + It("uses the local proxy URL instead of the cluster host", func() { + var requestCount int32 + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&requestCount, 1) + w.Header().Set("Content-Type", "application/json") + w.Write(summaryData) + })) + defer server.Close() + + node := newTestNode("local-proxy-node", "10.0.0.5", 10250) + cache := &singleNodeCache{nodes: []*v1.Node{node}} + + client := NewClient(server.Client(), 0) + config := NodeClientConfig{ + ClusterName: "test-cluster", + ConcurrentPollers: 2, + DirectNodeClient: Client{client: &failingHTTPClient{}, retries: 0}, + InClusterClient: client, + ProxyConfig: NodeClientProxyConfig{ + LocalProxy: server.URL, + ForceKubeProxy: true, + }, + } + + nssc := &NodeStatsSummaryClient{ + config: config, + cache: cache, + endpoint: "stats/summary", + clusterHostUrl: "https://should-not-be-used:6443", + bearerTokenFile: tempBearerFile, + } + + data, err := nssc.GetNodeData() + Expect(err).ToNot(HaveOccurred()) + Expect(data).To(HaveLen(1)) + Expect(atomic.LoadInt32(&requestCount)).To(BeNumerically(">=", 1)) + }) + }) + + Context("concurrent node collection", func() { + It("collects data from multiple nodes concurrently via proxy", func() { + var requestCount int32 + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&requestCount, 1) + time.Sleep(10 * time.Millisecond) + w.Header().Set("Content-Type", "application/json") + w.Write(summaryData) + })) + defer server.Close() + + testNodes := make([]*v1.Node, 5) + for i := range 5 { + testNodes[i] = newTestNode( + fmt.Sprintf("node-%d", i), + fmt.Sprintf("10.0.0.%d", i+1), + 10250, + ) + } + cache := &singleNodeCache{nodes: testNodes} + + client := NewClient(server.Client(), 0) + config := NodeClientConfig{ + ClusterName: "test-cluster", + ConcurrentPollers: 3, + InClusterClient: client, + ProxyConfig: NodeClientProxyConfig{ForceKubeProxy: true}, + } + + nssc := &NodeStatsSummaryClient{ + config: config, + cache: cache, + endpoint: "stats/summary", + clusterHostUrl: server.URL, + bearerTokenFile: tempBearerFile, + } + + data, err := nssc.GetNodeData() + Expect(err).ToNot(HaveOccurred()) + Expect(data).To(HaveLen(5)) + Expect(atomic.LoadInt32(&requestCount)).To(BeNumerically("==", 5)) + }) + }) + + Context("partial failure handling", func() { + It("returns partial results and errors when some nodes fail", func() { + callCount := int32(0) + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + count := atomic.AddInt32(&callCount, 1) + if count%2 == 0 { + w.WriteHeader(http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json") + w.Write(summaryData) + })) + defer server.Close() + + testNodes := make([]*v1.Node, 4) + for i := range 4 { + testNodes[i] = newTestNode( + fmt.Sprintf("node-%d", i), + fmt.Sprintf("10.0.0.%d", i+1), + 10250, + ) + } + cache := &singleNodeCache{nodes: testNodes} + + client := NewClient(server.Client(), 0) + config := NodeClientConfig{ + ClusterName: "test-cluster", + ConcurrentPollers: 4, + InClusterClient: client, + ProxyConfig: NodeClientProxyConfig{ForceKubeProxy: true}, + } + + nssc := &NodeStatsSummaryClient{ + config: config, + cache: cache, + endpoint: "stats/summary", + clusterHostUrl: server.URL, + bearerTokenFile: tempBearerFile, + } + + data, err := nssc.GetNodeData() + Expect(data).To(HaveLen(2)) + Expect(err).To(HaveOccurred()) + Expect(err).To(BeUnwrappableErrorWith(HaveLen(2))) + }) + }) + + Context("Fargate node detection", func() { + It("skips direct connection for Fargate nodes and uses proxy", func() { + var proxyHit bool + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + Expect(r.URL.Path).To(ContainSubstring("/api/v1/nodes/fargate-node/proxy/")) + proxyHit = true + w.Header().Set("Content-Type", "application/json") + w.Write(summaryData) + })) + defer server.Close() + + node := newTestNode("fargate-node", "10.0.0.1", 10250) + node.Labels["eks.amazonaws.com/compute-type"] = "fargate" + cache := &singleNodeCache{nodes: []*v1.Node{node}} + + client := NewClient(server.Client(), 0) + config := NodeClientConfig{ + ClusterName: "test-cluster", + ConcurrentPollers: 2, + DirectNodeClient: client, + InClusterClient: client, + } + + nssc := &NodeStatsSummaryClient{ + config: config, + cache: cache, + endpoint: "stats/summary", + clusterHostUrl: server.URL, + bearerTokenFile: tempBearerFile, + } + + data, err := nssc.GetNodeData() + Expect(err).ToNot(HaveOccurred()) + Expect(data).To(HaveLen(1)) + Expect(proxyHit).To(BeTrue(), "Proxy endpoint should be used for Fargate nodes") + }) + }) + + Context("bearer token handling", func() { + It("fails when bearer token file does not exist", func() { + node := newTestNode("token-node", "10.0.0.1", 10250) + cache := &singleNodeCache{nodes: []*v1.Node{node}} + + config := NodeClientConfig{ + ClusterName: "test-cluster", + ConcurrentPollers: 1, + DirectNodeClient: NewClient(&http.Client{}, 0), + InClusterClient: NewClient(&http.Client{}, 0), + } + + nssc := &NodeStatsSummaryClient{ + config: config, + cache: cache, + endpoint: "stats/summary", + clusterHostUrl: "https://localhost:6443", + bearerTokenFile: "/nonexistent/path/token", + } + + data, err := nssc.GetNodeData() + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("could not read bearer token")) + Expect(data).To(BeNil()) + }) + + It("sends bearer token in Authorization header", func() { + var receivedAuth string + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + receivedAuth = r.Header.Get("Authorization") + w.Header().Set("Content-Type", "application/json") + w.Write(summaryData) + })) + defer server.Close() + + node := newTestNode("auth-node", "10.0.0.1", 10250) + cache := &singleNodeCache{nodes: []*v1.Node{node}} + + client := NewClient(server.Client(), 0) + config := NodeClientConfig{ + ClusterName: "test-cluster", + ConcurrentPollers: 1, + DirectNodeClient: Client{client: &failingHTTPClient{}, retries: 0}, + InClusterClient: client, + ProxyConfig: NodeClientProxyConfig{ForceKubeProxy: true}, + } + + nssc := &NodeStatsSummaryClient{ + config: config, + cache: cache, + endpoint: "stats/summary", + clusterHostUrl: server.URL, + bearerTokenFile: tempBearerFile, + } + + data, err := nssc.GetNodeData() + Expect(err).ToNot(HaveOccurred()) + Expect(data).To(HaveLen(1)) + Expect(receivedAuth).To(Equal("bearer test-token")) + }) + }) + + Context("empty cluster", func() { + It("returns nil when no ready nodes exist", func() { + cache := &singleNodeCache{nodes: []*v1.Node{}} + + config := NodeClientConfig{ + ClusterName: "test-cluster", + ConcurrentPollers: 1, + } + + nssc := &NodeStatsSummaryClient{ + config: config, + cache: cache, + endpoint: "stats/summary", + clusterHostUrl: "https://localhost:6443", + bearerTokenFile: tempBearerFile, + } + + data, err := nssc.GetNodeData() + Expect(err).To(BeNil()) + Expect(data).To(BeNil()) + }) + }) +}) diff --git a/pkg/nodes/collection_test.go b/pkg/nodes/collection_test.go index 01867691..4ebdd787 100644 --- a/pkg/nodes/collection_test.go +++ b/pkg/nodes/collection_test.go @@ -60,7 +60,7 @@ var _ = Describe("Raw node data", func() { Expect(err).To(HaveOccurred()) Expect(err).To( BeUnwrappableErrorWith( - HaveLen(4), + HaveLen(5), HaveEach(MatchError(ContainSubstring("error retrieving node data")))), ) Expect(len(data)).To(BeNumerically("==", 0)) @@ -79,7 +79,7 @@ var _ = Describe("Raw node data", func() { } nodes := getReadyNodes(mockNcs.cache) - Expect(len(nodes)).To(BeNumerically("==", 4)) + Expect(len(nodes)).To(BeNumerically("==", 5)) // Note: Nodes.jsonl isn't in any order Expect(nodes[0].ObjectMeta.Name).Should(Equal("nodename4")) }) diff --git a/pkg/nodes/endpoint_test.go b/pkg/nodes/endpoint_test.go new file mode 100644 index 00000000..52c92827 --- /dev/null +++ b/pkg/nodes/endpoint_test.go @@ -0,0 +1,104 @@ +package nodes + +import ( + "net/http" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var _ = Describe("Endpoint formatting", func() { + Context("directNode.formatEndpoint", func() { + It("formats an IPv4 address correctly", func() { + d := directNode{ip: "10.0.0.1", port: 10250} + url := d.formatEndpoint("stats/summary") + Expect(url).To(Equal("https://10.0.0.1:10250/stats/summary")) + }) + + It("formats an IPv6 address with brackets per RFC 3986", func() { + d := directNode{ip: "fd00::1", port: 10250} + url := d.formatEndpoint("stats/summary") + Expect(url).To(Equal("https://[fd00::1]:10250/stats/summary")) + }) + + It("formats a full IPv6 address with brackets", func() { + d := directNode{ip: "2001:0db8:85a3:0000:0000:8a2e:0370:7334", port: 10250} + url := d.formatEndpoint("stats/summary") + Expect(url).To(Equal("https://[2001:0db8:85a3:0000:0000:8a2e:0370:7334]:10250/stats/summary")) + }) + }) + + Context("proxyAPI.formatEndpoint", func() { + It("formats the proxy URL correctly", func() { + p := proxyAPI{clusterHostURL: "https://kubernetes.default", nodeName: "node-1"} + url := p.formatEndpoint("stats/summary") + Expect(url).To(Equal("https://kubernetes.default/api/v1/nodes/node-1/proxy/stats/summary")) + }) + }) +}) + +var _ = Describe("NodeAddress", func() { + It("returns IP and port for a node with InternalIP", func() { + node := newTestNode("addr-node", "192.168.1.50", 10250) + ip, port, err := NodeAddress(node) + Expect(err).ToNot(HaveOccurred()) + Expect(ip).To(Equal("192.168.1.50")) + Expect(port).To(Equal(int32(10250))) + }) + + It("returns error for a node without InternalIP", func() { + node := &v1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: "no-ip-node"}, + Status: v1.NodeStatus{ + Addresses: []v1.NodeAddress{ + {Type: v1.NodeExternalIP, Address: "1.2.3.4"}, + }, + }, + } + _, _, err := NodeAddress(node) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("could not find internal IP")) + }) +}) + +var _ = Describe("connectionOptions", func() { + It("returns both direct and proxy methods by default", func() { + node := *newTestNode("test-node", "10.0.0.1", 10250) + config := NodeClientConfig{ + DirectNodeClient: NewClient(&http.Client{}, 0), + InClusterClient: NewClient(&http.Client{}, 0), + } + + nd := nodeFetchData{nodeName: "test-node", ClusterHostURL: "https://k8s:6443"} + methods := config.connectionOptions(node, nd) + Expect(methods).To(HaveLen(2)) + }) + + It("returns only proxy when ForceKubeProxy is true", func() { + node := *newTestNode("test-node", "10.0.0.1", 10250) + config := NodeClientConfig{ + DirectNodeClient: NewClient(&http.Client{}, 0), + InClusterClient: NewClient(&http.Client{}, 0), + ProxyConfig: NodeClientProxyConfig{ForceKubeProxy: true}, + } + + nd := nodeFetchData{nodeName: "test-node", ClusterHostURL: "https://k8s:6443"} + methods := config.connectionOptions(node, nd) + Expect(methods).To(HaveLen(1)) + }) + + It("returns only proxy for Fargate nodes", func() { + node := *newTestNode("fargate", "10.0.0.1", 10250) + node.Labels["eks.amazonaws.com/compute-type"] = "fargate" + config := NodeClientConfig{ + DirectNodeClient: NewClient(&http.Client{}, 0), + InClusterClient: NewClient(&http.Client{}, 0), + } + + nd := nodeFetchData{nodeName: "fargate", ClusterHostURL: "https://k8s:6443"} + methods := config.connectionOptions(node, nd) + Expect(methods).To(HaveLen(1)) + }) +}) diff --git a/pkg/nodes/helpers_test.go b/pkg/nodes/helpers_test.go new file mode 100644 index 00000000..b0201636 --- /dev/null +++ b/pkg/nodes/helpers_test.go @@ -0,0 +1,79 @@ +package nodes + +import ( + "fmt" + "net" + "net/http" + "strings" + + "github.com/ibm/finops-agent/pkg/cluster" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + stats "k8s.io/kubelet/pkg/apis/stats/v1alpha1" +) + +// singleNodeCache is a minimal ClusterCache returning a fixed list of nodes. +type singleNodeCache struct { + cluster.ClusterCache + nodes []*v1.Node +} + +func (s *singleNodeCache) GetAllNodes() []*v1.Node { + return s.nodes +} + +// failingHTTPClient always returns an error. +type failingHTTPClient struct{} + +func (f *failingHTTPClient) Do(_ *http.Request) (*http.Response, error) { + return nil, fmt.Errorf("connection refused") +} + +// mockStatsSummaryClient allows injecting a custom function for GetNodeData. +type mockStatsSummaryClient struct { + fn func() ([]*stats.Summary, error) +} + +func (m *mockStatsSummaryClient) GetNodeData() ([]*stats.Summary, error) { + return m.fn() +} + +func newTestNode(name, ip string, port int32) *v1.Node { + return &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Labels: map[string]string{}, + }, + Status: v1.NodeStatus{ + Conditions: []v1.NodeCondition{ + {Type: v1.NodeReady, Status: v1.ConditionTrue}, + }, + Addresses: []v1.NodeAddress{ + {Type: v1.NodeInternalIP, Address: ip}, + }, + DaemonEndpoints: v1.NodeDaemonEndpoints{ + KubeletEndpoint: v1.DaemonEndpoint{Port: port}, + }, + }, + } +} + +// hostPort extracts host and port from a URL like "https://127.0.0.1:12345". +func hostPort(url string) (string, int32) { + addr := strings.TrimPrefix(url, "https://") + addr = strings.TrimPrefix(addr, "http://") + host, portStr, err := net.SplitHostPort(addr) + if err != nil { + panic(fmt.Sprintf("hostPort: failed to split host and port from %q: %v", url, err)) + } + if host == "" || portStr == "" { + panic(fmt.Sprintf("hostPort: invalid host/port extracted from %q", url)) + } + + var port int32 + n, err := fmt.Sscanf(portStr, "%d", &port) + if err != nil || n != 1 { + panic(fmt.Sprintf("hostPort: failed to parse port %q from %q: %v", portStr, url, err)) + } + return host, port +} diff --git a/pkg/nodes/nodestats_integration_test.go b/pkg/nodes/nodestats_integration_test.go new file mode 100644 index 00000000..0e48aab8 --- /dev/null +++ b/pkg/nodes/nodestats_integration_test.go @@ -0,0 +1,114 @@ +package nodes + +import ( + "fmt" + "sync/atomic" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + stats "k8s.io/kubelet/pkg/apis/stats/v1alpha1" +) + +var _ = Describe("NodeStatsSummaryProvider integration", func() { + It("collects and caches node stats on an interval", func() { + callCount := int32(0) + mockClient := &mockStatsSummaryClient{ + fn: func() ([]*stats.Summary, error) { + count := atomic.AddInt32(&callCount, 1) + return []*stats.Summary{ + {Node: stats.NodeStats{NodeName: fmt.Sprintf("node-call-%d", count)}}, + }, nil + }, + } + + provider := NewNodeStatsSummaryProvider(mockClient) + started := provider.Start(100 * time.Millisecond) + Expect(started).To(BeTrue()) + + // Initial synchronous call happens immediately + data, err := provider.GetNodeData() + Expect(err).ToNot(HaveOccurred()) + Expect(data).To(HaveLen(1)) + Expect(data[0].Node.NodeName).To(Equal("node-call-1")) + + // Poll until background refresh has fired at least once more + Eventually(func() int32 { + return atomic.LoadInt32(&callCount) + }, "2s", "50ms").Should(BeNumerically(">=", 2)) + + data, err = provider.GetNodeData() + Expect(err).ToNot(HaveOccurred()) + Expect(data).To(HaveLen(1)) + + provider.Stop() + + // After stop, data should still be available (cached) + data, err = provider.GetNodeData() + Expect(err).ToNot(HaveOccurred()) + Expect(data).To(HaveLen(1)) + }) + + It("does not overwrite data on total failure", func() { + callCount := int32(0) + mockClient := &mockStatsSummaryClient{ + fn: func() ([]*stats.Summary, error) { + count := atomic.AddInt32(&callCount, 1) + if count == 1 { + return []*stats.Summary{ + {Node: stats.NodeStats{NodeName: "good-node"}}, + }, nil + } + return nil, fmt.Errorf("all nodes unreachable") + }, + } + + provider := NewNodeStatsSummaryProvider(mockClient) + started := provider.Start(50 * time.Millisecond) + Expect(started).To(BeTrue()) + + // Poll until the failing call has been attempted, then verify cached data survived + Eventually(func() int32 { + return atomic.LoadInt32(&callCount) + }, "2s", "50ms").Should(BeNumerically(">=", 2)) + + data, err := provider.GetNodeData() + Expect(err).ToNot(HaveOccurred()) + Expect(data).To(HaveLen(1)) + Expect(data[0].Node.NodeName).To(Equal("good-node")) + + provider.Stop() + }) + + It("returns error when no data has been recorded", func() { + mockClient := &mockStatsSummaryClient{ + fn: func() ([]*stats.Summary, error) { + return nil, fmt.Errorf("cannot reach any nodes") + }, + } + + provider := NewNodeStatsSummaryProvider(mockClient) + started := provider.Start(time.Hour) + Expect(started).To(BeTrue()) + + data, err := provider.GetNodeData() + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("no node stats summary data has been recorded")) + Expect(data).To(BeNil()) + + provider.Stop() + }) + + It("prevents double-start", func() { + mockClient := &mockStatsSummaryClient{ + fn: func() ([]*stats.Summary, error) { + return []*stats.Summary{{Node: stats.NodeStats{NodeName: "n"}}}, nil + }, + } + + provider := NewNodeStatsSummaryProvider(mockClient) + Expect(provider.Start(time.Hour)).To(BeTrue()) + Expect(provider.Start(time.Hour)).To(BeFalse()) + provider.Stop() + }) +}) diff --git a/pkg/nodes/request.go b/pkg/nodes/request.go index 0c346468..5cd8fc91 100644 --- a/pkg/nodes/request.go +++ b/pkg/nodes/request.go @@ -4,6 +4,7 @@ import ( "fmt" "io" "math" + "net" "net/http" "strconv" "time" @@ -92,7 +93,7 @@ type directNode struct { } func (d directNode) formatEndpoint(s string) string { - return fmt.Sprintf("https://%s:%v/%s", d.ip, d.port, s) + return fmt.Sprintf("https://%s/%s", net.JoinHostPort(d.ip, strconv.FormatInt(d.port, 10)), s) } // setupDirectNodeAPI retrieves node stats directly from the node api diff --git a/pkg/nodes/request_test.go b/pkg/nodes/request_test.go new file mode 100644 index 00000000..4c3ff0c8 --- /dev/null +++ b/pkg/nodes/request_test.go @@ -0,0 +1,165 @@ +package nodes + +import ( + "net/url" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("directNode formatEndpoint", func() { + Context("IPv4 addresses", func() { + It("should format IPv4 address correctly", func() { + node := directNode{ + ip: "10.128.15.239", + port: 10250, + } + endpoint := node.formatEndpoint("stats/summary") + Expect(endpoint).To(Equal("https://10.128.15.239:10250/stats/summary")) + + // Verify URL is parseable + parsedURL, err := url.Parse(endpoint) + Expect(err).ToNot(HaveOccurred()) + Expect(parsedURL.Scheme).To(Equal("https")) + Expect(parsedURL.Host).To(Equal("10.128.15.239:10250")) + Expect(parsedURL.Path).To(Equal("/stats/summary")) + }) + + It("should handle different ports", func() { + node := directNode{ + ip: "192.168.1.1", + port: 443, + } + endpoint := node.formatEndpoint("metrics") + Expect(endpoint).To(Equal("https://192.168.1.1:443/metrics")) + + parsedURL, err := url.Parse(endpoint) + Expect(err).ToNot(HaveOccurred()) + Expect(parsedURL.Host).To(Equal("192.168.1.1:443")) + }) + }) + + Context("IPv6 addresses", func() { + It("should wrap IPv6 address in brackets", func() { + node := directNode{ + ip: "2a05:d014:1314:704::a282", + port: 10250, + } + endpoint := node.formatEndpoint("stats/summary") + Expect(endpoint).To(Equal("https://[2a05:d014:1314:704::a282]:10250/stats/summary")) + + // Verify URL is parseable + parsedURL, err := url.Parse(endpoint) + Expect(err).ToNot(HaveOccurred()) + Expect(parsedURL.Scheme).To(Equal("https")) + Expect(parsedURL.Host).To(Equal("[2a05:d014:1314:704::a282]:10250")) + Expect(parsedURL.Path).To(Equal("/stats/summary")) + }) + + It("should handle IPv6 loopback address", func() { + node := directNode{ + ip: "::1", + port: 10250, + } + endpoint := node.formatEndpoint("stats/summary") + Expect(endpoint).To(Equal("https://[::1]:10250/stats/summary")) + + parsedURL, err := url.Parse(endpoint) + Expect(err).ToNot(HaveOccurred()) + Expect(parsedURL.Host).To(Equal("[::1]:10250")) + }) + + It("should handle private IPv6 address", func() { + node := directNode{ + ip: "fd00::1", + port: 443, + } + endpoint := node.formatEndpoint("metrics") + Expect(endpoint).To(Equal("https://[fd00::1]:443/metrics")) + + parsedURL, err := url.Parse(endpoint) + Expect(err).ToNot(HaveOccurred()) + Expect(parsedURL.Host).To(Equal("[fd00::1]:443")) + }) + + It("should handle full IPv6 address", func() { + node := directNode{ + ip: "2001:db8:85a3:0:0:8a2e:370:7334", + port: 8080, + } + endpoint := node.formatEndpoint("api/v1/health") + Expect(endpoint).To(Equal("https://[2001:db8:85a3:0:0:8a2e:370:7334]:8080/api/v1/health")) + + parsedURL, err := url.Parse(endpoint) + Expect(err).ToNot(HaveOccurred()) + Expect(parsedURL.Scheme).To(Equal("https")) + Expect(parsedURL.Host).To(Equal("[2001:db8:85a3:0:0:8a2e:370:7334]:8080")) + Expect(parsedURL.Path).To(Equal("/api/v1/health")) + }) + + It("should handle compressed IPv6 address", func() { + node := directNode{ + ip: "2001:db8::1", + port: 10250, + } + endpoint := node.formatEndpoint("stats/summary") + Expect(endpoint).To(Equal("https://[2001:db8::1]:10250/stats/summary")) + + parsedURL, err := url.Parse(endpoint) + Expect(err).ToNot(HaveOccurred()) + Expect(parsedURL.Host).To(Equal("[2001:db8::1]:10250")) + }) + }) + + Context("Edge cases", func() { + It("should handle empty path", func() { + node := directNode{ + ip: "10.0.0.1", + port: 443, + } + endpoint := node.formatEndpoint("") + Expect(endpoint).To(Equal("https://10.0.0.1:443/")) + + _, err := url.Parse(endpoint) + Expect(err).ToNot(HaveOccurred()) + }) + + It("should handle path with leading slash", func() { + node := directNode{ + ip: "192.168.1.1", + port: 8080, + } + endpoint := node.formatEndpoint("/api/v1/nodes") + Expect(endpoint).To(Equal("https://192.168.1.1:8080//api/v1/nodes")) + + _, err := url.Parse(endpoint) + Expect(err).ToNot(HaveOccurred()) + }) + + It("should handle different port values", func() { + node := directNode{ + ip: "10.0.0.1", + port: 1, + } + endpoint := node.formatEndpoint("test") + Expect(endpoint).To(Equal("https://10.0.0.1:1/test")) + + _, err := url.Parse(endpoint) + Expect(err).ToNot(HaveOccurred()) + }) + + It("should handle large port numbers", func() { + node := directNode{ + ip: "10.0.0.1", + port: 65535, + } + endpoint := node.formatEndpoint("test") + Expect(endpoint).To(Equal("https://10.0.0.1:65535/test")) + + _, err := url.Parse(endpoint) + Expect(err).ToNot(HaveOccurred()) + }) + }) +}) + +// Made with Bob diff --git a/pkg/nodes/testdata/nodes.jsonl b/pkg/nodes/testdata/nodes.jsonl index 2f46105c..1a36f27d 100644 --- a/pkg/nodes/testdata/nodes.jsonl +++ b/pkg/nodes/testdata/nodes.jsonl @@ -2,3 +2,5 @@ {"metadata":{"name":"nodename1","uid":"3e6eb4ef-2ecd-4bd9-8488-5661d46ab6d0","resourceVersion":"377577972","creationTimestamp":"2025-02-27T21:52:16Z","labels":{"beta.kubernetes.io/arch":"amd64","beta.kubernetes.io/instance-type":"e2-medium","beta.kubernetes.io/os":"linux","cloud.google.com/gke-boot-disk":"pd-balanced","cloud.google.com/gke-container-runtime":"containerd","cloud.google.com/gke-cpu-scaling-level":"2","cloud.google.com/gke-logging-variant":"DEFAULT","cloud.google.com/gke-max-pods-per-node":"100","cloud.google.com/gke-memory-gb-scaling-level":"4","cloud.google.com/gke-nodepool":"default-pool","cloud.google.com/gke-os-distribution":"cos","cloud.google.com/gke-provisioning":"standard","cloud.google.com/gke-stack-type":"IPV4","cloud.google.com/machine-family":"e2","cloud.google.com/private-node":"false","failure-domain.beta.kubernetes.io/region":"us-central1","failure-domain.beta.kubernetes.io/zone":"us-central1-c","kubernetes.io/arch":"amd64","kubernetes.io/hostname":"nodename1","kubernetes.io/os":"linux","node.kubernetes.io/instance-type":"e2-medium","topology.gke.io/zone":"us-central1-c","topology.kubernetes.io/region":"us-central1","topology.kubernetes.io/zone":"us-central1-c"},"annotations":{"container.googleapis.com/instance_id":"4057732895025750155","csi.volume.kubernetes.io/nodeid":"{\"pd.csi.storage.gke.io\":\"projects/containers-183923/zones/us-central1-c/instances/nodename1\"}","node.alpha.kubernetes.io/ttl":"0","node.gke.io/last-applied-node-labels":"cloud.google.com/gke-boot-disk=pd-balanced,cloud.google.com/gke-container-runtime=containerd,cloud.google.com/gke-cpu-scaling-level=2,cloud.google.com/gke-logging-variant=DEFAULT,cloud.google.com/gke-max-pods-per-node=100,cloud.google.com/gke-memory-gb-scaling-level=4,cloud.google.com/gke-nodepool=default-pool,cloud.google.com/gke-os-distribution=cos,cloud.google.com/gke-provisioning=standard,cloud.google.com/gke-stack-type=IPV4,cloud.google.com/machine-family=e2,cloud.google.com/private-node=false","node.gke.io/last-applied-node-taints":"","volumes.kubernetes.io/controller-managed-attach-detach":"true"},"managedFields":[{"manager":"cloud-controller-manager","operation":"Update","apiVersion":"v1","time":"2025-02-27T21:52:16Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:labels":{"f:beta.kubernetes.io/instance-type":{},"f:failure-domain.beta.kubernetes.io/region":{},"f:failure-domain.beta.kubernetes.io/zone":{},"f:node.kubernetes.io/instance-type":{},"f:topology.kubernetes.io/region":{},"f:topology.kubernetes.io/zone":{}}},"f:spec":{"f:podCIDR":{},"f:podCIDRs":{".":{},"v:\"10.84.1.0/24\"":{}},"f:providerID":{}}}},{"manager":"cloud-controller-manager","operation":"Update","apiVersion":"v1","time":"2025-02-27T21:52:16Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:addresses":{".":{},"k:{\"type\":\"ExternalIP\"}":{".":{},"f:address":{},"f:type":{}},"k:{\"type\":\"InternalIP\"}":{".":{},"f:address":{},"f:type":{}}},"f:conditions":{"k:{\"type\":\"NetworkUnavailable\"}":{".":{},"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}}}}},"subresource":"status"},{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2025-02-27T21:52:16Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{"f:node.alpha.kubernetes.io/ttl":{}}}}},{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2025-02-27T21:52:16Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:volumes.kubernetes.io/controller-managed-attach-detach":{}},"f:labels":{".":{},"f:beta.kubernetes.io/arch":{},"f:beta.kubernetes.io/os":{},"f:cloud.google.com/gke-boot-disk":{},"f:cloud.google.com/gke-container-runtime":{},"f:cloud.google.com/gke-cpu-scaling-level":{},"f:cloud.google.com/gke-logging-variant":{},"f:cloud.google.com/gke-max-pods-per-node":{},"f:cloud.google.com/gke-memory-gb-scaling-level":{},"f:cloud.google.com/gke-nodepool":{},"f:cloud.google.com/gke-os-distribution":{},"f:cloud.google.com/gke-provisioning":{},"f:cloud.google.com/gke-stack-type":{},"f:cloud.google.com/machine-family":{},"f:cloud.google.com/private-node":{},"f:kubernetes.io/arch":{},"f:kubernetes.io/hostname":{},"f:kubernetes.io/os":{}}}}},{"manager":"gcp-controller-manager","operation":"Update","apiVersion":"v1","time":"2025-02-27T21:52:17Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{"f:container.googleapis.com/instance_id":{},"f:node.gke.io/last-applied-node-labels":{},"f:node.gke.io/last-applied-node-taints":{}}}}},{"manager":"node-problem-detector","operation":"Update","apiVersion":"v1","time":"2025-03-01T23:37:52Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"CorruptDockerOverlay2\"}":{".":{},"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"DeprecatedAuthsFieldInContainerdConfiguration\"}":{".":{},"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"DeprecatedConfigsFieldInContainerdConfiguration\"}":{".":{},"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"DeprecatedMirrorsFieldInContainerdConfiguration\"}":{".":{},"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"DeprecatedOtherContainerdFeatures\"}":{".":{},"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"DeprecatedPullingSchemaV1Image\"}":{".":{},"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"DeprecatedUsingV1Alpha2Cri\"}":{".":{},"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"FrequentContainerdRestart\"}":{".":{},"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"FrequentDockerRestart\"}":{".":{},"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"FrequentKubeletRestart\"}":{".":{},"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"FrequentUnregisterNetDevice\"}":{".":{},"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"KernelDeadlock\"}":{".":{},"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"ReadonlyFilesystem\"}":{".":{},"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"SysctlChanged\"}":{".":{},"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}}}}},"subresource":"status"},{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2025-03-01T23:38:43Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{"f:csi.volume.kubernetes.io/nodeid":{}},"f:labels":{"f:topology.gke.io/zone":{}}},"f:status":{"f:allocatable":{"f:ephemeral-storage":{}},"f:conditions":{"k:{\"type\":\"DiskPressure\"}":{"f:lastHeartbeatTime":{}},"k:{\"type\":\"MemoryPressure\"}":{"f:lastHeartbeatTime":{}},"k:{\"type\":\"PIDPressure\"}":{"f:lastHeartbeatTime":{}},"k:{\"type\":\"Ready\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}}},"f:images":{}}},"subresource":"status"}]},"spec":{"podCIDR":"10.84.1.0/24","podCIDRs":["10.84.1.0/24"],"providerID":"gce://containers-183923/us-central1-c/nodename1"},"status":{"capacity":{"cpu":"2","ephemeral-storage":"98831908Ki","hugepages-1Gi":"0","hugepages-2Mi":"0","memory":"4018124Ki","pods":"100"},"allocatable":{"cpu":"940m","ephemeral-storage":"47060071478","hugepages-1Gi":"0","hugepages-2Mi":"0","memory":"2872268Ki","pods":"100"},"conditions":[{"type":"FrequentUnregisterNetDevice","status":"False","lastHeartbeatTime":"2025-03-01T23:37:52Z","lastTransitionTime":"2025-02-27T21:52:16Z","reason":"NoFrequentUnregisterNetDevice","message":"node is functioning properly"},{"type":"FrequentContainerdRestart","status":"False","lastHeartbeatTime":"2025-03-01T23:37:52Z","lastTransitionTime":"2025-02-27T21:52:16Z","reason":"NoFrequentContainerdRestart","message":"containerd is functioning properly"},{"type":"SysctlChanged","status":"False","lastHeartbeatTime":"2025-03-01T23:37:52Z","lastTransitionTime":"2025-02-27T21:52:16Z","reason":"SysctlNotChanged","message":"Default sysctls are in effect, no unexpected sysctl changes"},{"type":"CorruptDockerOverlay2","status":"False","lastHeartbeatTime":"2025-03-01T23:37:52Z","lastTransitionTime":"2025-02-27T21:52:16Z","reason":"NoCorruptDockerOverlay2","message":"docker overlay2 is functioning properly"},{"type":"DeprecatedOtherContainerdFeatures","status":"False","lastHeartbeatTime":"2025-03-01T23:37:52Z","lastTransitionTime":"2025-02-27T21:52:16Z","reason":"DeprecatedOtherContainerdFeaturesNotDetected","message":"No deprecation risk: did not find any deprecations other than 3 configs fields (auths/configs/mirrors), pulling schema v1 images and using v1alpha2 CRI."},{"type":"KernelDeadlock","status":"False","lastHeartbeatTime":"2025-03-01T23:37:52Z","lastTransitionTime":"2025-02-27T21:52:16Z","reason":"KernelHasNoDeadlock","message":"kernel has no deadlock"},{"type":"DeprecatedUsingV1Alpha2Cri","status":"False","lastHeartbeatTime":"2025-03-01T23:37:52Z","lastTransitionTime":"2025-02-27T21:52:16Z","reason":"DeprecatedUsingV1Alpha2CriNotDetected","message":"No deprecation risk: did not use v1alpha2 CRI"},{"type":"DeprecatedConfigsFieldInContainerdConfiguration","status":"False","lastHeartbeatTime":"2025-03-01T23:37:52Z","lastTransitionTime":"2025-02-27T21:52:16Z","reason":"DeprecatedConfigsFieldInContainerdConfigurationNotDetected","message":"No deprecation risk: did not find any deprecated 'configs' field in containerd's config"},{"type":"ReadonlyFilesystem","status":"False","lastHeartbeatTime":"2025-03-01T23:37:52Z","lastTransitionTime":"2025-02-27T21:52:16Z","reason":"FilesystemIsNotReadOnly","message":"Filesystem is not read-only"},{"type":"DeprecatedAuthsFieldInContainerdConfiguration","status":"False","lastHeartbeatTime":"2025-03-01T23:37:52Z","lastTransitionTime":"2025-02-27T21:52:16Z","reason":"DeprecatedAuthsFieldInContainerdConfigurationNotDetected","message":"No deprecation risk: did not find any deprecated 'auths' field in containerd's config"},{"type":"FrequentKubeletRestart","status":"False","lastHeartbeatTime":"2025-03-01T23:37:52Z","lastTransitionTime":"2025-02-27T21:52:16Z","reason":"NoFrequentKubeletRestart","message":"kubelet is functioning properly"},{"type":"DeprecatedPullingSchemaV1Image","status":"False","lastHeartbeatTime":"2025-03-01T23:37:52Z","lastTransitionTime":"2025-02-27T21:52:16Z","reason":"DeprecatedPullingSchemaV1ImageDetected","message":"No deprecation risk: did not pull any schema v1 images"},{"type":"FrequentDockerRestart","status":"False","lastHeartbeatTime":"2025-03-01T23:37:52Z","lastTransitionTime":"2025-02-27T21:52:16Z","reason":"NoFrequentDockerRestart","message":"docker is functioning properly"},{"type":"DeprecatedMirrorsFieldInContainerdConfiguration","status":"False","lastHeartbeatTime":"2025-03-01T23:37:52Z","lastTransitionTime":"2025-02-27T21:52:16Z","reason":"DeprecatedMirrorsFieldInContainerdConfigurationNotDetected","message":"No deprecation risk: did not find any deprecated 'mirrors' field in containerd's config"},{"type":"NetworkUnavailable","status":"False","lastHeartbeatTime":"2025-02-27T21:52:16Z","lastTransitionTime":"2025-02-27T21:52:16Z","reason":"RouteCreated","message":"NodeController create implicit route"},{"type":"MemoryPressure","status":"False","lastHeartbeatTime":"2025-03-01T23:38:43Z","lastTransitionTime":"2025-02-27T21:52:14Z","reason":"KubeletHasSufficientMemory","message":"kubelet has sufficient memory available"},{"type":"DiskPressure","status":"False","lastHeartbeatTime":"2025-03-01T23:38:43Z","lastTransitionTime":"2025-02-27T21:52:14Z","reason":"KubeletHasNoDiskPressure","message":"kubelet has no disk pressure"},{"type":"PIDPressure","status":"False","lastHeartbeatTime":"2025-03-01T23:38:43Z","lastTransitionTime":"2025-02-27T21:52:14Z","reason":"KubeletHasSufficientPID","message":"kubelet has sufficient PID available"},{"type":"Ready","status":"True","lastHeartbeatTime":"2025-03-01T23:38:43Z","lastTransitionTime":"2025-02-27T21:52:16Z","reason":"KubeletReady","message":"kubelet is posting ready status"}],"addresses":[{"type":"InternalIP","address":"10.128.0.10"},{"type":"ExternalIP","address":"34.56.7.14"}],"daemonEndpoints":{"kubeletEndpoint":{"Port":10250}},"nodeInfo":{"machineID":"332251a3d8788e57bb7d4c394caf043d","systemUUID":"332251a3-d878-8e57-bb7d-4c394caf043d","bootID":"a800d944-b897-4b98-a322-a3e2ef9cd7ed","kernelVersion":"6.1.123+","osImage":"Container-Optimized OS from Google","containerRuntimeVersion":"containerd://1.7.24","kubeletVersion":"v1.30.9-gke.1127000","kubeProxyVersion":"v1.30.9-gke.1127000","operatingSystem":"linux","architecture":"amd64"},"images":[{"names":["asia.gcr.io/gke-release-staging/cilium/cilium@sha256:a69b8b0cab532f641f2bcdd823842599b852666f1bcc763200b9cf4b3f40537f","eu.gcr.io/gke-release-staging/cilium/cilium@sha256:a69b8b0cab532f641f2bcdd823842599b852666f1bcc763200b9cf4b3f40537f","gcr.io/gke-release-staging/cilium/cilium@sha256:a69b8b0cab532f641f2bcdd823842599b852666f1bcc763200b9cf4b3f40537f","asia.gke.gcr.io/cilium/cilium:v1.14.13-gke1.30-gke.64","eu.gke.gcr.io/cilium/cilium:v1.14.13-gke1.30-gke.64"],"sizeBytes":171632673},{"names":["gke.gcr.io/prometheus-engine/prometheus@sha256:3e6493d4b01ab583382731491d980bc164873ad4969e92c0bdd0da278359ccac"],"sizeBytes":113010021},{"names":["gke.gcr.io/fluent-bit@sha256:bd48755377a723a91bc3fd96f67796ba3a3023808fd69b37d97cb1aa5985d9ae"],"sizeBytes":93330926},{"names":["gke.gcr.io/kube-proxy-amd64:v1.30.9-gke.1127000","gke.gcr.io/kube-proxy-amd64:v1.30.9-gke.500","k8s.gcr.io/kube-proxy-amd64:v1.30.9-gke.500"],"sizeBytes":88273080},{"names":["gke.gcr.io/prometheus-engine/operator@sha256:b06adf14b06c9fc809d4b8db41329e4f3c34d9b1baa2abd45542ad817aed3917"],"sizeBytes":84461449},{"names":["gke.gcr.io/gcp-compute-persistent-disk-csi-driver@sha256:bbbf275b1482cf3fc658f0bde28b1fbd24054451b5a97e23812821c82374a2b2"],"sizeBytes":60814920},{"names":["gke.gcr.io/prometheus-engine/config-reloader@sha256:d199f266545ee281fa51d30e0a5f9c4da27da23055b153ca93adbf7483d19633"],"sizeBytes":59834302},{"names":["asia.gcr.io/gke-release-staging/cilium/certgen@sha256:73529e65f241d2b390d4eadb4b886ea9de8c3ef27ae3854733d6a9cbefd6d0b0","eu.gcr.io/gke-release-staging/cilium/certgen@sha256:73529e65f241d2b390d4eadb4b886ea9de8c3ef27ae3854733d6a9cbefd6d0b0","gcr.io/gke-release-staging/cilium/certgen@sha256:73529e65f241d2b390d4eadb4b886ea9de8c3ef27ae3854733d6a9cbefd6d0b0","asia.gke.gcr.io/cilium/certgen:v0.1.13-gke.14","eu.gke.gcr.io/cilium/certgen:v0.1.13-gke.14"],"sizeBytes":40760357},{"names":["gke.gcr.io/k8s-dns-dnsmasq-nanny@sha256:e178b753d49a90ec32f1f45e0f52ce64019641d3fd45d8deadcf08cb73b8c840"],"sizeBytes":37001839},{"names":["gke.gcr.io/prometheus-to-sd@sha256:443ce4937d4bc893e894cd740c730e9039ccc76e0bd7494d8bf8725486658e10"],"sizeBytes":35881529},{"names":["asia.gcr.io/gke-release-staging/cilium/slim-daemon/anet-agent@sha256:1a871e457e08d4e3e06a079f76e6a5a67c5869797ae305a6fb7f1e2c4712f0cd","eu.gcr.io/gke-release-staging/cilium/slim-daemon/anet-agent@sha256:1a871e457e08d4e3e06a079f76e6a5a67c5869797ae305a6fb7f1e2c4712f0cd","gcr.io/gke-release-staging/cilium/slim-daemon/anet-agent@sha256:1a871e457e08d4e3e06a079f76e6a5a67c5869797ae305a6fb7f1e2c4712f0cd","asia.gke.gcr.io/cilium/slim-daemon/anet-agent:v1.14.13-gke1.30-gke.64","eu.gke.gcr.io/cilium/slim-daemon/anet-agent:v1.14.13-gke1.30-gke.64"],"sizeBytes":33172919},{"names":["gke.gcr.io/fluent-bit-gke-exporter@sha256:0ed31fb2cc1b2b747a1e304a5530124c97909a0eb17774c7ce80595abf73d1e6"],"sizeBytes":32969391},{"names":["gke.gcr.io/k8s-dns-kube-dns@sha256:b609a51c8aa4add2d1d0811737f177b4e944ea0781a48eead0d804722787f96f"],"sizeBytes":32667404},{"names":["gke.gcr.io/k8s-dns-sidecar@sha256:9e60f83b54d010a7dd7e5a868a6713ad410442c72f0b7540cda010c50651c0bc"],"sizeBytes":29112653},{"names":["gke.gcr.io/gke-metrics-agent@sha256:be215beaf8f2acd5c4f64ff3d28e6dc60071ef6f0da5eb9774fda69148c38ba4"],"sizeBytes":27003958},{"names":["gke.gcr.io/gke-metrics-collector@sha256:3d76420863be0cdbdf5f9a512e032d9b20ae8fd7be4b5eca22ecdb1e867f23bf"],"sizeBytes":25809971},{"names":["gke.gcr.io/gke-metrics-collector@sha256:360c05bb437d77b55090ba512bddfda52f62166c78171593d1152327db2a914f"],"sizeBytes":25582287},{"names":["gke.gcr.io/gke-metrics-collector@sha256:d460e6b5088332f62b990f8a1f7bf6d9eca7c3f41cb974e3db493d6b0fc4ad70"],"sizeBytes":24425624},{"names":["gke.gcr.io/gke-metrics-collector@sha256:463e73163c4d343b8a3327e0d2e8e955d22434e9005a1a188275ac55b8cfebb4"],"sizeBytes":24343841},{"names":["gke.gcr.io/event-exporter@sha256:c67b696cbda0df01639bfe360e58870185fafc57bb60168be54264c19d2b0e76"],"sizeBytes":24294749},{"names":["asia.gcr.io/gke-release-staging/cilium/hubble-cli@sha256:327d40413ece3e0cf8e14cebd5d24b15a1987bf8a4e5ec369dabdb17203e62c2","eu.gcr.io/gke-release-staging/cilium/hubble-cli@sha256:327d40413ece3e0cf8e14cebd5d24b15a1987bf8a4e5ec369dabdb17203e62c2","gcr.io/gke-release-staging/cilium/hubble-cli@sha256:327d40413ece3e0cf8e14cebd5d24b15a1987bf8a4e5ec369dabdb17203e62c2","asia.gke.gcr.io/cilium/hubble-cli:v0.13.5-gke.14","eu.gke.gcr.io/cilium/hubble-cli:v0.13.5-gke.14"],"sizeBytes":24123625},{"names":["asia.gcr.io/gke-release-staging/anthos-networking/anetd-sidecar@sha256:2a788f8b3fc1112f706a7b22f13fb0e430d76c5743e6f8a6c2c350345f3a006d","eu.gcr.io/gke-release-staging/anthos-networking/anetd-sidecar@sha256:2a788f8b3fc1112f706a7b22f13fb0e430d76c5743e6f8a6c2c350345f3a006d","gcr.io/gke-release-staging/anthos-networking/anetd-sidecar@sha256:2a788f8b3fc1112f706a7b22f13fb0e430d76c5743e6f8a6c2c350345f3a006d","asia.gke.gcr.io/anthos-networking/anetd-sidecar:v2.9.77","eu.gke.gcr.io/anthos-networking/anetd-sidecar:v2.9.77"],"sizeBytes":23881188},{"names":["asia.gcr.io/gke-release-staging/gke-metrics-collector@sha256:1593f1e9570b99b1843cc886b723d4e291e2281fe2b9e45f565064cac379e4cf","eu.gcr.io/gke-release-staging/gke-metrics-collector@sha256:1593f1e9570b99b1843cc886b723d4e291e2281fe2b9e45f565064cac379e4cf","gcr.io/gke-release-staging/gke-metrics-collector@sha256:1593f1e9570b99b1843cc886b723d4e291e2281fe2b9e45f565064cac379e4cf","asia.gke.gcr.io/gke-metrics-collector:20240508_2300_RC0","eu.gke.gcr.io/gke-metrics-collector:20240508_2300_RC0"],"sizeBytes":23823718},{"names":["asia.gcr.io/gke-release-staging/gke-metrics-collector@sha256:545f5594f9a6ee714b315897f92edeaa858b1074e4c6a38894353214907e4fe5","eu.gcr.io/gke-release-staging/gke-metrics-collector@sha256:545f5594f9a6ee714b315897f92edeaa858b1074e4c6a38894353214907e4fe5","gcr.io/gke-release-staging/gke-metrics-collector@sha256:545f5594f9a6ee714b315897f92edeaa858b1074e4c6a38894353214907e4fe5","asia.gke.gcr.io/gke-metrics-collector:20240425_2300_RC0","eu.gke.gcr.io/gke-metrics-collector:20240425_2300_RC0"],"sizeBytes":23717101},{"names":["asia.gcr.io/gke-release-staging/netd@sha256:6ca0892cc32f66705087257c12ef97f3f27b270402e03299450b19ecb93e48e6","eu.gcr.io/gke-release-staging/netd@sha256:6ca0892cc32f66705087257c12ef97f3f27b270402e03299450b19ecb93e48e6","gcr.io/gke-release-staging/netd@sha256:6ca0892cc32f66705087257c12ef97f3f27b270402e03299450b19ecb93e48e6","asia.gke.gcr.io/netd:v0.8.4-gke.18","eu.gke.gcr.io/netd:v0.8.4-gke.18"],"sizeBytes":23427157}]}} {"metadata":{"name":"nodename2","uid":"934ae34e-a4bc-4936-bbfe-5b63f8a092aa","resourceVersion":"377577810","creationTimestamp":"2025-02-27T21:56:56Z","labels":{"beta.kubernetes.io/arch":"amd64","beta.kubernetes.io/instance-type":"e2-medium","beta.kubernetes.io/os":"linux","cloud.google.com/gke-boot-disk":"pd-balanced","cloud.google.com/gke-container-runtime":"containerd","cloud.google.com/gke-cpu-scaling-level":"2","cloud.google.com/gke-logging-variant":"DEFAULT","cloud.google.com/gke-max-pods-per-node":"100","cloud.google.com/gke-memory-gb-scaling-level":"4","cloud.google.com/gke-nodepool":"default-pool","cloud.google.com/gke-os-distribution":"cos","cloud.google.com/gke-provisioning":"standard","cloud.google.com/gke-stack-type":"IPV4","cloud.google.com/machine-family":"e2","cloud.google.com/private-node":"false","failure-domain.beta.kubernetes.io/region":"us-central1","failure-domain.beta.kubernetes.io/zone":"us-central1-c","kubernetes.io/arch":"amd64","kubernetes.io/hostname":"nodename2","kubernetes.io/os":"linux","node.kubernetes.io/instance-type":"e2-medium","topology.gke.io/zone":"us-central1-c","topology.kubernetes.io/region":"us-central1","topology.kubernetes.io/zone":"us-central1-c"},"annotations":{"container.googleapis.com/instance_id":"8761733261181149550","csi.volume.kubernetes.io/nodeid":"{\"pd.csi.storage.gke.io\":\"projects/containers-183923/zones/us-central1-c/instances/nodename2\"}","node.alpha.kubernetes.io/ttl":"0","node.gke.io/last-applied-node-labels":"cloud.google.com/gke-boot-disk=pd-balanced,cloud.google.com/gke-container-runtime=containerd,cloud.google.com/gke-cpu-scaling-level=2,cloud.google.com/gke-logging-variant=DEFAULT,cloud.google.com/gke-max-pods-per-node=100,cloud.google.com/gke-memory-gb-scaling-level=4,cloud.google.com/gke-nodepool=default-pool,cloud.google.com/gke-os-distribution=cos,cloud.google.com/gke-provisioning=standard,cloud.google.com/gke-stack-type=IPV4,cloud.google.com/machine-family=e2,cloud.google.com/private-node=false","node.gke.io/last-applied-node-taints":"","volumes.kubernetes.io/controller-managed-attach-detach":"true"},"managedFields":[{"manager":"cloud-controller-manager","operation":"Update","apiVersion":"v1","time":"2025-02-27T21:56:56Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:labels":{"f:beta.kubernetes.io/instance-type":{},"f:failure-domain.beta.kubernetes.io/region":{},"f:failure-domain.beta.kubernetes.io/zone":{},"f:node.kubernetes.io/instance-type":{},"f:topology.kubernetes.io/region":{},"f:topology.kubernetes.io/zone":{}}},"f:spec":{"f:podCIDR":{},"f:podCIDRs":{".":{},"v:\"10.84.3.0/24\"":{}},"f:providerID":{}}}},{"manager":"cloud-controller-manager","operation":"Update","apiVersion":"v1","time":"2025-02-27T21:56:56Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:addresses":{".":{},"k:{\"type\":\"ExternalIP\"}":{".":{},"f:address":{},"f:type":{}},"k:{\"type\":\"InternalIP\"}":{".":{},"f:address":{},"f:type":{}}},"f:conditions":{"k:{\"type\":\"NetworkUnavailable\"}":{".":{},"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}}}}},"subresource":"status"},{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2025-02-27T21:56:56Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{"f:node.alpha.kubernetes.io/ttl":{}}}}},{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2025-02-27T21:56:56Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:volumes.kubernetes.io/controller-managed-attach-detach":{}},"f:labels":{".":{},"f:beta.kubernetes.io/arch":{},"f:beta.kubernetes.io/os":{},"f:cloud.google.com/gke-boot-disk":{},"f:cloud.google.com/gke-container-runtime":{},"f:cloud.google.com/gke-cpu-scaling-level":{},"f:cloud.google.com/gke-logging-variant":{},"f:cloud.google.com/gke-max-pods-per-node":{},"f:cloud.google.com/gke-memory-gb-scaling-level":{},"f:cloud.google.com/gke-nodepool":{},"f:cloud.google.com/gke-os-distribution":{},"f:cloud.google.com/gke-provisioning":{},"f:cloud.google.com/gke-stack-type":{},"f:cloud.google.com/machine-family":{},"f:cloud.google.com/private-node":{},"f:kubernetes.io/arch":{},"f:kubernetes.io/hostname":{},"f:kubernetes.io/os":{}}}}},{"manager":"gcp-controller-manager","operation":"Update","apiVersion":"v1","time":"2025-02-27T21:56:57Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{"f:container.googleapis.com/instance_id":{},"f:node.gke.io/last-applied-node-labels":{},"f:node.gke.io/last-applied-node-taints":{}}}}},{"manager":"node-problem-detector","operation":"Update","apiVersion":"v1","time":"2025-03-01T23:37:39Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"CorruptDockerOverlay2\"}":{".":{},"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"DeprecatedAuthsFieldInContainerdConfiguration\"}":{".":{},"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"DeprecatedConfigsFieldInContainerdConfiguration\"}":{".":{},"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"DeprecatedMirrorsFieldInContainerdConfiguration\"}":{".":{},"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"DeprecatedOtherContainerdFeatures\"}":{".":{},"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"DeprecatedPullingSchemaV1Image\"}":{".":{},"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"DeprecatedUsingV1Alpha2Cri\"}":{".":{},"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"FrequentContainerdRestart\"}":{".":{},"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"FrequentDockerRestart\"}":{".":{},"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"FrequentKubeletRestart\"}":{".":{},"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"FrequentUnregisterNetDevice\"}":{".":{},"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"KernelDeadlock\"}":{".":{},"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"ReadonlyFilesystem\"}":{".":{},"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"SysctlChanged\"}":{".":{},"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}}}}},"subresource":"status"},{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2025-03-01T23:38:27Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{"f:csi.volume.kubernetes.io/nodeid":{}},"f:labels":{"f:topology.gke.io/zone":{}}},"f:status":{"f:conditions":{"k:{\"type\":\"DiskPressure\"}":{"f:lastHeartbeatTime":{}},"k:{\"type\":\"MemoryPressure\"}":{"f:lastHeartbeatTime":{}},"k:{\"type\":\"PIDPressure\"}":{"f:lastHeartbeatTime":{}},"k:{\"type\":\"Ready\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}}},"f:images":{}}},"subresource":"status"}]},"spec":{"podCIDR":"10.84.3.0/24","podCIDRs":["10.84.3.0/24"],"providerID":"gce://containers-183923/us-central1-c/nodename2"},"status":{"capacity":{"cpu":"2","ephemeral-storage":"98831908Ki","hugepages-1Gi":"0","hugepages-2Mi":"0","memory":"4018124Ki","pods":"100"},"allocatable":{"cpu":"940m","ephemeral-storage":"47060071478","hugepages-1Gi":"0","hugepages-2Mi":"0","memory":"2872268Ki","pods":"100"},"conditions":[{"type":"SysctlChanged","status":"False","lastHeartbeatTime":"2025-03-01T23:37:39Z","lastTransitionTime":"2025-02-27T21:56:56Z","reason":"SysctlNotChanged","message":"Default sysctls are in effect, no unexpected sysctl changes"},{"type":"FrequentUnregisterNetDevice","status":"False","lastHeartbeatTime":"2025-03-01T23:37:39Z","lastTransitionTime":"2025-02-27T21:56:56Z","reason":"NoFrequentUnregisterNetDevice","message":"node is functioning properly"},{"type":"FrequentKubeletRestart","status":"False","lastHeartbeatTime":"2025-03-01T23:37:39Z","lastTransitionTime":"2025-02-27T21:56:56Z","reason":"NoFrequentKubeletRestart","message":"kubelet is functioning properly"},{"type":"FrequentDockerRestart","status":"False","lastHeartbeatTime":"2025-03-01T23:37:39Z","lastTransitionTime":"2025-02-27T21:56:56Z","reason":"NoFrequentDockerRestart","message":"docker is functioning properly"},{"type":"DeprecatedUsingV1Alpha2Cri","status":"False","lastHeartbeatTime":"2025-03-01T23:37:39Z","lastTransitionTime":"2025-02-27T21:56:56Z","reason":"DeprecatedUsingV1Alpha2CriNotDetected","message":"No deprecation risk: did not use v1alpha2 CRI"},{"type":"CorruptDockerOverlay2","status":"False","lastHeartbeatTime":"2025-03-01T23:37:39Z","lastTransitionTime":"2025-02-27T21:56:56Z","reason":"NoCorruptDockerOverlay2","message":"docker overlay2 is functioning properly"},{"type":"DeprecatedPullingSchemaV1Image","status":"False","lastHeartbeatTime":"2025-03-01T23:37:39Z","lastTransitionTime":"2025-02-27T21:56:56Z","reason":"DeprecatedPullingSchemaV1ImageDetected","message":"No deprecation risk: did not pull any schema v1 images"},{"type":"DeprecatedMirrorsFieldInContainerdConfiguration","status":"False","lastHeartbeatTime":"2025-03-01T23:37:39Z","lastTransitionTime":"2025-02-27T21:56:56Z","reason":"DeprecatedMirrorsFieldInContainerdConfigurationNotDetected","message":"No deprecation risk: did not find any deprecated 'mirrors' field in containerd's config"},{"type":"DeprecatedAuthsFieldInContainerdConfiguration","status":"False","lastHeartbeatTime":"2025-03-01T23:37:39Z","lastTransitionTime":"2025-02-27T21:56:56Z","reason":"DeprecatedAuthsFieldInContainerdConfigurationNotDetected","message":"No deprecation risk: did not find any deprecated 'auths' field in containerd's config"},{"type":"KernelDeadlock","status":"False","lastHeartbeatTime":"2025-03-01T23:37:39Z","lastTransitionTime":"2025-02-27T21:56:56Z","reason":"KernelHasNoDeadlock","message":"kernel has no deadlock"},{"type":"DeprecatedOtherContainerdFeatures","status":"False","lastHeartbeatTime":"2025-03-01T23:37:39Z","lastTransitionTime":"2025-02-27T21:56:56Z","reason":"DeprecatedOtherContainerdFeaturesNotDetected","message":"No deprecation risk: did not find any deprecations other than 3 configs fields (auths/configs/mirrors), pulling schema v1 images and using v1alpha2 CRI."},{"type":"DeprecatedConfigsFieldInContainerdConfiguration","status":"False","lastHeartbeatTime":"2025-03-01T23:37:39Z","lastTransitionTime":"2025-02-27T21:56:56Z","reason":"DeprecatedConfigsFieldInContainerdConfigurationNotDetected","message":"No deprecation risk: did not find any deprecated 'configs' field in containerd's config"},{"type":"FrequentContainerdRestart","status":"False","lastHeartbeatTime":"2025-03-01T23:37:39Z","lastTransitionTime":"2025-02-27T21:56:56Z","reason":"NoFrequentContainerdRestart","message":"containerd is functioning properly"},{"type":"ReadonlyFilesystem","status":"False","lastHeartbeatTime":"2025-03-01T23:37:39Z","lastTransitionTime":"2025-02-27T21:56:56Z","reason":"FilesystemIsNotReadOnly","message":"Filesystem is not read-only"},{"type":"NetworkUnavailable","status":"False","lastHeartbeatTime":"2025-02-27T21:56:56Z","lastTransitionTime":"2025-02-27T21:56:56Z","reason":"RouteCreated","message":"NodeController create implicit route"},{"type":"MemoryPressure","status":"False","lastHeartbeatTime":"2025-03-01T23:38:27Z","lastTransitionTime":"2025-02-27T21:56:54Z","reason":"KubeletHasSufficientMemory","message":"kubelet has sufficient memory available"},{"type":"DiskPressure","status":"False","lastHeartbeatTime":"2025-03-01T23:38:27Z","lastTransitionTime":"2025-02-27T21:56:54Z","reason":"KubeletHasNoDiskPressure","message":"kubelet has no disk pressure"},{"type":"PIDPressure","status":"False","lastHeartbeatTime":"2025-03-01T23:38:27Z","lastTransitionTime":"2025-02-27T21:56:54Z","reason":"KubeletHasSufficientPID","message":"kubelet has sufficient PID available"},{"type":"Ready","status":"True","lastHeartbeatTime":"2025-03-01T23:38:27Z","lastTransitionTime":"2025-02-27T21:56:56Z","reason":"KubeletReady","message":"kubelet is posting ready status"}],"addresses":[{"type":"InternalIP","address":"10.128.15.218"},{"type":"ExternalIP","address":"104.154.120.113"}],"daemonEndpoints":{"kubeletEndpoint":{"Port":10250}},"nodeInfo":{"machineID":"0a8dd5524cb9aff0eea913210e98dab7","systemUUID":"0a8dd552-4cb9-aff0-eea9-13210e98dab7","bootID":"a2e09d0b-ad28-471c-a375-3959a856c779","kernelVersion":"6.1.123+","osImage":"Container-Optimized OS from Google","containerRuntimeVersion":"containerd://1.7.24","kubeletVersion":"v1.30.9-gke.1127000","kubeProxyVersion":"v1.30.9-gke.1127000","operatingSystem":"linux","architecture":"amd64"},"images":[{"names":["asia.gcr.io/gke-release-staging/cilium/cilium@sha256:a69b8b0cab532f641f2bcdd823842599b852666f1bcc763200b9cf4b3f40537f","eu.gcr.io/gke-release-staging/cilium/cilium@sha256:a69b8b0cab532f641f2bcdd823842599b852666f1bcc763200b9cf4b3f40537f","gcr.io/gke-release-staging/cilium/cilium@sha256:a69b8b0cab532f641f2bcdd823842599b852666f1bcc763200b9cf4b3f40537f","asia.gke.gcr.io/cilium/cilium:v1.14.13-gke1.30-gke.64","eu.gke.gcr.io/cilium/cilium:v1.14.13-gke1.30-gke.64"],"sizeBytes":171632673},{"names":["gke.gcr.io/prometheus-engine/prometheus@sha256:3e6493d4b01ab583382731491d980bc164873ad4969e92c0bdd0da278359ccac"],"sizeBytes":113010021},{"names":["gke.gcr.io/fluent-bit@sha256:bd48755377a723a91bc3fd96f67796ba3a3023808fd69b37d97cb1aa5985d9ae"],"sizeBytes":93330926},{"names":["gke.gcr.io/kube-proxy-amd64:v1.30.9-gke.1127000","gke.gcr.io/kube-proxy-amd64:v1.30.9-gke.500","k8s.gcr.io/kube-proxy-amd64:v1.30.9-gke.500"],"sizeBytes":88273080},{"names":["gke.gcr.io/gcp-compute-persistent-disk-csi-driver@sha256:bbbf275b1482cf3fc658f0bde28b1fbd24054451b5a97e23812821c82374a2b2"],"sizeBytes":60814920},{"names":["gke.gcr.io/prometheus-engine/config-reloader@sha256:d199f266545ee281fa51d30e0a5f9c4da27da23055b153ca93adbf7483d19633"],"sizeBytes":59834302},{"names":["asia.gcr.io/gke-release-staging/cilium/certgen@sha256:73529e65f241d2b390d4eadb4b886ea9de8c3ef27ae3854733d6a9cbefd6d0b0","eu.gcr.io/gke-release-staging/cilium/certgen@sha256:73529e65f241d2b390d4eadb4b886ea9de8c3ef27ae3854733d6a9cbefd6d0b0","gcr.io/gke-release-staging/cilium/certgen@sha256:73529e65f241d2b390d4eadb4b886ea9de8c3ef27ae3854733d6a9cbefd6d0b0","asia.gke.gcr.io/cilium/certgen:v0.1.13-gke.14","eu.gke.gcr.io/cilium/certgen:v0.1.13-gke.14"],"sizeBytes":40760357},{"names":["gke.gcr.io/k8s-dns-dnsmasq-nanny@sha256:e178b753d49a90ec32f1f45e0f52ce64019641d3fd45d8deadcf08cb73b8c840"],"sizeBytes":37001839},{"names":["gke.gcr.io/prometheus-to-sd@sha256:443ce4937d4bc893e894cd740c730e9039ccc76e0bd7494d8bf8725486658e10"],"sizeBytes":35881529},{"names":["docker.io/cloudability/metrics-agent@sha256:88da6d41e0cd48be4e273fc4bb640c98883259c3b8e450175725e75150b075dd","docker.io/cloudability/metrics-agent:latest"],"sizeBytes":35846897},{"names":["asia.gcr.io/gke-release-staging/cilium/slim-daemon/anet-agent@sha256:1a871e457e08d4e3e06a079f76e6a5a67c5869797ae305a6fb7f1e2c4712f0cd","eu.gcr.io/gke-release-staging/cilium/slim-daemon/anet-agent@sha256:1a871e457e08d4e3e06a079f76e6a5a67c5869797ae305a6fb7f1e2c4712f0cd","gcr.io/gke-release-staging/cilium/slim-daemon/anet-agent@sha256:1a871e457e08d4e3e06a079f76e6a5a67c5869797ae305a6fb7f1e2c4712f0cd","asia.gke.gcr.io/cilium/slim-daemon/anet-agent:v1.14.13-gke1.30-gke.64","eu.gke.gcr.io/cilium/slim-daemon/anet-agent:v1.14.13-gke1.30-gke.64"],"sizeBytes":33172919},{"names":["gke.gcr.io/fluent-bit-gke-exporter@sha256:0ed31fb2cc1b2b747a1e304a5530124c97909a0eb17774c7ce80595abf73d1e6"],"sizeBytes":32969391},{"names":["gke.gcr.io/k8s-dns-kube-dns@sha256:b609a51c8aa4add2d1d0811737f177b4e944ea0781a48eead0d804722787f96f"],"sizeBytes":32667404},{"names":["gke.gcr.io/k8s-dns-sidecar@sha256:9e60f83b54d010a7dd7e5a868a6713ad410442c72f0b7540cda010c50651c0bc"],"sizeBytes":29112653},{"names":["gke.gcr.io/gke-metrics-agent@sha256:be215beaf8f2acd5c4f64ff3d28e6dc60071ef6f0da5eb9774fda69148c38ba4"],"sizeBytes":27003958},{"names":["gke.gcr.io/gke-metrics-collector@sha256:3d76420863be0cdbdf5f9a512e032d9b20ae8fd7be4b5eca22ecdb1e867f23bf"],"sizeBytes":25809971},{"names":["gke.gcr.io/gke-metrics-collector@sha256:360c05bb437d77b55090ba512bddfda52f62166c78171593d1152327db2a914f"],"sizeBytes":25582287},{"names":["gke.gcr.io/gke-metrics-collector@sha256:d460e6b5088332f62b990f8a1f7bf6d9eca7c3f41cb974e3db493d6b0fc4ad70"],"sizeBytes":24425624},{"names":["gke.gcr.io/gke-metrics-collector@sha256:463e73163c4d343b8a3327e0d2e8e955d22434e9005a1a188275ac55b8cfebb4"],"sizeBytes":24343841},{"names":["asia.gcr.io/gke-release-staging/cilium/hubble-cli@sha256:327d40413ece3e0cf8e14cebd5d24b15a1987bf8a4e5ec369dabdb17203e62c2","eu.gcr.io/gke-release-staging/cilium/hubble-cli@sha256:327d40413ece3e0cf8e14cebd5d24b15a1987bf8a4e5ec369dabdb17203e62c2","gcr.io/gke-release-staging/cilium/hubble-cli@sha256:327d40413ece3e0cf8e14cebd5d24b15a1987bf8a4e5ec369dabdb17203e62c2","asia.gke.gcr.io/cilium/hubble-cli:v0.13.5-gke.14","eu.gke.gcr.io/cilium/hubble-cli:v0.13.5-gke.14"],"sizeBytes":24123625},{"names":["asia.gcr.io/gke-release-staging/anthos-networking/anetd-sidecar@sha256:2a788f8b3fc1112f706a7b22f13fb0e430d76c5743e6f8a6c2c350345f3a006d","eu.gcr.io/gke-release-staging/anthos-networking/anetd-sidecar@sha256:2a788f8b3fc1112f706a7b22f13fb0e430d76c5743e6f8a6c2c350345f3a006d","gcr.io/gke-release-staging/anthos-networking/anetd-sidecar@sha256:2a788f8b3fc1112f706a7b22f13fb0e430d76c5743e6f8a6c2c350345f3a006d","asia.gke.gcr.io/anthos-networking/anetd-sidecar:v2.9.77","eu.gke.gcr.io/anthos-networking/anetd-sidecar:v2.9.77"],"sizeBytes":23881188},{"names":["asia.gcr.io/gke-release-staging/gke-metrics-collector@sha256:1593f1e9570b99b1843cc886b723d4e291e2281fe2b9e45f565064cac379e4cf","eu.gcr.io/gke-release-staging/gke-metrics-collector@sha256:1593f1e9570b99b1843cc886b723d4e291e2281fe2b9e45f565064cac379e4cf","gcr.io/gke-release-staging/gke-metrics-collector@sha256:1593f1e9570b99b1843cc886b723d4e291e2281fe2b9e45f565064cac379e4cf","asia.gke.gcr.io/gke-metrics-collector:20240508_2300_RC0","eu.gke.gcr.io/gke-metrics-collector:20240508_2300_RC0"],"sizeBytes":23823718},{"names":["asia.gcr.io/gke-release-staging/gke-metrics-collector@sha256:545f5594f9a6ee714b315897f92edeaa858b1074e4c6a38894353214907e4fe5","eu.gcr.io/gke-release-staging/gke-metrics-collector@sha256:545f5594f9a6ee714b315897f92edeaa858b1074e4c6a38894353214907e4fe5","gcr.io/gke-release-staging/gke-metrics-collector@sha256:545f5594f9a6ee714b315897f92edeaa858b1074e4c6a38894353214907e4fe5","asia.gke.gcr.io/gke-metrics-collector:20240425_2300_RC0","eu.gke.gcr.io/gke-metrics-collector:20240425_2300_RC0"],"sizeBytes":23717101},{"names":["asia.gcr.io/gke-release-staging/netd@sha256:6ca0892cc32f66705087257c12ef97f3f27b270402e03299450b19ecb93e48e6","eu.gcr.io/gke-release-staging/netd@sha256:6ca0892cc32f66705087257c12ef97f3f27b270402e03299450b19ecb93e48e6","gcr.io/gke-release-staging/netd@sha256:6ca0892cc32f66705087257c12ef97f3f27b270402e03299450b19ecb93e48e6","asia.gke.gcr.io/netd:v0.8.4-gke.18","eu.gke.gcr.io/netd:v0.8.4-gke.18"],"sizeBytes":23427157},{"names":["asia.gcr.io/gke-release-staging/cilium/hubble-relay@sha256:365232539d4c473b20544833b79b2fe448190dc2a5280e6bd38b672a2d7ad121","eu.gcr.io/gke-release-staging/cilium/hubble-relay@sha256:365232539d4c473b20544833b79b2fe448190dc2a5280e6bd38b672a2d7ad121","gcr.io/gke-release-staging/cilium/hubble-relay@sha256:365232539d4c473b20544833b79b2fe448190dc2a5280e6bd38b672a2d7ad121","asia.gke.gcr.io/cilium/hubble-relay:v1.14.13-gke1.30-gke.64","eu.gke.gcr.io/cilium/hubble-relay:v1.14.13-gke1.30-gke.64"],"sizeBytes":21850766}]}} {"metadata":{"name":"nodename3","uid":"e11a5a98-56c9-4bc9-b8f7-1e951f1f2140","resourceVersion":"377579378","creationTimestamp":"2025-02-27T21:49:29Z","labels":{"beta.kubernetes.io/arch":"amd64","beta.kubernetes.io/instance-type":"e2-medium","beta.kubernetes.io/os":"linux","cloud.google.com/gke-boot-disk":"pd-balanced","cloud.google.com/gke-container-runtime":"containerd","cloud.google.com/gke-cpu-scaling-level":"2","cloud.google.com/gke-logging-variant":"DEFAULT","cloud.google.com/gke-max-pods-per-node":"100","cloud.google.com/gke-memory-gb-scaling-level":"4","cloud.google.com/gke-nodepool":"default-pool","cloud.google.com/gke-os-distribution":"cos","cloud.google.com/gke-provisioning":"standard","cloud.google.com/gke-stack-type":"IPV4","cloud.google.com/machine-family":"e2","cloud.google.com/private-node":"false","failure-domain.beta.kubernetes.io/region":"us-central1","failure-domain.beta.kubernetes.io/zone":"us-central1-c","kubernetes.io/arch":"amd64","kubernetes.io/hostname":"nodename3","kubernetes.io/os":"linux","node.kubernetes.io/instance-type":"e2-medium","topology.gke.io/zone":"us-central1-c","topology.kubernetes.io/region":"us-central1","topology.kubernetes.io/zone":"us-central1-c"},"annotations":{"container.googleapis.com/instance_id":"2148973188718879535","csi.volume.kubernetes.io/nodeid":"{\"pd.csi.storage.gke.io\":\"projects/containers-183923/zones/us-central1-c/instances/nodename3\"}","node.alpha.kubernetes.io/ttl":"0","node.gke.io/last-applied-node-labels":"cloud.google.com/gke-boot-disk=pd-balanced,cloud.google.com/gke-container-runtime=containerd,cloud.google.com/gke-cpu-scaling-level=2,cloud.google.com/gke-logging-variant=DEFAULT,cloud.google.com/gke-max-pods-per-node=100,cloud.google.com/gke-memory-gb-scaling-level=4,cloud.google.com/gke-nodepool=default-pool,cloud.google.com/gke-os-distribution=cos,cloud.google.com/gke-provisioning=standard,cloud.google.com/gke-stack-type=IPV4,cloud.google.com/machine-family=e2,cloud.google.com/private-node=false","node.gke.io/last-applied-node-taints":"","volumes.kubernetes.io/controller-managed-attach-detach":"true"},"managedFields":[{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2025-02-27T21:49:29Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{".":{},"f:volumes.kubernetes.io/controller-managed-attach-detach":{}},"f:labels":{".":{},"f:beta.kubernetes.io/arch":{},"f:beta.kubernetes.io/os":{},"f:cloud.google.com/gke-boot-disk":{},"f:cloud.google.com/gke-container-runtime":{},"f:cloud.google.com/gke-cpu-scaling-level":{},"f:cloud.google.com/gke-logging-variant":{},"f:cloud.google.com/gke-max-pods-per-node":{},"f:cloud.google.com/gke-memory-gb-scaling-level":{},"f:cloud.google.com/gke-nodepool":{},"f:cloud.google.com/gke-os-distribution":{},"f:cloud.google.com/gke-provisioning":{},"f:cloud.google.com/gke-stack-type":{},"f:cloud.google.com/machine-family":{},"f:cloud.google.com/private-node":{},"f:kubernetes.io/arch":{},"f:kubernetes.io/hostname":{},"f:kubernetes.io/os":{}}}}},{"manager":"cloud-controller-manager","operation":"Update","apiVersion":"v1","time":"2025-02-27T21:49:30Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:labels":{"f:beta.kubernetes.io/instance-type":{},"f:failure-domain.beta.kubernetes.io/region":{},"f:failure-domain.beta.kubernetes.io/zone":{},"f:node.kubernetes.io/instance-type":{},"f:topology.kubernetes.io/region":{},"f:topology.kubernetes.io/zone":{}}},"f:spec":{"f:podCIDR":{},"f:podCIDRs":{".":{},"v:\"10.84.4.0/24\"":{}},"f:providerID":{}}}},{"manager":"cloud-controller-manager","operation":"Update","apiVersion":"v1","time":"2025-02-27T21:49:30Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:addresses":{".":{},"k:{\"type\":\"ExternalIP\"}":{".":{},"f:address":{},"f:type":{}},"k:{\"type\":\"InternalIP\"}":{".":{},"f:address":{},"f:type":{}}},"f:conditions":{"k:{\"type\":\"NetworkUnavailable\"}":{".":{},"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}}}}},"subresource":"status"},{"manager":"kube-controller-manager","operation":"Update","apiVersion":"v1","time":"2025-02-27T21:49:30Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{"f:node.alpha.kubernetes.io/ttl":{}}}}},{"manager":"gcp-controller-manager","operation":"Update","apiVersion":"v1","time":"2025-02-27T21:49:31Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{"f:container.googleapis.com/instance_id":{},"f:node.gke.io/last-applied-node-labels":{},"f:node.gke.io/last-applied-node-taints":{}}}}},{"manager":"node-problem-detector","operation":"Update","apiVersion":"v1","time":"2025-03-01T23:40:09Z","fieldsType":"FieldsV1","fieldsV1":{"f:status":{"f:conditions":{"k:{\"type\":\"CorruptDockerOverlay2\"}":{".":{},"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"DeprecatedAuthsFieldInContainerdConfiguration\"}":{".":{},"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"DeprecatedConfigsFieldInContainerdConfiguration\"}":{".":{},"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"DeprecatedMirrorsFieldInContainerdConfiguration\"}":{".":{},"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"DeprecatedOtherContainerdFeatures\"}":{".":{},"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"DeprecatedPullingSchemaV1Image\"}":{".":{},"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"DeprecatedUsingV1Alpha2Cri\"}":{".":{},"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"FrequentContainerdRestart\"}":{".":{},"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"FrequentDockerRestart\"}":{".":{},"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"FrequentKubeletRestart\"}":{".":{},"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"FrequentUnregisterNetDevice\"}":{".":{},"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"KernelDeadlock\"}":{".":{},"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"ReadonlyFilesystem\"}":{".":{},"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}},"k:{\"type\":\"SysctlChanged\"}":{".":{},"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{},"f:type":{}}}}},"subresource":"status"},{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2025-03-01T23:41:05Z","fieldsType":"FieldsV1","fieldsV1":{"f:metadata":{"f:annotations":{"f:csi.volume.kubernetes.io/nodeid":{}},"f:labels":{"f:topology.gke.io/zone":{}}},"f:status":{"f:conditions":{"k:{\"type\":\"DiskPressure\"}":{"f:lastHeartbeatTime":{}},"k:{\"type\":\"MemoryPressure\"}":{"f:lastHeartbeatTime":{}},"k:{\"type\":\"PIDPressure\"}":{"f:lastHeartbeatTime":{}},"k:{\"type\":\"Ready\"}":{"f:lastHeartbeatTime":{},"f:lastTransitionTime":{},"f:message":{},"f:reason":{},"f:status":{}}},"f:images":{}}},"subresource":"status"}]},"spec":{"podCIDR":"10.84.4.0/24","podCIDRs":["10.84.4.0/24"],"providerID":"gce://containers-183923/us-central1-c/nodename3"},"status":{"capacity":{"cpu":"2","ephemeral-storage":"98831908Ki","hugepages-1Gi":"0","hugepages-2Mi":"0","memory":"4018132Ki","pods":"100"},"allocatable":{"cpu":"940m","ephemeral-storage":"47060071478","hugepages-1Gi":"0","hugepages-2Mi":"0","memory":"2872276Ki","pods":"100"},"conditions":[{"type":"DeprecatedUsingV1Alpha2Cri","status":"False","lastHeartbeatTime":"2025-03-01T23:40:09Z","lastTransitionTime":"2025-02-27T21:49:29Z","reason":"DeprecatedUsingV1Alpha2CriNotDetected","message":"No deprecation risk: did not use v1alpha2 CRI"},{"type":"CorruptDockerOverlay2","status":"False","lastHeartbeatTime":"2025-03-01T23:40:09Z","lastTransitionTime":"2025-02-27T21:49:29Z","reason":"NoCorruptDockerOverlay2","message":"docker overlay2 is functioning properly"},{"type":"SysctlChanged","status":"False","lastHeartbeatTime":"2025-03-01T23:40:09Z","lastTransitionTime":"2025-02-27T21:49:29Z","reason":"SysctlNotChanged","message":"Default sysctls are in effect, no unexpected sysctl changes"},{"type":"DeprecatedAuthsFieldInContainerdConfiguration","status":"False","lastHeartbeatTime":"2025-03-01T23:40:09Z","lastTransitionTime":"2025-02-27T21:49:29Z","reason":"DeprecatedAuthsFieldInContainerdConfigurationNotDetected","message":"No deprecation risk: did not find any deprecated 'auths' field in containerd's config"},{"type":"FrequentUnregisterNetDevice","status":"False","lastHeartbeatTime":"2025-03-01T23:40:09Z","lastTransitionTime":"2025-02-27T21:49:29Z","reason":"NoFrequentUnregisterNetDevice","message":"node is functioning properly"},{"type":"FrequentContainerdRestart","status":"False","lastHeartbeatTime":"2025-03-01T23:40:09Z","lastTransitionTime":"2025-02-27T21:49:29Z","reason":"NoFrequentContainerdRestart","message":"containerd is functioning properly"},{"type":"DeprecatedPullingSchemaV1Image","status":"False","lastHeartbeatTime":"2025-03-01T23:40:09Z","lastTransitionTime":"2025-02-27T21:49:29Z","reason":"DeprecatedPullingSchemaV1ImageDetected","message":"No deprecation risk: did not pull any schema v1 images"},{"type":"FrequentDockerRestart","status":"False","lastHeartbeatTime":"2025-03-01T23:40:09Z","lastTransitionTime":"2025-02-27T21:49:29Z","reason":"NoFrequentDockerRestart","message":"docker is functioning properly"},{"type":"DeprecatedOtherContainerdFeatures","status":"False","lastHeartbeatTime":"2025-03-01T23:40:09Z","lastTransitionTime":"2025-02-27T21:49:29Z","reason":"DeprecatedOtherContainerdFeaturesNotDetected","message":"No deprecation risk: did not find any deprecations other than 3 configs fields (auths/configs/mirrors), pulling schema v1 images and using v1alpha2 CRI."},{"type":"DeprecatedConfigsFieldInContainerdConfiguration","status":"False","lastHeartbeatTime":"2025-03-01T23:40:09Z","lastTransitionTime":"2025-02-27T21:49:30Z","reason":"DeprecatedConfigsFieldInContainerdConfigurationNotDetected","message":"No deprecation risk: did not find any deprecated 'configs' field in containerd's config"},{"type":"ReadonlyFilesystem","status":"False","lastHeartbeatTime":"2025-03-01T23:40:09Z","lastTransitionTime":"2025-02-27T21:49:29Z","reason":"FilesystemIsNotReadOnly","message":"Filesystem is not read-only"},{"type":"DeprecatedMirrorsFieldInContainerdConfiguration","status":"False","lastHeartbeatTime":"2025-03-01T23:40:09Z","lastTransitionTime":"2025-02-27T21:49:29Z","reason":"DeprecatedMirrorsFieldInContainerdConfigurationNotDetected","message":"No deprecation risk: did not find any deprecated 'mirrors' field in containerd's config"},{"type":"KernelDeadlock","status":"False","lastHeartbeatTime":"2025-03-01T23:40:09Z","lastTransitionTime":"2025-02-27T21:49:29Z","reason":"KernelHasNoDeadlock","message":"kernel has no deadlock"},{"type":"FrequentKubeletRestart","status":"False","lastHeartbeatTime":"2025-03-01T23:40:09Z","lastTransitionTime":"2025-02-27T21:49:29Z","reason":"NoFrequentKubeletRestart","message":"kubelet is functioning properly"},{"type":"NetworkUnavailable","status":"False","lastHeartbeatTime":"2025-02-27T21:49:30Z","lastTransitionTime":"2025-02-27T21:49:30Z","reason":"RouteCreated","message":"NodeController create implicit route"},{"type":"MemoryPressure","status":"False","lastHeartbeatTime":"2025-03-01T23:41:05Z","lastTransitionTime":"2025-02-27T21:49:27Z","reason":"KubeletHasSufficientMemory","message":"kubelet has sufficient memory available"},{"type":"DiskPressure","status":"False","lastHeartbeatTime":"2025-03-01T23:41:05Z","lastTransitionTime":"2025-02-27T21:49:27Z","reason":"KubeletHasNoDiskPressure","message":"kubelet has no disk pressure"},{"type":"PIDPressure","status":"False","lastHeartbeatTime":"2025-03-01T23:41:05Z","lastTransitionTime":"2025-02-27T21:49:27Z","reason":"KubeletHasSufficientPID","message":"kubelet has sufficient PID available"},{"type":"Ready","status":"True","lastHeartbeatTime":"2025-03-01T23:41:05Z","lastTransitionTime":"2025-02-27T21:49:30Z","reason":"KubeletReady","message":"kubelet is posting ready status"}],"addresses":[{"type":"InternalIP","address":"10.128.15.217"},{"type":"ExternalIP","address":"34.56.144.234"}],"daemonEndpoints":{"kubeletEndpoint":{"Port":10250}},"nodeInfo":{"machineID":"97fb66702ba318020f503a2cb0c3d8e7","systemUUID":"97fb6670-2ba3-1802-0f50-3a2cb0c3d8e7","bootID":"22f0f516-6608-4f80-bfb8-d2529fe5b7d7","kernelVersion":"6.1.123+","osImage":"Container-Optimized OS from Google","containerRuntimeVersion":"containerd://1.7.24","kubeletVersion":"v1.30.9-gke.1127000","kubeProxyVersion":"v1.30.9-gke.1127000","operatingSystem":"linux","architecture":"amd64"},"images":[{"names":["asia.gcr.io/gke-release-staging/cilium/cilium@sha256:a69b8b0cab532f641f2bcdd823842599b852666f1bcc763200b9cf4b3f40537f","eu.gcr.io/gke-release-staging/cilium/cilium@sha256:a69b8b0cab532f641f2bcdd823842599b852666f1bcc763200b9cf4b3f40537f","gcr.io/gke-release-staging/cilium/cilium@sha256:a69b8b0cab532f641f2bcdd823842599b852666f1bcc763200b9cf4b3f40537f","asia.gke.gcr.io/cilium/cilium:v1.14.13-gke1.30-gke.64","eu.gke.gcr.io/cilium/cilium:v1.14.13-gke1.30-gke.64"],"sizeBytes":171632673},{"names":["gke.gcr.io/prometheus-engine/prometheus@sha256:3e6493d4b01ab583382731491d980bc164873ad4969e92c0bdd0da278359ccac"],"sizeBytes":113010021},{"names":["gke.gcr.io/fluent-bit@sha256:bd48755377a723a91bc3fd96f67796ba3a3023808fd69b37d97cb1aa5985d9ae"],"sizeBytes":93330926},{"names":["gke.gcr.io/kube-proxy-amd64:v1.30.9-gke.1127000","gke.gcr.io/kube-proxy-amd64:v1.30.9-gke.500","k8s.gcr.io/kube-proxy-amd64:v1.30.9-gke.500"],"sizeBytes":88273080},{"names":["gke.gcr.io/gcp-compute-persistent-disk-csi-driver@sha256:bbbf275b1482cf3fc658f0bde28b1fbd24054451b5a97e23812821c82374a2b2"],"sizeBytes":60814920},{"names":["gke.gcr.io/prometheus-engine/config-reloader@sha256:d199f266545ee281fa51d30e0a5f9c4da27da23055b153ca93adbf7483d19633"],"sizeBytes":59834302},{"names":["asia.gcr.io/gke-release-staging/cilium/certgen@sha256:73529e65f241d2b390d4eadb4b886ea9de8c3ef27ae3854733d6a9cbefd6d0b0","eu.gcr.io/gke-release-staging/cilium/certgen@sha256:73529e65f241d2b390d4eadb4b886ea9de8c3ef27ae3854733d6a9cbefd6d0b0","gcr.io/gke-release-staging/cilium/certgen@sha256:73529e65f241d2b390d4eadb4b886ea9de8c3ef27ae3854733d6a9cbefd6d0b0","asia.gke.gcr.io/cilium/certgen:v0.1.13-gke.14","eu.gke.gcr.io/cilium/certgen:v0.1.13-gke.14"],"sizeBytes":40760357},{"names":["docker.io/artichoke111/metrics-agent@sha256:3cbe3fbd370ee77dc47fc4a918b14822811d22585566791a9f4ecc569d9c29c9","docker.io/artichoke111/metrics-agent:metrics-agent-2.11.17-dirty"],"sizeBytes":35927772},{"names":["docker.io/cloudability/metrics-agent@sha256:88da6d41e0cd48be4e273fc4bb640c98883259c3b8e450175725e75150b075dd","docker.io/cloudability/metrics-agent:latest"],"sizeBytes":35846897},{"names":["asia.gcr.io/gke-release-staging/cilium/slim-daemon/anet-agent@sha256:1a871e457e08d4e3e06a079f76e6a5a67c5869797ae305a6fb7f1e2c4712f0cd","eu.gcr.io/gke-release-staging/cilium/slim-daemon/anet-agent@sha256:1a871e457e08d4e3e06a079f76e6a5a67c5869797ae305a6fb7f1e2c4712f0cd","gcr.io/gke-release-staging/cilium/slim-daemon/anet-agent@sha256:1a871e457e08d4e3e06a079f76e6a5a67c5869797ae305a6fb7f1e2c4712f0cd","asia.gke.gcr.io/cilium/slim-daemon/anet-agent:v1.14.13-gke1.30-gke.64","eu.gke.gcr.io/cilium/slim-daemon/anet-agent:v1.14.13-gke1.30-gke.64"],"sizeBytes":33172919},{"names":["gke.gcr.io/fluent-bit-gke-exporter@sha256:0ed31fb2cc1b2b747a1e304a5530124c97909a0eb17774c7ce80595abf73d1e6"],"sizeBytes":32969391},{"names":["gke.gcr.io/gke-metrics-agent@sha256:be215beaf8f2acd5c4f64ff3d28e6dc60071ef6f0da5eb9774fda69148c38ba4"],"sizeBytes":27003958},{"names":["gke.gcr.io/gke-metrics-collector@sha256:3d76420863be0cdbdf5f9a512e032d9b20ae8fd7be4b5eca22ecdb1e867f23bf"],"sizeBytes":25809971},{"names":["gke.gcr.io/gke-metrics-collector@sha256:d460e6b5088332f62b990f8a1f7bf6d9eca7c3f41cb974e3db493d6b0fc4ad70"],"sizeBytes":24425624},{"names":["gke.gcr.io/gke-metrics-collector@sha256:463e73163c4d343b8a3327e0d2e8e955d22434e9005a1a188275ac55b8cfebb4"],"sizeBytes":24343841},{"names":["asia.gcr.io/gke-release-staging/cilium/hubble-cli@sha256:327d40413ece3e0cf8e14cebd5d24b15a1987bf8a4e5ec369dabdb17203e62c2","eu.gcr.io/gke-release-staging/cilium/hubble-cli@sha256:327d40413ece3e0cf8e14cebd5d24b15a1987bf8a4e5ec369dabdb17203e62c2","gcr.io/gke-release-staging/cilium/hubble-cli@sha256:327d40413ece3e0cf8e14cebd5d24b15a1987bf8a4e5ec369dabdb17203e62c2","asia.gke.gcr.io/cilium/hubble-cli:v0.13.5-gke.14","eu.gke.gcr.io/cilium/hubble-cli:v0.13.5-gke.14"],"sizeBytes":24123625},{"names":["asia.gcr.io/gke-release-staging/anthos-networking/anetd-sidecar@sha256:2a788f8b3fc1112f706a7b22f13fb0e430d76c5743e6f8a6c2c350345f3a006d","eu.gcr.io/gke-release-staging/anthos-networking/anetd-sidecar@sha256:2a788f8b3fc1112f706a7b22f13fb0e430d76c5743e6f8a6c2c350345f3a006d","gcr.io/gke-release-staging/anthos-networking/anetd-sidecar@sha256:2a788f8b3fc1112f706a7b22f13fb0e430d76c5743e6f8a6c2c350345f3a006d","asia.gke.gcr.io/anthos-networking/anetd-sidecar:v2.9.77","eu.gke.gcr.io/anthos-networking/anetd-sidecar:v2.9.77"],"sizeBytes":23881188},{"names":["asia.gcr.io/gke-release-staging/gke-metrics-collector@sha256:1593f1e9570b99b1843cc886b723d4e291e2281fe2b9e45f565064cac379e4cf","eu.gcr.io/gke-release-staging/gke-metrics-collector@sha256:1593f1e9570b99b1843cc886b723d4e291e2281fe2b9e45f565064cac379e4cf","gcr.io/gke-release-staging/gke-metrics-collector@sha256:1593f1e9570b99b1843cc886b723d4e291e2281fe2b9e45f565064cac379e4cf","asia.gke.gcr.io/gke-metrics-collector:20240508_2300_RC0","eu.gke.gcr.io/gke-metrics-collector:20240508_2300_RC0"],"sizeBytes":23823718},{"names":["asia.gcr.io/gke-release-staging/gke-metrics-collector@sha256:545f5594f9a6ee714b315897f92edeaa858b1074e4c6a38894353214907e4fe5","eu.gcr.io/gke-release-staging/gke-metrics-collector@sha256:545f5594f9a6ee714b315897f92edeaa858b1074e4c6a38894353214907e4fe5","gcr.io/gke-release-staging/gke-metrics-collector@sha256:545f5594f9a6ee714b315897f92edeaa858b1074e4c6a38894353214907e4fe5","asia.gke.gcr.io/gke-metrics-collector:20240425_2300_RC0","eu.gke.gcr.io/gke-metrics-collector:20240425_2300_RC0"],"sizeBytes":23805895},{"names":["asia.gcr.io/gke-release-staging/netd@sha256:6ca0892cc32f66705087257c12ef97f3f27b270402e03299450b19ecb93e48e6","eu.gcr.io/gke-release-staging/netd@sha256:6ca0892cc32f66705087257c12ef97f3f27b270402e03299450b19ecb93e48e6","gcr.io/gke-release-staging/netd@sha256:6ca0892cc32f66705087257c12ef97f3f27b270402e03299450b19ecb93e48e6","asia.gke.gcr.io/netd:v0.8.4-gke.18","eu.gke.gcr.io/netd:v0.8.4-gke.18"],"sizeBytes":23427157},{"names":["asia.gcr.io/gke-release-staging/cilium/hubble-relay@sha256:365232539d4c473b20544833b79b2fe448190dc2a5280e6bd38b672a2d7ad121","eu.gcr.io/gke-release-staging/cilium/hubble-relay@sha256:365232539d4c473b20544833b79b2fe448190dc2a5280e6bd38b672a2d7ad121","gcr.io/gke-release-staging/cilium/hubble-relay@sha256:365232539d4c473b20544833b79b2fe448190dc2a5280e6bd38b672a2d7ad121","asia.gke.gcr.io/cilium/hubble-relay:v1.14.13-gke1.30-gke.64","eu.gke.gcr.io/cilium/hubble-relay:v1.14.13-gke1.30-gke.64"],"sizeBytes":21850766},{"names":["gke.gcr.io/gke-distroless/bash@sha256:12d99a6a72f4fecd689ead5d93001c1f3acea08ec72a55bbdfc070e0edc30fa4"],"sizeBytes":18654784},{"names":["gke.gcr.io/gke-distroless/bash@sha256:ec5022c67b5316ae07f44ed374894e9bb55d548884d293da6b0d350a46dff2df"],"sizeBytes":18373482},{"names":["asia.gcr.io/gke-release-staging/ip-masq-agent@sha256:4035e9a6996d792ded07e94aab4e0a54bd3a3d58dddbbd0fb4edc8898a4c4d71","eu.gcr.io/gke-release-staging/ip-masq-agent@sha256:4035e9a6996d792ded07e94aab4e0a54bd3a3d58dddbbd0fb4edc8898a4c4d71","gcr.io/gke-release-staging/ip-masq-agent@sha256:4035e9a6996d792ded07e94aab4e0a54bd3a3d58dddbbd0fb4edc8898a4c4d71","asia.gke.gcr.io/ip-masq-agent:v2.11.0-gke.30","eu.gke.gcr.io/ip-masq-agent:v2.11.0-gke.30"],"sizeBytes":16420466},{"names":["asia.gcr.io/gke-release-staging/netd-init@sha256:96c1dd2984b3d3f7029242b9248a0dcbdde2714e7d92bdbc51e9a85f60f0e9f3","eu.gcr.io/gke-release-staging/netd-init@sha256:96c1dd2984b3d3f7029242b9248a0dcbdde2714e7d92bdbc51e9a85f60f0e9f3","gcr.io/gke-release-staging/netd-init@sha256:96c1dd2984b3d3f7029242b9248a0dcbdde2714e7d92bdbc51e9a85f60f0e9f3","asia.gke.gcr.io/netd-init:v0.8.4-gke.18","eu.gke.gcr.io/netd-init:v0.8.4-gke.18"],"sizeBytes":15416418}]}} + +{"metadata":{"name":"nodename5","uid":"f8a3c2d1-9b4e-4c5a-a1f2-3d6e7b8c9a0b","resourceVersion":"377580000","creationTimestamp":"2025-02-27T22:30:00Z","labels":{"beta.kubernetes.io/arch":"amd64","beta.kubernetes.io/instance-type":"e2-medium","beta.kubernetes.io/os":"linux","cloud.google.com/gke-boot-disk":"pd-balanced","cloud.google.com/gke-container-runtime":"containerd","cloud.google.com/gke-cpu-scaling-level":"2","cloud.google.com/gke-logging-variant":"DEFAULT","cloud.google.com/gke-max-pods-per-node":"100","cloud.google.com/gke-memory-gb-scaling-level":"4","cloud.google.com/gke-nodepool":"ipv6-pool","cloud.google.com/gke-os-distribution":"cos","cloud.google.com/gke-provisioning":"standard","cloud.google.com/gke-stack-type":"IPV4_IPV6","cloud.google.com/machine-family":"e2","cloud.google.com/private-node":"false","failure-domain.beta.kubernetes.io/region":"us-central1","failure-domain.beta.kubernetes.io/zone":"us-central1-c","kubernetes.io/arch":"amd64","kubernetes.io/hostname":"nodename5","kubernetes.io/os":"linux","node.kubernetes.io/instance-type":"e2-medium","topology.gke.io/zone":"us-central1-c","topology.kubernetes.io/region":"us-central1","topology.kubernetes.io/zone":"us-central1-c"},"annotations":{"container.googleapis.com/instance_id":"1234567890123456789","csi.volume.kubernetes.io/nodeid":"{\"pd.csi.storage.gke.io\":\"projects/containers-183923/zones/us-central1-c/instances/nodename5\"}","node.alpha.kubernetes.io/ttl":"0","volumes.kubernetes.io/controller-managed-attach-detach":"true"},"managedFields":[{"manager":"kubelet","operation":"Update","apiVersion":"v1","time":"2025-02-27T22:30:00Z","fieldsType":"FieldsV1"}]},"spec":{"podCIDR":"10.84.5.0/24","podCIDRs":["10.84.5.0/24","2600:1900:4000::/64"],"providerID":"gce://containers-183923/us-central1-c/nodename5"},"status":{"capacity":{"cpu":"2","ephemeral-storage":"98831908Ki","hugepages-1Gi":"0","hugepages-2Mi":"0","memory":"4018132Ki","pods":"100"},"allocatable":{"cpu":"940m","ephemeral-storage":"47060071478","hugepages-1Gi":"0","hugepages-2Mi":"0","memory":"2872276Ki","pods":"100"},"conditions":[{"type":"Ready","status":"True","lastHeartbeatTime":"2025-03-01T23:45:00Z","lastTransitionTime":"2025-02-27T22:30:00Z","reason":"KubeletReady","message":"kubelet is posting ready status"}],"addresses":[{"type":"InternalIP","address":"2001:db8::1"},{"type":"InternalIP","address":"10.128.0.50"},{"type":"ExternalIP","address":"2001:db8:85a3::8a2e:370:7334"}],"daemonEndpoints":{"kubeletEndpoint":{"Port":10250}},"nodeInfo":{"machineID":"a1b2c3d4e5f6g7h8i9j0k1l2m3n4o5p6","systemUUID":"a1b2c3d4-e5f6-g7h8-i9j0-k1l2m3n4o5p6","bootID":"b1c2d3e4-f5g6-h7i8-j9k0-l1m2n3o4p5q6","kernelVersion":"6.1.123+","osImage":"Container-Optimized OS from Google","containerRuntimeVersion":"containerd://1.7.24","kubeletVersion":"v1.30.9-gke.1127000","kubeProxyVersion":"v1.30.9-gke.1127000","operatingSystem":"linux","architecture":"amd64"},"images":[]}}