Capture pod events just before deleting the agent

src/main/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesEventsException.java

@@ -0,0 +1,25 @@
+package org.csanchez.jenkins.plugins.kubernetes;
+
+import io.fabric8.kubernetes.api.model.Event;
+
+import java.util.List;
+
+/**
+ * Custom exception which adds information about Kubernetes events.
+ */
+public class KubernetesEventsException extends Exception {
+    public KubernetesEventsException(List<Event> events) {
+        super(toMessage(events));
+    }
+
+    private static String toMessage(List<Event> events) {
+        StringBuilder sb = new StringBuilder("Events follow:\n");
+        for (Event event : events) {
+            String[] lines = event.getMessage().split("\n");
+            for (String line : lines) {
+                sb.append(String.format("[%s][%s/%s][%s] %s%n", event.getType(), event.getInvolvedObject().getNamespace(), event.getInvolvedObject().getName(), event.getReason(), line));
+            }
+        }
+        return sb.toString();
+    }
+}

src/main/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesLauncher.java

@@ -39,9 +39,11 @@
 import javax.annotation.CheckForNull;
 
 import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
+import io.fabric8.kubernetes.api.model.Event;
 import io.fabric8.kubernetes.client.KubernetesClientException;
 import jenkins.metrics.api.Metrics;
 import org.apache.commons.lang.StringUtils;
+import org.jenkinsci.plugins.kubernetes.auth.KubernetesAuthException;
 import org.kohsuke.stapler.DataBoundConstructor;
 
 import hudson.model.TaskListener;
@@ -251,6 +253,14 @@ else if (httpCode == 409 && e.getMessage().contains("Operation cannot be fulfill
             }
             Metrics.metricRegistry().counter(MetricNames.PODS_LAUNCHED).inc();
         } catch (Throwable ex) {
+            try {
+                List<Event> podEvents = ((KubernetesComputer) computer).getPodEvents();
+                if (!podEvents.isEmpty()) {
+                    ex.addSuppressed(new KubernetesEventsException(podEvents));
+                }
+            } catch (KubernetesAuthException | IOException e) {
+                LOGGER.log(Level.FINE, "Unable to add Kubernetes events");
+            }
             setProblem(ex);
             LOGGER.log(Level.WARNING, String.format("Error in provisioning; agent=%s, template=%s", node, template), ex);
             LOGGER.log(Level.FINER, "Removing Jenkins node: {0}", node.getNodeName());

src/test/java/org/csanchez/jenkins/plugins/kubernetes/pipeline/KubernetesPipelineTest.java

@@ -24,75 +24,79 @@
 
 package org.csanchez.jenkins.plugins.kubernetes.pipeline;
 
-import static org.csanchez.jenkins.plugins.kubernetes.KubernetesTestUtil.CONTAINER_ENV_VAR_FROM_SECRET_VALUE;
-import static org.csanchez.jenkins.plugins.kubernetes.KubernetesTestUtil.POD_ENV_VAR_FROM_SECRET_VALUE;
-import static org.csanchez.jenkins.plugins.kubernetes.KubernetesTestUtil.assumeWindows;
-import static org.csanchez.jenkins.plugins.kubernetes.KubernetesTestUtil.deletePods;
-import static org.csanchez.jenkins.plugins.kubernetes.KubernetesTestUtil.getLabels;
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.Matchers.emptyIterable;
-import static org.hamcrest.Matchers.hasEntry;
-import static org.hamcrest.Matchers.hasSize;
-import static org.hamcrest.Matchers.oneOf;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assume.*;
-
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import hudson.model.Computer;
 import com.gargoylesoftware.htmlunit.html.DomNodeUtil;
 import com.gargoylesoftware.htmlunit.html.HtmlElement;
 import com.gargoylesoftware.htmlunit.html.HtmlPage;
+import hudson.model.Computer;
 import hudson.model.Label;
+import hudson.model.Result;
 import hudson.model.Run;
 import hudson.slaves.SlaveComputer;
 import hudson.util.VersionNumber;
 import io.fabric8.kubernetes.api.model.ObjectMeta;
 import io.fabric8.kubernetes.api.model.Pod;
 import io.fabric8.kubernetes.api.model.PodList;
 import io.fabric8.kubernetes.client.KubernetesClientException;
+import io.fabric8.kubernetes.client.KubernetesClientTimeoutException;
 import jenkins.metrics.api.Metrics;
 import jenkins.model.Jenkins;
 import org.csanchez.jenkins.plugins.kubernetes.ContainerTemplate;
+import org.csanchez.jenkins.plugins.kubernetes.KubernetesLauncher;
 import org.csanchez.jenkins.plugins.kubernetes.KubernetesSlave;
 import org.csanchez.jenkins.plugins.kubernetes.MetricNames;
 import org.csanchez.jenkins.plugins.kubernetes.PodAnnotation;
 import org.csanchez.jenkins.plugins.kubernetes.PodTemplate;
 import org.csanchez.jenkins.plugins.kubernetes.PodTemplateUtils;
+import org.hamcrest.CustomTypeSafeMatcher;
 import org.hamcrest.MatcherAssert;
 import org.hamcrest.Matchers;
+import org.jenkinsci.plugins.workflow.flow.FlowDurabilityHint;
+import org.jenkinsci.plugins.workflow.flow.GlobalDefaultFlowDurabilityLevel;
 import org.jenkinsci.plugins.workflow.job.WorkflowRun;
 import org.jenkinsci.plugins.workflow.support.steps.ExecutorStepExecution;
 import org.jenkinsci.plugins.workflow.test.steps.SemaphoreStep;
 import org.junit.After;
 import org.junit.Before;
+import org.junit.Ignore;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
+import org.jvnet.hudson.test.FlagRule;
 import org.jvnet.hudson.test.Issue;
 import org.jvnet.hudson.test.JenkinsRule;
 import org.jvnet.hudson.test.JenkinsRuleNonLocalhost;
 import org.jvnet.hudson.test.LoggerRule;
+import org.jvnet.hudson.test.MockAuthorizationStrategy;
 
-import hudson.model.Result;
+import java.io.File;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
 import java.util.Locale;
+import java.util.Map;
+import java.util.Optional;
+import java.util.logging.Level;
+import java.util.logging.Logger;
 import java.util.stream.Collectors;
 
-import org.jenkinsci.plugins.workflow.flow.FlowDurabilityHint;
-import org.jenkinsci.plugins.workflow.flow.GlobalDefaultFlowDurabilityLevel;
-import org.junit.Ignore;
-import org.jvnet.hudson.test.FlagRule;
-import org.jvnet.hudson.test.MockAuthorizationStrategy;
+import static org.csanchez.jenkins.plugins.kubernetes.KubernetesTestUtil.CONTAINER_ENV_VAR_FROM_SECRET_VALUE;
+import static org.csanchez.jenkins.plugins.kubernetes.KubernetesTestUtil.POD_ENV_VAR_FROM_SECRET_VALUE;
+import static org.csanchez.jenkins.plugins.kubernetes.KubernetesTestUtil.assumeWindows;
+import static org.csanchez.jenkins.plugins.kubernetes.KubernetesTestUtil.deletePods;
+import static org.csanchez.jenkins.plugins.kubernetes.KubernetesTestUtil.getLabels;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.any;
+import static org.hamcrest.Matchers.emptyIterable;
+import static org.hamcrest.Matchers.hasEntry;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.oneOf;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assume.assumeNoException;
+import static org.junit.Assume.assumeNotNull;
 
 public class KubernetesPipelineTest extends AbstractKubernetesPipelineTest {
 
@@ -678,6 +682,29 @@ public void invalidPodGetsCancelled() throws Exception {
         r.assertLogContains("ERROR: Queue task was cancelled", b);
     }
 
+    @Test
+    public void podEventsPrinted() throws Exception {
+        warnings.record(KubernetesLauncher.class.getName(), Level.WARNING).capture(1000);
+        Thread.sleep(20000);
+        b.doKill();
+        r.assertBuildStatus(Result.ABORTED, r.waitForCompletion(b));
+        assertThat(warnings, LoggerRule.recorded(Level.WARNING, any(String.class),
+                new CustomTypeSafeMatcher<Throwable>("has \"Readiness probe failed\" in suppressed exceptions") {
+            @Override
+            protected boolean matchesSafely(Throwable item) {
+                if (item != null) {
+                    for (Throwable t : item.getSuppressed()) {
+                        if (t.getMessage().contains("Readiness probe failed")) {
+                            return true;
+                        }
+                    }
+                    return false;
+                }
+                return false;
+            }
+        }));
+    }
+
     @Issue("SECURITY-1646")
     @Test
     public void substituteEnv() throws Exception {

src/test/resources/org/csanchez/jenkins/plugins/kubernetes/pipeline/podEventsPrinted.groovy

@@ -0,0 +1,16 @@
+podTemplate(slaveConnectTimeout:10, yaml:'''
+  spec:
+    containers:
+    - name: jnlp
+      readinessProbe:
+        exec:
+          command:
+          - cat
+          - /tmp/healthy
+        initialDelaySeconds: 1
+        periodSeconds: 1
+''') {
+  node(POD_LABEL) {
+    sh 'true'
+  }
+}