diff --git a/src/main/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesEventsException.java b/src/main/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesEventsException.java new file mode 100644 index 0000000000..4381cea5f1 --- /dev/null +++ b/src/main/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesEventsException.java @@ -0,0 +1,25 @@ +package org.csanchez.jenkins.plugins.kubernetes; + +import io.fabric8.kubernetes.api.model.Event; + +import java.util.List; + +/** + * Custom exception which adds information about Kubernetes events. + */ +public class KubernetesEventsException extends Exception { + public KubernetesEventsException(List events) { + super(toMessage(events)); + } + + private static String toMessage(List events) { + StringBuilder sb = new StringBuilder("Events follow:\n"); + for (Event event : events) { + String[] lines = event.getMessage().split("\n"); + for (String line : lines) { + sb.append(String.format("[%s][%s/%s][%s] %s%n", event.getType(), event.getInvolvedObject().getNamespace(), event.getInvolvedObject().getName(), event.getReason(), line)); + } + } + return sb.toString(); + } +} diff --git a/src/main/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesLauncher.java b/src/main/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesLauncher.java index c27e697ce6..e46ef981ad 100644 --- a/src/main/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesLauncher.java +++ b/src/main/java/org/csanchez/jenkins/plugins/kubernetes/KubernetesLauncher.java @@ -39,9 +39,11 @@ import javax.annotation.CheckForNull; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; +import io.fabric8.kubernetes.api.model.Event; import io.fabric8.kubernetes.client.KubernetesClientException; import jenkins.metrics.api.Metrics; import org.apache.commons.lang.StringUtils; +import org.jenkinsci.plugins.kubernetes.auth.KubernetesAuthException; import org.kohsuke.stapler.DataBoundConstructor; import hudson.model.TaskListener; @@ -251,6 +253,14 @@ else if (httpCode == 409 && e.getMessage().contains("Operation cannot be fulfill } Metrics.metricRegistry().counter(MetricNames.PODS_LAUNCHED).inc(); } catch (Throwable ex) { + try { + List podEvents = ((KubernetesComputer) computer).getPodEvents(); + if (!podEvents.isEmpty()) { + ex.addSuppressed(new KubernetesEventsException(podEvents)); + } + } catch (KubernetesAuthException | IOException e) { + LOGGER.log(Level.FINE, "Unable to add Kubernetes events"); + } setProblem(ex); LOGGER.log(Level.WARNING, String.format("Error in provisioning; agent=%s, template=%s", node, template), ex); LOGGER.log(Level.FINER, "Removing Jenkins node: {0}", node.getNodeName()); diff --git a/src/test/java/org/csanchez/jenkins/plugins/kubernetes/pipeline/KubernetesPipelineTest.java b/src/test/java/org/csanchez/jenkins/plugins/kubernetes/pipeline/KubernetesPipelineTest.java index 0d32b511c9..87ead6e22b 100644 --- a/src/test/java/org/csanchez/jenkins/plugins/kubernetes/pipeline/KubernetesPipelineTest.java +++ b/src/test/java/org/csanchez/jenkins/plugins/kubernetes/pipeline/KubernetesPipelineTest.java @@ -24,36 +24,12 @@ package org.csanchez.jenkins.plugins.kubernetes.pipeline; -import static org.csanchez.jenkins.plugins.kubernetes.KubernetesTestUtil.CONTAINER_ENV_VAR_FROM_SECRET_VALUE; -import static org.csanchez.jenkins.plugins.kubernetes.KubernetesTestUtil.POD_ENV_VAR_FROM_SECRET_VALUE; -import static org.csanchez.jenkins.plugins.kubernetes.KubernetesTestUtil.assumeWindows; -import static org.csanchez.jenkins.plugins.kubernetes.KubernetesTestUtil.deletePods; -import static org.csanchez.jenkins.plugins.kubernetes.KubernetesTestUtil.getLabels; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.emptyIterable; -import static org.hamcrest.Matchers.hasEntry; -import static org.hamcrest.Matchers.hasSize; -import static org.hamcrest.Matchers.oneOf; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assume.*; - -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.logging.Level; -import java.util.logging.Logger; - -import hudson.model.Computer; import com.gargoylesoftware.htmlunit.html.DomNodeUtil; import com.gargoylesoftware.htmlunit.html.HtmlElement; import com.gargoylesoftware.htmlunit.html.HtmlPage; +import hudson.model.Computer; import hudson.model.Label; +import hudson.model.Result; import hudson.model.Run; import hudson.slaves.SlaveComputer; import hudson.util.VersionNumber; @@ -61,38 +37,66 @@ import io.fabric8.kubernetes.api.model.Pod; import io.fabric8.kubernetes.api.model.PodList; import io.fabric8.kubernetes.client.KubernetesClientException; +import io.fabric8.kubernetes.client.KubernetesClientTimeoutException; import jenkins.metrics.api.Metrics; import jenkins.model.Jenkins; import org.csanchez.jenkins.plugins.kubernetes.ContainerTemplate; +import org.csanchez.jenkins.plugins.kubernetes.KubernetesLauncher; import org.csanchez.jenkins.plugins.kubernetes.KubernetesSlave; import org.csanchez.jenkins.plugins.kubernetes.MetricNames; import org.csanchez.jenkins.plugins.kubernetes.PodAnnotation; import org.csanchez.jenkins.plugins.kubernetes.PodTemplate; import org.csanchez.jenkins.plugins.kubernetes.PodTemplateUtils; +import org.hamcrest.CustomTypeSafeMatcher; import org.hamcrest.MatcherAssert; import org.hamcrest.Matchers; +import org.jenkinsci.plugins.workflow.flow.FlowDurabilityHint; +import org.jenkinsci.plugins.workflow.flow.GlobalDefaultFlowDurabilityLevel; import org.jenkinsci.plugins.workflow.job.WorkflowRun; import org.jenkinsci.plugins.workflow.support.steps.ExecutorStepExecution; import org.jenkinsci.plugins.workflow.test.steps.SemaphoreStep; import org.junit.After; import org.junit.Before; +import org.junit.Ignore; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import org.jvnet.hudson.test.FlagRule; import org.jvnet.hudson.test.Issue; import org.jvnet.hudson.test.JenkinsRule; import org.jvnet.hudson.test.JenkinsRuleNonLocalhost; import org.jvnet.hudson.test.LoggerRule; +import org.jvnet.hudson.test.MockAuthorizationStrategy; -import hudson.model.Result; +import java.io.File; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.logging.Level; +import java.util.logging.Logger; import java.util.stream.Collectors; -import org.jenkinsci.plugins.workflow.flow.FlowDurabilityHint; -import org.jenkinsci.plugins.workflow.flow.GlobalDefaultFlowDurabilityLevel; -import org.junit.Ignore; -import org.jvnet.hudson.test.FlagRule; -import org.jvnet.hudson.test.MockAuthorizationStrategy; +import static org.csanchez.jenkins.plugins.kubernetes.KubernetesTestUtil.CONTAINER_ENV_VAR_FROM_SECRET_VALUE; +import static org.csanchez.jenkins.plugins.kubernetes.KubernetesTestUtil.POD_ENV_VAR_FROM_SECRET_VALUE; +import static org.csanchez.jenkins.plugins.kubernetes.KubernetesTestUtil.assumeWindows; +import static org.csanchez.jenkins.plugins.kubernetes.KubernetesTestUtil.deletePods; +import static org.csanchez.jenkins.plugins.kubernetes.KubernetesTestUtil.getLabels; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.any; +import static org.hamcrest.Matchers.emptyIterable; +import static org.hamcrest.Matchers.hasEntry; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.oneOf; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeNoException; +import static org.junit.Assume.assumeNotNull; public class KubernetesPipelineTest extends AbstractKubernetesPipelineTest { @@ -678,6 +682,29 @@ public void invalidPodGetsCancelled() throws Exception { r.assertLogContains("ERROR: Queue task was cancelled", b); } + @Test + public void podEventsPrinted() throws Exception { + warnings.record(KubernetesLauncher.class.getName(), Level.WARNING).capture(1000); + Thread.sleep(20000); + b.doKill(); + r.assertBuildStatus(Result.ABORTED, r.waitForCompletion(b)); + assertThat(warnings, LoggerRule.recorded(Level.WARNING, any(String.class), + new CustomTypeSafeMatcher("has \"Readiness probe failed\" in suppressed exceptions") { + @Override + protected boolean matchesSafely(Throwable item) { + if (item != null) { + for (Throwable t : item.getSuppressed()) { + if (t.getMessage().contains("Readiness probe failed")) { + return true; + } + } + return false; + } + return false; + } + })); + } + @Issue("SECURITY-1646") @Test public void substituteEnv() throws Exception { diff --git a/src/test/resources/org/csanchez/jenkins/plugins/kubernetes/pipeline/podEventsPrinted.groovy b/src/test/resources/org/csanchez/jenkins/plugins/kubernetes/pipeline/podEventsPrinted.groovy new file mode 100644 index 0000000000..2d629f8050 --- /dev/null +++ b/src/test/resources/org/csanchez/jenkins/plugins/kubernetes/pipeline/podEventsPrinted.groovy @@ -0,0 +1,16 @@ +podTemplate(slaveConnectTimeout:10, yaml:''' + spec: + containers: + - name: jnlp + readinessProbe: + exec: + command: + - cat + - /tmp/healthy + initialDelaySeconds: 1 + periodSeconds: 1 +''') { + node(POD_LABEL) { + sh 'true' + } +}