Add ability to pass a custom message format with job name and parameters for triggering.

Author: Jesse Zoldak

src/main/java/com/base2services/jenkins/SqsQueueHandler.java

@@ -10,6 +10,7 @@
 import hudson.util.SequentialExecutionQueue;
 import hudson.util.TimeUnit2;
 
+import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.Executors;
 import java.util.logging.Level;
@@ -43,7 +44,7 @@ protected void doRun() throws Exception {
                 }
             }
         } else {
-            logger.fine("Currently Waiting for Messages from Queues");
+            LOGGER.fine("Currently Waiting for Messages from Queues");
         }
     }
 
@@ -66,15 +67,23 @@ public void run() {
             TriggerProcessor processor = profile.getTriggerProcessor();
             ReceiveMessageRequest receiveMessageRequest = new ReceiveMessageRequest(queueUrl);
             receiveMessageRequest.setWaitTimeSeconds(20);
-            List<Message> messages = sqs.receiveMessage(receiveMessageRequest).getMessages();
+            List<Message> messages = new ArrayList<Message>();
+            // Try to pick up the messages from SQS, and log if an error was encountered,
+            // for example a 403 Access to the resource is denied.
+            try {
+                messages = sqs.receiveMessage(receiveMessageRequest).getMessages();
+
+            } catch (Exception ex) {
+                LOGGER.warning("Unable to retrieve messages from the queue. " + ex.getMessage());
+            }
             for(Message message : messages) {
-                //Process the message payload, it needs to conform to the GitHub Web-Hook JSON format
+                //Process the message payload
                 try {
-                    logger.fine("got payload\n" + message.getBody());
+                    LOGGER.fine("got payload\n" + message.getBody());
                     processor.trigger(message.getBody());
 
                 } catch (Exception ex) {
-                    logger.log(Level.SEVERE,"unable to trigger builds " + ex.getMessage(),ex);
+                    LOGGER.log(Level.SEVERE,"unable to trigger builds " + ex.getMessage(),ex);
                 } finally {
                     //delete the message even if it failed
                     sqs.deleteMessage(new DeleteMessageRequest()

src/main/java/com/base2services/jenkins/github/GitHubTriggerProcessor.java

@@ -5,13 +5,29 @@
 import com.cloudbees.jenkins.GitHubRepositoryName;
 import com.cloudbees.jenkins.GitHubTrigger;
 import hudson.model.AbstractProject;
+import hudson.model.BooleanParameterValue;
+import hudson.model.Cause;
+import hudson.model.CauseAction;
 import hudson.model.Hudson;
+import hudson.model.Job;
+import hudson.model.ParametersAction;
+import hudson.model.ParameterValue;
+import hudson.model.StringParameterValue;
+import hudson.model.queue.QueueTaskFuture;
 import hudson.security.ACL;
 import hudson.triggers.Trigger;
+import hudson.triggers.TriggerDescriptor;
+import jenkins.model.Jenkins;
+import jenkins.model.ParameterizedJobMixIn;
+import net.sf.json.JSONArray;
+import net.sf.json.JSONException;
 import net.sf.json.JSONObject;
 import org.acegisecurity.Authentication;
 import org.acegisecurity.context.SecurityContextHolder;
 
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
 import java.util.logging.Logger;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -27,7 +43,22 @@ public class GitHubTriggerProcessor implements TriggerProcessor {
     private static final Logger LOGGER = Logger.getLogger(GitHubTriggerProcessor.class.getName());
 
     public void trigger(String payload) {
-        processGitHubPayload(payload, SqsBuildTrigger.class);
+        JSONObject json = extractJsonFromPayload(payload);
+        // You can signal through the payload that you will be using a custom format
+        // by including a root object named "custom_format" of any type and value.
+        if (json.has("custom_format")) {
+            processCustomPayload(json, SqsBuildTrigger.class);
+        }
+        // The default format, e.g. payload passed directly from a GitHub webhook,
+        // always includes a "repository" object.
+        else if (json.has("repository")) {
+            // Note that the payload will again be extracted from JSON at the start of the processGitHubPayload function.
+            // Leaving it this way so as not to change the contract, to be backwards compatible with any integrations.
+            processGitHubPayload(payload, SqsBuildTrigger.class);
+        }
+        else {
+            LOGGER.warning("Unable to determine the format of the SQS message.");
+        }
     }
 
     public void processGitHubPayload(String payload, Class<? extends Trigger> triggerClass) {
@@ -71,22 +102,117 @@ public void processGitHubPayload(String payload, Class<? extends Trigger> trigge
         }
     }
 
-    private JSONObject extractJsonFromPayload(String payload) {
-        JSONObject repository = null;
+    public JSONObject extractJsonFromPayload(String payload) {
         JSONObject json = JSONObject.fromObject(payload);
         if(json.has("Type")) {
             String msg = json.getString("Message");
             if(msg != null) {
                 char ch[] = msg.toCharArray();
                 if((ch[0] == '"') && (ch[msg.length()-1]) == '"') {
-                   msg = msg.substring(1,msg.length()-1); //remove the leading and trailing double quotes
+                    msg = msg.substring(1,msg.length()-1); //remove the leading and trailing double quotes
                 }
                 return JSONObject.fromObject(msg);
             }
-        } else if (json.has("repository")){
-            return json;
+        }
+        return json;
+    }
+
+    public void processCustomPayload(JSONObject json, Class<? extends Trigger> triggerClass) {
+        // Note that custom payloads will only trigger jobs that are configured with this SQS trigger.
+        // The custom payload must contain a root object named "job" of type string.
+        String jobToTrigger = json.getString("job");
+        if (jobToTrigger == null) {
+            LOGGER.warning("Custom sqs message payload does not contain information about which job to trigger.");
+            return;
+        }
+        // The custom payload can contain a root object named "parameters"
+        // that contains a list of parameters to pass to the job when scheduled.
+        // Each parameter object should contain information about its type, name, and value.
+        // TODO this will only work with string or boolean parameters,
+        // and you need to pass in ALL parameters for the job,
+        // as the scheduled job will not fill in any defaults for you.
+        List<ParameterValue> parameters = getParamsFromJson(json);
+
+        Authentication old = SecurityContextHolder.getContext().getAuthentication();
+        SecurityContextHolder.getContext().setAuthentication(ACL.SYSTEM);
+        try {
+            Jenkins jenkins = Jenkins.getInstance();
+            for (Job job: jenkins.getAllItems(Job.class)) {
+                String jobName = job.getDisplayName();
+                if (jobName.equals(jobToTrigger)) {
+                    // Custom triggers operate on Parameterized jobs only
+                    if (job instanceof ParameterizedJobMixIn.ParameterizedJob) {
+                        // Make sure the job is configured to use the SQS trigger
+                        ParameterizedJobMixIn.ParameterizedJob pJob = (ParameterizedJobMixIn.ParameterizedJob) job;
+                        final Map<TriggerDescriptor, Trigger<?>> pJobTriggers = pJob.getTriggers();
+                        SqsBuildTrigger.DescriptorImpl descriptor = jenkins.getDescriptorByType(SqsBuildTrigger.DescriptorImpl.class);
+                        if (!pJobTriggers.containsKey(descriptor)) {
+                            LOGGER.warning("The job " + jobName + " is not configured to use the SQS trigger.");
+                        } else {
+                            final Job theJob = job;
+                            ParameterizedJobMixIn mixin = new ParameterizedJobMixIn() {
+                                @Override
+                                protected Job asJob() {
+                                    return theJob;
+                                }
+                            };
+                            Cause cause = new Cause.RemoteCause("SQS", "Triggered by SQS.");
+                            CauseAction cAction = new CauseAction(cause);
+                            ParametersAction pAction = new ParametersAction(parameters);
+                            final QueueTaskFuture queueTaskFuture = mixin.scheduleBuild2(0, cAction, pAction);
+                            if (queueTaskFuture == null) {
+                                LOGGER.warning("Unable to schedule the job " + jobName);
+                            }
+                        }
+                    } else {
+                        LOGGER.warning("The job " + jobName + " is not configured as a Parameterized Job.");
+                    }
+                }
+            }
+        } finally {
+            SecurityContextHolder.getContext().setAuthentication(old);
+        }
+    }
+
+    private Boolean isValidParameterJson(JSONObject param) {
+        if (param.has("name") && param.has("value") && param.has("type")) {
+            if (param.getString("type").matches("string|boolean")) {
+                return true;
+            } else {
+                LOGGER.warning("'string' and 'boolean' are the only supported parameter types.");
+                return false;
+            }
+        } else {
+            LOGGER.warning("Parameters must contain key/value pairs for 'name', 'value', and 'type'.");
+            return false;
+        }
+    }
 
+    public List<ParameterValue> getParamsFromJson(JSONObject json) {
+        List<ParameterValue> params = new ArrayList<ParameterValue>();
+        if (json.has("parameters")) {
+            try {
+                JSONArray parameters = json.getJSONArray("parameters");
+                for (int i = 0; i < parameters.size(); i++) {
+                    JSONObject param = (JSONObject) parameters.get(i);
+                    if (isValidParameterJson(param)) {
+                        String name = param.getString("name");
+                        String type = param.getString("type");
+                        if (type.equals("boolean")) {
+                            Boolean value = param.getBoolean("value");
+                            BooleanParameterValue parameterValue = new BooleanParameterValue(name, value);
+                            params.add(parameterValue);
+                        } else {
+                            String value = param.getString("value");
+                            StringParameterValue parameterValue = new StringParameterValue(name, value);
+                            params.add(parameterValue);
+                        }
+                    }
+                }
+            } catch (JSONException e) {
+                LOGGER.warning("Parameters must be passed as a JSONArray in the SQS message.");
+            }
         }
-        return null;
+        return params;
     }
 }

src/main/resources/com/base2services/jenkins/SqsBuildTrigger/help-manual.jelly

@@ -1,9 +1,36 @@
 <j:jelly xmlns:j="jelly:core" xmlns:st="jelly:stapler" xmlns:d="jelly:define" xmlns:l="/lib/layout" xmlns:t="/lib/hudson" xmlns:f="/lib/form">
     <l:ajax>
         <div>
-            Don't let Jenkins talk to GitHub and manage the SQS Queue hook, and opt to do it manually.
-            In this mode, in addition to configure projects with "Build when a message is published to an SQS Queue",
-            you need to manually add the GitHub SQS Hook
+            Don't let Jenkins talk to GitHub and manage the SQS Queue hook; instead opt to do it manually.
+            <p>There are two cases in which you would want to do this:</p>
+            <ol>
+                <li>You want to control the GitHub hook yourself.</li>
+                <li>You are sending a custom JSON payload.</li>
+            </ol>
+            <p>In both these cases, in addition to either manually adding the GitHub SQS Hook in the GitHub settings of
+                your repo (default message payload) or setting up your own application that will publish messages (custom payload),
+                you also need to configure your project(s) to "Build when a message is published to an SQS Queue."</p>
+            <p>Custom payload technical details:</p>
+            <ul>
+            <li>Signal to the plugin that you are using a custom payload by including a root object named
+                "custom_format" of any type and value.</li>
+            <li>As noted above, custom payloads will only trigger jobs that are configured to use the SQS trigger.</li>
+            <li>The custom payload must contain a root object named "job" of type string which contains the name of the job to trigger.</li>
+            <li>The custom payload may contain a root object named "parameters" that contains a list of parameters to
+                pass to the job when scheduled. Each parameter object should contain information about its type, name, and value.</li>
+            <li>Valid parameter types are "string" and "boolean". These will get cast as
+                <a href="http://javadoc.jenkins-ci.org/hudson/model/StringParameterValue.html">StringParameterValue</a> and
+                <a href="http://javadoc.jenkins-ci.org/hudson/model/BooleanParameterValue.html">BooleanParameterValue</a>, respectively.
+            </li>
+            <li>You must pass all parameter values that you want to use for the job. Default values will <b>not</b> be determined and
+                filled in automatically.</li>
+            </ul>
+            <p>Custom payload examples:</p>
+            <ul>
+                <li><pre>{"custom_format": true, "job": "myProject"}</pre></li>
+                <li><pre>{"custom_format": true, "job": "myProject", "parameters": [{"name": "foo", "value": true, "type": "boolean"}]}</pre></li>
+                <li><pre>{"custom_format": true, "job": "myProject", "parameters": [{"name": "foo", "value": true, "type": "boolean"}, {"name": "bar", "value": "baz", "type": "string"}]}</pre></li>
+            </ul>
         </div>
     </l:ajax>
 </j:jelly>

src/test/java/com/base2services/jenkins/GetParamsFromJsonTest.java

@@ -0,0 +1,107 @@
+package com.base2services.jenkins;
+
+import com.base2services.jenkins.github.GitHubTriggerProcessor;
+import hudson.model.BooleanParameterValue;
+import hudson.model.ParameterValue;
+import hudson.model.StringParameterValue;
+import net.sf.json.JSONObject;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.List;
+
+public class GetParamsFromJsonTest {
+
+    private String payload;
+    private GitHubTriggerProcessor gtp;
+
+    @Before
+    public void setUp() throws Exception {
+        gtp = new GitHubTriggerProcessor();
+    }
+
+    @Test
+    public void testEmptyParams() throws Exception {
+        payload = "{'job': 'testProject'}".replace("'", "\"");
+        JSONObject json = gtp.extractJsonFromPayload(payload);
+        List<ParameterValue> parameterValues = gtp.getParamsFromJson(json);
+        assert parameterValues.size() == 0;
+    }
+
+    @Test
+    public void testNonArrayParams() throws Exception {
+        payload = "{'parameters': 'foo'}".replace("'", "\"");
+        JSONObject json = gtp.extractJsonFromPayload(payload);
+        List<ParameterValue> parameterValues = gtp.getParamsFromJson(json);
+        assert parameterValues.size() == 0;
+    }
+
+    @Test
+    public void testEmptyParamsArray() throws Exception {
+        payload = "{'parameters': []}".replace("'", "\"");
+        JSONObject json = gtp.extractJsonFromPayload(payload);
+        List<ParameterValue> parameterValues = gtp.getParamsFromJson(json);
+        assert parameterValues.size() == 0;
+    }
+
+    @Test
+    public void testInvalidParameterType() throws Exception {
+        payload = "{'parameters': [{'name': 'foo', 'value': 'bar', 'type': 'blah'}]}".replace("'", "\"");
+        GitHubTriggerProcessor gtp = new GitHubTriggerProcessor();
+        JSONObject json = gtp.extractJsonFromPayload(payload);
+        List<ParameterValue> parameterValues = gtp.getParamsFromJson(json);
+        assert parameterValues.size() == 0;
+    }
+
+    @Test
+    public void testStringParameter() throws Exception {
+        payload = "{'parameters': [{'name': 'foo', 'value': 'bar', 'type': 'string'}]}".replace("'", "\"");
+        GitHubTriggerProcessor gtp = new GitHubTriggerProcessor();
+        JSONObject json = gtp.extractJsonFromPayload(payload);
+        List<ParameterValue> parameterValues = gtp.getParamsFromJson(json);
+        assert parameterValues.size() == 1;
+        assert parameterValues.get(0).getClass().equals(StringParameterValue.class);
+        assert parameterValues.get(0).getName().equals("foo");
+        assert parameterValues.get(0).getValue().equals("bar");
+    }
+
+    @Test
+    public void testBooleanParameter() throws Exception {
+        payload = "{'parameters': [{'name': 'foo', 'value': true, 'type': 'boolean'}]}".replace("'", "\"");
+        GitHubTriggerProcessor gtp = new GitHubTriggerProcessor();
+        JSONObject json = gtp.extractJsonFromPayload(payload);
+        List<ParameterValue> parameterValues = gtp.getParamsFromJson(json);
+        assert parameterValues.size() == 1;
+        assert parameterValues.get(0).getClass().equals(BooleanParameterValue.class);
+        assert parameterValues.get(0).getName().equals("foo");
+        assert parameterValues.get(0).getValue().equals(true);
+    }
+
+    @Test
+    public void testMultipleParameters() throws Exception {
+        String payload_string = String.format("{'parameters': [%s]}",
+            "{'name': 'foo', 'value': 'bar', 'type': 'string'}, {'name': 'hello', 'value': 'world', 'type': 'string'}");
+        payload = payload_string.replace("'", "\"");
+        GitHubTriggerProcessor gtp = new GitHubTriggerProcessor();
+        JSONObject json = gtp.extractJsonFromPayload(payload);
+        List<ParameterValue> parameterValues = gtp.getParamsFromJson(json);
+        assert parameterValues.size() == 2;
+        assert parameterValues.get(0).getName().equals("foo");
+        assert parameterValues.get(0).getValue().equals("bar");
+        assert parameterValues.get(1).getName().equals("hello");
+        assert parameterValues.get(1).getValue().equals("world");
+    }
+
+    @Test
+    public void testInvalidParametersAreIgnored() throws Exception {
+        String payload_string = String.format("{'parameters': [%s]}",
+            "{'ihavenoname': 'blah'},{'name': 'foo', 'value': 'bar', 'type': 'string'}");
+        payload = payload_string.replace("'", "\"");
+        GitHubTriggerProcessor gtp = new GitHubTriggerProcessor();
+        JSONObject json = gtp.extractJsonFromPayload(payload);
+        List<ParameterValue> parameterValues = gtp.getParamsFromJson(json);
+        assert parameterValues.size() == 1;
+        assert parameterValues.get(0).getName().equals("foo");
+        assert parameterValues.get(0).getValue().equals("bar");
+    }
+}

src/test/java/com/base2services/jenkins/GitHubTriggerProcessorTest.java

@@ -72,7 +72,7 @@ public void shouldNotTriggerWithAFork() throws Exception {
     public void shouldNotTriggerWithBadPayload() throws Exception {
         payload = "{'noRepoInfo': {'empty': 'https://github.com/foo/bar'}}".replace("'", "\"");
         GitHubTriggerProcessor gtp = new GitHubTriggerProcessor();
-        gtp.processGitHubPayload(payload, sbt.getClass());
+        gtp.trigger(payload);
         verify(sbt, never()).onPost();
     }