Merge pull request #2 from d-smith/kinesis-function

Added lambda function to write kinesis data to Sumo. NOTE: this function does not break up Kinesis messages like the Sumo Kinesis Connector.

Author: duchatran

kinesis/README.md

@@ -0,0 +1,54 @@
+# Kinesis to Sumologic
+
+This function reads messages from a Kinesis
+stream and posts them to a SumoLogic hosted HTTP collector.
+
+## Lambda configuration
+
+There are no module dependencies for this code, so you can paste it into the
+lambda console directly. Note you must set the collector host and the
+path that includes your secret key in options for this to work.
+
+For the Sumo collector configuration, do not enable multiline processing or
+one message per request -- the idea is to send as many messages in one request
+as possible to Sumo and let Sumo break them apart as needed.
+
+In the AWS console, use a code entry type of 'Edit code inline' and paste in the
+code (doublecheck the hostname and path as per your collector setup).
+
+In configuration specify index.handler as the Handler. Specify a Role that has
+sufficient privileges to read from the kinesis stream, invoke a lambda
+function, and write cloud watch logs. I tested with this policy, which is
+too loose for production.
+
+<pre>
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": [
+        "lambda:InvokeFunction"
+      ],
+      "Resource": [
+        "*"
+      ]
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "kinesis:GetRecords",
+        "kinesis:GetShardIterator",
+        "kinesis:DescribeStream",
+        "kinesis:ListStreams",
+        "logs:CreateLogGroup",
+        "logs:CreateLogStream",
+        "logs:PutLogEvents"
+      ],
+      "Resource": "*"
+    }
+  ]
+}
+</pre>
+
+For the Event Source, pick the stream containing the data you want to send to Sumo.

kinesis/k2sl_lambda.js

@@ -0,0 +1,34 @@
+var https = require('https');
+
+///////////////////////////////////////////////////////////////////////////////////////////////////////////
+// Remember to change the hostname and path to match your collection API and specific HTTP-source endpoint
+// See more at: https://service.sumologic.com/help/Default.htm#Collector_Management_API.htm
+///////////////////////////////////////////////////////////////////////////////////////////////////////////
+var options = { 'hostname': 'endpoint1.collection.us2.sumologic.com',
+  'path': 'https://endpoint1.collection.us2.sumologic.com/receiver/v1/http/XXXXX',
+  'method': 'POST'
+};
+
+exports.handler = function(event, context) {
+
+  var req = https.request(options, function(res) {
+      var body = '';
+      console.log('Status:', res.statusCode);
+      res.setEncoding('utf8');
+      res.on('data', function(chunk) { body += chunk; });
+      res.on('end', function() {
+        console.log('Successfully processed HTTPS response');
+        context.succeed(); });
+    });
+
+  options.agent = new https.Agent(options);
+
+  req.on('error', context.fail);
+
+  event.Records.forEach(function(record) {
+    var payload = new Buffer(record.kinesis.data, 'base64').toString('ascii');
+    req.write(payload + '\n');
+  })
+  req.end();
+
+}