C# Rendler implementation

README.md

@@ -39,12 +39,13 @@ You can see that 1 slave is registered and you've got some idle CPUs and Memory.
 ### Run RENDLER in the `mesos-demo` VM
 
 Check implementations of the RENDLER scheduler in the `python`, `go`,
-`scala`, and `cpp` directories. Run instructions are here:
+`scala`, `cpp` and `csharp` directories. Run instructions are here:
 
 - [Python RENDLER framework](python/README.md)
 - [Go RENDLER framework](go/README.md)
 - [Scala RENDLER framework](scala/README.md)
 - [C++ RENDLER framework](cpp/README.md)
+- [C# RENDLER framework](csharp/README.md)
 
 Feel free to contribute your own!
 

csharp/.gitignore

@@ -0,0 +1,10 @@
+*.suo
+*.user
+*.sdf
+*.userprefs
+.vs
+bin
+obj
+*.so
+*.nupkg
+*.lock.json

csharp/NuGet.Config

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8"?>
+<configuration>
+  <activePackageSource>
+    <add key="dev_Rendler" value="dev_nuget_feed" />
+  </activePackageSource>
+  <packageSources>
+    <add key="dev_Rendler" value="dev_nuget_feed" />
+  </packageSources>
+</configuration>

csharp/README.md

@@ -0,0 +1,14 @@
+## C# Rendler Framework
+
+### Preparation
+
+1. The implementation uses the [mesos-clr](https://github.com/bcrusu/mesos-clr) library which needs to be build and placed to the `ext` directory before building the Rendler project.
+2. Pack the Rendler binaries to `rendler.tar.gz` and place the archive inside the [frameworks_home](http://mesos.apache.org/documentation/latest/configuration/) directory.
+
+### Running
+
+To start the Rendler framework, run the command: 
+```bash
+mono rendler.exe -scheduler -master=MASTER_ADDRESS -output=RENDLER_OUTPUT_DIR [-starturl=CRAWL_START_URL] [-user=RUN_AS_USER]
+```
+

csharp/Rendler.sln

@@ -0,0 +1,32 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio 14
+VisualStudioVersion = 14.0.25123.0
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{5D5A1D30-E747-47E8-8C4F-AAC48FAFA9C4}"
+EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{1B43699B-D07A-403E-BA95-8F1EB4E4A590}"
+	ProjectSection(SolutionItems) = preProject
+		global.json = global.json
+	EndProjectSection
+EndProject
+Project("{8BB2217D-0F2D-49D1-97BC-3654ED321F3B}") = "Rendler", "src\Rendler\Rendler.xproj", "{36A62F5A-1F76-494D-9377-2595AE03C598}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{36A62F5A-1F76-494D-9377-2595AE03C598}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{36A62F5A-1F76-494D-9377-2595AE03C598}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{36A62F5A-1F76-494D-9377-2595AE03C598}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{36A62F5A-1F76-494D-9377-2595AE03C598}.Release|Any CPU.Build.0 = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(NestedProjects) = preSolution
+		{36A62F5A-1F76-494D-9377-2595AE03C598} = {5D5A1D30-E747-47E8-8C4F-AAC48FAFA9C4}
+	EndGlobalSection
+EndGlobal

csharp/dev_nuget_feed/readme

@@ -0,0 +1 @@
+place 'mesosclr.x.x.x.nupkg' in this directory

csharp/global.json

@@ -0,0 +1,6 @@
+{
+  "projects": [ "src", "test" ],
+  "sdk": {
+    "version": "1.0.0-preview1-002702"
+  }
+}

csharp/src/Rendler/DotHelper.cs

@@ -0,0 +1,65 @@
+using System.Collections.Generic;
+using System.IO;
+using System.Text;
+
+namespace Rendler
+{
+    internal static class DotHelper
+    {
+        public static void Write(string outputPath, IDictionary<string, List<string>> nodeToChildNodes,
+            IDictionary<string, string> nodeImageFileName)
+        {
+            var nodeNames = new Dictionary<string, string>();
+            var nodeIdCounter = 0;
+
+            using (var fs = new FileStream(outputPath, FileMode.CreateNew, FileAccess.Write, FileShare.Write))
+            using (var writer = new StreamWriter(fs, Encoding.UTF8))
+            {
+                writer.WriteLine("digraph G {");
+                writer.WriteLine("\tnode [shape=box];");
+
+                foreach (var node in nodeToChildNodes)
+                {
+                    var url = node.Key;
+                    var nodeName = "url_" + (++nodeIdCounter);
+                    nodeNames[url] = nodeName;
+
+                    writer.Write("\t");
+                    writer.Write(nodeName);
+
+                    string imageFileName;
+					if (nodeImageFileName.TryGetValue(url, out imageFileName))
+                    {
+                        writer.Write(" [label=\"\" image=\"");
+                        writer.Write(imageFileName);
+                    }
+                    else
+                    {
+                        writer.Write(" [label=\"");
+                        writer.Write(url);
+                    }
+
+                    writer.WriteLine("\"];");
+                }
+
+                writer.WriteLine();
+
+                foreach (var node in nodeToChildNodes)
+                {
+                    var nodeName = nodeNames[node.Key];
+                    foreach (var childNode in node.Value)
+                    {
+                        var childNodeName = nodeNames[childNode];
+                        writer.Write("\t");
+                        writer.Write(nodeName);
+                        writer.Write(" -> ");
+                        writer.Write(childNodeName);
+                        writer.WriteLine(";");
+                    }
+                }
+
+                writer.WriteLine("}");
+            }
+        }
+    }
+}

csharp/src/Rendler/Executors/CrawlExecutor.cs

@@ -0,0 +1,106 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Net;
+using System.Text;
+using System.Text.RegularExpressions;
+using System.Threading.Tasks;
+using mesos;
+using mesosclr;
+using Rendler.Executors.Messages;
+using System.Net.Http;
+
+namespace Rendler.Executors
+{
+    internal class CrawlExecutor : ExecutorBase
+    {
+        private static readonly Regex ExtractLinksRegex = new Regex("<a[^>]+href=[\"']?(?<link>[^\"'>]+)[\"']?[^>]*>(.+?)</a>", RegexOptions.Compiled | RegexOptions.IgnoreCase);
+
+        public override void Registered(IExecutorDriver driver, ExecutorInfo executorInfo, FrameworkInfo frameworkInfo, SlaveInfo slaveInfo)
+        {
+            Console.WriteLine($"Registered executor on '{slaveInfo.hostname}'.");
+        }
+
+        public override void LaunchTask(IExecutorDriver driver, TaskInfo taskInfo)
+        {
+            Console.WriteLine($"Launching crawl task '{taskInfo.task_id.value}'...");
+
+            Task.Factory.StartNew(async () =>
+            {
+                try
+                {
+                    await RunTask(driver, taskInfo);
+                }
+                catch (Exception e)
+                {
+                    Console.WriteLine($"Exception during crawl operation: {e}");
+                    driver.SendTaskErrorStatus(taskInfo.task_id);
+                }
+            });
+        }
+
+        private static async Task RunTask(IExecutorDriver driver, TaskInfo taskInfo)
+        {
+            driver.SendTaskRunningStatus(taskInfo.task_id);
+
+            var url = Encoding.UTF8.GetString(taskInfo.data);
+
+            var htmlContent = await GetUrlContent(url);
+            if (htmlContent != null)
+            {
+                var links = ExtractLinks(htmlContent);
+                links = links
+                    .Select(x => x.ToLower())
+                    .Distinct(StringComparer.CurrentCultureIgnoreCase);
+
+                if (links.Any())
+                    SendCrawlResultMessage(driver, url, links.ToArray());
+            }
+
+            driver.SendTaskFinishedStatus(taskInfo.task_id);
+        }
+
+        private static IEnumerable<string> ExtractLinks(string htmlContent)
+        {
+            var match = ExtractLinksRegex.Match(htmlContent);
+            while (match.Success)
+            {
+                yield return match.Groups["link"].Value.Trim();
+                match = match.NextMatch();
+            }
+        }
+
+        private static async Task<string> GetUrlContent(string url)
+        {
+            using (var client = new HttpClient())
+            {
+                client.DefaultRequestHeaders.Add("X-PoweredBy", "minions");
+
+                try
+                {
+                    return await client.GetStringAsync(url);
+                }
+                catch (WebException e)
+                {
+                    Console.WriteLine($"Error fetching url '{url}'; Error: {e}");
+                    return null;
+                }
+            }
+        }
+
+        private static void SendCrawlResultMessage(IExecutorDriver driver, string url, string[] links)
+        {
+            var message = new Message
+            {
+                Type = "CrawlResult",
+                Body = JsonHelper.Serialize(new CrawlResultMessage
+                {
+                    Url = url,
+                    Links = links
+                })
+            };
+
+            driver.SendFrameworkMessage(JsonHelper.Serialize(message));
+        }
+    }
+}

csharp/src/Rendler/Executors/ExecutorBase.cs

@@ -0,0 +1,42 @@
+using System;
+using mesos;
+using mesosclr;
+
+namespace Rendler.Executors
+{
+    abstract class ExecutorBase : IExecutor
+    {
+        public virtual void Registered(IExecutorDriver driver, ExecutorInfo executorInfo, FrameworkInfo frameworkInfo, SlaveInfo slaveInfo)
+        {
+        }
+
+        public virtual void Reregistered(IExecutorDriver driver, SlaveInfo slaveInfo)
+        {
+        }
+
+        public virtual void Disconnected(IExecutorDriver driver)
+        {
+        }
+
+        public virtual void LaunchTask(IExecutorDriver driver, TaskInfo taskInfo)
+        {
+        }
+
+        public virtual void KillTask(IExecutorDriver driver, TaskID taskId)
+        {
+        }
+
+        public virtual void FrameworkMessage(IExecutorDriver driver, byte[] data)
+        {
+        }
+
+        public virtual void Shutdown(IExecutorDriver driver)
+        {
+        }
+
+        public virtual void Error(IExecutorDriver driver, string message)
+        {
+            Console.WriteLine($"Error: '{message}'.");
+        }
+    }
+}

csharp/src/Rendler/Executors/Messages/CrawlResultMessage.cs

@@ -0,0 +1,14 @@
+using System.Runtime.Serialization;
+
+namespace Rendler.Executors.Messages
+{
+    [DataContract]
+    public class CrawlResultMessage
+    {
+        [DataMember]
+        public string Url { get; set; }
+
+        [DataMember]
+        public string[] Links { get; set; }
+    }
+}

csharp/src/Rendler/Executors/Messages/Message.cs

@@ -0,0 +1,14 @@
+using System.Runtime.Serialization;
+
+namespace Rendler.Executors.Messages
+{
+    [DataContract]
+    internal class Message
+    {
+        [DataMember]
+        public string Type { get; set; }
+
+        [DataMember]
+        public byte[] Body { get; set; }
+    }
+}

csharp/src/Rendler/Executors/Messages/RenderResultMessage.cs

@@ -0,0 +1,14 @@
+using System.Runtime.Serialization;
+
+namespace Rendler.Executors.Messages
+{
+    [DataContract]
+    public class RenderResultMessage
+    {
+        [DataMember]
+        public string Url { get; set; }
+
+        [DataMember]
+        public string FileName { get; set; }
+    }
+}