Tentatively, dynamically loaded plugins seem to work. (re: #288)

- Added a 'Plugin' manager object that dynamically loads a jar, parses out plugin metadata, and in
- Added --plugin/-P command-line option to load plugins at startup
- Added a vizier-plugins option to the Vizier properties file to load plugins at startup
- Fixed typo in capitalization of Class__L__oader in ClassLoaderUtils

Author: okennedy

plugins/mimir-pip.jar

@@ -137,6 +137,16 @@ class Config(arguments: Seq[String])
     descr = "Set the SparkSQL warehouse directory (default: {cache-dir}/spark-warehouse)"
   )
 
+  val extraPlugins = opt[List[File]]("plugin",
+    short = 'P',
+    descr = "Enable a plugin for this session"
+  )
+
+  lazy val plugins:Seq[File] = 
+    Option(defaults.getProperty("vizier-plugins"))
+        .map { _.split(":").map { new File(_) }.toSeq }
+        .getOrElse { Seq() } ++ extraPlugins()
+
   def workingDirectoryFile: File = 
     new File(
       workingDirectory

vizier/backend/src/info/vizierdb/Config.scala

@@ -0,0 +1,85 @@
+package info.vizierdb
+
+import play.api.libs.json._
+import java.net.URLClassLoader
+import java.net.URL
+import java.io.File
+import java.io.FileNotFoundException
+import com.typesafe.scalalogging.LazyLogging
+import scala.collection.mutable
+import info.vizierdb.util.ClassLoaderUtils
+
+case class Plugin(
+  name: String,
+  schema_version: Int,
+  plugin_class: String,
+  description: String,
+  documentation: Option[String],
+)
+{
+} 
+
+object Plugin
+  extends LazyLogging
+{
+
+  val loaded = mutable.Map[String, Plugin]()
+
+
+  implicit val pluginFormat: Format[Plugin] = Json.format
+
+  def load(file: File): Plugin =
+  {
+    val jar = 
+      if(file.isAbsolute){ file }
+      else { 
+        Vizier.config.workingDirectoryFile.toPath
+              .resolve(file.toPath)
+              .toFile 
+      }
+
+    if(!jar.exists()){ throw new FileNotFoundException(jar.getAbsoluteFile.toString) }
+    val url = jar.getAbsoluteFile().toURI().toURL()
+
+    // It feels like a bit of a hack to just stack plugins onto the currently running
+    // ClassLoader... but this seems to be the only way to make the loaded classes visible
+    // to the running spark instance.
+    val loader = new URLClassLoader(
+      Array(jar.toURI.toURL),
+      Thread.currentThread().getContextClassLoader()
+    )
+    Thread.currentThread().setContextClassLoader(loader)
+
+    val plugin = 
+      Json.parse(
+        loader.getResourceAsStream("vizier-plugin.json")
+      ).asOpt[Plugin]
+       .getOrElse { 
+          throw new RuntimeException(s"$jar is not a valid Vizier Plugin")
+       }
+ 
+    val detail = s"${plugin.name} [$jar]"
+
+    assert(
+      plugin.schema_version > 0 && plugin.schema_version <= 1,
+      s"Unsupported version '${plugin.schema_version} for plugin $detail"
+    )
+
+    assert(
+      !loaded.contains(plugin.name),
+      s"Plugin ${plugin.name} is already loaded."
+    )
+
+    loaded.put(plugin.name, plugin)
+
+    val clazz = Class.forName(plugin.plugin_class, true, loader)
+    val singleton = clazz.getDeclaredField("MODULE$").get()
+    val initMethod = clazz.getMethod("init", Vizier.sparkSession.getClass)
+
+    ClassLoaderUtils.withContextClassLoader(loader) {
+      initMethod.invoke(singleton, Vizier.sparkSession)
+    }
+
+    return plugin
+  }
+}

vizier/backend/src/info/vizierdb/Plugin.scala

@@ -159,6 +159,15 @@ object Vizier
     }
   }
 
+  def loadPlugins(plugins: Seq[File]): Unit =
+  {
+    for(p <- plugins){ 
+      println(s"  ...loading plugin $p")
+      val plugin = Plugin.load(p)
+      println(s"    ...loaded ${plugin.name}")
+    }
+  }
+
   def setWorkingDirectory(): Unit =
   {
     if(config.workingDirectory.isDefined){
@@ -200,6 +209,12 @@ object Vizier
     println("Starting Spark...")
     initSpark()
 
+    // Set up plugins
+    if(!config.plugins.isEmpty){
+      println("Loading plugins...")
+      loadPlugins(config.plugins)
+    }
+
     config.subcommand match {
       //////////////// HANDLE SPECIAL COMMANDS //////////////////
       case Some(subcommand) => 

vizier/backend/src/info/vizierdb/Vizier.scala

@@ -16,7 +16,7 @@ package info.vizierdb.util
 
 object ClassLoaderUtils
 {
-  def withContextClassloader[T](classloader: ClassLoader)(f: => T): T =
+  def withContextClassLoader[T](classloader: ClassLoader)(f: => T): T =
   {
     val originalClassloader = Thread.currentThread().getContextClassLoader()
     Thread.currentThread().setContextClassLoader(classloader)

vizier/backend/src/info/vizierdb/util/ClassLoaderUtils.scala

@@ -80,7 +80,7 @@ class RunningCell(
 
   def exec(): Boolean =
   {
-    ClassLoaderUtils.withContextClassloader(workflowTask.classloader) {
+    ClassLoaderUtils.withContextClassLoader(workflowTask.classloader) {
       try {
         processSynchronously match {
           case _ if aborted.get => 

vizier/backend/src/info/vizierdb/viztrails/RunningCell.scala

@@ -62,7 +62,7 @@ class RunningWorkflow(workflow: Workflow, val classloader: ClassLoader)
 
   def exec: Boolean =
   {
-    ClassLoaderUtils.withContextClassloader[Boolean](classloader){
+    ClassLoaderUtils.withContextClassLoader[Boolean](classloader){
       val ret = 
         try { 
           logger.info(s"Starting execution of Workflow ${workflow.id}")

vizier/backend/src/info/vizierdb/viztrails/RunningWorkflow.scala

@@ -0,0 +1,29 @@
+package info.vizierdb
+
+import org.specs2.mutable.Specification
+import org.specs2.specification.BeforeAll
+import info.vizierdb.test.SharedTestResources
+import java.io.File
+import info.vizierdb.util.ClassLoaderUtils
+
+class PluginTest extends Specification with BeforeAll
+{
+  def beforeAll(): Unit = 
+  {
+    SharedTestResources.init()
+  }
+
+  "Load Class Jars" >>
+  {
+    val jar = new File("plugins/mimir-pip.jar")
+
+    val plugin = Plugin.load(jar)
+
+    val df = Vizier.sparkSession.sql("""
+      SELECT gaussian(5.0, 1.0)
+    """)
+
+    ok
+  }
+
+}