Merge pull request #41 from sievericcardo/triple-store

#40 Fuseki Triple store

Author: Edkamb

DockerfileFuseki

@@ -0,0 +1,35 @@
+# syntax=docker/dockerfile:1.3-labs
+
+# To build the container:
+#     docker build -f DockerfileFuseki -t fuseki .
+# To run smol in the current directory:
+#     docker run -it --rm -v "$PWD":/root/smol -p 3030:3030 fuseki
+
+# Use a base image, for example, Ubuntu
+FROM ubuntu:latest
+
+RUN <<EOF
+    apt-get -y update
+    DEBIAN_FRONTEND=noninteractive apt-get -y install openjdk-17-jdk-headless python3 python3-pip liblapack3 wget
+    rm -rf /var/lib/apt/lists/*
+    pip3 install pyzmq
+EOF
+COPY . /usr/local/src/smol
+COPY src/test/resources/docker_fuseki_config.ttl /usr/local/src/smol
+COPY src/test/resources/tree_shapes.ttl /usr/local/src/smol
+
+# Set the working directory
+WORKDIR /usr/local/src/smol
+
+# Download and copy the tar.gz file into the container
+ADD https://dlcdn.apache.org/jena/binaries/apache-jena-fuseki-4.10.0.tar.gz /usr/local/src/smol
+
+# Extract the tar.gz file
+RUN tar -xvzf apache-jena-fuseki-4.10.0.tar.gz
+
+# Execute the command
+CMD /usr/local/src/smol/apache-jena-fuseki-4.10.0/fuseki-server --update --config docker_fuseki_config.ttl
+
+# Expose port 3030
+EXPOSE 3030
+

build.gradle

@@ -31,6 +31,7 @@ dependencies {
     testImplementation 'io.kotest:kotest-runner-junit5:5.2.3'
     testImplementation 'org.jetbrains.kotlin:kotlin-test-junit'
     testImplementation 'org.apache.commons:commons-lang3:3.12.0'
+    testImplementation 'org.apache.jena:jena-fuseki-main:3.16.0'
     implementation 'com.github.ajalt.clikt:clikt:3.4.2'
     implementation 'org.antlr:antlr4:4.8'
     antlr 'org.antlr:antlr4:4.8'

fuseki_server_test.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# Build the fuseki image
+docker build -f DockerfileFuseki -t fuseki .
+
+# Run the fuseki image
+docker run  -d --name fuseki_container -it --rm -v "$PWD":/root/smol -p 3030:3030 fuseki
+
+# Define the FUSEKI_SERVER environment variable
+export FUSEKI_DOCKER=true
+
+# Execute the test
+./gradlew --stop
+./gradlew test --tests no.uio.microobject.test.data.TripleManagerTest
+
+# Stop the fuseki server
+kill $(ps aux | grep '[f]useki' | awk '{print $2}')
+
+# Stop the Docker container when done testing (optional)
+docker stop fuseki_container
+docker kill fuseki_container # to kill the container if docker stop doesn't work
+docker rm fuseki_container
+
+# Delete the fuseki image
+docker rmi fuseki

src/main/kotlin/no/uio/microobject/data/TripleManager.kt

@@ -5,8 +5,10 @@ import com.github.owlcs.ontapi.OntologyManager
 import com.github.owlcs.ontapi.config.OntLoaderConfiguration
 import no.uio.microobject.ast.expr.LiteralExpr
 import no.uio.microobject.ast.expr.TRUEEXPR
+import no.uio.microobject.main.ReasonerMode
 import java.io.*
 import no.uio.microobject.main.Settings
+import no.uio.microobject.main.testModel
 import no.uio.microobject.runtime.*
 import no.uio.microobject.type.*
 import org.apache.commons.io.IOUtils
@@ -18,7 +20,9 @@ import org.apache.jena.graph.Node_URI
 import org.apache.jena.graph.NodeFactory
 import org.apache.jena.graph.Triple
 import org.apache.jena.graph.compose.MultiUnion
+import org.apache.jena.query.*
 import org.apache.jena.rdf.model.*
+import org.apache.jena.rdfconnection.RDFConnectionFactory
 import org.apache.jena.reasoner.Reasoner
 import org.apache.jena.reasoner.ReasonerRegistry
 import org.apache.jena.util.iterator.ExtendedIterator
@@ -28,14 +32,16 @@ import org.semanticweb.owlapi.model.OWLOntology
 import java.net.URL
 import java.util.*
 import kotlin.collections.HashMap
+import kotlin.system.exitProcess
 
 
 // Settings controlling the TripleManager.
 data class TripleSettings(
     val sources: HashMap<String,Boolean>, // Which sources to include
     val guards: HashMap<String,Boolean>, // If true, then guard clauses are used.
     var virtualization: HashMap<String,Boolean>, // If true, virtualization is used. Otherwise, naive method is used.
-    var jenaReasoner: String, // Must be either off, rdfs or owl
+    var jenaReasoner: ReasonerMode, // Must be either off, rdfs or owl
+    var fusekiModel: Model? = null // If given, then this model is used instead of the FusekiGraph
 )
 
 // Class managing triples from all the different sources, how to reason over them, and how to query them using SPARQL or DL queries.
@@ -44,10 +50,11 @@ class TripleManager(private val settings: Settings, val staticTable: StaticTable
 
     // Default settings. These can be changed with REPL commands.
     var currentTripleSettings = TripleSettings(
-        sources = hashMapOf("heap" to true, "staticTable" to true, "vocabularyFile" to true, "fmos" to true, "externalOntology" to (settings.background != "")),
+        sources = hashMapOf("heap" to true, "staticTable" to true, "vocabularyFile" to true, "fmos" to true, "externalOntology" to (settings.background != ""), "urlOntology" to (settings.tripleStore != "")),
         guards = hashMapOf("heap" to true, "staticTable" to true),
         virtualization = hashMapOf("heap" to true, "staticTable" to true, "fmos" to true),
-        jenaReasoner = "owl"
+        jenaReasoner = settings.reasoner,
+        fusekiModel = null
     )
 
 
@@ -127,6 +134,11 @@ class TripleManager(private val settings: Settings, val staticTable: StaticTable
         if (tripleSettings.sources.getOrDefault("externalOntology", false)) {
             includedGraphs.add(getExternalOntologyAsModel().graph)
         }
+        if (tripleSettings.sources.getOrDefault("urlOntology", false)) {
+            if (tripleSettings.fusekiModel == null)
+                tripleSettings.fusekiModel = getTripleStoreOntologyAsModel()
+            includedGraphs.add(tripleSettings.fusekiModel!!.graph)
+        }
         val model = ModelFactory.createModelForGraph(MultiUnion(includedGraphs.toTypedArray()))
         for ((key, value) in prefixMap) model.setNsPrefix(key, value)  // Adding prefixes
         return model
@@ -146,6 +158,21 @@ class TripleManager(private val settings: Settings, val staticTable: StaticTable
         return model
     }
 
+    private fun getTripleStoreOntologyAsModel(): Model {
+        // Test case only
+        if (testModel != null) return testModel as Model
+        // Normal behaviour with a Fuseki environment
+        return RDFConnectionFactory.connect(settings.tripleStore + "/data").fetch()
+    }
+
+    /**
+     * Regenerate the triple store model. We'll do so by fetching again the data
+     * This will be called when the triple store is updated, and we want to update the model.
+     */
+    fun regenerateTripleStoreModel(): Unit {
+        currentTripleSettings.fusekiModel = getTripleStoreOntologyAsModel()
+    }
+
     // Returns the Jena model containing statements from vocab.owl
     private fun getVocabularyModel(): Model {
         val vocabularyModel = ModelFactory.createDefaultModel()
@@ -157,15 +184,13 @@ class TripleManager(private val settings: Settings, val staticTable: StaticTable
         return vocabularyModel.read(iStream, null, "TTL")
     }
 
-
     // Get the requested Jena reasoner
     private fun getJenaReasoner(tripleSettings: TripleSettings): Reasoner? {
         when (tripleSettings.jenaReasoner) {
-            "off" -> { return null }
-            "owl" -> { return ReasonerRegistry.getOWLReasoner() }
-            "rdfs" -> { return ReasonerRegistry.getRDFSReasoner() }
+            ReasonerMode.off -> { return null }
+            ReasonerMode.owl -> { return ReasonerRegistry.getOWLReasoner() }
+            ReasonerMode.rdfs -> { return ReasonerRegistry.getRDFSReasoner() }
         }
-        return null
     }
 
     // A custom type of (nice)iterator which takes a list as input and iterates over them.

src/main/kotlin/no/uio/microobject/main/MainKt.kt

@@ -5,20 +5,33 @@ import com.github.ajalt.clikt.parameters.options.*
 import com.github.ajalt.clikt.parameters.types.path
 import org.jline.reader.LineReaderBuilder
 import no.uio.microobject.runtime.REPL
+import org.apache.jena.query.QueryFactory
+import org.apache.jena.query.ResultSet
+import org.apache.jena.rdf.model.Model
+import org.apache.jena.rdfconnection.RDFConnectionFactory
 import java.io.File
 import java.nio.file.Paths
 import kotlin.system.exitProcess
 
+// test cases only
+var testModel : Model? = null
+
+enum class ReasonerMode {
+    off, rdfs, owl
+}
+
 data class Settings(var verbose : Boolean,      //Verbosity
                     val materialize : Boolean,  //Materialize
                     var outdir : String,        //path of temporary outputs
+                    val tripleStore : String,   // url for the triple store database
                     val background : String,    //owl background knowledge
                     val domainPrefix : String,  //prefix used in the domain model (domain:)
                     val progPrefix : String = "https://github.com/Edkamb/SemanticObjects/Program#",    //prefix for the program (prog:)
                     val runPrefix : String  = "https://github.com/Edkamb/SemanticObjects/Run${System.currentTimeMillis()}#",    //prefix for this run (run:)
                     val langPrefix : String = "https://github.com/Edkamb/SemanticObjects#",
                     val extraPrefixes : HashMap<String, String>,
-                    val useQueryType : Boolean = false
+                    val useQueryType : Boolean = false,
+                    val reasoner : ReasonerMode = ReasonerMode.owl
                     ){
     private var prefixMapCache: HashMap<String, String>? = null
     fun prefixMap() : HashMap<String, String> {
@@ -78,6 +91,8 @@ class Main : CliktCommand() {
         "--load" to "repl",       "-l" to "repl",
     ).default("repl")
 
+    private val reasoner by option("--jenaReasoner", "-j", help="Set value of the internally used reasoner to 'off', 'rdfs', or 'owl' (default -> 'owl')").default("owl")
+    private val tripleStore by option("--sparqlEndpoint", "-s",  help="url for SPARQL endpoint")
     private val back         by option("--back",      "-b",  help="path to a file containing OWL class definitions as background knowledge.").path()
     private val domainPrefix by option("--domain",    "-d",  help="prefix for domain:.").default("https://github.com/Edkamb/SemanticObjects/ontologies/default#")
     private val input        by option("--input",     "-i",  help="path to a .smol file which is loaded on startup.").path()
@@ -94,18 +109,53 @@ class Main : CliktCommand() {
         //check that background knowledge exists
         var backgr = ""
         if(back != null){
+            assert(tripleStore == null)
             val file = File(back.toString())
             if(file.exists()){
                 backgr = file.readText()
             }else println("Could not find file for background knowledge: ${file.path}")
         }
 
+        var tripleStoreUrl = ""
+        if (tripleStore != null){
+            assert(back == null)
+
+            val url = tripleStore.toString() + "/query"
+
+            // We check if the connection exists by querying the triple store for a single element
+            // If the query fails, we exit the program. There might be a more elegant way of doing it
+            val conn = RDFConnectionFactory.connect(url)
+
+            val query = QueryFactory.create("SELECT * WHERE { ?s ?p ?o } LIMIT 1")
+            val qexec = conn.query(query)
+            val result: ResultSet = qexec.execSelect()
+
+            // check that we retrieved something
+            if (!result.hasNext()) {
+                println("Error: the url for the triple store is not valid.")
+                exitProcess(-1)
+            } else {
+                tripleStoreUrl = tripleStore.toString()
+                conn.close()
+            }
+        }
+
+        val reasonerMode = when(reasoner){
+            "off" -> ReasonerMode.off
+            "rdfs" -> ReasonerMode.rdfs
+            "owl" -> ReasonerMode.owl
+            else -> {
+                println("Error: the reasoner mode is not valid.")
+                exitProcess(-1)
+            }
+        }
+
         if (input == null && mainMode != "repl"){
             println("Error: please specify an input .smol file using \"--input\".")
             exitProcess(-1)
         }
 
-        val repl = REPL( Settings(verbose, materialize, outdir.toString(), backgr, domainPrefix, extraPrefixes=HashMap(extra), useQueryType = queryType))
+        val repl = REPL( Settings(verbose, materialize, outdir.toString(), tripleStoreUrl, backgr, domainPrefix, extraPrefixes=HashMap(extra), useQueryType = queryType, reasoner = reasonerMode))
         if(input != null){
             repl.command("read", input.toString())
         }

src/main/kotlin/no/uio/microobject/runtime/REPL.kt

@@ -11,6 +11,7 @@ import no.uio.microobject.antlr.WhileParser
 import no.uio.microobject.ast.Expression
 import no.uio.microobject.ast.Translate
 import no.uio.microobject.data.TripleManager
+import no.uio.microobject.main.ReasonerMode
 import no.uio.microobject.main.Settings
 import no.uio.microobject.type.TypeChecker
 import org.antlr.v4.runtime.CharStreams
@@ -277,7 +278,7 @@ class REPL(private val settings: Settings) {
                 else {
                     if  (!allowedParameters.contains(p[0])) { printRepl("\nParameter must one of: $allowedParameters") }
                     else {
-                        interpreter!!.tripleManager.currentTripleSettings.jenaReasoner = p[0]
+                        interpreter!!.tripleManager.currentTripleSettings.jenaReasoner = ReasonerMode.valueOf(p[0])
                         printRepl("Reasoner changed to: ${p[0]}")
                     }
                 }

src/test/kotlin/no/uio/microobject/test/MicroObjectTest.kt

@@ -22,6 +22,7 @@ import org.apache.commons.lang3.SystemUtils.IS_OS_WINDOWS
 
 open class MicroObjectTest : StringSpec() {
     protected enum class StringLoad {STMT, CLASS, PRG, PATH, RES}
+    private val IS_FUSEKI_DOCKER = System.getenv("FUSEKI_DOCKER") == "true"
 
     val fmuNeedsWindows: (TestCase) -> Enabled = {
         if (IS_OS_WINDOWS) Enabled.enabled
@@ -35,12 +36,17 @@ open class MicroObjectTest : StringSpec() {
         if (IS_OS_LINUX) Enabled.enabled
         else Enabled.disabled("The FMU of this test needs to run on Linux.")
     }
+    val tripleStoreToTest: (TestCase) -> Enabled = {
+        if (IS_FUSEKI_DOCKER) Enabled.enabled
+        else Enabled.disabled("The triple store needs to be running on a docker container for this test.")
+    }
 
     private var settings = Settings(false, false,  "/tmp/mo","","","urn:", extraPrefixes = hashMapOf())
+    private var tripleStoreSettings = Settings(false, false,  "/tmp/mo","http://localhost:3030/ds","","urn:", extraPrefixes = hashMapOf())
     protected fun loadBackground(path : String, domainPrefix : String = ""){
         val file = File(path)
         val backgr = file.readText()
-        settings = Settings(false, false,  "/tmp/mo",backgr,domainPrefix,"urn:", extraPrefixes = hashMapOf())
+        settings = Settings(false, false,  "/tmp/mo","",backgr,domainPrefix,"urn:", extraPrefixes = hashMapOf())
     }
     private fun loadString(program : String) : WhileParser.ProgramContext{
         val stdLib = this::class.java.classLoader.getResource("StdLib.smol").readText() + "\n\n"
@@ -111,6 +117,38 @@ open class MicroObjectTest : StringSpec() {
         return Pair(interpreter, tC)
     }
 
+    protected fun initTripleStoreInterpreter(str : String, loadAs : StringLoad = StringLoad.PATH) : Pair<Interpreter, TypeChecker> {
+        val ast = when(loadAs){
+            StringLoad.STMT -> loadStatement(str)
+            StringLoad.PRG -> loadString(str)
+            StringLoad.CLASS -> loadClass(str)
+            StringLoad.PATH -> loadPath(str)
+            StringLoad.RES -> loadPath(this::class.java.classLoader.getResource("$str.smol").file)
+        }
+        val visitor = Translate()
+        val pair = visitor.generateStatic(ast)
+
+        val tripleManager = TripleManager(tripleStoreSettings, pair.second, null)
+
+        val tC = TypeChecker(ast, tripleStoreSettings, tripleManager)
+        tC.collect()
+        var rules = ""
+
+
+        val initGlobalStore: GlobalMemory = mutableMapOf(Pair(pair.first.obj, mutableMapOf()))
+
+        val initStack = Stack<StackEntry>()
+        initStack.push(pair.first)
+        val interpreter = Interpreter(
+            initStack,
+            initGlobalStore,
+            mutableMapOf(),
+            pair.second,
+            tripleStoreSettings
+        )
+        return Pair(interpreter, tC)
+    }
+
     protected fun initTc(str : String, loadAs : StringLoad = StringLoad.PATH) : Pair<TypeChecker, WhileParser.ProgramContext> {
 
         val ast = when(loadAs){

src/test/kotlin/no/uio/microobject/test/basic/BasicTest.kt

@@ -30,7 +30,7 @@ class BasicTest : StringSpec() {
         val pair = visitor.generateStatic(tree)
 
 
-        val settings = Settings(false,false,  "/tmp/mo","","urn:", "", extraPrefixes = hashMapOf())
+        val settings = Settings(false,false,  "/tmp/mo", "","","urn:", "", extraPrefixes = hashMapOf())
         val tripleManager = TripleManager(settings, pair.second, null)
 
         val tC = TypeChecker(tree, settings, tripleManager)