start refactor and doc core.worker and worker.springboot

dbpedia · Jun 6, 2023 · 591c4bb · 591c4bb
1 parent 7a9eedd
commit 591c4bb
Show file tree

Hide file tree

Showing 79 changed files with 719 additions and 831 deletions.
diff --git a/databus-mods-lib/databus-mods-core/pom.xml b/databus-mods-lib/databus-mods-core/pom.xml
@@ -16,6 +16,15 @@
     </parent>
 
     <dependencies>
+        <!-- https://mvnrepository.com/artifact/org.scalatest/scalatest -->
+        <dependency>
+            <groupId>org.scalatest</groupId>
+            <artifactId>scalatest_2.12</artifactId>
+            <version>3.2.16</version>
+            <scope>test</scope>
+        </dependency>
+
+
         <!-- JENA -->
         <dependency>
             <groupId>org.apache.jena</groupId>

diff --git a/...mods/core/databus/DatabusIdentifier.scala → ...ia/databus/dataid/DatabusIdentifier.scala b/...mods/core/databus/DatabusIdentifier.scala → ...ia/databus/dataid/DatabusIdentifier.scala
@@ -1,6 +1,4 @@
-package org.dbpedia.databus.mods.core.databus
-
-import java.util.regex.Pattern
+package org.dbpedia.databus.dataid
 
 import scala.util.matching.Regex
 

diff --git a/.../core/databus/DatabusIdentifierType.scala → ...atabus/dataid/DatabusIdentifierType.scala b/.../core/databus/DatabusIdentifierType.scala → ...atabus/dataid/DatabusIdentifierType.scala
@@ -1,4 +1,4 @@
-package org.dbpedia.databus.mods.core.databus
+package org.dbpedia.databus.dataid
 
 object DatabusIdentifierType extends Enumeration {
   type DatabusIdentifierType = Value

diff --git a/...a/databus/mods/core/util/Compressor.scala → ...ia/databus/mods/core/io/Compression.scala b/...a/databus/mods/core/util/Compressor.scala → ...ia/databus/mods/core/io/Compression.scala
@@ -1,12 +1,13 @@
-package org.dbpedia.databus.mods.core.util
+package org.dbpedia.databus.mods.core.io
 
 import org.apache.commons.compress.compressors.{CompressorException, CompressorStreamFactory}
 
 import java.io.{BufferedInputStream, InputStream}
 
-object Compressor {
+object Compression {
+
   def decompress(bis: BufferedInputStream): InputStream = {
-    //Welche Funktion hat actualDecompressConcatenated?
+    // TODO what does actualDecompressConcatenated?
     try {
 
       new CompressorStreamFactory().createCompressorInputStream(

diff --git a/...edia/databus/mods/core/util/UriUtil.scala → ...pedia/databus/mods/core/io/DataUtil.scala b/...edia/databus/mods/core/util/UriUtil.scala → ...pedia/databus/mods/core/io/DataUtil.scala
@@ -1,18 +1,17 @@
-package org.dbpedia.databus.mods.core.util
-
-import java.io.{File, FileInputStream, InputStream}
-import java.net.URI
+package org.dbpedia.databus.mods.core.io
 
 import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream
 import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream
 import org.apache.http.client.methods.HttpGet
 import org.apache.http.impl.client.HttpClientBuilder
 
-// TODO visitor pattern
-object UriUtil {
+import java.io.{File, FileInputStream, InputStream}
+import java.net.URI
 
-  def openStream(uri: URI): InputStream = {
+object DataUtil {
 
+  def openStream(uri: URI): InputStream = {
+    // TODO
     val is = uri.getScheme match {
       case "http" | "https" =>
         val client = HttpClientBuilder.create().build()

diff --git a/...ia/databus/mods/core/util/IORdfUtil.scala → .../dbpedia/databus/mods/core/io/RdfIO.scala b/...ia/databus/mods/core/util/IORdfUtil.scala → .../dbpedia/databus/mods/core/io/RdfIO.scala
@@ -1,16 +1,16 @@
-package org.dbpedia.databus.mods.core.util
+package org.dbpedia.databus.mods.core.io
 
 import org.apache.jena.graph.Triple
 import org.apache.jena.riot.lang.{PipedRDFIterator, PipedRDFStream, PipedTriplesStream}
 import org.apache.jena.riot.{Lang, RDFParser}
 
 import java.io.{BufferedInputStream, InputStream}
 
-object IORdfUtil {
+object RdfIO {
 
   def toPipedRDF(inputStream: InputStream): PipedRDFIterator[Triple] = {
     val bis = new BufferedInputStream(inputStream)
-    val in = Compressor.decompress(bis)
+    val in = Compression.decompress(bis)
 
     //    val lang = org.dbpedia.databus.util.MimeTypeGetter.getRDFFormat(item.downloadURL)
     val pipedRDF: PipedRDFIterator[Triple] = new PipedRDFIterator[Triple]()

diff --git a/...edia/databus/mods/model/ModActivity.scala → ...databus/mods/core/model/ModActivity.scala b/...edia/databus/mods/model/ModActivity.scala → ...databus/mods/core/model/ModActivity.scala
@@ -1,4 +1,4 @@
-package org.dbpedia.databus.mods.model
+package org.dbpedia.databus.mods.core.model
 
 abstract class ModActivity {
 

diff --git a/...abus/mods/model/ModActivityMetadata.scala → ...mods/core/model/ModActivityMetadata.scala b/...abus/mods/model/ModActivityMetadata.scala → ...mods/core/model/ModActivityMetadata.scala
@@ -1,12 +1,13 @@
-package org.dbpedia.databus.mods.model
+package org.dbpedia.databus.mods.core.model
 
 import org.apache.jena.datatypes.xsd.XSDDateTime
 import org.apache.jena.rdf.model.{Model, ModelFactory, ResourceFactory}
 import org.apache.jena.vocabulary.RDF
 import org.dbpedia.databus.mods.core.MetadataType
-import org.dbpedia.databus.mods.model.vocabulary.{MOD, PROV}
+import org.dbpedia.databus.mods.core.model.vocabulary.{MOD, PROV}
 
 import java.io.ByteArrayInputStream
+import java.net.URI
 import java.nio.charset.StandardCharsets
 
 // TODO try: generated ++ statSummary >= 1
@@ -27,12 +28,13 @@ class ModActivityMetadata(
   val statSummary: Option[String],
 ) {
 
-  def createRdfModel(base: String): Model = {
+  def createRdfModel(base: URI): Model = {
+
     val model = ModelFactory.createDefaultModel()
 
     // TODO efficient?
     val triples =
-      s"""@base <$base> .
+      s"""@base <${base.toString}> .
          |
          |@prefix prov: <http://www.w3.org/ns/prov#> .
          |@prefix xsd: <http://www.w3.org/2001/XMLSchema#> .
@@ -45,19 +47,20 @@ class ModActivityMetadata(
          |<activity> <${MOD.version}> "$version" .
          |""".stripMargin
 
-    model.read(new ByteArrayInputStream(triples.getBytes(StandardCharsets.UTF_8)), base, "TURTLE")
+    model.read(new ByteArrayInputStream(triples.getBytes(StandardCharsets.UTF_8)), base.toString, "TURTLE")
 
     // add generated results
     // TODO better
     generated.foreach({
       modResult =>
+        val modResultUri = base.relativize(modResult.id).toString
         model.add(
           ResourceFactory.createResource(base + "activity"),
           ResourceFactory.createProperty(PROV.generated),
-          ResourceFactory.createResource(base + modResult.name)
+          ResourceFactory.createResource(modResultUri)
         )
         model.add(
-          ResourceFactory.createResource(base + modResult.name),
+          ResourceFactory.createResource(modResultUri),
           ResourceFactory.createProperty(
             modResult.metadataType match {
               case MetadataType.Enrichment => MOD.enrichmentDerivedFrom
@@ -68,7 +71,7 @@ class ModActivityMetadata(
           ResourceFactory.createResource(base + "activity"),
         )
         model.add(
-          ResourceFactory.createResource(base + modResult.name),
+          ResourceFactory.createResource(modResultUri),
           RDF.`type`,
           ResourceFactory.createResource(
             modResult.metadataType match {
@@ -91,13 +94,13 @@ class ModActivityMetadata(
         )
     })
 
-    return model
+    model
   }
 }
 
 object ModActivityMetadata {
 
-  def builder(used: String = ""): ModActivityMetadataBuilder = {
+  def builder(used: String = null): ModActivityMetadataBuilder = {
      ModActivityMetadataBuilder(used = used)
   }
 }
diff --git a/...ds/model/ModActivityMetadataBuilder.scala → ...re/model/ModActivityMetadataBuilder.scala b/...ds/model/ModActivityMetadataBuilder.scala → ...re/model/ModActivityMetadataBuilder.scala
@@ -1,7 +1,7 @@
-package org.dbpedia.databus.mods.model
+package org.dbpedia.databus.mods.core.model
 
 import org.apache.jena.datatypes.xsd.XSDDateTime
-import org.dbpedia.databus.mods.model.vocabulary.MOD
+import org.dbpedia.databus.mods.core.model.vocabulary.MOD
 
 import java.time.Instant
 import java.util.Calendar

diff --git a/...bus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/model/ModActivityRequest.scala b/...bus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/model/ModActivityRequest.scala
@@ -0,0 +1,3 @@
+package org.dbpedia.databus.mods.core.model
+
+case class ModActivityRequest(dataId: String, accessUri: Option[String])
diff --git a/...us-mods-core/src/main/scala/org/dbpedia/databus/mods/core/model/ModActivityResponse.scala b/...us-mods-core/src/main/scala/org/dbpedia/databus/mods/core/model/ModActivityResponse.scala
@@ -0,0 +1,3 @@
+package org.dbpedia.databus.mods.core.model
+
+case class ModActivityResponse(baseUri: String, data: Array[Byte])
diff --git a/...-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/model/ModResult.scala b/...-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/model/ModResult.scala
@@ -0,0 +1,21 @@
+package org.dbpedia.databus.mods.core.model
+
+import org.dbpedia.databus.dataid.Part
+import org.dbpedia.databus.mods.core.MetadataType
+
+import java.net.URI
+
+/**
+ * Data artifact created by a mod activity
+ *
+ * @param used the data artifact used as input
+ * @param id
+ * @param metadataType
+ */
+class ModResult(
+  val id: URI,
+  val suffix: String,
+  val used: URI,
+  val metadataType: MetadataType
+)
+
diff --git a/...tabus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/model/ModResultFactory.scala b/...tabus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/model/ModResultFactory.scala
@@ -0,0 +1,21 @@
+package org.dbpedia.databus.mods.core.model
+
+import org.dbpedia.databus.dataid.Part
+import org.dbpedia.databus.mods.core.MetadataType
+
+import java.net.URI
+
+object ModResultFactory {
+
+  def enrichment(used: URI): ModResult = {
+    new ModResult(null, null, used, MetadataType.Enrichment)
+  }
+
+  def statistics(used: URI): ModResult = {
+    new ModResult(null, null, used,MetadataType.Statistics)
+  }
+
+  def summary(used: URI): ModResult = {
+    new ModResult(null, null, used,MetadataType.Summary)
+  }
+}
diff --git a/...a/databus/mods/model/vocabulary/MOD.scala → ...abus/mods/core/model/vocabulary/MOD.scala b/...a/databus/mods/model/vocabulary/MOD.scala → ...abus/mods/core/model/vocabulary/MOD.scala
@@ -1,4 +1,4 @@
-package org.dbpedia.databus.mods.model.vocabulary
+package org.dbpedia.databus.mods.core.model.vocabulary
 
 object MOD {
 

diff --git a/.../databus/mods/model/vocabulary/PROV.scala → ...bus/mods/core/model/vocabulary/PROV.scala b/.../databus/mods/model/vocabulary/PROV.scala → ...bus/mods/core/model/vocabulary/PROV.scala
@@ -1,4 +1,4 @@
-package org.dbpedia.databus.mods.model.vocabulary
+package org.dbpedia.databus.mods.core.model.vocabulary
 
 object PROV {
 

diff --git a/...s-mods-core/src/main/scala/org/dbpedia/databus/mods/core/util/DatabusModVocabHelper.scala b/...s-mods-core/src/main/scala/org/dbpedia/databus/mods/core/util/DatabusModVocabHelper.scala
diff --git a/...atabus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/util/ModActivityUtils.scala b/...atabus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/util/ModActivityUtils.scala
@@ -0,0 +1,19 @@
+package org.dbpedia.databus.mods.core.util
+
+import org.dbpedia.databus.mods.core.io.DataUtil
+import org.dbpedia.databus.mods.core.model.ModActivityRequest
+
+import java.io.InputStream
+import java.net.URI
+
+object ModActivityUtils {
+
+  def openInputStream(modActivityRequest: ModActivityRequest): InputStream = {
+    modActivityRequest.accessUri match {
+      case Some(accessUri) =>
+        DataUtil.openStream(new URI(accessUri))
+      case None =>
+        DataUtil.openStream(new URI(modActivityRequest.dataId))
+    }
+  }
+}
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		package org.dbpedia.databus.mods.core.model

		case class ModActivityRequest(dataId: String, accessUri: Option[String])
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		package org.dbpedia.databus.mods.core.model

		case class ModActivityResponse(baseUri: String, data: Array[Byte])