diff --git a/databus-mods-lib/databus-mods-core/pom.xml b/databus-mods-lib/databus-mods-core/pom.xml index 10c0207..a36effa 100644 --- a/databus-mods-lib/databus-mods-core/pom.xml +++ b/databus-mods-lib/databus-mods-core/pom.xml @@ -16,6 +16,15 @@ + + + org.scalatest + scalatest_2.12 + 3.2.16 + test + + + org.apache.jena diff --git a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/databus/DatabusIdentifier.scala b/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/dataid/DatabusIdentifier.scala similarity index 97% rename from databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/databus/DatabusIdentifier.scala rename to databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/dataid/DatabusIdentifier.scala index b1e1ea8..136184e 100644 --- a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/databus/DatabusIdentifier.scala +++ b/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/dataid/DatabusIdentifier.scala @@ -1,6 +1,4 @@ -package org.dbpedia.databus.mods.core.databus - -import java.util.regex.Pattern +package org.dbpedia.databus.dataid import scala.util.matching.Regex diff --git a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/databus/DatabusIdentifierType.scala b/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/dataid/DatabusIdentifierType.scala similarity index 77% rename from databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/databus/DatabusIdentifierType.scala rename to databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/dataid/DatabusIdentifierType.scala index 64ee7f6..899317b 100644 --- a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/databus/DatabusIdentifierType.scala +++ b/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/dataid/DatabusIdentifierType.scala @@ -1,4 +1,4 @@ -package org.dbpedia.databus.mods.core.databus +package org.dbpedia.databus.dataid object DatabusIdentifierType extends Enumeration { type DatabusIdentifierType = Value diff --git a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/util/Compressor.scala b/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/io/Compression.scala similarity index 83% rename from databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/util/Compressor.scala rename to databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/io/Compression.scala index 1e8c207..985216b 100644 --- a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/util/Compressor.scala +++ b/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/io/Compression.scala @@ -1,12 +1,13 @@ -package org.dbpedia.databus.mods.core.util +package org.dbpedia.databus.mods.core.io import org.apache.commons.compress.compressors.{CompressorException, CompressorStreamFactory} import java.io.{BufferedInputStream, InputStream} -object Compressor { +object Compression { + def decompress(bis: BufferedInputStream): InputStream = { - //Welche Funktion hat actualDecompressConcatenated? + // TODO what does actualDecompressConcatenated? try { new CompressorStreamFactory().createCompressorInputStream( diff --git a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/util/UriUtil.scala b/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/io/DataUtil.scala similarity index 91% rename from databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/util/UriUtil.scala rename to databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/io/DataUtil.scala index ac9ab2c..9c1b083 100644 --- a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/util/UriUtil.scala +++ b/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/io/DataUtil.scala @@ -1,18 +1,17 @@ -package org.dbpedia.databus.mods.core.util - -import java.io.{File, FileInputStream, InputStream} -import java.net.URI +package org.dbpedia.databus.mods.core.io import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream import org.apache.http.client.methods.HttpGet import org.apache.http.impl.client.HttpClientBuilder -// TODO visitor pattern -object UriUtil { +import java.io.{File, FileInputStream, InputStream} +import java.net.URI - def openStream(uri: URI): InputStream = { +object DataUtil { + def openStream(uri: URI): InputStream = { + // TODO val is = uri.getScheme match { case "http" | "https" => val client = HttpClientBuilder.create().build() diff --git a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/util/IORdfUtil.scala b/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/io/RdfIO.scala similarity index 91% rename from databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/util/IORdfUtil.scala rename to databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/io/RdfIO.scala index cdb2a8e..e303cf6 100644 --- a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/util/IORdfUtil.scala +++ b/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/io/RdfIO.scala @@ -1,4 +1,4 @@ -package org.dbpedia.databus.mods.core.util +package org.dbpedia.databus.mods.core.io import org.apache.jena.graph.Triple import org.apache.jena.riot.lang.{PipedRDFIterator, PipedRDFStream, PipedTriplesStream} @@ -6,11 +6,11 @@ import org.apache.jena.riot.{Lang, RDFParser} import java.io.{BufferedInputStream, InputStream} -object IORdfUtil { +object RdfIO { def toPipedRDF(inputStream: InputStream): PipedRDFIterator[Triple] = { val bis = new BufferedInputStream(inputStream) - val in = Compressor.decompress(bis) + val in = Compression.decompress(bis) // val lang = org.dbpedia.databus.util.MimeTypeGetter.getRDFFormat(item.downloadURL) val pipedRDF: PipedRDFIterator[Triple] = new PipedRDFIterator[Triple]() diff --git a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/model/ModActivity.scala b/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/model/ModActivity.scala similarity index 75% rename from databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/model/ModActivity.scala rename to databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/model/ModActivity.scala index c5e01a0..e4123fc 100644 --- a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/model/ModActivity.scala +++ b/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/model/ModActivity.scala @@ -1,4 +1,4 @@ -package org.dbpedia.databus.mods.model +package org.dbpedia.databus.mods.core.model abstract class ModActivity { diff --git a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/model/ModActivityMetadata.scala b/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/model/ModActivityMetadata.scala similarity index 83% rename from databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/model/ModActivityMetadata.scala rename to databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/model/ModActivityMetadata.scala index d9cd24d..c8b024e 100644 --- a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/model/ModActivityMetadata.scala +++ b/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/model/ModActivityMetadata.scala @@ -1,12 +1,13 @@ -package org.dbpedia.databus.mods.model +package org.dbpedia.databus.mods.core.model import org.apache.jena.datatypes.xsd.XSDDateTime import org.apache.jena.rdf.model.{Model, ModelFactory, ResourceFactory} import org.apache.jena.vocabulary.RDF import org.dbpedia.databus.mods.core.MetadataType -import org.dbpedia.databus.mods.model.vocabulary.{MOD, PROV} +import org.dbpedia.databus.mods.core.model.vocabulary.{MOD, PROV} import java.io.ByteArrayInputStream +import java.net.URI import java.nio.charset.StandardCharsets // TODO try: generated ++ statSummary >= 1 @@ -27,12 +28,13 @@ class ModActivityMetadata( val statSummary: Option[String], ) { - def createRdfModel(base: String): Model = { + def createRdfModel(base: URI): Model = { + val model = ModelFactory.createDefaultModel() // TODO efficient? val triples = - s"""@base <$base> . + s"""@base <${base.toString}> . | |@prefix prov: . |@prefix xsd: . @@ -45,19 +47,20 @@ class ModActivityMetadata( | <${MOD.version}> "$version" . |""".stripMargin - model.read(new ByteArrayInputStream(triples.getBytes(StandardCharsets.UTF_8)), base, "TURTLE") + model.read(new ByteArrayInputStream(triples.getBytes(StandardCharsets.UTF_8)), base.toString, "TURTLE") // add generated results // TODO better generated.foreach({ modResult => + val modResultUri = base.relativize(modResult.id).toString model.add( ResourceFactory.createResource(base + "activity"), ResourceFactory.createProperty(PROV.generated), - ResourceFactory.createResource(base + modResult.name) + ResourceFactory.createResource(modResultUri) ) model.add( - ResourceFactory.createResource(base + modResult.name), + ResourceFactory.createResource(modResultUri), ResourceFactory.createProperty( modResult.metadataType match { case MetadataType.Enrichment => MOD.enrichmentDerivedFrom @@ -68,7 +71,7 @@ class ModActivityMetadata( ResourceFactory.createResource(base + "activity"), ) model.add( - ResourceFactory.createResource(base + modResult.name), + ResourceFactory.createResource(modResultUri), RDF.`type`, ResourceFactory.createResource( modResult.metadataType match { @@ -91,13 +94,13 @@ class ModActivityMetadata( ) }) - return model + model } } object ModActivityMetadata { - def builder(used: String = ""): ModActivityMetadataBuilder = { + def builder(used: String = null): ModActivityMetadataBuilder = { ModActivityMetadataBuilder(used = used) } } diff --git a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/model/ModActivityMetadataBuilder.scala b/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/model/ModActivityMetadataBuilder.scala similarity index 95% rename from databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/model/ModActivityMetadataBuilder.scala rename to databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/model/ModActivityMetadataBuilder.scala index 79d917a..7a7bb03 100644 --- a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/model/ModActivityMetadataBuilder.scala +++ b/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/model/ModActivityMetadataBuilder.scala @@ -1,7 +1,7 @@ -package org.dbpedia.databus.mods.model +package org.dbpedia.databus.mods.core.model import org.apache.jena.datatypes.xsd.XSDDateTime -import org.dbpedia.databus.mods.model.vocabulary.MOD +import org.dbpedia.databus.mods.core.model.vocabulary.MOD import java.time.Instant import java.util.Calendar diff --git a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/model/ModActivityRequest.scala b/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/model/ModActivityRequest.scala new file mode 100644 index 0000000..a9287ea --- /dev/null +++ b/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/model/ModActivityRequest.scala @@ -0,0 +1,3 @@ +package org.dbpedia.databus.mods.core.model + +case class ModActivityRequest(dataId: String, accessUri: Option[String]) \ No newline at end of file diff --git a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/model/ModActivityResponse.scala b/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/model/ModActivityResponse.scala new file mode 100644 index 0000000..7058a29 --- /dev/null +++ b/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/model/ModActivityResponse.scala @@ -0,0 +1,3 @@ +package org.dbpedia.databus.mods.core.model + +case class ModActivityResponse(baseUri: String, data: Array[Byte]) diff --git a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/model/ModResult.scala b/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/model/ModResult.scala new file mode 100644 index 0000000..c1dc088 --- /dev/null +++ b/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/model/ModResult.scala @@ -0,0 +1,21 @@ +package org.dbpedia.databus.mods.core.model + +import org.dbpedia.databus.dataid.Part +import org.dbpedia.databus.mods.core.MetadataType + +import java.net.URI + +/** + * Data artifact created by a mod activity + * + * @param used the data artifact used as input + * @param id + * @param metadataType + */ +class ModResult( + val id: URI, + val suffix: String, + val used: URI, + val metadataType: MetadataType +) + diff --git a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/model/ModResultFactory.scala b/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/model/ModResultFactory.scala new file mode 100644 index 0000000..63cf63b --- /dev/null +++ b/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/model/ModResultFactory.scala @@ -0,0 +1,21 @@ +package org.dbpedia.databus.mods.core.model + +import org.dbpedia.databus.dataid.Part +import org.dbpedia.databus.mods.core.MetadataType + +import java.net.URI + +object ModResultFactory { + + def enrichment(used: URI): ModResult = { + new ModResult(null, null, used, MetadataType.Enrichment) + } + + def statistics(used: URI): ModResult = { + new ModResult(null, null, used,MetadataType.Statistics) + } + + def summary(used: URI): ModResult = { + new ModResult(null, null, used,MetadataType.Summary) + } +} diff --git a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/model/vocabulary/MOD.scala b/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/model/vocabulary/MOD.scala similarity index 92% rename from databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/model/vocabulary/MOD.scala rename to databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/model/vocabulary/MOD.scala index 1c2d847..2455ca5 100644 --- a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/model/vocabulary/MOD.scala +++ b/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/model/vocabulary/MOD.scala @@ -1,4 +1,4 @@ -package org.dbpedia.databus.mods.model.vocabulary +package org.dbpedia.databus.mods.core.model.vocabulary object MOD { diff --git a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/model/vocabulary/PROV.scala b/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/model/vocabulary/PROV.scala similarity index 81% rename from databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/model/vocabulary/PROV.scala rename to databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/model/vocabulary/PROV.scala index 0891c00..6e5291a 100644 --- a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/model/vocabulary/PROV.scala +++ b/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/model/vocabulary/PROV.scala @@ -1,4 +1,4 @@ -package org.dbpedia.databus.mods.model.vocabulary +package org.dbpedia.databus.mods.core.model.vocabulary object PROV { diff --git a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/util/DatabusModVocabHelper.scala b/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/util/DatabusModVocabHelper.scala deleted file mode 100644 index ed8df60..0000000 --- a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/util/DatabusModVocabHelper.scala +++ /dev/null @@ -1,74 +0,0 @@ -package org.dbpedia.databus.mods.core.util - -import java.net.{MalformedURLException, URL} - -import org.apache.jena.rdf.model.{Model, ModelFactory, ResourceFactory} - -class DatabusModVocabHelper(modName: String) { - - private val modVocabModel = ModelFactory.createDefaultModel() - - object Prefixes { - val owl = "http://www.w3.org/2002/07/owl#" - val mod = "http://dataid.dbpedia.org/ns/mod.ttl#" - val rdfs = "http://www.w3.org/2000/01/rdf-schema#" - val prov = "http://www.w3.org/ns/prov#" - } - - private val uriByPrefix: Map[String, String] = Map( - "rdfs" -> Prefixes.rdfs, - "mod" -> Prefixes.mod, - "owl" -> Prefixes.owl, - "prov" -> Prefixes.prov - ) - - import scala.collection.JavaConverters.mapAsJavaMapConverter - - modVocabModel.setNsPrefixes(uriByPrefix.asJava) - - addStmtToModVocab(s"#${modName}", "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", s"${Prefixes.owl}Class") - addStmtToModVocab(s"#${modName}", s"${Prefixes.rdfs}subClassOf", s"${Prefixes.mod}DatabusMod") - addStmtToModVocab(s"#${modName}", s"${Prefixes.rdfs}label", "") - - /** - * add statement to mod vocabulary model - * - * @param s subject - * @param p predicate - * @param o object - */ - private def addStmtToModVocab(s: String, p: String, o: Object): Unit = { - modVocabModel.add( - ResourceFactory.createStatement( - ResourceFactory.createResource(s), - ResourceFactory.createProperty(p), - try { - new URL(o.toString) - ResourceFactory.createResource(o.toString) - } catch { - case malformedURL: MalformedURLException => ResourceFactory.createTypedLiteral(o) - }) - ) - } - - def addFileTypeToModVocab(fileType: String, label: String, comment: String): Unit = { - val subj = s"${Prefixes.mod}${fileType.toLowerCase}DerivedFrom" - addStmtToModVocab(subj, "http://www.w3.org/1999/02/22-rdf-syntax-ns#type", s"${Prefixes.owl}ObjectProperty") - addStmtToModVocab(subj, s"${Prefixes.rdfs}subPropertyOf", s"${Prefixes.prov}wasDerivedFrom") - addStmtToModVocab(subj, s"${Prefixes.rdfs}subPropertyOf", s"${Prefixes.mod}wasDerivedFrom") - addStmtToModVocab(subj, s"${Prefixes.rdfs}domain", s"${Prefixes.mod}${fileType.toUpperCase}_Summary") - addStmtToModVocab(subj, s"${Prefixes.rdfs}label", label) - addStmtToModVocab(subj, s"${Prefixes.rdfs}comment", comment) - } - - - /** - * get model of ModVocabHelper - * - * @return model - */ - def getModel(): Model = { - modVocabModel - } - -} diff --git a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/util/ModActivityUtils.scala b/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/util/ModActivityUtils.scala new file mode 100644 index 0000000..1080e1c --- /dev/null +++ b/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/util/ModActivityUtils.scala @@ -0,0 +1,19 @@ +package org.dbpedia.databus.mods.core.util + +import org.dbpedia.databus.mods.core.io.DataUtil +import org.dbpedia.databus.mods.core.model.ModActivityRequest + +import java.io.InputStream +import java.net.URI + +object ModActivityUtils { + + def openInputStream(modActivityRequest: ModActivityRequest): InputStream = { + modActivityRequest.accessUri match { + case Some(accessUri) => + DataUtil.openStream(new URI(accessUri)) + case None => + DataUtil.openStream(new URI(modActivityRequest.dataId)) + } + } +} diff --git a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/util/ModApiUtil.scala b/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/util/ModApiUtil.scala deleted file mode 100644 index d422a34..0000000 --- a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/util/ModApiUtil.scala +++ /dev/null @@ -1,81 +0,0 @@ -package org.dbpedia.databus.mods.core.util - -import java.io.ByteArrayOutputStream -import java.net.{URI, URL} - -import org.apache.http.StatusLine -import org.apache.http.client.methods.{CloseableHttpResponse, HttpGet, HttpPost} -import org.apache.http.impl.client.HttpClientBuilder -import org.apache.jena.rdf.model.{Model, ModelFactory} -import org.apache.jena.riot.{Lang, RDFDataMgr} -import org.slf4j.LoggerFactory - -class ModApiUtil -object ModApiUtil { - - private val log = LoggerFactory.getLogger(classOf[ModApiUtil]) - - private val defaultRetrySeconds = 200 - private val maxRetrySeconds = 60 * 60 * 1000 - - // def get(): Unit = { - // val response: Response = Request.Get("").execute() - // } - // - // def post(): Unit = { - // - // } - // - // def delete(): Unit = { - // - // - // } - - def submitAndPoll(uri: URI, defaultRetryDelay: Int = 0): (Array[Byte],URI) = { - var follow: Option[URI] = None - val bos = new ByteArrayOutputStream() - var baseURI = uri - var retryDelay = defaultRetryDelay - - val httpClient = HttpClientBuilder.create().disableRedirectHandling().build() - val postResponse = httpClient.execute(new HttpPost(uri)) - postResponse match { - case ok if ok.getStatusLine.getStatusCode == 200 => - ok.getEntity.writeTo(bos) - case followRequest if needsFollow(followRequest.getStatusLine) => - val location = getLocationHeader(postResponse) - follow = if(location.isEmpty) None else Some(new URI(uri.getScheme+"://"+uri.getHost+":"+uri.getPort+location.get)) - case errorResponse => throw new Exception(errorResponse.getStatusLine.toString) - } - - while (follow.isDefined) { - baseURI = follow.get - Thread.sleep(retryDelay) - val getResponse = httpClient.execute(new HttpGet(follow.get)) - getResponse match { - case ok if ok.getStatusLine.getStatusCode == 200 => - ok.getEntity.writeTo(bos) - follow = None - case followRequest if needsFollow(followRequest.getStatusLine) => - val location = getLocationHeader(getResponse) - follow = if(location.isEmpty) None else Some(new URI(uri.getScheme+"://"+uri.getHost+":"+uri.getPort+location.get)) - case _ => throw new Exception("") - } - } - (bos.toByteArray,baseURI) - } - - private def getLocationHeader(response: CloseableHttpResponse): Option[String] = { - val a = response.getFirstHeader("Location").getElements - Some(a(0).toString) - } - - private def needsFollow(line: StatusLine): Boolean = { - val code = line.getStatusCode - code match { - case accepted if accepted == 202 => true - case redirect if redirect >= 300 && redirect < 400 => true - case _ => false - } - } -} diff --git a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/util/ModelUtil.scala b/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/util/ModelUtil.scala index f026f62..b9b6093 100644 --- a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/util/ModelUtil.scala +++ b/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/util/ModelUtil.scala @@ -1,9 +1,10 @@ package org.dbpedia.databus.mods.core.util import java.net.{MalformedURLException, URL} - import org.apache.jena.rdf.model.{Model, Resource, ResourceFactory} +import scala.language.implicitConversions + object ModelUtil { sealed trait SimpleNodeWrapper diff --git a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/util/RdfFileHelpers.scala b/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/util/RdfFileHelpers.scala index 26cf825..03c3d2d 100644 --- a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/util/RdfFileHelpers.scala +++ b/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/util/RdfFileHelpers.scala @@ -5,6 +5,7 @@ import java.util.concurrent.{ExecutorService, Executors} import org.apache.jena.graph.Triple import org.apache.jena.riot.lang.{PipedRDFIterator, PipedRDFStream, PipedTriplesStream} import org.apache.jena.riot.{Lang, RDFParser} +import org.dbpedia.databus.mods.core.io.Compression object RdfFileHelpers { @@ -33,7 +34,7 @@ object RdfFileHelpers { // iter val bis = new BufferedInputStream(new FileInputStream(file)) - val in = Compressor.decompress(bis) + val in = Compression.decompress(bis) // val lang = org.dbpedia.databus.util.MimeTypeGetter.getRDFFormat(item.downloadURL) val iter: PipedRDFIterator[Triple] = new PipedRDFIterator[Triple]() diff --git a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/validate/MetadataValidator.scala b/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/validate/MetadataValidator.scala deleted file mode 100644 index 2ae02e5..0000000 --- a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/validate/MetadataValidator.scala +++ /dev/null @@ -1,23 +0,0 @@ -package org.dbpedia.databus.mods.core.validate - -import org.apache.jena.riot.{Lang, RDFDataMgr} -import org.apache.jena.shacl.lib.ShLib -import org.apache.jena.shacl.{ShaclValidator, Shapes} - -object MetadataValidator extends App { - - scala.util.Properties.setProp("scala.time","true") - - val SHAPES = "dataid-shacl.ttl" - val DATA = "dataid-unexpanded.ttl" - - val shapesGraph = RDFDataMgr.loadGraph(SHAPES) - val dataGraph = RDFDataMgr.loadGraph(DATA) - - val shapes = Shapes.parse(shapesGraph) - - val report = ShaclValidator.get.validate(shapes,dataGraph) - ShLib.printReport(report) - - RDFDataMgr.write(System.out,report.getModel,Lang.TTL) -} diff --git a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/vocab/ModVocabModelBuilder.scala b/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/vocab/ModVocabModelBuilder.scala deleted file mode 100644 index a0ba108..0000000 --- a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/vocab/ModVocabModelBuilder.scala +++ /dev/null @@ -1,25 +0,0 @@ -package org.dbpedia.databus.mods.core.vocab - -import org.apache.jena.rdf.model.{Model, ModelFactory} - -class ModVocabModelBuilder { - - private val model = ModelFactory.createDefaultModel() - - def inheritModStatisticsDerivedFrom() : Unit = { - - } - - def inheritModEnrichmentDerivedFrom(): Unit = { - - } - - def inheritModStatistics(): Unit = { - - } - - def build: Model = { - - model - } -} diff --git a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/worker/api/ModActivityApiHttp.scala b/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/worker/api/ModActivityApiHttp.scala new file mode 100644 index 0000000..946fb64 --- /dev/null +++ b/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/worker/api/ModActivityApiHttp.scala @@ -0,0 +1,10 @@ +package org.dbpedia.databus.mods.core.worker.api + +import org.dbpedia.databus.mods.core.worker.exec.ActivityExecution + +class ModActivityApiHttp(activityExecution: ActivityExecution) { + + def handleRequest(): Unit = { + // TODO + } +} diff --git a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/worker/api/ModActivityApiHttpPoll.scala b/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/worker/api/ModActivityApiHttpPoll.scala new file mode 100644 index 0000000..e57badd --- /dev/null +++ b/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/worker/api/ModActivityApiHttpPoll.scala @@ -0,0 +1,86 @@ +package org.dbpedia.databus.mods.core.worker.api + +import org.apache.commons.io.IOUtils +import org.apache.jena.riot.{Lang, RDFWriter, RIOT} +import org.dbpedia.databus.dataid.Part +import org.dbpedia.databus.mods.core.worker.exec.ActivityExecution +import org.dbpedia.databus.mods.core.model.ModActivityRequest + +import java.net.URI +import java.nio.charset.StandardCharsets +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicLong +import javax.servlet.http.{HttpServletRequest, HttpServletResponse} + +/** + * An Implementation of the Worker Mod Activity HTTP API + */ +class ModActivityApiHttpPoll(activityExecution: ActivityExecution) { + + val dataIdByJob = new ConcurrentHashMap[Long,String]() + val currentJobId = new AtomicLong(0) + + def handleRequest(request: HttpServletRequest, response: HttpServletResponse): Unit = { + + request.getMethod match { + case "POST" => + handlePOST(request, response) + case "GET" => + handleGET(request, response) + } + } + + private def handlePOST(request: HttpServletRequest, response: HttpServletResponse): Unit = { + + createModActivityRequest(request) match { + case Some(modActivityRequest: ModActivityRequest) => + response.setStatus(202) + activityExecution.submit(modActivityRequest) + val jobId = currentJobId.getAndIncrement() + dataIdByJob.put(jobId,modActivityRequest.dataId) + response.setHeader("location",s"/${jobId.toString}/activity") + case None => + response.setStatus(400) + } + } + + private def handleGET(request: HttpServletRequest, response: HttpServletResponse): Unit = { + + val jobId = request.getRequestURI.split("/").reverse(1) + + val dataId = dataIdByJob.get(jobId.toLong) + if(null == dataId) { + response.setStatus(404) + } else { + activityExecution.get(dataId) match { + case None => + response.setStatus(404) + case Some(fMam) => + if (fMam.isDone) { + // can throw an exception which is caught in WorkerApi abstract + val mam = fMam.get() + response.setStatus(200) + val os = response.getOutputStream + RDFWriter.create() + .set(RIOT.symTurtleDirectiveStyle, "n3") + .set(RIOT.symTurtleOmitBase, "false") + .lang(Lang.TRIG) + .base("http://example.org/space/") + .source(mam.createRdfModel(new URI("http://example.org/space/"))) + .output(os); + os.close() + } else { + response.setStatus(202) + response.setHeader("location", request.getRequestURI) + response.setHeader("retry-after", 0.toString) + } + } + } + } + + def createModActivityRequest(request: HttpServletRequest): Option[ModActivityRequest] = { + val dataId = request.getParameter("dataId") + val accessUri = request.getParameter("accessUri") + Some(ModActivityRequest(dataId, Some(accessUri))) + } +} diff --git a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/worker/api/ModActivityClientHttp.scala b/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/worker/api/ModActivityClientHttp.scala new file mode 100644 index 0000000..788fd86 --- /dev/null +++ b/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/worker/api/ModActivityClientHttp.scala @@ -0,0 +1,88 @@ +package org.dbpedia.databus.mods.core.worker.api + +import org.apache.http.client.entity.UrlEncodedFormEntity +import org.apache.http.{NameValuePair, StatusLine} +import org.apache.http.client.methods.{CloseableHttpResponse, HttpGet, HttpPost} +import org.apache.http.impl.client.HttpClientBuilder +import org.apache.http.message.BasicNameValuePair +import org.apache.jena.sparql.function.library.leviathan.log +import org.dbpedia.databus.mods.core.model.ModActivityResponse +import org.slf4j.LoggerFactory + +import java.io.ByteArrayOutputStream +import java.net.{URI, URL} +import java.nio.charset.StandardCharsets +import java.util +import scala.util.Try + +class ModActivityClientHttp { + + private val log = LoggerFactory.getLogger(classOf[ModActivityClientHttp]) + + def send(endpoint: URI, dataId: URI, accessUri: Option[URI] = None, minDelay: Int = 0): ModActivityResponse = { + + var follow: Option[URI] = None + val bos = new ByteArrayOutputStream() + + val httpClient = HttpClientBuilder.create().disableRedirectHandling().build() + val httpPost = new HttpPost(endpoint) + val postParameters = new util.ArrayList[NameValuePair]() + postParameters.add(new BasicNameValuePair("dataId", dataId.toString)) + httpPost.setEntity(new UrlEncodedFormEntity(postParameters, StandardCharsets.UTF_8.name())) + + httpClient.execute(httpPost) match { + case okResponse if okResponse.getStatusLine.getStatusCode == 200 => + log.debug(s"status ${okResponse.getStatusLine}") + okResponse.getEntity.writeTo(bos) + case followableResponse if needsFollow(followableResponse.getStatusLine) => + log.debug(s"status ${followableResponse.getStatusLine}") + val location = getLocationHeader(followableResponse) + follow = + if (location.isEmpty) None // TODO Exception + else Some(new URL(endpoint.toURL,location.get).toURI) + case errorResponse => + log.debug(s"status ${errorResponse.getStatusLine}") + throw new Exception(errorResponse.getStatusLine.toString) + } + + while (follow.isDefined) { + Thread.sleep(minDelay) + val getResponse = httpClient.execute(new HttpGet(follow.get)) + getResponse match { + case okResponse if okResponse.getStatusLine.getStatusCode == 200 => + log.debug(s"status ${okResponse.getStatusLine}") + okResponse.getEntity.writeTo(bos) + follow = None + case followRequest if needsFollow(followRequest.getStatusLine) => + log.debug(s"status ${followRequest.getStatusLine}") + val location = getLocationHeader(getResponse) + follow = + if (location.isEmpty) None // TODO Exception + else Some(new URL(endpoint.toURL,location.get).toURI) + case errorResponse => + log.debug(s"status ${errorResponse.getStatusLine}") + throw new Exception(errorResponse.getStatusLine.toString) + } + } + ModActivityResponse(dataId.toString, bos.toByteArray) + } + + private def getLocationHeader(response: CloseableHttpResponse): Option[String] = Try { + val locationValues = response.getFirstHeader("Location").getElements + locationValues(0).toString + }.toOption + + private def getRetryAfterHeader(response: CloseableHttpResponse): Option[String] = Try { + val retryAfterValues = response.getFirstHeader("Retry-After").getElements + retryAfterValues(0).toString + }.toOption + + private def needsFollow(line: StatusLine): Boolean = { + val code = line.getStatusCode + code match { + case accepted if accepted == 202 => true + case redirect if redirect >= 300 && redirect < 400 => true + case _ => false + } + } +} diff --git a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/worker/controller/impl/ModApiController.scala b/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/worker/controller/impl/ModApiController.scala deleted file mode 100644 index 2c74b72..0000000 --- a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/worker/controller/impl/ModApiController.scala +++ /dev/null @@ -1,5 +0,0 @@ -package org.dbpedia.databus.mods.core.worker.controller.impl - -class ModApiController { - -} diff --git a/databus-mods-lib/databus-mods-spring-boot-starter/src/main/scala/org/dbpedia/databus/mods/worker/springboot/service/ActivityService.scala b/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/worker/exec/ActivityExecution.scala similarity index 55% rename from databus-mods-lib/databus-mods-spring-boot-starter/src/main/scala/org/dbpedia/databus/mods/worker/springboot/service/ActivityService.scala rename to databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/worker/exec/ActivityExecution.scala index 498f787..a075332 100644 --- a/databus-mods-lib/databus-mods-spring-boot-starter/src/main/scala/org/dbpedia/databus/mods/worker/springboot/service/ActivityService.scala +++ b/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/worker/exec/ActivityExecution.scala @@ -1,16 +1,11 @@ -package org.dbpedia.databus.mods.worker.springboot.service +package org.dbpedia.databus.mods.core.worker.exec -import org.dbpedia.databus.mods.model.{ModActivity, ModActivityMetadata, ModActivityMetadataBuilder, ModActivityRequest} -import org.springframework.beans.factory.annotation.Autowired -import org.springframework.stereotype.Service +import org.dbpedia.databus.mods.core.model.{ModActivity, ModActivityMetadata, ModActivityRequest} -import java.util.concurrent._ +import java.util.concurrent.{ConcurrentHashMap, Future, LinkedBlockingQueue, ThreadPoolExecutor, TimeUnit} -@Service -class ActivityService() { - - @Autowired - private var modActivity: ModActivity = _ +// https://stackoverflow.com/questions/8905780/thread-pool-handling-duplicate-tasks +class ActivityExecution(modActivity: ModActivity) { private val executor = new ThreadPoolExecutor( 1, 1, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue[Runnable]()) @@ -20,9 +15,7 @@ class ActivityService() { // GET Request current // TODO implement def get(id: String): Option[Future[ModActivityMetadata]] = synchronized { - val a = activities.get(id) - println(a) - a match { + activities.get(id) match { case null => None case futureActivityMetadata: Future[ModActivityMetadata] => @@ -33,9 +26,9 @@ class ActivityService() { def submit(activityRequest: ModActivityRequest): Unit = synchronized { val future: Future[ModActivityMetadata] = executor.submit( - new ActivityRunner(activityRequest, modActivity) + new ActivityTask(activityRequest, modActivity) ) // TODO better put? - activities.put(activityRequest.id, future) + activities.put(activityRequest.dataId, future) } } diff --git a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/worker/exec/ActivityTask.scala b/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/worker/exec/ActivityTask.scala new file mode 100644 index 0000000..95552b5 --- /dev/null +++ b/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/worker/exec/ActivityTask.scala @@ -0,0 +1,25 @@ +package org.dbpedia.databus.mods.core.worker.exec + +import org.apache.jena.datatypes.xsd.XSDDateTime +import org.dbpedia.databus.mods.core.model.vocabulary.MOD +import org.dbpedia.databus.mods.core.model.{ModActivity, ModActivityMetadata, ModActivityRequest} + +import java.util.Calendar +import java.util.concurrent.Callable + +class ActivityTask ( + activityRequest: ModActivityRequest, + modActivity: ModActivity + ) extends Callable[ModActivityMetadata] { + + override def call(): ModActivityMetadata = { + + val builder = + ModActivityMetadata.builder(activityRequest.dataId) + .withType(MOD.DatabusMod) + .withVersion("1.0") + .withStartedAtTime(new XSDDateTime(Calendar.getInstance())) + + modActivity.perform(activityRequest, builder) + } +} diff --git a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/worker/execution/ModProcessor.scala b/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/worker/execution/ModProcessor.scala deleted file mode 100644 index 14f2408..0000000 --- a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/worker/execution/ModProcessor.scala +++ /dev/null @@ -1,6 +0,0 @@ -package org.dbpedia.databus.mods.core.worker.execution - -trait ModProcessor { - - def process(extension: Extension) -} diff --git a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/worker/execution/ModResultFile.scala b/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/worker/execution/ModResultFile.scala deleted file mode 100644 index 69d5a33..0000000 --- a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/worker/execution/ModResultFile.scala +++ /dev/null @@ -1,5 +0,0 @@ -package org.dbpedia.databus.mods.core.worker.execution - -import java.io.File - -case class ModResultFile(file: File) \ No newline at end of file diff --git a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/model/ModActivityRequest.scala b/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/model/ModActivityRequest.scala deleted file mode 100644 index 2d44b98..0000000 --- a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/model/ModActivityRequest.scala +++ /dev/null @@ -1,3 +0,0 @@ -package org.dbpedia.databus.mods.model - -case class ModActivityRequest(id: String, source: String) \ No newline at end of file diff --git a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/model/ModMetadata.scala b/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/model/ModMetadata.scala deleted file mode 100644 index cdd904a..0000000 --- a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/model/ModMetadata.scala +++ /dev/null @@ -1,3 +0,0 @@ -package org.dbpedia.databus.mods.model - -trait ModMetadata diff --git a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/model/ModProcess.scala b/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/model/ModProcess.scala deleted file mode 100644 index ab9cb71..0000000 --- a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/model/ModProcess.scala +++ /dev/null @@ -1,6 +0,0 @@ -package org.dbpedia.databus.mods.model - -trait ModProcess { - - def performActivity -} diff --git a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/model/ModResult.scala b/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/model/ModResult.scala deleted file mode 100644 index 1cb58f8..0000000 --- a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/model/ModResult.scala +++ /dev/null @@ -1,19 +0,0 @@ -package org.dbpedia.databus.mods.model - -import org.dbpedia.databus.dataid.Part -import org.dbpedia.databus.mods.core.MetadataType - -/** - * Files created by the mod activity - * - * @param metadataType - * @param property - */ -class ModResult( - val used: Part, - val metadataType: MetadataType, - val name: String -) extends ModMetadata { - -} - diff --git a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/model/ModResultFactory.scala b/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/model/ModResultFactory.scala deleted file mode 100644 index cd78dec..0000000 --- a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/model/ModResultFactory.scala +++ /dev/null @@ -1,19 +0,0 @@ -package org.dbpedia.databus.mods.model - -import org.dbpedia.databus.dataid.Part -import org.dbpedia.databus.mods.core.MetadataType - -object ModResultFactory { - - def enrichment(used: String, name: String): ModResult = { - new ModResult(Part(used),MetadataType.Enrichment, name) - } - - def statistics(used: String, name: String): ModResult = { - new ModResult(Part(used),MetadataType.Statistics, name) - } - - def summary(used: String, name: String): ModResult = { - new ModResult(Part(used),MetadataType.Summary, name) - } -} diff --git a/databus-mods-lib/databus-mods-core/src/test/scala/org/dbpedia/databus/mods/core/CoreTests.scala b/databus-mods-lib/databus-mods-core/src/test/scala/org/dbpedia/databus/mods/core/CoreTests.scala new file mode 100644 index 0000000..663a6c6 --- /dev/null +++ b/databus-mods-lib/databus-mods-core/src/test/scala/org/dbpedia/databus/mods/core/CoreTests.scala @@ -0,0 +1,10 @@ +package org.dbpedia.databus.mods.core + +import org.dbpedia.databus.mods.core.model.ModActivityMetadata +import org.scalatest.funsuite.AnyFunSuite + +import java.net.URI + +class CoreTests extends AnyFunSuite { + +} diff --git a/databus-mods-lib/databus-mods-dummy/README.md b/databus-mods-lib/databus-mods-dummy/README.md new file mode 100644 index 0000000..e69de29 diff --git a/databus-mods-lib/databus-mods-dummy/pom.xml b/databus-mods-lib/databus-mods-dummy/pom.xml index 57f8b2f..fe0e5a2 100644 --- a/databus-mods-lib/databus-mods-dummy/pom.xml +++ b/databus-mods-lib/databus-mods-dummy/pom.xml @@ -32,6 +32,12 @@ databus-mods-spring-boot-starter 1.0-SNAPSHOT + + org.dbpedia.databus-mods + databus-mods-core + 1.0-SNAPSHOT + + org.springframework.boot spring-boot-starter-test diff --git a/databus-mods-lib/databus-mods-dummy/src/main/resources/banner.txt b/databus-mods-lib/databus-mods-dummy/src/main/resources/banner.txt new file mode 100644 index 0000000..557c7c8 --- /dev/null +++ b/databus-mods-lib/databus-mods-dummy/src/main/resources/banner.txt @@ -0,0 +1,5 @@ + ___ __ __ _ + | \ _ _ _ __ _ __ _ \ \ / /__ _ _| |_____ _ _ + | |) | || | ' \| ' \ || \ \/\/ / _ \ '_| / / -_) '_| + |___/ \_,_|_|_|_|_|_|_\_, |\_/\_/\___/_| |_\_\___|_| + |__/ \ No newline at end of file diff --git a/databus-mods-lib/databus-mods-dummy/src/main/scala/org/dbpedia/databus/mods/worker/dummy/DummyDatabusModWorker.scala b/databus-mods-lib/databus-mods-dummy/src/main/scala/org/dbpedia/databus/mods/worker/dummy/DummyDatabusModWorker.scala deleted file mode 100644 index 24c3307..0000000 --- a/databus-mods-lib/databus-mods-dummy/src/main/scala/org/dbpedia/databus/mods/worker/dummy/DummyDatabusModWorker.scala +++ /dev/null @@ -1,33 +0,0 @@ -package org.dbpedia.databus.mods.worker.dummy - -import org.apache.jena.datatypes.xsd.XSDDateTime -import org.dbpedia.databus.mods.model.{ModActivity, ModActivityMetadata, ModActivityMetadataBuilder, ModActivityRequest, ModResultFactory} -import org.dbpedia.databus.mods.worker.springboot.EnableModWorkerApi -import org.dbpedia.databus.mods.worker.springboot.controller.WorkerApiProfile -import org.springframework.boot.SpringApplication -import org.springframework.boot.autoconfigure.SpringBootApplication -import org.springframework.context.annotation.Bean - -import java.util.Calendar -import scala.util.Random; - -@SpringBootApplication -class DummyDatabusModWorker { - - @Bean - @EnableModWorkerApi(version = "1.0.0", profile = WorkerApiProfile.Polling) - def getModActivity: ModActivity = new ModActivity { - override def perform(request: ModActivityRequest, builder: ModActivityMetadataBuilder): ModActivityMetadata = { - - builder.addGenerated(ModResultFactory.enrichment(request.id,"result.ttl")) - builder.addGenerated(ModResultFactory.enrichment(request.id,"result2.ttl")) - - builder.withStatSummary((Random.nextInt(100)/100.0).toString).build() - } - } -} - - -object DummyDatabusModWorker extends App { - SpringApplication.run(classOf[DummyDatabusModWorker], args: _*) -} \ No newline at end of file diff --git a/databus-mods-lib/databus-mods-dummy/src/main/scala/org/dbpedia/databus/mods/worker/dummy/DummyWorker.scala b/databus-mods-lib/databus-mods-dummy/src/main/scala/org/dbpedia/databus/mods/worker/dummy/DummyWorker.scala new file mode 100644 index 0000000..611e7be --- /dev/null +++ b/databus-mods-lib/databus-mods-dummy/src/main/scala/org/dbpedia/databus/mods/worker/dummy/DummyWorker.scala @@ -0,0 +1,27 @@ +package org.dbpedia.databus.mods.worker.dummy + +import org.dbpedia.databus.mods.core.model.{ModActivity, ModActivityMetadata, ModActivityMetadataBuilder, ModActivityRequest} +import org.dbpedia.databus.mods.worker.springboot.{EnableModWorkerApi, ModWorkerApiProfile} +import org.springframework.boot.SpringApplication +import org.springframework.boot.autoconfigure.SpringBootApplication +import org.springframework.context.annotation.Bean + +import scala.util.Random; + +@SpringBootApplication +class DummyWorker { + + @Bean + @EnableModWorkerApi(version = "1.0.0", profile = ModWorkerApiProfile.HttpPoll) + def getModActivity: ModActivity = new ModActivity { + override def perform(request: ModActivityRequest, builder: ModActivityMetadataBuilder): ModActivityMetadata = { + Thread.sleep(1000) + builder.withStatSummary((Random.nextInt(100)/100.0).toString).build() + } + } +} + + +object DummyWorker extends App { + SpringApplication.run(classOf[DummyWorker], args: _*) +} \ No newline at end of file diff --git a/databus-mods-lib/databus-mods-dummy/src/main/scala/org/dbpedia/databus/mods/worker/dummy/FakeDatabusImpl.scala b/databus-mods-lib/databus-mods-dummy/src/main/scala/org/dbpedia/databus/mods/worker/dummy/FakeDatabusImpl.scala new file mode 100644 index 0000000..85362ee --- /dev/null +++ b/databus-mods-lib/databus-mods-dummy/src/main/scala/org/dbpedia/databus/mods/worker/dummy/FakeDatabusImpl.scala @@ -0,0 +1,20 @@ +package org.dbpedia.databus.mods.worker.dummy + +import org.apache.commons.io.IOUtils +import org.springframework.stereotype.Controller +import org.springframework.web.bind.annotation.RequestMapping + +import java.nio.charset.StandardCharsets +import javax.servlet.http.{HttpServletRequest, HttpServletResponse} + +@Controller +class FakeDatabusImpl { + + @RequestMapping(value = Array("/{publisher}/{group}/{version}/{file}")) + def databusFile(httpServletRequest: HttpServletRequest, httpServletResponse: HttpServletResponse): Unit = { + httpServletResponse.setStatus(200) + val os = httpServletResponse.getOutputStream + IOUtils.write("Hello World",os,StandardCharsets.UTF_8) + os.close() + } +} diff --git a/databus-mods-lib/databus-mods-dummy/src/test/scala/org/dbpedia/databus/mods/worker/dummy/DummyTest.scala b/databus-mods-lib/databus-mods-dummy/src/test/scala/org/dbpedia/databus/mods/worker/dummy/DummyTest.scala index 1461093..b280a40 100644 --- a/databus-mods-lib/databus-mods-dummy/src/test/scala/org/dbpedia/databus/mods/worker/dummy/DummyTest.scala +++ b/databus-mods-lib/databus-mods-dummy/src/test/scala/org/dbpedia/databus/mods/worker/dummy/DummyTest.scala @@ -1,8 +1,10 @@ package org.dbpedia.databus.mods.worker.dummy -import org.dbpedia.databus.mods.model.ModActivity -import org.dbpedia.databus.mods.worker.springboot.controller.{PollingBasedWorkerApi, WorkerApi} -import org.dbpedia.databus.mods.worker.springboot.service.ActivityService +import org.apache.commons.io.IOUtils +import org.dbpedia.databus.mods.core.model.ModActivity +import org.dbpedia.databus.mods.core.worker.api.ModActivityClientHttp +import org.dbpedia.databus.mods.worker.springboot.controller.{ActivityController, ActivityControllerPollImpl} +import org.dbpedia.databus.mods.worker.springboot.service.ActivityExecutionService import org.junit.jupiter.api.Test import org.springframework.beans.factory.annotation.Autowired import org.springframework.boot.test.context.SpringBootTest @@ -12,10 +14,12 @@ import org.springframework.boot.web.server.LocalServerPort import org.springframework.http.{HttpEntity, HttpHeaders, HttpMethod, MediaType, ResponseEntity} import org.springframework.util.LinkedMultiValueMap +import java.net.URI + @SpringBootTest( webEnvironment = WebEnvironment.RANDOM_PORT, - classes = Array(classOf[DummyDatabusModWorker]) + classes = Array(classOf[DummyWorker]) ) class DummyTest { @@ -23,13 +27,13 @@ class DummyTest { private var port: Integer = _ @Autowired - private var restTemplate: TestRestTemplate = _ + private var activityService: ActivityExecutionService = _ @Autowired - private var activityService: ActivityService = _ + private var workerApi: ActivityController = _ @Autowired - private var workerApi: WorkerApi = _ + private var databusImpl: FakeDatabusImpl = _ @Autowired private var modActivity: ModActivity = _ @@ -38,41 +42,22 @@ class DummyTest { def contextLoad(): Unit = { assert(activityService != null) assert(workerApi != null) - assert(workerApi.isInstanceOf[PollingBasedWorkerApi]) + assert(workerApi.isInstanceOf[ActivityControllerPollImpl]) assert(modActivity != null); + assert(databusImpl != null) } @Test def test(): Unit = { - val path = "/vehnem/paper-supplements/demo-graph/20210301/demo-graph.nt.gz"; - - val headers = new HttpHeaders(); - headers.setContentType(MediaType.APPLICATION_FORM_URLENCODED); - - val map= new LinkedMultiValueMap[String,String](); - map.add("source", "file:///proc/cpuinfo"); - - var lastStatusCode = 0; - - val rePOST = this.restTemplate.exchange( - "http://localhost:"+port+path+"/activity", - HttpMethod.POST, - new HttpEntity(map,headers), - classOf[String]) - lastStatusCode = rePOST.getStatusCodeValue - assert(lastStatusCode == 202,"wrong status code response POST") + val client = new ModActivityClientHttp - var body = ""; - while (lastStatusCode == 202) { - val reGET = this.restTemplate.getForEntity( - "http://localhost:"+port+path+"/activity", - classOf[String]) - lastStatusCode = reGET.getStatusCodeValue - body = reGET.getBody; - } + val result = client.send( + new URI(s"http://localhost:${port}/activity"), + new URI(s"http://localhost:${port}/publisher/group/artifact/version/file"), + minDelay = 1000 + ) - System.out.println(body); - assert(lastStatusCode == 200); + IOUtils.write(result.data,System.out) } } diff --git a/databus-mods-lib/databus-mods-spring-boot-starter/src/main/java/org.dbpedia.databus.mods.worker.springboot/EnableModWorkerApi.java b/databus-mods-lib/databus-mods-spring-boot-starter/src/main/java/org.dbpedia.databus.mods.worker.springboot/EnableModWorkerApi.java index 71b97f3..c51b87f 100644 --- a/databus-mods-lib/databus-mods-spring-boot-starter/src/main/java/org.dbpedia.databus.mods.worker.springboot/EnableModWorkerApi.java +++ b/databus-mods-lib/databus-mods-spring-boot-starter/src/main/java/org.dbpedia.databus.mods.worker.springboot/EnableModWorkerApi.java @@ -1,7 +1,5 @@ package org.dbpedia.databus.mods.worker.springboot; -import org.dbpedia.databus.mods.worker.springboot.controller.WorkerApiProfile; - import java.lang.annotation.*; @Target(ElementType.TYPE) @@ -11,7 +9,5 @@ String version() default ""; - String apiEndpoint() default ""; - - WorkerApiProfile profile() default WorkerApiProfile.Basic; + ModWorkerApiProfile profile() default ModWorkerApiProfile.Http; } diff --git a/databus-mods-lib/databus-mods-spring-boot-starter/src/main/java/org.dbpedia.databus.mods.worker.springboot/ModWorkerProcessor.java b/databus-mods-lib/databus-mods-spring-boot-starter/src/main/java/org.dbpedia.databus.mods.worker.springboot/ModWorkerProcessor.java deleted file mode 100644 index ec4fc74..0000000 --- a/databus-mods-lib/databus-mods-spring-boot-starter/src/main/java/org.dbpedia.databus.mods.worker.springboot/ModWorkerProcessor.java +++ /dev/null @@ -1,23 +0,0 @@ -package org.dbpedia.databus.mods.worker.springboot; - -import org.dbpedia.databus.mods.worker.springboot.controller.WorkerApiProfile; -import org.springframework.context.annotation.ImportSelector; -import org.springframework.core.annotation.AnnotationAttributes; -import org.springframework.core.type.AnnotationMetadata; - -public class ModWorkerProcessor implements ImportSelector { - - @Override - public String[] selectImports(AnnotationMetadata annotationMetadata) { - AnnotationAttributes attributes = - AnnotationAttributes.fromMap( - annotationMetadata.getAnnotationAttributes - (EnableModWorkerApi.class.getName(), false)); - String criteria = attributes.getString("profile"); - if (criteria.equals(WorkerApiProfile.Basic)) { - return new String[]{"enableannot.selector.SomeBeanConfigurationDefault"}; - } else { - return new String[]{"enableannot.selector.SomeBeanConfigurationType1"}; - } - } -} \ No newline at end of file diff --git a/databus-mods-lib/databus-mods-spring-boot-starter/src/main/java/org/dbpedia/databus/mods/worker/springboot/ModWorkerApiProfile.java b/databus-mods-lib/databus-mods-spring-boot-starter/src/main/java/org/dbpedia/databus/mods/worker/springboot/ModWorkerApiProfile.java new file mode 100644 index 0000000..2482377 --- /dev/null +++ b/databus-mods-lib/databus-mods-spring-boot-starter/src/main/java/org/dbpedia/databus/mods/worker/springboot/ModWorkerApiProfile.java @@ -0,0 +1,5 @@ +package org.dbpedia.databus.mods.worker.springboot; + +public enum ModWorkerApiProfile { + Http, HttpPoll +} diff --git a/databus-mods-lib/databus-mods-spring-boot-starter/src/main/java/org/dbpedia/databus/mods/worker/springboot/controller/WorkerApiProfile.java b/databus-mods-lib/databus-mods-spring-boot-starter/src/main/java/org/dbpedia/databus/mods/worker/springboot/controller/WorkerApiProfile.java deleted file mode 100644 index e4cb029..0000000 --- a/databus-mods-lib/databus-mods-spring-boot-starter/src/main/java/org/dbpedia/databus/mods/worker/springboot/controller/WorkerApiProfile.java +++ /dev/null @@ -1,5 +0,0 @@ -package org.dbpedia.databus.mods.worker.springboot.controller; - -public enum WorkerApiProfile { - Basic, Polling -} diff --git a/databus-mods-lib/databus-mods-spring-boot-starter/src/main/scala/org/dbpedia/databus/mods/worker/springboot/config/ModWorkerApiAutoConfig.scala b/databus-mods-lib/databus-mods-spring-boot-starter/src/main/scala/org/dbpedia/databus/mods/worker/springboot/config/ModWorkerApiAutoConfig.scala index b218bb6..8722692 100644 --- a/databus-mods-lib/databus-mods-spring-boot-starter/src/main/scala/org/dbpedia/databus/mods/worker/springboot/config/ModWorkerApiAutoConfig.scala +++ b/databus-mods-lib/databus-mods-spring-boot-starter/src/main/scala/org/dbpedia/databus/mods/worker/springboot/config/ModWorkerApiAutoConfig.scala @@ -1,9 +1,9 @@ package org.dbpedia.databus.mods.worker.springboot.config -import org.dbpedia.databus.mods.model.ModActivity -import org.dbpedia.databus.mods.worker.springboot.EnableModWorkerApi -import org.dbpedia.databus.mods.worker.springboot.controller.{BasicWorkerApi, PollingBasedWorkerApi, WorkerApi, WorkerApiProfile} -import org.dbpedia.databus.mods.worker.springboot.service.{ActivityService, DefaultModActivity, LocalResultService, ResultService} +import org.dbpedia.databus.mods.core.model.{ModActivity, ModActivityMetadataBuilder, ModActivityRequest} +import org.dbpedia.databus.mods.worker.springboot.controller.{ActivityController, ActivityControllerImpl, ActivityControllerPollImpl} +import org.dbpedia.databus.mods.worker.springboot.service.{ActivityExecutionService, LocalResultService, ResultService} +import org.dbpedia.databus.mods.worker.springboot.{EnableModWorkerApi, ModWorkerApiProfile} import org.slf4j.LoggerFactory import org.springframework.beans.factory.annotation.Value import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean @@ -11,6 +11,8 @@ import org.springframework.context.ApplicationContext import org.springframework.context.annotation.{Bean, Configuration} import java.io.File +import scala.collection.JavaConverters._ + @Configuration class ModWorkerApiAutoConfig() { @@ -19,18 +21,21 @@ class ModWorkerApiAutoConfig() { @Bean @ConditionalOnMissingBean(Array(classOf[ModActivity])) - def getModActivity(): ModActivity = { - new DefaultModActivity() + def getModActivity: ModActivity = (request: ModActivityRequest, builder: ModActivityMetadataBuilder) => { + builder.withStatSummary("1.0").build() } @Bean - def getActivityService(): ActivityService = { - new ActivityService + def getActivityService(modActivity: ModActivity): ActivityExecutionService = { + new ActivityExecutionService(modActivity) } @Value("${result.dir:files}") var baseDirPath: String = _ + @Value("${api.http.retry-after:0}") + var retryAfter: Int = 0 + @Bean @ConditionalOnMissingBean(Array(classOf[ResultService])) def resultService(): ResultService = { @@ -38,8 +43,10 @@ class ModWorkerApiAutoConfig() { } @Bean - def defString(context: ApplicationContext): WorkerApi = { - import scala.collection.JavaConverters._ + def getActivityController( + context: ApplicationContext, + activityExecutionService: ActivityExecutionService + ): ActivityController = { val annotation = context.getBeansWithAnnotation(classOf[EnableModWorkerApi]).keySet().asScala.map( key => context.findAnnotationOnBean(key, classOf[EnableModWorkerApi]) @@ -47,16 +54,11 @@ class ModWorkerApiAutoConfig() { if (annotation.isDefined) { annotation.get.profile() match { - case WorkerApiProfile.Basic => new BasicWorkerApi - case WorkerApiProfile.Polling => new PollingBasedWorkerApi(getActivityService()) + case ModWorkerApiProfile.Http => new ActivityControllerImpl + case ModWorkerApiProfile.HttpPoll => new ActivityControllerPollImpl(activityExecutionService) } } else { - new BasicWorkerApi + new ActivityControllerImpl } } - - // @Bean - // def workerApi(): WorkerApi = { - // new PollingBasedWorkerApi(activityService()) - // } } diff --git a/databus-mods-lib/databus-mods-spring-boot-starter/src/main/scala/org/dbpedia/databus/mods/worker/springboot/controller/ActivityController.scala b/databus-mods-lib/databus-mods-spring-boot-starter/src/main/scala/org/dbpedia/databus/mods/worker/springboot/controller/ActivityController.scala new file mode 100644 index 0000000..571b637 --- /dev/null +++ b/databus-mods-lib/databus-mods-spring-boot-starter/src/main/scala/org/dbpedia/databus/mods/worker/springboot/controller/ActivityController.scala @@ -0,0 +1,83 @@ +package org.dbpedia.databus.mods.worker.springboot.controller + +import org.dbpedia.databus.dataid.Part +import org.dbpedia.databus.mods.core.model.ModActivityRequest +import org.slf4j.LoggerFactory +import org.springframework.stereotype.Controller +import org.springframework.web.bind.annotation.{PathVariable, RequestMapping, RequestMethod, RequestParam} + +import java.io.PrintWriter +import java.nio.charset.StandardCharsets +import javax.servlet.http.{HttpServletRequest, HttpServletResponse} + +@Controller +abstract class ActivityController { + + /** + * todo multiple APIs for one application + */ + private val log = LoggerFactory.getLogger(classOf[ActivityController]) + + @RequestMapping( + value = Array("/activity"), + method = Array(RequestMethod.GET, RequestMethod.POST)) + def activity( + request: HttpServletRequest, response: HttpServletResponse + ): Unit = { + + try { + handleRequest(request, response) + } catch { + case e: Exception => + response.setStatus(500) + val os = response.getOutputStream + val pw = new PrintWriter(os, true, StandardCharsets.UTF_8) + e.printStackTrace(pw) + pw.close() + os.close() + } + } + + + @RequestMapping(value = Array("/{jobId}/activity"), method = Array(RequestMethod.GET)) + def job( + @PathVariable jobId: String, + request: HttpServletRequest, response: HttpServletResponse + ): Unit = { + handleRequest(request,response) + } + +// @RequestMapping( +// value = Array("{publisher}/{group}/{artifact}/{version}/{file}/activity"), +// method = Array(RequestMethod.GET, RequestMethod.POST)) +// def activity( +// @PathVariable publisher: String, +// @PathVariable group: String, +// @PathVariable artifact: String, +// @PathVariable version: String, +// @PathVariable file: String, +// request: HttpServletRequest, response: HttpServletResponse +// ): Unit = { +// +// // TODO +// val didPartUri = s"https://databus.dbpedia.org/$publisher/$group/$artifact/$version/$file" +// +// val didPart = Part.apply(didPartUri) +// try { +// handleRequest(didPart, request, response) +// } catch { +// case e: Exception => +// response.setStatus(500) +// val os = response.getOutputStream +// val pw = new PrintWriter(os, true, StandardCharsets.UTF_8) +// e.printStackTrace(pw) +// pw.close() +// os.close() +// } +// } + + def handleRequest( + httpServletRequest: HttpServletRequest, + httpServletResponse: HttpServletResponse + ): Unit +} diff --git a/databus-mods-lib/databus-mods-spring-boot-starter/src/main/scala/org/dbpedia/databus/mods/worker/springboot/controller/BasicWorkerApi.scala b/databus-mods-lib/databus-mods-spring-boot-starter/src/main/scala/org/dbpedia/databus/mods/worker/springboot/controller/ActivityControllerImpl.scala similarity index 61% rename from databus-mods-lib/databus-mods-spring-boot-starter/src/main/scala/org/dbpedia/databus/mods/worker/springboot/controller/BasicWorkerApi.scala rename to databus-mods-lib/databus-mods-spring-boot-starter/src/main/scala/org/dbpedia/databus/mods/worker/springboot/controller/ActivityControllerImpl.scala index 14215ad..b09aba8 100644 --- a/databus-mods-lib/databus-mods-spring-boot-starter/src/main/scala/org/dbpedia/databus/mods/worker/springboot/controller/BasicWorkerApi.scala +++ b/databus-mods-lib/databus-mods-spring-boot-starter/src/main/scala/org/dbpedia/databus/mods/worker/springboot/controller/ActivityControllerImpl.scala @@ -2,18 +2,22 @@ package org.dbpedia.databus.mods.worker.springboot.controller import org.apache.commons.io.IOUtils import org.dbpedia.databus.dataid.Part +import org.dbpedia.databus.mods.core.model.ModActivityRequest import org.slf4j.LoggerFactory import org.springframework.web.bind.annotation.GetMapping import java.nio.charset.StandardCharsets import javax.servlet.http.{HttpServletRequest, HttpServletResponse} -class BasicWorkerApi extends WorkerApi { +class ActivityControllerImpl extends ActivityController { - private val log = LoggerFactory.getLogger(classOf[BasicWorkerApi]) + private val log = LoggerFactory.getLogger(classOf[ActivityControllerImpl]) @GetMapping - override def handleRequest(didPart: Part, request: HttpServletRequest, response: HttpServletResponse): Unit = { + override def handleRequest( + request: HttpServletRequest, + response: HttpServletResponse + ): Unit = { response.setStatus(404) val os = response.getOutputStream IOUtils.write("TODO", os, StandardCharsets.UTF_8) diff --git a/databus-mods-lib/databus-mods-spring-boot-starter/src/main/scala/org/dbpedia/databus/mods/worker/springboot/controller/ActivityControllerPollImpl.scala b/databus-mods-lib/databus-mods-spring-boot-starter/src/main/scala/org/dbpedia/databus/mods/worker/springboot/controller/ActivityControllerPollImpl.scala new file mode 100644 index 0000000..da0ed77 --- /dev/null +++ b/databus-mods-lib/databus-mods-spring-boot-starter/src/main/scala/org/dbpedia/databus/mods/worker/springboot/controller/ActivityControllerPollImpl.scala @@ -0,0 +1,26 @@ +package org.dbpedia.databus.mods.worker.springboot.controller + +import org.apache.commons.io.IOUtils +import org.apache.jena.riot.{Lang, RDFDataMgr, RDFWriter, RIOT} +import org.dbpedia.databus.dataid.Part +import org.dbpedia.databus.mods.core.model.{ModActivity, ModActivityRequest} +import org.dbpedia.databus.mods.core.worker.api.ModActivityApiHttpPoll +import org.dbpedia.databus.mods.worker.springboot.service.ActivityExecutionService +import org.slf4j.LoggerFactory +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.web.bind.annotation.{PathVariable, RequestMapping, RequestMethod} + +import java.nio.charset.StandardCharsets +import javax.servlet.http.{HttpServletRequest, HttpServletResponse} + +class ActivityControllerPollImpl(activityExecutionService: ActivityExecutionService) extends ActivityController { + + private val api = new ModActivityApiHttpPoll(activityExecutionService) + + override def handleRequest( + request: HttpServletRequest, + response: HttpServletResponse + ): Unit = { + api.handleRequest(request,response) + } +} diff --git a/databus-mods-lib/databus-mods-spring-boot-starter/src/main/scala/org/dbpedia/databus/mods/worker/springboot/controller/BasicFileAccessController.scala b/databus-mods-lib/databus-mods-spring-boot-starter/src/main/scala/org/dbpedia/databus/mods/worker/springboot/controller/BasicResultFileController.scala similarity index 95% rename from databus-mods-lib/databus-mods-spring-boot-starter/src/main/scala/org/dbpedia/databus/mods/worker/springboot/controller/BasicFileAccessController.scala rename to databus-mods-lib/databus-mods-spring-boot-starter/src/main/scala/org/dbpedia/databus/mods/worker/springboot/controller/BasicResultFileController.scala index 625a513..ad0d41d 100644 --- a/databus-mods-lib/databus-mods-spring-boot-starter/src/main/scala/org/dbpedia/databus/mods/worker/springboot/controller/BasicFileAccessController.scala +++ b/databus-mods-lib/databus-mods-spring-boot-starter/src/main/scala/org/dbpedia/databus/mods/worker/springboot/controller/BasicResultFileController.scala @@ -9,7 +9,7 @@ import org.springframework.web.bind.annotation.{PathVariable, RequestMapping, Re import javax.servlet.http.{HttpServletRequest, HttpServletResponse} @Controller -class BasicFileAccessController(resultService: ResultService) { +class BasicResultFileController(resultService: ResultService) { @RequestMapping( value = Array("{publisher}/{group}/{artifact}/{version}/{file}/{result}"), diff --git a/databus-mods-lib/databus-mods-spring-boot-starter/src/main/scala/org/dbpedia/databus/mods/worker/springboot/controller/PollingBasedWorkerApi.scala b/databus-mods-lib/databus-mods-spring-boot-starter/src/main/scala/org/dbpedia/databus/mods/worker/springboot/controller/PollingBasedWorkerApi.scala deleted file mode 100644 index 4501b7f..0000000 --- a/databus-mods-lib/databus-mods-spring-boot-starter/src/main/scala/org/dbpedia/databus/mods/worker/springboot/controller/PollingBasedWorkerApi.scala +++ /dev/null @@ -1,71 +0,0 @@ -package org.dbpedia.databus.mods.worker.springboot.controller - -import org.apache.commons.io.IOUtils -import org.apache.jena.riot.{Lang, RDFDataMgr, RDFWriter, RIOT} -import org.dbpedia.databus.dataid.Part -import org.dbpedia.databus.mods.model.{ModActivity, ModActivityRequest} -import org.dbpedia.databus.mods.worker.springboot.service.ActivityService -import org.slf4j.LoggerFactory -import org.springframework.beans.factory.annotation.Autowired - -import java.nio.charset.StandardCharsets -import javax.servlet.http.{HttpServletRequest, HttpServletResponse} - -class PollingBasedWorkerApi(activityService: ActivityService) extends WorkerApi { - - private val log = LoggerFactory.getLogger(classOf[PollingBasedWorkerApi]) - - override def handleRequest(didPart: Part, request: HttpServletRequest, response: HttpServletResponse): Unit = { - - request.getMethod match { - case "POST" => - handlePOST(didPart, request, response) - case "GET" => - handleGET(didPart, request, response) - } - } - - private def handlePOST(didPart: Part, request: HttpServletRequest, response: HttpServletResponse): Unit = { - val os = response.getOutputStream - - val source = request.getParameter("source") - if (null == source) { - response.setStatus(500) - IOUtils.write("param missing", os, StandardCharsets.UTF_8) - } else { - didPart.downloadURL = source - val activityRequest = ModActivityRequest(didPart.uri,didPart.downloadURL) - activityService.submit(activityRequest) - response.setStatus(202) - response.setHeader("location", request.getRequestURI) - } - os.close() - } - - private def handleGET(didPart: Part, request: HttpServletRequest, response: HttpServletResponse): Unit = { - - activityService.get(didPart.uri) match { - case None => - response.setStatus(404) - case Some(fMam) => - if (fMam.isDone) { - // can throw an exception which is caught in WorkerApi abstract - val mam = fMam.get() - response.setStatus(200) - val os = response.getOutputStream - RDFWriter.create() - .set(RIOT.symTurtleDirectiveStyle, "n3") - .set(RIOT.symTurtleOmitBase,"false") - .lang(Lang.TRIG) - .base("http://example.org/space/") - .source(mam.createRdfModel("http://example.org/space/")) - .output(os); - os.close() - } else { - response.setStatus(202) - response.setHeader("location", request.getRequestURI) - } - } - } - -} diff --git a/databus-mods-lib/databus-mods-spring-boot-starter/src/main/scala/org/dbpedia/databus/mods/worker/springboot/controller/WorkerApi.scala b/databus-mods-lib/databus-mods-spring-boot-starter/src/main/scala/org/dbpedia/databus/mods/worker/springboot/controller/WorkerApi.scala deleted file mode 100644 index 33acf38..0000000 --- a/databus-mods-lib/databus-mods-spring-boot-starter/src/main/scala/org/dbpedia/databus/mods/worker/springboot/controller/WorkerApi.scala +++ /dev/null @@ -1,50 +0,0 @@ -package org.dbpedia.databus.mods.worker.springboot.controller - -import org.dbpedia.databus.dataid.Part -import org.slf4j.LoggerFactory -import org.springframework.stereotype.Controller -import org.springframework.web.bind.annotation.{PathVariable, RequestMapping, RequestMethod} - -import java.io.PrintWriter -import java.nio.charset.StandardCharsets -import javax.servlet.http.{HttpServletRequest, HttpServletResponse} - -@Controller -abstract class WorkerApi { - - /** - * todo multiple APIs for one application - */ - private val log = LoggerFactory.getLogger(classOf[WorkerApi]) - - @RequestMapping( - value = Array("{publisher}/{group}/{artifact}/{version}/{file}/activity"), - method = Array(RequestMethod.GET, RequestMethod.POST)) - def activity( - @PathVariable publisher: String, - @PathVariable group: String, - @PathVariable artifact: String, - @PathVariable version: String, - @PathVariable file: String, - request: HttpServletRequest, response: HttpServletResponse - ): Unit = { - - // TODO - val didPartUri = s"https://databus.dbpedia.org/$publisher/$group/$artifact/$version/$file" - - val didPart = Part.apply(didPartUri) - try { - handleRequest(didPart, request, response) - } catch { - case e: Exception => - response.setStatus(500) - val os = response.getOutputStream - val pw = new PrintWriter(os, true, StandardCharsets.UTF_8) - e.printStackTrace(pw) - pw.close() - os.close() - } - } - - def handleRequest(didPart: Part, request: HttpServletRequest, response: HttpServletResponse): Unit -} diff --git a/databus-mods-lib/databus-mods-spring-boot-starter/src/main/scala/org/dbpedia/databus/mods/worker/springboot/service/ActivityExecutionService.scala b/databus-mods-lib/databus-mods-spring-boot-starter/src/main/scala/org/dbpedia/databus/mods/worker/springboot/service/ActivityExecutionService.scala new file mode 100644 index 0000000..f9a0270 --- /dev/null +++ b/databus-mods-lib/databus-mods-spring-boot-starter/src/main/scala/org/dbpedia/databus/mods/worker/springboot/service/ActivityExecutionService.scala @@ -0,0 +1,11 @@ +package org.dbpedia.databus.mods.worker.springboot.service + +import org.dbpedia.databus.mods.core.model.{ModActivity, ModActivityMetadata, ModActivityMetadataBuilder, ModActivityRequest} +import org.dbpedia.databus.mods.core.worker.exec.ActivityExecution +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.stereotype.Service + +import java.util.concurrent._ + +@Service +class ActivityExecutionService(modActivity: ModActivity) extends ActivityExecution(modActivity) {} \ No newline at end of file diff --git a/databus-mods-lib/databus-mods-spring-boot-starter/src/main/scala/org/dbpedia/databus/mods/worker/springboot/service/ActivityRunner.scala b/databus-mods-lib/databus-mods-spring-boot-starter/src/main/scala/org/dbpedia/databus/mods/worker/springboot/service/ActivityRunner.scala deleted file mode 100644 index e7627c4..0000000 --- a/databus-mods-lib/databus-mods-spring-boot-starter/src/main/scala/org/dbpedia/databus/mods/worker/springboot/service/ActivityRunner.scala +++ /dev/null @@ -1,26 +0,0 @@ -package org.dbpedia.databus.mods.worker.springboot.service - -import org.apache.jena.datatypes.xsd.XSDDateTime -import org.apache.jena.vocabulary.XSD -import org.dbpedia.databus.mods.model.vocabulary.MOD -import org.dbpedia.databus.mods.model.{ModActivity, ModActivityMetadata, ModActivityRequest} - -import java.util.Calendar -import java.util.concurrent.Callable - -class ActivityRunner( - activityRequest: ModActivityRequest, - modActivity: ModActivity -) extends Callable[ModActivityMetadata] { - - override def call(): ModActivityMetadata = { - - val builder = - ModActivityMetadata.builder(activityRequest.id) - .withType(MOD.DatabusMod) - .withVersion("1.0") - .withStartedAtTime(new XSDDateTime(Calendar.getInstance())) - - modActivity.perform(activityRequest, builder) - } -} diff --git a/databus-mods-lib/databus-mods-spring-boot-starter/src/main/scala/org/dbpedia/databus/mods/worker/springboot/service/DefaultModActivity.scala b/databus-mods-lib/databus-mods-spring-boot-starter/src/main/scala/org/dbpedia/databus/mods/worker/springboot/service/DefaultModActivity.scala deleted file mode 100644 index 3064aaa..0000000 --- a/databus-mods-lib/databus-mods-spring-boot-starter/src/main/scala/org/dbpedia/databus/mods/worker/springboot/service/DefaultModActivity.scala +++ /dev/null @@ -1,12 +0,0 @@ -package org.dbpedia.databus.mods.worker.springboot.service - -import org.dbpedia.databus.mods.model.{ModActivity, ModActivityMetadata, ModActivityMetadataBuilder, ModActivityRequest} -import org.springframework.stereotype.Component - -@Component -class DefaultModActivity extends ModActivity { - - override def perform(request: ModActivityRequest, builder: ModActivityMetadataBuilder): ModActivityMetadata = { - builder.withStatSummary("1.0").build() - } -} diff --git a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/worker/AsyncWorker.scala b/databus-mods-lib/databus-mods-spring-boot-starter/worker/AsyncWorkerConfiguration.scala similarity index 75% rename from databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/worker/AsyncWorker.scala rename to databus-mods-lib/databus-mods-spring-boot-starter/worker/AsyncWorkerConfiguration.scala index 7e5e978..15936b9 100644 --- a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/worker/AsyncWorker.scala +++ b/databus-mods-lib/databus-mods-spring-boot-starter/worker/AsyncWorkerConfiguration.scala @@ -1,4 +1,4 @@ -package org.dbpedia.databus.mods.core.worker +package org.dbpedia.databus.mods.worker.springboot.worker import org.springframework.context.annotation.{ComponentScan, Configuration} @@ -8,6 +8,6 @@ import org.springframework.context.annotation.{ComponentScan, Configuration} "org.dbpedia.databus_mods.lib.worker.service", "org.dbpedia.databus_mods.lib.worker.execution" )) -class AsyncWorker { +class AsyncWorkerConfiguration { } diff --git a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/worker/controller/impl/ModApiControllerAsync.scala b/databus-mods-lib/databus-mods-spring-boot-starter/worker/controller/impl/ModApiControllerAsync.scala similarity index 93% rename from databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/worker/controller/impl/ModApiControllerAsync.scala rename to databus-mods-lib/databus-mods-spring-boot-starter/worker/controller/impl/ModApiControllerAsync.scala index bdd20fe..a1e1907 100644 --- a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/worker/controller/impl/ModApiControllerAsync.scala +++ b/databus-mods-lib/databus-mods-spring-boot-starter/worker/controller/impl/ModApiControllerAsync.scala @@ -1,4 +1,4 @@ -package org.dbpedia.databus.mods.core.worker.controller.impl +package org.dbpedia.databus.mods.worker.springboot.worker.controller.impl import java.io.FileInputStream import java.net.URI @@ -6,8 +6,8 @@ import java.nio.charset.StandardCharsets import java.util.Optional import javax.servlet.http.{HttpServletRequest, HttpServletResponse} import org.apache.commons.io.IOUtils -import org.dbpedia.databus.mods.core.worker.execution.ModRequest -import org.dbpedia.databus.mods.core.worker.service.{ExecutionService, FileService} +import org.dbpedia.databus.mods.worker.springboot.worker.execution.ModRequest +import org.dbpedia.databus.mods.worker.springboot.worker.service.{ExecutionService, FileService} import org.slf4j.LoggerFactory import org.springframework.web.bind.annotation.{PathVariable, RequestMapping, RequestMethod, RequestParam, RestController} diff --git a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/worker/execution/Extension.scala b/databus-mods-lib/databus-mods-spring-boot-starter/worker/execution/Extension.scala similarity index 92% rename from databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/worker/execution/Extension.scala rename to databus-mods-lib/databus-mods-spring-boot-starter/worker/execution/Extension.scala index af5cd79..f8c3c86 100644 --- a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/worker/execution/Extension.scala +++ b/databus-mods-lib/databus-mods-spring-boot-starter/worker/execution/Extension.scala @@ -1,10 +1,10 @@ -package org.dbpedia.databus.mods.core.worker.execution +package org.dbpedia.databus.mods.worker.springboot.worker.execution import org.apache.jena.datatypes.xsd.XSDDateTime import org.apache.jena.rdf.model.{Model, ModelFactory} import org.apache.jena.vocabulary.{RDF, XSD} import org.dbpedia.databus.mods.core.util.ModelUtil.ModelWrapper -import org.dbpedia.databus.mods.core.worker.service.FileService +import org.dbpedia.databus.mods.worker.springboot.worker.service.FileService import java.io.{FileOutputStream, OutputStream} import java.util.Calendar diff --git a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/worker/execution/ModExecutor.scala b/databus-mods-lib/databus-mods-spring-boot-starter/worker/execution/ModExecutor.scala similarity index 90% rename from databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/worker/execution/ModExecutor.scala rename to databus-mods-lib/databus-mods-spring-boot-starter/worker/execution/ModExecutor.scala index 56357b5..d084760 100644 --- a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/worker/execution/ModExecutor.scala +++ b/databus-mods-lib/databus-mods-spring-boot-starter/worker/execution/ModExecutor.scala @@ -1,6 +1,6 @@ -package org.dbpedia.databus.mods.core.worker.execution +package org.dbpedia.databus.mods.worker.springboot.worker.execution -import org.dbpedia.databus.mods.core.worker.service.FileService +import org.dbpedia.databus.mods.worker.springboot.worker.service.FileService import org.slf4j.LoggerFactory import java.io.{FileOutputStream, PrintWriter} diff --git a/databus-mods-lib/databus-mods-spring-boot-starter/worker/execution/ModProcessor.scala b/databus-mods-lib/databus-mods-spring-boot-starter/worker/execution/ModProcessor.scala new file mode 100644 index 0000000..17d3546 --- /dev/null +++ b/databus-mods-lib/databus-mods-spring-boot-starter/worker/execution/ModProcessor.scala @@ -0,0 +1,6 @@ +package org.dbpedia.databus.mods.worker.springboot.worker.execution + +trait ModProcessor { + + def process(extension: Extension) +} diff --git a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/worker/execution/ModQueue.scala b/databus-mods-lib/databus-mods-spring-boot-starter/worker/execution/ModQueue.scala similarity index 90% rename from databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/worker/execution/ModQueue.scala rename to databus-mods-lib/databus-mods-spring-boot-starter/worker/execution/ModQueue.scala index 29ef819..778a31c 100644 --- a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/worker/execution/ModQueue.scala +++ b/databus-mods-lib/databus-mods-spring-boot-starter/worker/execution/ModQueue.scala @@ -1,4 +1,4 @@ -package org.dbpedia.databus.mods.core.worker.execution +package org.dbpedia.databus.mods.worker.springboot.worker.execution import java.util import java.util.concurrent.LinkedBlockingQueue diff --git a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/worker/execution/ModRequest.scala b/databus-mods-lib/databus-mods-spring-boot-starter/worker/execution/ModRequest.scala similarity index 68% rename from databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/worker/execution/ModRequest.scala rename to databus-mods-lib/databus-mods-spring-boot-starter/worker/execution/ModRequest.scala index 6c20f14..f69696a 100644 --- a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/worker/execution/ModRequest.scala +++ b/databus-mods-lib/databus-mods-spring-boot-starter/worker/execution/ModRequest.scala @@ -1,4 +1,4 @@ -package org.dbpedia.databus.mods.core.worker.execution +package org.dbpedia.databus.mods.worker.springboot.worker.execution import java.net.URI diff --git a/databus-mods-lib/databus-mods-spring-boot-starter/worker/execution/ModResultFile.scala b/databus-mods-lib/databus-mods-spring-boot-starter/worker/execution/ModResultFile.scala new file mode 100644 index 0000000..bc36e31 --- /dev/null +++ b/databus-mods-lib/databus-mods-spring-boot-starter/worker/execution/ModResultFile.scala @@ -0,0 +1,5 @@ +package org.dbpedia.databus.mods.worker.springboot.worker.execution + +import java.io.File + +case class ModResultFile(file: File) \ No newline at end of file diff --git a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/worker/service/ExecutionService.scala b/databus-mods-lib/databus-mods-spring-boot-starter/worker/service/ExecutionService.scala similarity index 79% rename from databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/worker/service/ExecutionService.scala rename to databus-mods-lib/databus-mods-spring-boot-starter/worker/service/ExecutionService.scala index 5d0a7b7..1190271 100644 --- a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/worker/service/ExecutionService.scala +++ b/databus-mods-lib/databus-mods-spring-boot-starter/worker/service/ExecutionService.scala @@ -1,6 +1,6 @@ -package org.dbpedia.databus.mods.core.worker.service +package org.dbpedia.databus.mods.worker.springboot.worker.service -import org.dbpedia.databus.mods.core.worker.execution.{ModExecutor, ModProcessor, ModQueue, ModRequest} +import org.dbpedia.databus.mods.worker.springboot.worker.execution.{ModExecutor, ModProcessor, ModQueue, ModRequest} import org.springframework.stereotype.Service import java.util.concurrent.Executors diff --git a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/worker/service/FileService.scala b/databus-mods-lib/databus-mods-spring-boot-starter/worker/service/FileService.scala similarity index 95% rename from databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/worker/service/FileService.scala rename to databus-mods-lib/databus-mods-spring-boot-starter/worker/service/FileService.scala index 41972ef..e4bcd70 100644 --- a/databus-mods-lib/databus-mods-core/src/main/scala/org/dbpedia/databus/mods/core/worker/service/FileService.scala +++ b/databus-mods-lib/databus-mods-spring-boot-starter/worker/service/FileService.scala @@ -1,4 +1,4 @@ -package org.dbpedia.databus.mods.core.worker.service +package org.dbpedia.databus.mods.worker.springboot.worker.service import better.files.File import org.apache.commons.io.FileUtils diff --git a/databus-mods-lib/databus-mods-validate/pom.xml b/databus-mods-lib/databus-mods-validate/pom.xml index 08982c2..597eb83 100644 --- a/databus-mods-lib/databus-mods-validate/pom.xml +++ b/databus-mods-lib/databus-mods-validate/pom.xml @@ -21,6 +21,11 @@ + + org.dbpedia.databus-mods + databus-mods-core + 1.0-SNAPSHOT + info.picocli picocli diff --git a/databus-mods-lib/databus-mods-validate/src/main/resources/mod-shapes.ttl b/databus-mods-lib/databus-mods-validate/src/main/resources/mod-shapes.ttl new file mode 100644 index 0000000..e69de29 diff --git a/databus-mods-lib/databus-mods-validate/src/main/scala/org/dbpedia/databus/mods/validate/ApiHttpValidator.scala b/databus-mods-lib/databus-mods-validate/src/main/scala/org/dbpedia/databus/mods/validate/ApiHttpValidator.scala new file mode 100644 index 0000000..a7757a4 --- /dev/null +++ b/databus-mods-lib/databus-mods-validate/src/main/scala/org/dbpedia/databus/mods/validate/ApiHttpValidator.scala @@ -0,0 +1,29 @@ +package org.dbpedia.databus.mods.validate + +import org.apache.commons.io.IOUtils +import org.dbpedia.databus.mods.core.worker.api.ModActivityClientHttp +import picocli.CommandLine.{Command, Option} + +import java.net.URI +import java.nio.charset.StandardCharsets +import java.util.concurrent.Callable + +@Command(name = "httpapi", mixinStandardHelpOptions = true) +class ApiHttpValidator extends Callable[Int] { + + @Option(names = Array("-w", "--worker"), required = true) + var endpoint: String = _ + + @Option(names = Array("-i", "--dataId"), required = true) + var dataId: String = _ + + @Option(names = Array("--accessUri"), required = false) + var accessUri: String = _ + + override def call(): Int = { + val client = new ModActivityClientHttp + val modActivityResponse = client.send(new URI(endpoint), new URI(dataId), minDelay = 200) + println(IOUtils.toString(modActivityResponse.data,StandardCharsets.UTF_8.name())) + 0 + } +} diff --git a/databus-mods-lib/databus-mods-validate/src/main/scala/org/dbpedia/databus/mods/validate/MetadataValidator.scala b/databus-mods-lib/databus-mods-validate/src/main/scala/org/dbpedia/databus/mods/validate/MetadataValidator.scala index 1bb6690..a9598a7 100644 --- a/databus-mods-lib/databus-mods-validate/src/main/scala/org/dbpedia/databus/mods/validate/MetadataValidator.scala +++ b/databus-mods-lib/databus-mods-validate/src/main/scala/org/dbpedia/databus/mods/validate/MetadataValidator.scala @@ -1,15 +1,19 @@ -//package org.dbpedia.databus.mods.validate -// -//import org.apache.jena.riot.{Lang, RDFDataMgr} -//import org.apache.jena.shacl.lib.ShLib -//import org.apache.jena.shacl.{ShaclValidator, Shapes} -// -//object MetadataValidator extends App { -// -// scala.util.Properties.setProp("scala.time","true") -// -// val SHAPES = "dataid-shacl.ttl" -// val DATA = "dataid-unexpanded.ttl" +package org.dbpedia.databus.mods.validate + +import org.apache.jena.riot.{Lang, RDFDataMgr} +import org.apache.jena.shacl.lib.ShLib +import org.apache.jena.shacl.{ShaclValidator, Shapes} +import picocli.CommandLine.Command + +import java.util.concurrent.Callable + +@Command(name = "metadata", mixinStandardHelpOptions = true) +class MetadataValidator extends Callable[Int] { + + scala.util.Properties.setProp("scala.time","true") + +// val SHAPES = null +// val DATA = null // // val shapesGraph = RDFDataMgr.loadGraph(SHAPES) // val dataGraph = RDFDataMgr.loadGraph(DATA) @@ -20,5 +24,9 @@ // ShLib.printReport(report) // // RDFDataMgr.write(System.out,report.getModel,Lang.TTL) -//} -// + + override def call(): Int = { + 0 // SUCCESS + } +} + diff --git a/databus-mods-lib/databus-mods-validate/src/main/scala/org/dbpedia/databus/mods/validate/ValidationCLI.scala b/databus-mods-lib/databus-mods-validate/src/main/scala/org/dbpedia/databus/mods/validate/ValidationCLI.scala index 32f1ba7..0d400c2 100644 --- a/databus-mods-lib/databus-mods-validate/src/main/scala/org/dbpedia/databus/mods/validate/ValidationCLI.scala +++ b/databus-mods-lib/databus-mods-validate/src/main/scala/org/dbpedia/databus/mods/validate/ValidationCLI.scala @@ -1,6 +1,8 @@ package org.dbpedia.databus.mods.validate -import org.dbpedia.databus.mods.validate.worker.ValidateWorker +import ch.qos.logback.classic.{Level, Logger} +import org.dbpedia.databus.mods.core.worker.api.ModActivityClientHttp +import org.slf4j.LoggerFactory import picocli.AutoComplete.GenerateCompletion import picocli.CommandLine import picocli.CommandLine.{Command, Option} @@ -8,15 +10,23 @@ import picocli.CommandLine.{Command, Option} import java.util.concurrent.Callable object ValidationCLI extends App { - val exitCode = new CommandLine(new ValidationCLI).execute(args: _*) + + val apacheLog = LoggerFactory.getLogger("org.apache.http").asInstanceOf[Logger] + apacheLog.setLevel(Level.OFF) + + val exitCode = { + if (0 == args.length) + new CommandLine(new ValidationCLI).execute("--help") + else + new CommandLine(new ValidationCLI).execute(args: _*) + } System.exit(exitCode) } -@Command(name= "validate", mixinStandardHelpOptions = true, subcommands = Array(classOf[ValidateWorker],classOf[GenerateCompletion])) +@Command(name = "validate", mixinStandardHelpOptions = true, subcommands = Array(classOf[ApiHttpValidator], classOf[GenerateCompletion])) class ValidationCLI extends Callable[Int] { override def call(): Int = { - val exitCode = 0 - exitCode + 0 // SUCCESS } } \ No newline at end of file diff --git a/databus-mods-lib/databus-mods-validate/src/main/scala/org/dbpedia/databus/mods/validate/worker/Request.scala b/databus-mods-lib/databus-mods-validate/src/main/scala/org/dbpedia/databus/mods/validate/worker/Request.scala deleted file mode 100644 index 4e1f37c..0000000 --- a/databus-mods-lib/databus-mods-validate/src/main/scala/org/dbpedia/databus/mods/validate/worker/Request.scala +++ /dev/null @@ -1,49 +0,0 @@ -//package org.dbpedia.databus.mods.validate.worker -// -//import org.apache.http.impl.client.{HttpClientBuilder, LaxRedirectStrategy} -//import org.springframework.http.client.{ClientHttpRequestFactory, HttpComponentsClientHttpRequestFactory} -//import org.springframework.http.{HttpEntity, HttpHeaders, HttpMethod, MediaType} -//import org.springframework.util.LinkedMultiValueMap -//import org.springframework.web.client.RestTemplate -// -//class Request(endpoint: String) extends { -// -// val factory = new HttpComponentsClientHttpRequestFactory( -// HttpClientBuilder.create(). -// setRedirectStrategy(new LaxRedirectStrategy) -// .build() -// ) -// val req = new RestTemplate(factory) -// -// def check(databusURI: String): Unit = { -// val fileID = databusURI.split("/").takeRight(5).mkString("/") -// -// val map= new LinkedMultiValueMap[String,String](); -// map.add("source", databusURI); -// -// val headers = new HttpHeaders -// headers.setContentType(MediaType.APPLICATION_FORM_URLENCODED) -// -// val response = req.exchange( -// endpoint+"/"+fileID+"/activity", -// HttpMethod.POST, -// new HttpEntity(map,headers), -// classOf[String] -// ) -// -// String = -// -// -//// lastStatusCode = rePOST.getStatusCodeValue(); -//// assert(lastStatusCode == 202); -//// -//// String body = ""; -//// while (lastStatusCode == 202) { -//// ResponseEntity reGET = this.restTemplate.getForEntity( -//// "http://localhost:"+port+path+"/activity", -//// String.class); -//// lastStatusCode = reGET.getStatusCodeValue(); -//// body = reGET.getBody(); -//// } -// } -//} diff --git a/databus-mods-lib/databus-mods-validate/src/main/scala/org/dbpedia/databus/mods/validate/worker/Result.scala b/databus-mods-lib/databus-mods-validate/src/main/scala/org/dbpedia/databus/mods/validate/worker/Result.scala deleted file mode 100644 index 3e5bc8a..0000000 --- a/databus-mods-lib/databus-mods-validate/src/main/scala/org/dbpedia/databus/mods/validate/worker/Result.scala +++ /dev/null @@ -1,3 +0,0 @@ -//package org.dbpedia.databus.mods.validate.worker -// -//case class Result(apiType: String) diff --git a/databus-mods-lib/databus-mods-validate/src/main/scala/org/dbpedia/databus/mods/validate/worker/ValidateWorker.scala b/databus-mods-lib/databus-mods-validate/src/main/scala/org/dbpedia/databus/mods/validate/worker/ValidateWorker.scala deleted file mode 100644 index 9b5f1e6..0000000 --- a/databus-mods-lib/databus-mods-validate/src/main/scala/org/dbpedia/databus/mods/validate/worker/ValidateWorker.scala +++ /dev/null @@ -1,78 +0,0 @@ -package org.dbpedia.databus.mods.validate.worker - -import org.apache.http.impl.client.{HttpClientBuilder, LaxRedirectStrategy} -import org.springframework.http.client.{ClientHttpRequestFactory, HttpComponentsClientHttpRequestFactory} -import org.springframework.http.{HttpEntity, HttpHeaders, HttpMethod, MediaType, RequestEntity, ResponseEntity} -import org.springframework.util.{LinkedMultiValueMap, MultiValueMap} -import org.springframework.web.client.RestTemplate -import picocli.CommandLine.{Command, Option, Parameters} - -import java.io.File -import java.util.concurrent.Callable -import scala.util.{Failure, Success, Try} - -@Command(name = "worker", mixinStandardHelpOptions = true) -class ValidateWorker extends Callable[Int] { - - @Option(names = Array("-e", "--endpoint"), required = true) - var endpoint: String = _ - - @Option(names = Array("-i", "--input"), required = true) - var use: File = _ - - @Option(names = Array("--test"), required = false) - var test: java.net.InetAddress = _ - - val factory: ClientHttpRequestFactory = new HttpComponentsClientHttpRequestFactory( - HttpClientBuilder.create(). - setRedirectStrategy(new LaxRedirectStrategy) - .build() - ) - val req = new RestTemplate(factory) - - override def call(): Int = { - - Try { - val path: String = use.getAbsolutePath.split("/").takeRight(5).mkString("/") - - val headers: HttpHeaders = new HttpHeaders - headers.setContentType(MediaType.APPLICATION_FORM_URLENCODED) - - val map: MultiValueMap[String, String] = new LinkedMultiValueMap[String, String] - map.add("source", use.getAbsolutePath) - - var lastStatusCode: Int = 0 - - val rePOST: ResponseEntity[String] = - req.exchange( - endpoint + path + "/activity", - HttpMethod.POST, - new HttpEntity[MultiValueMap[String, String]](map, headers), - classOf[String] - ) - - lastStatusCode = rePOST.getStatusCodeValue - assert(lastStatusCode == 202) - - var body: String = rePOST.getBody - - while (lastStatusCode == 202) { - val reGET: ResponseEntity[String] = - req.getForEntity( - endpoint + path + "/activity", - classOf[String] - ) - lastStatusCode = reGET.getStatusCodeValue - body = reGET.getBody - } - - System.out.println(body) - assert(lastStatusCode == 200) - } match { - case Success(value) => 0 - case Failure(exception) => - exception.printStackTrace() - 1 - } - } -} diff --git a/databus-mods/databus-mods-void/src/main/scala/org/dbpedia/databus_mods/void/ApiDocConfig.scala b/databus-mods/databus-mods-void/src/main/scala/org/dbpedia/databus_mods/void/ApiDocConfig.scala deleted file mode 100644 index fb5589a..0000000 --- a/databus-mods/databus-mods-void/src/main/scala/org/dbpedia/databus_mods/void/ApiDocConfig.scala +++ /dev/null @@ -1,32 +0,0 @@ -//package org.dbpedia.databus_mods.void -// -//import org.springframework.context.annotation.{Bean, Configuration} -// -//@Configuration -//@EnableSwagger2WebMvc -//class ApiDocConfig { -// -// def apiInfo: ApiInfo = { -// new ApiInfo( -// "Databus Mod Worker", -// "Databus Mod Worker", -// "1.0", -// "termsOfServiceUrl", -// "contactName or Concact", -// "license", -// "licenseUrl" -// ) -// } -// -// @Bean -// def api(): Docket = { -// new Docket(DocumentationType.SWAGGER_2) -// .useDefaultResponseMessages(false) -// .select() -// .apis(RequestHandlerSelectors.basePackage("org.dbpedia.databus_mods.void")) -// .paths(PathSelectors.any()) -// .build() -// .pathMapping("/") -// .apiInfo(apiInfo) -// } -//} \ No newline at end of file diff --git a/databus-mods/databus-mods-void/src/main/scala/org/dbpedia/databus_mods/void/VoidProcess.scala b/databus-mods/databus-mods-void/src/main/scala/org/dbpedia/databus_mods/void/RDFVoidUtil.scala similarity index 74% rename from databus-mods/databus-mods-void/src/main/scala/org/dbpedia/databus_mods/void/VoidProcess.scala rename to databus-mods/databus-mods-void/src/main/scala/org/dbpedia/databus_mods/void/RDFVoidUtil.scala index 892b38e..82f7a18 100644 --- a/databus-mods/databus-mods-void/src/main/scala/org/dbpedia/databus_mods/void/VoidProcess.scala +++ b/databus-mods/databus-mods-void/src/main/scala/org/dbpedia/databus_mods/void/RDFVoidUtil.scala @@ -12,27 +12,7 @@ import org.springframework.stereotype.Component import java.net.URI import scala.collection.mutable -@Component -class VoidProcess extends ModProcessor { - - private val log = LoggerFactory.getLogger(classOf[VoidProcess]) - - def process(ext: Extension): Unit = { - ext.setType("https://mods.tools.dbpedia.org/ns/rdf#VoidMod") - ext.addPrefix("", "https://mods.tools.dbpedia.org/ns/rdf#") - val is = UriUtil.openStream(new URI(ext.source)) - val pipedRDF = IORdfUtil.toPipedRDF(is) - - if (pipedRDF.hasNext) { - val (classPartitionMap, propertyPartitionMap) = calculateVoIDPartitions(pipedRDF) - val voidModel = toJenaModel(classPartitionMap, propertyPartitionMap) - voidModel.setNsPrefix("void", "http://rdfs.org/ns/void#") - voidModel.write(ext.createModResult("rdfVoid.ttl", "http://dataid.dbpedia.org/ns/mods#statisticsDerivedFrom"), "TURTLE") - } else { - log.warn(s"empty iterator") - } - is.close() - } +object RDFVoidUtil { def calculateVoIDPartitions(iter: PipedRDFIterator[Triple]): (mutable.HashMap[String, Int], mutable.HashMap[String, Int]) = { diff --git a/databus-mods/databus-mods-void/src/main/scala/org/dbpedia/databus_mods/void/RDFVoidWorker.scala b/databus-mods/databus-mods-void/src/main/scala/org/dbpedia/databus_mods/void/RDFVoidWorker.scala new file mode 100644 index 0000000..8aa9b26 --- /dev/null +++ b/databus-mods/databus-mods-void/src/main/scala/org/dbpedia/databus_mods/void/RDFVoidWorker.scala @@ -0,0 +1,50 @@ +package org.dbpedia.databus_mods.void + +import org.apache.tomcat.util.buf.UriUtil +import org.dbpedia.databus.mods.core.io.DataUtil +import org.dbpedia.databus.mods.core.io.RdfIO +import org.dbpedia.databus.mods.core.model.{ModActivity, ModActivityMetadata, ModActivityMetadataBuilder, ModActivityRequest} +import org.dbpedia.databus.mods.worker.springboot.EnableModWorkerApi +import org.dbpedia.databus.mods.worker.springboot.ModWorkerApiProfile +import org.slf4j.LoggerFactory +import org.springframework.boot.SpringApplication +import org.springframework.boot.autoconfigure.SpringBootApplication +import org.springframework.context.annotation.{Bean, Import} + +import java.net.URI + +@SpringBootApplication +class RDFVoidWorker { + + private val log = LoggerFactory.getLogger(classOf[RDFVoidWorker]) + + @Bean + @EnableModWorkerApi(version = "1.0.0", profile = ModWorkerApiProfile.HttpPoll) + def getModActivity() = new ModActivity { + + override def perform(request: ModActivityRequest, builder: ModActivityMetadataBuilder): ModActivityMetadata = { + builder.withType("https://mods.tools.dbpedia.org/ns/rdf#VoidMod") + +// val is = DataUtil.openStream(request.accessUri)) + + val pipedRDF = RdfIO.toPipedRDF(is) + + if (pipedRDF.hasNext) { + val (classPartitionMap, propertyPartitionMap) = RDFVoidUtil.calculateVoIDPartitions(pipedRDF) + val voidModel = RDFVoidUtil.toJenaModel(classPartitionMap, propertyPartitionMap) + voidModel.setNsPrefix("void", "http://rdfs.org/ns/void#") + voidModel.write(builder.createModResult("rdfVoid.ttl", "http://dataid.dbpedia.org/ns/mods#statisticsDerivedFrom"), "TURTLE") + } else { + log.warn(s"empty iterator") + } + is.close() + + builder.build() + } + } +} + +object RDFVoidWorker extends App { + SpringApplication.run(classOf[RDFVoidWorker], args: _*) +} + diff --git a/databus-mods/databus-mods-void/src/main/scala/org/dbpedia/databus_mods/void/Worker.scala b/databus-mods/databus-mods-void/src/main/scala/org/dbpedia/databus_mods/void/Worker.scala deleted file mode 100644 index 5a12b21..0000000 --- a/databus-mods/databus-mods-void/src/main/scala/org/dbpedia/databus_mods/void/Worker.scala +++ /dev/null @@ -1,15 +0,0 @@ -package org.dbpedia.databus_mods.void - -import org.dbpedia.databus.mods.core.worker.AsyncWorker -import org.springframework.boot.SpringApplication -import org.springframework.boot.autoconfigure.SpringBootApplication -import org.springframework.context.annotation.Import - -@SpringBootApplication -@Import(value = Array(classOf[AsyncWorker])) -class Worker {} - -object Worker extends App { - SpringApplication.run(classOf[Worker], args: _*) -} -