diff --git a/databus-mods-lib/databus-mods-dummy/pom.xml b/databus-mods-lib/databus-mods-dummy/pom.xml index fe0e5a2..13afc06 100644 --- a/databus-mods-lib/databus-mods-dummy/pom.xml +++ b/databus-mods-lib/databus-mods-dummy/pom.xml @@ -37,7 +37,6 @@ databus-mods-core 1.0-SNAPSHOT - org.springframework.boot spring-boot-starter-test diff --git a/databus-mods-lib/databus-mods-dummy/src/main/scala/org/dbpedia/databus/mods/worker/dummy/DummyWorker.scala b/databus-mods-lib/databus-mods-dummy/src/main/scala/org/dbpedia/databus/mods/worker/dummy/DummyWorker.scala index 611e7be..ee85fa2 100644 --- a/databus-mods-lib/databus-mods-dummy/src/main/scala/org/dbpedia/databus/mods/worker/dummy/DummyWorker.scala +++ b/databus-mods-lib/databus-mods-dummy/src/main/scala/org/dbpedia/databus/mods/worker/dummy/DummyWorker.scala @@ -1,7 +1,9 @@ package org.dbpedia.databus.mods.worker.dummy +import ch.qos.logback.classic.{Level, Logger} import org.dbpedia.databus.mods.core.model.{ModActivity, ModActivityMetadata, ModActivityMetadataBuilder, ModActivityRequest} import org.dbpedia.databus.mods.worker.springboot.{EnableModWorkerApi, ModWorkerApiProfile} +import org.slf4j.LoggerFactory import org.springframework.boot.SpringApplication import org.springframework.boot.autoconfigure.SpringBootApplication import org.springframework.context.annotation.Bean @@ -9,19 +11,21 @@ import org.springframework.context.annotation.Bean import scala.util.Random; @SpringBootApplication +@EnableModWorkerApi(version = "1.0", profile = ModWorkerApiProfile.HttpPoll) class DummyWorker { @Bean - @EnableModWorkerApi(version = "1.0.0", profile = ModWorkerApiProfile.HttpPoll) - def getModActivity: ModActivity = new ModActivity { + def dummyActivity = new ModActivity { override def perform(request: ModActivityRequest, builder: ModActivityMetadataBuilder): ModActivityMetadata = { Thread.sleep(1000) - builder.withStatSummary((Random.nextInt(100)/100.0).toString).build() + builder.withStatSummary((Random.nextInt(100) / 100.0).toString).build() } } } - object DummyWorker extends App { + val log = LoggerFactory.getLogger("org.spring").asInstanceOf[Logger] + log.setLevel(Level.DEBUG) SpringApplication.run(classOf[DummyWorker], args: _*) -} \ No newline at end of file +} + diff --git a/databus-mods-lib/databus-mods-spring-boot-starter/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports b/databus-mods-lib/databus-mods-spring-boot-starter/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports new file mode 100644 index 0000000..c04ebe8 --- /dev/null +++ b/databus-mods-lib/databus-mods-spring-boot-starter/src/main/resources/META-INF/spring/org.springframework.boot.autoconfigure.AutoConfiguration.imports @@ -0,0 +1 @@ +org.dbpedia.databus.mods.worker.springboot.config.ModWorkerApiAutoConfig \ No newline at end of file diff --git a/databus-mods-lib/databus-mods-spring-boot-starter/src/main/scala/org/dbpedia/databus/mods/worker/springboot/config/ModWorkerApiAutoConfig.scala b/databus-mods-lib/databus-mods-spring-boot-starter/src/main/scala/org/dbpedia/databus/mods/worker/springboot/config/ModWorkerApiAutoConfig.scala index 8722692..bf67cb5 100644 --- a/databus-mods-lib/databus-mods-spring-boot-starter/src/main/scala/org/dbpedia/databus/mods/worker/springboot/config/ModWorkerApiAutoConfig.scala +++ b/databus-mods-lib/databus-mods-spring-boot-starter/src/main/scala/org/dbpedia/databus/mods/worker/springboot/config/ModWorkerApiAutoConfig.scala @@ -18,6 +18,7 @@ import scala.collection.JavaConverters._ class ModWorkerApiAutoConfig() { private val log = LoggerFactory.getLogger(classOf[ModWorkerApiAutoConfig]) + log.info("Starting Mod Worker Interface") @Bean @ConditionalOnMissingBean(Array(classOf[ModActivity])) @@ -54,8 +55,12 @@ class ModWorkerApiAutoConfig() { if (annotation.isDefined) { annotation.get.profile() match { - case ModWorkerApiProfile.Http => new ActivityControllerImpl - case ModWorkerApiProfile.HttpPoll => new ActivityControllerPollImpl(activityExecutionService) + case ModWorkerApiProfile.Http => + log.info("Started Mod Worker with Http API") + new ActivityControllerImpl + case ModWorkerApiProfile.HttpPoll => + log.info("Started Mod Worker with HttpPoll API") + new ActivityControllerPollImpl(activityExecutionService) } } else { new ActivityControllerImpl diff --git a/databus-mods-lib/databus-mods-validate/src/main/scala/org/dbpedia/databus/mods/validate/ApiHttpValidator.scala b/databus-mods-lib/databus-mods-validate/src/main/scala/org/dbpedia/databus/mods/validate/ApiHttpValidator.scala index a7757a4..90aa4b0 100644 --- a/databus-mods-lib/databus-mods-validate/src/main/scala/org/dbpedia/databus/mods/validate/ApiHttpValidator.scala +++ b/databus-mods-lib/databus-mods-validate/src/main/scala/org/dbpedia/databus/mods/validate/ApiHttpValidator.scala @@ -11,13 +11,25 @@ import java.util.concurrent.Callable @Command(name = "httpapi", mixinStandardHelpOptions = true) class ApiHttpValidator extends Callable[Int] { - @Option(names = Array("-w", "--worker"), required = true) + @Option( + names = Array("-w", "--worker-endpoint"), + required = true, + description = Array("The URL of the mod worker API endpoint '.../activity'") + ) var endpoint: String = _ - @Option(names = Array("-i", "--dataId"), required = true) + @Option( + names = Array("-i", "--dataId"), + required = true, + description = Array("A (Databus) DataID IRI") + ) var dataId: String = _ - @Option(names = Array("--accessUri"), required = false) + @Option( + names = Array("--accessUri"), + required = false, + description = Array("An alternative access URI of the file referenced by the given DataID") + ) var accessUri: String = _ override def call(): Int = { diff --git a/databus-mods-server/application.yml b/databus-mods-server/application.yml deleted file mode 100755 index 8d6f63c..0000000 --- a/databus-mods-server/application.yml +++ /dev/null @@ -1,95 +0,0 @@ -spring: - datasource: - driver-class-name: 'org.apache.derby.jdbc.EmbeddedDriver' - username: 'mysqluser' - password: 'mysqlpass' - url: 'jdbc:derby:derby;create=true' - jpa: - hibernate: - ddl-auto: 'create-drop' -# ddl-auto: 'update' -server: - port: 9000 -vaadin: - urlMapping: '/cmdr/*' -database: - databaseUrl: 'jdbc:derby:./derby' -fileCache: - maxNumberOfFiles: 20 - volume: '/home/marvin/src/github.com/dbpedia/databus-mods/databus-mods-server/volumes/fileCache' -extServer: - sparql: - databaseUrl: 'jdbc:virtuoso://localhost:1111/charset=UTF-8/' - databaseUsr: 'dba' - databasePsw: 'myDbaPassword' - endpoint: 'http://localhost:8890/sparql' - http: - volume: '/home/marvin/src/github.com/dbpedia/databus-mods/databus-mods-server/volumes/www' - baseUrl: 'http://localhost:8900/' -mods: - - name: 'SPOMod' - accepts: 'file' - links: - - api: 'http://localhost:9004/a' - fileCache: '/home/marvin/src/github.com/dbpedia/databus-mods/databus-mods-server/volumes/fileCache' - localRepo: '/home/marvin/src/github.com/dbpedia/databus-mods/databus-mods-server/volumes/fileCache/localRepo' - mountRepo: '/home/marvin/src/github.com/dbpedia/databus-mods/databus-mods-server/volumes/SPO' - query: ' -PREFIX dataid: -PREFIX dct: -PREFIX dcat: -PREFIX db: -PREFIX rdf: -PREFIX rdfs: -SELECT DISTINCT ?file ?sha256sum ?downloadURL WHERE { - ?dataset dcat:distribution ?distribution . - ?distribution dataid:file ?file . - ?distribution dataid:sha256sum ?sha256sum . - ?distribution dcat:downloadURL ?downloadURL . - ?dataset dataid:artifact . -} -' -# - name: 'FileMetrics' -# accepts: 'file' -# links: -# - api: 'http://localhost:9001/a' -# fileCache: '/workspace/fileCache' -# localRepo: '/workspace/localRepo' -# mountRepo: '/home/marvin/src/github.com/dbpedia/databus-mods/databus-mods-server/volumes/FileMetrics' -# query: ' -#PREFIX dataid: -#PREFIX dct: -#PREFIX dcat: -#PREFIX db: -#PREFIX rdf: -#PREFIX rdfs: -#SELECT DISTINCT ?file ?sha256sum ?downloadURL WHERE { -# ?dataset dcat:distribution ?distribution . -# ?distribution dataid:file ?file . -# ?distribution dataid:sha256sum ?sha256sum . -# ?distribution dcat:downloadURL ?downloadURL . -# ?dataset dataid:artifact . -#} -#' -# - name: 'VoidVocab' -# accepts: 'file' -# links: -# - api: 'http://localhost:9002/a' -# fileCache: '/workspace/fileCache' -# localRepo: '/workspace/localRepo' -# mountRepo: '/home/marvin/src/github.com/dbpedia/databus-mods/databus-mods-server/volumes/VoidVocab' -# query: ' -#PREFIX dataid: -#PREFIX dct: -#PREFIX dcat: -#PREFIX db: -#PREFIX rdf: -#PREFIX rdfs: -#SELECT DISTINCT ?file ?sha256sum ?downloadURL WHERE { -# ?dataset dcat:distribution ?distribution . -# ?distribution dataid:file ?file . -# ?distribution dataid:sha256sum ?sha256sum . -# ?distribution dcat:downloadURL ?downloadURL . -# ?dataset dataid:artifact . -#} -#' \ No newline at end of file diff --git a/databus-mods-server/databus-mods-server-core/application.yml b/databus-mods-server/databus-mods-server-core/application.yml deleted file mode 100644 index 359e4a2..0000000 --- a/databus-mods-server/databus-mods-server-core/application.yml +++ /dev/null @@ -1,61 +0,0 @@ -server: - port: 9050 -spring: - datasource: - driver-class-name: 'org.apache.derby.jdbc.EmbeddedDriver' - username: 'mysqluser' - password: 'mysqlpass' - url: 'jdbc:derby:derby;create=true' - jpa: - hibernate: - ddl-auto: 'create-drop' -# ddl-auto: 'update' - mvc: - pathmatch: - matching-strategy: ant_path_matcher - -mod-server: - databus: "https://databus.dbpedia.org/repo/sparql" - volume: ./target/repo - web-base: "https://mods.tools.dbpedia.org/main/" - triple-store: - db-url: 'jdbc:virtuoso://localhost:11111/charset=UTF-8/' - db-user: 'dba' - db-password: 'DasIstDasHausVomNikolaus' - endpoint: 'http://localhost:8890/sparql' - schedule: - online-period: 60000 -# online-check: 5000 - online-check: 3600000 -# task-updates: -1 - task-updates: 3600000 - -mods: - - name: 'void' - accepts: - - 'file' - selects: - - 'rdfVoid.ttl' - workers: - - 'http://localhost:9000/modApi' - query: ' -PREFIX xsd: -PREFIX dataid: -PREFIX dct: -PREFIX dcat: -PREFIX db: -PREFIX dataid-cv: - -SELECT ?file ?issued ?sha256sum ?downloadURL WHERE { - { SELECT ?file ?issued ?sha256sum ?downloadURL WHERE { - ?s dct:publisher . - ?s dataid:file ?file . - ?s dct:issued ?issued . - ?s dataid:sha256sum ?sha256sum . - ?s dcat:downloadURL ?downloadURL . - ?s dataid-cv:type ?cv . - FILTER( str(?cv) = "parsed") - ?s dataid:formatExtension ?ext . - FILTER( str(?ext) = "nt" ) - } LIMIT 1 } -}' diff --git a/databus-mods-server/databus-mods-server-core/pom.xml b/databus-mods-server/databus-mods-server-core/pom.xml index 557b4cd..140cc32 100644 --- a/databus-mods-server/databus-mods-server-core/pom.xml +++ b/databus-mods-server/databus-mods-server-core/pom.xml @@ -84,13 +84,20 @@ derby - - javax.persistence - javax.persistence-api - 2.2 + com.h2database + h2 + 2.1.214 + runtime + + + + + + + com.openlink.virtuoso virtjdbc4-1 diff --git a/databus-mods-server/databus-mods-server-core/src/main/resources/application.yml b/databus-mods-server/databus-mods-server-core/src/main/resources/application.yml index 65c710a..aba3282 100644 --- a/databus-mods-server/databus-mods-server-core/src/main/resources/application.yml +++ b/databus-mods-server/databus-mods-server-core/src/main/resources/application.yml @@ -1,15 +1,28 @@ server: port: 9050 spring: +# datasource: +# url: jdbc:h2:mem:mydb +# username: sa +# password: password +# driver-class-name: org.h2.Driver +# jpa: +# spring.jpa.database-platform: org.hibernate.dialect.H2Dialect +# hibernate: +# ddl-auto: 'create-drop' datasource: driver-class-name: 'org.apache.derby.jdbc.EmbeddedDriver' username: 'mysqluser' password: 'mysqlpass' url: 'jdbc:derby:derby;create=true' + hikari: + validationTimeout: 60000 jpa: + properties: + hibernate: + dialect: 'org.hibernate.dialect.DerbyDialect' hibernate: -# ddl-auto: 'update' - ddl-auto: 'create-drop' + ddl-auto: 'update' vaadin: urlMapping: '/demo/*' @@ -43,28 +56,28 @@ tmp: # http: # volume: '/home/marvin/src/github.com/dbpedia/databus-mods/databus-mods-server/volumes/www' # baseUrl: 'http://localhost:8900/' -mods: - - name: 'DEMO' - accept: 'file' - select: - workers: - - 'http://localhost:32768/api' - query: ' -PREFIX dataid: -PREFIX dct: -PREFIX dcat: -PREFIX db: -PREFIX rdf: -PREFIX rdfs: -SELECT DISTINCT ?file ?sha256sum ?downloadURL ?issued WHERE { -{ SELECT DISTINCT ?file ?sha256sum ?downloadURL ?issued WHERE { - ?dataset dcat:distribution ?distribution . - ?distribution dct:issued ?issued . - ?distribution dataid:file ?file . - ?distribution dataid:sha256sum ?sha256sum . - ?distribution dcat:downloadURL ?downloadURL . - ?distribution dcat:byteSize ?size . - ?dataset dataid:artifact . - } ORDER BY ?size OFFSET 5 LIMIT 10 } -} -' \ No newline at end of file +#mods: +# - name: 'DEMO' +# accept: 'file' +# select: +# workers: +# - 'http://localhost:32768/api' +# query: ' +#PREFIX dataid: +#PREFIX dct: +#PREFIX dcat: +#PREFIX db: +#PREFIX rdf: +#PREFIX rdfs: +#SELECT DISTINCT ?file ?sha256sum ?downloadURL ?issued WHERE { +#{ SELECT DISTINCT ?file ?sha256sum ?downloadURL ?issued WHERE { +# ?dataset dcat:distribution ?distribution . +# ?distribution dct:issued ?issued . +# ?distribution dataid:file ?file . +# ?distribution dataid:sha256sum ?sha256sum . +# ?distribution dcat:downloadURL ?downloadURL . +# ?distribution dcat:byteSize ?size . +# ?dataset dataid:artifact . +# } ORDER BY ?size OFFSET 5 LIMIT 10 } +#} +#' \ No newline at end of file diff --git a/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/config/SchedulerConfig.scala b/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/config/SchedulerConfig.scala index 15247a7..883130a 100644 --- a/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/config/SchedulerConfig.scala +++ b/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/config/SchedulerConfig.scala @@ -1,21 +1,17 @@ package org.dbpedia.databus_mods.server.core.config -import java.util.Date -import org.dbpedia.databus_mods.server.core.mods.online.OnlineCheckService import org.dbpedia.databus_mods.server.core.service.TaskService -import org.springframework.beans.factory.annotation.{Autowired, Value} +import org.springframework.beans.factory.annotation.Value import org.springframework.context.annotation.Configuration -import org.springframework.scheduling.{Trigger, TriggerContext} import org.springframework.scheduling.annotation.SchedulingConfigurer import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler import org.springframework.scheduling.config.ScheduledTaskRegistrar -import java.time.Instant - @Configuration -class SchedulerConfig(onlineCheckService: OnlineCheckService, - taskService: TaskService, - @Value("${mod-server.schedule.task-updates}") updateDelay: String) extends SchedulingConfigurer { +class SchedulerConfig( + taskService: TaskService, + @Value("${mod-server.schedule.task-updates}") updateDelay: String +) extends SchedulingConfigurer { override def configureTasks(taskRegistrar: ScheduledTaskRegistrar): Unit = { val taskScheduler = new ThreadPoolTaskScheduler(); @@ -23,18 +19,18 @@ class SchedulerConfig(onlineCheckService: OnlineCheckService, taskScheduler.initialize(); taskRegistrar.setTaskScheduler(taskScheduler); // online-check - taskRegistrar.addTriggerTask(new Runnable { - override def run(): Unit = { - onlineCheckService.update() - } - }, - new Trigger { - override def nextExecution(triggerContext: TriggerContext): Instant = { - onlineCheckService.getNextExecutionTime(triggerContext.lastActualExecution()) - } - }) + // taskRegistrar.addTriggerTask(new Runnable { + // override def run(): Unit = { + // onlineCheckService.update() + // } + // }, + // new Trigger { + // override def nextExecution(triggerContext: TriggerContext): Instant = { + // onlineCheckService.getNextExecutionTime(triggerContext.lastActualExecution()) + // } + // }) // task updates - if(updateDelay.toInt > 0) { + if (updateDelay.toInt > 0) { taskRegistrar.addFixedDelayTask(new Runnable { override def run(): Unit = { taskService.update() @@ -44,3 +40,4 @@ class SchedulerConfig(onlineCheckService: OnlineCheckService, } } + diff --git a/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/mods/online/OnlineCheck.scala b/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/mods/online/OnlineCheck.scala index 039a7e6..7f9aa1d 100644 --- a/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/mods/online/OnlineCheck.scala +++ b/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/mods/online/OnlineCheck.scala @@ -1,46 +1,46 @@ -package org.dbpedia.databus_mods.server.core.mods.online - -import java.sql.Timestamp - -import com.fasterxml.jackson.annotation.JsonView -import javax.persistence.{Entity, FetchType, GeneratedValue, GenerationType, Id, JoinColumn, ManyToOne, Temporal, TemporalType} -import org.dbpedia.databus_mods.server.core.persistence.DatabusFile -import org.dbpedia.databus_mods.server.core.views.Views - -import scala.annotation.meta.field -import scala.beans.BeanProperty - -@Entity(name = "online_check") -class OnlineCheck -( - @(JsonView@field)(value = Array(classOf[Views.PublicOnlineCheck])) - @(ManyToOne@field)(fetch = FetchType.EAGER) - @(JoinColumn@field)(name = "databusFile_id") - @BeanProperty - var databusFile: DatabusFile, - @(JsonView@field)(value = Array(classOf[Views.PublicOnlineCheck])) - @BeanProperty - @Temporal(TemporalType.TIMESTAMP) - var timestamp: Timestamp, - @(JsonView@field)(value = Array(classOf[Views.PublicOnlineCheck])) - @BeanProperty - var status: Int -) { - @(Id@field) - @(GeneratedValue@field)(strategy = GenerationType.TABLE) - @BeanProperty - var id: Long = _ - - def this() { - this(null, null, 200) - } - - def this(databusFile: DatabusFile, status: Int) { - this(databusFile,new Timestamp(System.currentTimeMillis()),status) - } - - override def toString: String = { - s"""$timestamp; $status ;${databusFile.getDataIdSingleFile}""" - } -} - +//package org.dbpedia.databus_mods.server.core.mods.online +// +//import java.sql.Timestamp +// +//import com.fasterxml.jackson.annotation.JsonView +//import jakarta.persistence.{Entity, FetchType, GeneratedValue, GenerationType, Id, JoinColumn, ManyToOne, Temporal, TemporalType} +//import org.dbpedia.databus_mods.server.core.persistence.DatabusFile +//import org.dbpedia.databus_mods.server.core.views.Views +// +//import scala.annotation.meta.field +//import scala.beans.BeanProperty +// +//@Entity(name = "online_check") +//class OnlineCheck +//( +// @(JsonView@field)(value = Array(classOf[Views.PublicOnlineCheck])) +// @(ManyToOne@field)(fetch = FetchType.EAGER) +// @(JoinColumn@field)(name = "databusFile_id") +// @BeanProperty +// var databusFile: DatabusFile, +// @(JsonView@field)(value = Array(classOf[Views.PublicOnlineCheck])) +// @BeanProperty +// @Temporal(TemporalType.TIMESTAMP) +// var timestamp: Timestamp, +// @(JsonView@field)(value = Array(classOf[Views.PublicOnlineCheck])) +// @BeanProperty +// var status: Int +//) { +// @(Id@field) +// @(GeneratedValue@field)(strategy = GenerationType.TABLE) +// @BeanProperty +// var id: Long = _ +// +// def this() { +// this(null, null, 200) +// } +// +// def this(databusFile: DatabusFile, status: Int) { +// this(databusFile,new Timestamp(System.currentTimeMillis()),status) +// } +// +// override def toString: String = { +// s"""$timestamp; $status ;${databusFile.getDataIdSingleFile}""" +// } +//} +// diff --git a/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/mods/online/OnlineCheckController.scala b/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/mods/online/OnlineCheckController.scala index 46cf013..302b17a 100644 --- a/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/mods/online/OnlineCheckController.scala +++ b/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/mods/online/OnlineCheckController.scala @@ -1,27 +1,27 @@ -package org.dbpedia.databus_mods.server.core.mods.online - -import com.fasterxml.jackson.annotation.JsonView -import org.dbpedia.databus_mods.server.core.views.Views -import org.springframework.web.bind.annotation.{RequestMapping, RequestMethod, RequestParam, RestController} - -@RestController -@RequestMapping(value = Array("main/online")) -class OnlineCheckController(onlineCheckService: OnlineCheckService) { - - @RequestMapping(value = Array("config"),method = Array(RequestMethod.POST)) - def setRate(@RequestParam rate: String): Unit = { - onlineCheckService.setExecutionDelay(rate.toInt) - } - -// @JsonView(value = Array(classOf[Views.PublicOnlineCheck])) -// @RequestMapping(value = Array(), method = Array(RequestMethod.GET)) -// def list(): java.util.Iterator[OnlineCheck] = { -// onlineCheckService.getAll -// } +//package org.dbpedia.databus_mods.server.core.mods.online +// +//import com.fasterxml.jackson.annotation.JsonView +//import org.dbpedia.databus_mods.server.core.views.Views +//import org.springframework.web.bind.annotation.{RequestMapping, RequestMethod, RequestParam, RestController} // -// @JsonView(value = Array(classOf[Views.PublicOnlineCheck])) -// @RequestMapping(value = Array(), method = Array(RequestMethod.POST)) -// def update(): Unit = { -// onlineCheckService.update() +//@RestController +//@RequestMapping(value = Array("main/online")) +//class OnlineCheckController(onlineCheckService: OnlineCheckService) { +// +// @RequestMapping(value = Array("config"),method = Array(RequestMethod.POST)) +// def setRate(@RequestParam rate: String): Unit = { +// onlineCheckService.setExecutionDelay(rate.toInt) // } -} +// +//// @JsonView(value = Array(classOf[Views.PublicOnlineCheck])) +//// @RequestMapping(value = Array(), method = Array(RequestMethod.GET)) +//// def list(): java.util.Iterator[OnlineCheck] = { +//// onlineCheckService.getAll +//// } +//// +//// @JsonView(value = Array(classOf[Views.PublicOnlineCheck])) +//// @RequestMapping(value = Array(), method = Array(RequestMethod.POST)) +//// def update(): Unit = { +//// onlineCheckService.update() +//// } +//} diff --git a/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/mods/online/OnlineCheckRepository.scala b/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/mods/online/OnlineCheckRepository.scala index ee13684..ce6c4c8 100644 --- a/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/mods/online/OnlineCheckRepository.scala +++ b/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/mods/online/OnlineCheckRepository.scala @@ -1,17 +1,17 @@ -package org.dbpedia.databus_mods.server.core.mods.online - -import java.sql.Timestamp -import java.util.Optional - -import org.springframework.data.repository.CrudRepository -import org.springframework.stereotype.Repository - -@Repository -trait OnlineCheckRepository extends CrudRepository[OnlineCheck, Long] { - - - def findFirstByDatabusFileDataIdSingleFileOrderByTimestampDesc(databusFileDataIdSingleFile: String): Optional[OnlineCheck] - - - def findByTimestampGreaterThanOrderByTimestamp(timestamp: Timestamp): java.util.List[OnlineCheck] -} +//package org.dbpedia.databus_mods.server.core.mods.online +// +//import java.sql.Timestamp +//import java.util.Optional +// +//import org.springframework.data.repository.CrudRepository +//import org.springframework.stereotype.Repository +// +//@Repository +//trait OnlineCheckRepository extends CrudRepository[OnlineCheck, Long] { +// +// +// def findFirstByDatabusFileDataIdSingleFileOrderByTimestampDesc(databusFileDataIdSingleFile: String): Optional[OnlineCheck] +// +// +// def findByTimestampGreaterThanOrderByTimestamp(timestamp: Timestamp): java.util.List[OnlineCheck] +//} diff --git a/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/mods/online/OnlineCheckService.scala b/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/mods/online/OnlineCheckService.scala index ee62e1e..0a44431 100644 --- a/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/mods/online/OnlineCheckService.scala +++ b/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/mods/online/OnlineCheckService.scala @@ -1,186 +1,186 @@ -package org.dbpedia.databus_mods.server.core.mods.online - -import java.io.FileWriter -import java.net.URI -import java.sql.Timestamp -import java.util -import java.util.concurrent.atomic.{AtomicBoolean, AtomicInteger} -import java.util.{Calendar, Date, GregorianCalendar, Optional} -import org.apache.http.client.methods.HttpHead -import org.apache.http.impl.client.HttpClientBuilder -import org.dbpedia.databus_mods.server.core.persistence.DatabusFile -import org.dbpedia.databus_mods.server.core.service.{DatabusFileService, FileService} -import org.slf4j.LoggerFactory -import org.springframework.beans.factory.annotation.Value -import org.springframework.stereotype.Service - -import java.time.Instant -import scala.collection.JavaConversions._ - -@Service -class OnlineCheckService(databusFileService: DatabusFileService, - onlineCheckRepository: OnlineCheckRepository, - fileService: FileService, - @Value("${mod-server.schedule.online-check}") defaultExecutionDelay: String, - @Value("${mod-server.schedule.online-period}") onlinePeriod: String) { - - private val log = LoggerFactory.getLogger(classOf[OnlineCheckService]) - - private val executionDelay = new AtomicInteger(defaultExecutionDelay.toInt) - - def setExecutionDelay(millis: Int): Unit = { - if (millis > 15000) - executionDelay.set(millis) - else - "Error: rate <= 15000" - } - - def getNextExecutionTime(lastActualExecutionTime: Instant): Instant = { - val nextExecutionTime = new GregorianCalendar - nextExecutionTime.setTime(if (lastActualExecutionTime != null) Date.from(lastActualExecutionTime) else new Date()) - nextExecutionTime.add(Calendar.MILLISECOND, executionDelay.get()) //you can get the value from wherever you want - nextExecutionTime.getTime.toInstant - } - - private val updateLock = new AtomicBoolean(false) - - def update(): Unit = { - if (updateLock.get()) { - log.info("skip still running") - } else { - log.info("update") - databusFileService.getAll - // .filter(_.getPublisher == "ontologies") - .foreach({ - databusFile => - val url = new URI(databusFile.getDownloadUrl) - val client = HttpClientBuilder.create().build() - var code = -1 - try { - val response = client.execute(new HttpHead(url)) - code = response.getStatusLine.getStatusCode - response.close() - } catch { - case e: Exception => - log.warn(s"s error check $url") - } - val onlineCheck = new OnlineCheck(databusFile, code) - - val possibleLastOnlineCheck = - onlineCheckRepository.findFirstByDatabusFileDataIdSingleFileOrderByTimestampDesc(databusFile.getDataIdSingleFile) - - var append = false - if (possibleLastOnlineCheck.isPresent) { - if (possibleLastOnlineCheck.get.status != onlineCheck.status) { - onlineCheckRepository.save(onlineCheck) - append = true - } - } else { - onlineCheckRepository.save(onlineCheck) - append = true - } - updateMetadata(databusFile, onlineCheck, append, possibleLastOnlineCheck) - - client.close() - }) - log.info("updated") - } - } - - private def updateMetadata(databusFile: DatabusFile, latestCheck: OnlineCheck, append: Boolean, priorCheck: Optional[OnlineCheck]): Unit = { - - val csv = fileService.getOrCreate("online", databusFile.getDatabusPath, "online.csv") - if (append) { - val csvWriter = new FileWriter(csv, true) - val entry = latestCheck.getTimestamp + ";" + latestCheck.getStatus + "\n" - csvWriter.write(entry) - csvWriter.flush() - csvWriter.close() - } - - val minTimestamp = new Timestamp(latestCheck.timestamp.getTime - onlinePeriod.toLong) - val changes = onlineCheckRepository.findByTimestampGreaterThanOrderByTimestamp(minTimestamp) - - if (changes.isEmpty) { - changes.add(latestCheck) - } - - var current = minTimestamp.getTime - var up = 0L - changes.foreach({ - onlineChange => - if (onlineChange.status == 200) { - up += onlineChange.getTimestamp.getTime - current - } - current = onlineChange.getTimestamp.getTime - }) - if (latestCheck.getStatus == 200) - up += latestCheck.timestamp.getTime - current - val uptime: Double = up / onlinePeriod.toDouble - - // if(changes.size() == 1 && priorCheck.isPresent && (priorCheck.get().status == latestCheck.status)) { - // - // } else { - // TODO lower writes - val svg = fileService.getOrCreate("online", databusFile.getDatabusPath, "online.svg") - val svgWriter = new FileWriter(svg) - svgWriter.write(createSVG(uptime)) - svgWriter.flush() - svgWriter.close() - // } - } - - private def createSVG(stat: Double): String = { - val statString = (stat * 100).toString - val roundStat = statString.split("\\.").head - val color = { - if (stat > 0.99) { - "#97ca00" //green - } else if (stat > 0.95) { - "#007ec6" //blue - } else if (stat > 0.50) { - "#dfb317" //yellow - } else { - "#e05d44" //red - } - } - val Array(x, width1, width2, textLength) = { - if(stat == 1.0) { - Array(635,43,86,330) //100% - } else if(stat > 0.1) { - Array(595,35,78,250) //10% - } else { - Array(565,29,72,190) //1% - } - } - - s""" - | - | online: 100% - | - | - | - | - | - | - | - | - | - | - | - | - | - | - | online - | - | $roundStat% - | - |""".stripMargin - } - - def getAll: util.Iterator[OnlineCheck] = { - onlineCheckRepository.findAll().iterator() - } -} - - +//package org.dbpedia.databus_mods.server.core.mods.online +// +//import java.io.FileWriter +//import java.net.URI +//import java.sql.Timestamp +//import java.util +//import java.util.concurrent.atomic.{AtomicBoolean, AtomicInteger} +//import java.util.{Calendar, Date, GregorianCalendar, Optional} +//import org.apache.http.client.methods.HttpHead +//import org.apache.http.impl.client.HttpClientBuilder +//import org.dbpedia.databus_mods.server.core.persistence.DatabusFile +//import org.dbpedia.databus_mods.server.core.service.{DatabusFileService, FileService} +//import org.slf4j.LoggerFactory +//import org.springframework.beans.factory.annotation.Value +//import org.springframework.stereotype.Service +// +//import java.time.Instant +//import scala.collection.JavaConversions._ +// +//@Service +//class OnlineCheckService(databusFileService: DatabusFileService, +// onlineCheckRepository: OnlineCheckRepository, +// fileService: FileService, +// @Value("${mod-server.schedule.online-check}") defaultExecutionDelay: String, +// @Value("${mod-server.schedule.online-period}") onlinePeriod: String) { +// +// private val log = LoggerFactory.getLogger(classOf[OnlineCheckService]) +// +// private val executionDelay = new AtomicInteger(defaultExecutionDelay.toInt) +// +// def setExecutionDelay(millis: Int): Unit = { +// if (millis > 15000) +// executionDelay.set(millis) +// else +// "Error: rate <= 15000" +// } +// +// def getNextExecutionTime(lastActualExecutionTime: Instant): Instant = { +// val nextExecutionTime = new GregorianCalendar +// nextExecutionTime.setTime(if (lastActualExecutionTime != null) Date.from(lastActualExecutionTime) else new Date()) +// nextExecutionTime.add(Calendar.MILLISECOND, executionDelay.get()) //you can get the value from wherever you want +// nextExecutionTime.getTime.toInstant +// } +// +// private val updateLock = new AtomicBoolean(false) +// +// def update(): Unit = { +// if (updateLock.get()) { +// log.info("skip still running") +// } else { +// log.info("update") +// databusFileService.getAll +// // .filter(_.getPublisher == "ontologies") +// .foreach({ +// databusFile => +// val url = new URI(databusFile.getDownloadUrl) +// val client = HttpClientBuilder.create().build() +// var code = -1 +// try { +// val response = client.execute(new HttpHead(url)) +// code = response.getStatusLine.getStatusCode +// response.close() +// } catch { +// case e: Exception => +// log.warn(s"s error check $url") +// } +// val onlineCheck = new OnlineCheck(databusFile, code) +// +// val possibleLastOnlineCheck = +// onlineCheckRepository.findFirstByDatabusFileDataIdSingleFileOrderByTimestampDesc(databusFile.getDataIdSingleFile) +// +// var append = false +// if (possibleLastOnlineCheck.isPresent) { +// if (possibleLastOnlineCheck.get.status != onlineCheck.status) { +// onlineCheckRepository.save(onlineCheck) +// append = true +// } +// } else { +// onlineCheckRepository.save(onlineCheck) +// append = true +// } +// updateMetadata(databusFile, onlineCheck, append, possibleLastOnlineCheck) +// +// client.close() +// }) +// log.info("updated") +// } +// } +// +// private def updateMetadata(databusFile: DatabusFile, latestCheck: OnlineCheck, append: Boolean, priorCheck: Optional[OnlineCheck]): Unit = { +// +// val csv = fileService.getOrCreate("online", databusFile.getDatabusPath, "online.csv") +// if (append) { +// val csvWriter = new FileWriter(csv, true) +// val entry = latestCheck.getTimestamp + ";" + latestCheck.getStatus + "\n" +// csvWriter.write(entry) +// csvWriter.flush() +// csvWriter.close() +// } +// +// val minTimestamp = new Timestamp(latestCheck.timestamp.getTime - onlinePeriod.toLong) +// val changes = onlineCheckRepository.findByTimestampGreaterThanOrderByTimestamp(minTimestamp) +// +// if (changes.isEmpty) { +// changes.add(latestCheck) +// } +// +// var current = minTimestamp.getTime +// var up = 0L +// changes.foreach({ +// onlineChange => +// if (onlineChange.status == 200) { +// up += onlineChange.getTimestamp.getTime - current +// } +// current = onlineChange.getTimestamp.getTime +// }) +// if (latestCheck.getStatus == 200) +// up += latestCheck.timestamp.getTime - current +// val uptime: Double = up / onlinePeriod.toDouble +// +// // if(changes.size() == 1 && priorCheck.isPresent && (priorCheck.get().status == latestCheck.status)) { +// // +// // } else { +// // TODO lower writes +// val svg = fileService.getOrCreate("online", databusFile.getDatabusPath, "online.svg") +// val svgWriter = new FileWriter(svg) +// svgWriter.write(createSVG(uptime)) +// svgWriter.flush() +// svgWriter.close() +// // } +// } +// +// private def createSVG(stat: Double): String = { +// val statString = (stat * 100).toString +// val roundStat = statString.split("\\.").head +// val color = { +// if (stat > 0.99) { +// "#97ca00" //green +// } else if (stat > 0.95) { +// "#007ec6" //blue +// } else if (stat > 0.50) { +// "#dfb317" //yellow +// } else { +// "#e05d44" //red +// } +// } +// val Array(x, width1, width2, textLength) = { +// if(stat == 1.0) { +// Array(635,43,86,330) //100% +// } else if(stat > 0.1) { +// Array(595,35,78,250) //10% +// } else { +// Array(565,29,72,190) //1% +// } +// } +// +// s""" +// | +// | online: 100% +// | +// | +// | +// | +// | +// | +// | +// | +// | +// | +// | +// | +// | +// | +// | online +// | +// | $roundStat% +// | +// |""".stripMargin +// } +// +// def getAll: util.Iterator[OnlineCheck] = { +// onlineCheckRepository.findAll().iterator() +// } +//} +// +// diff --git a/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/persistence/DatabusFile.scala b/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/persistence/DatabusFile.scala index 66232bf..7be6181 100644 --- a/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/persistence/DatabusFile.scala +++ b/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/persistence/DatabusFile.scala @@ -3,7 +3,7 @@ package org.dbpedia.databus_mods.server.core.persistence import java.util import com.fasterxml.jackson.annotation.JsonView -import javax.persistence._ +import jakarta.persistence._ import org.dbpedia.databus_mods.server.core.views.Views import scala.annotation.meta.field @@ -12,9 +12,10 @@ import scala.beans.BeanProperty @Entity @Table( name = "databus_file", - uniqueConstraints = Array( - // TODO - new UniqueConstraint(columnNames = Array("dataIdSingleFile", "checkSum")))) +// uniqueConstraints = Array( +// // TODO +// new UniqueConstraint(columnNames = Array("dataIdSingleFile", "checkSum"))) +) class DatabusFile ( // TODO rename to databusFileID or fileID @@ -44,7 +45,7 @@ class DatabusFile ) { @(Id@field) - @(GeneratedValue@field)(strategy = GenerationType.TABLE) + @(GeneratedValue@field)(strategy = GenerationType.IDENTITY) @BeanProperty @(JsonView@field)(value = Array(classOf[Views.Default])) var id: Long = _ diff --git a/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/persistence/Mod.scala b/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/persistence/Mod.scala index c4bd724..e5e6088 100644 --- a/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/persistence/Mod.scala +++ b/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/persistence/Mod.scala @@ -1,20 +1,20 @@ package org.dbpedia.databus_mods.server.core.persistence -import java.util - import com.fasterxml.jackson.annotation.JsonView -import javax.persistence._ +import jakarta.persistence._ import org.dbpedia.databus_mods.server.core.views.Views import org.hibernate.annotations.{LazyCollection, LazyCollectionOption} +import java.util import scala.annotation.meta.field import scala.beans.BeanProperty @Entity @Table( name = "mod", - uniqueConstraints = Array( - new UniqueConstraint(columnNames = Array("name")))) +// uniqueConstraints = Array( +// new UniqueConstraint(columnNames = Array("name"))) +) class Mod ( @BeanProperty @@ -26,7 +26,7 @@ class Mod var query: String ) { @(Id@field) - @(GeneratedValue@field)(strategy = GenerationType.TABLE) + @(GeneratedValue@field)(strategy = GenerationType.IDENTITY) @(JsonView@field)(value = Array(classOf[Views.Default])) @BeanProperty var id: Long = _ diff --git a/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/persistence/Task.scala b/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/persistence/Task.scala index 4fa6ce4..6c9e98b 100644 --- a/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/persistence/Task.scala +++ b/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/persistence/Task.scala @@ -1,9 +1,8 @@ package org.dbpedia.databus_mods.server.core.persistence import com.fasterxml.jackson.annotation.JsonView -import javax.persistence._ +import jakarta.persistence._ import org.dbpedia.databus_mods.server.core.views.Views -import org.hibernate.annotations.{Generated, GenerationTime} import scala.annotation.meta.field import scala.beans.BeanProperty @@ -11,8 +10,9 @@ import scala.beans.BeanProperty @Entity @Table( name = "task", - uniqueConstraints = Array( - new UniqueConstraint(columnNames = Array("databusFile_id", "mod_id")))) +// uniqueConstraints = Array( +// new UniqueConstraint(columnNames = Array("databusFile_id", "mod_id"))) +) class Task ( @(ManyToOne@field)(fetch = FetchType.EAGER) @@ -22,11 +22,11 @@ class Task var databusFile: DatabusFile, @(ManyToOne@field)(fetch = FetchType.EAGER) @BeanProperty - @(JsonView@field)(value = Array(classOf[Views.PublicTaskView],classOf[Views.DatabusFileView])) + @(JsonView@field)(value = Array(classOf[Views.PublicTaskView], classOf[Views.DatabusFileView])) var mod: Mod ) { @Id - @GeneratedValue(strategy = GenerationType.TABLE) + @GeneratedValue(strategy = GenerationType.IDENTITY) @BeanProperty @(JsonView@field)(value = Array(classOf[Views.Default])) var id: Long = _ @@ -36,8 +36,8 @@ class Task databusFile.getTasks.remove(this) } -// @BeanProperty -// var uri: String = _ + // @BeanProperty + // var uri: String = _ @BeanProperty @(JsonView@field)(value = Array(classOf[Views.Default])) @@ -50,19 +50,19 @@ class Task var worker: Worker = _ // @Transient -// @Basic -// private var statusValue: Int = _ -// -// @PostLoad -// def fillTransient(): Unit = { -// this.state = Status(statusValue) -// } + // @Basic + // private var statusValue: Int = _ + // + // @PostLoad + // def fillTransient(): Unit = { + // this.state = Status(statusValue) + // } -// @PrePersist -// def fillPersist(): Unit = { -// println(state) -// this.statusValue = state.id -// } + // @PrePersist + // def fillPersist(): Unit = { + // println(state) + // this.statusValue = state.id + // } def copyOf(t: Task): Unit = { setId(t.getId) @@ -75,11 +75,11 @@ class Task this(null, null) } - override def toString: String = { - s"""TASK#$id( - | databusFile.id : ${databusFile.getId}, - | databusFile.dataIdSingleFile : ${databusFile.getDataIdSingleFile}, - | mod.id : ${mod.getId}, - | mod.name : ${mod.getName})""".stripMargin - } + override def toString: String = { + s"""TASK#$id( + | databusFile.id : ${databusFile.getId}, + | databusFile.dataIdSingleFile : ${databusFile.getDataIdSingleFile}, + | mod.id : ${mod.getId}, + | mod.name : ${mod.getName})""".stripMargin + } } diff --git a/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/persistence/Worker.scala b/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/persistence/Worker.scala index 2333778..355eb2c 100644 --- a/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/persistence/Worker.scala +++ b/databus-mods-server/databus-mods-server-core/src/main/scala/org/dbpedia/databus_mods/server/core/persistence/Worker.scala @@ -3,15 +3,16 @@ package org.dbpedia.databus_mods.server.core.persistence import com.fasterxml.jackson.annotation.JsonView import org.dbpedia.databus_mods.server.core.views.Views -import javax.persistence._ +import jakarta.persistence._ import scala.annotation.meta.field import scala.beans.BeanProperty @Entity @Table( name = "worker", - uniqueConstraints = Array( - new UniqueConstraint(columnNames = Array("url")))) +// uniqueConstraints = Array( +// new UniqueConstraint(columnNames = Array("url"))) + ) class Worker ( @(ManyToOne@field) @@ -24,7 +25,7 @@ class Worker var url: String ) { @(Id@field) - @(GeneratedValue@field)(strategy = GenerationType.TABLE) + @(GeneratedValue@field)(strategy = GenerationType.IDENTITY) @BeanProperty @(JsonView@field)(value = Array(classOf[Views.Default])) var id: Long = _ diff --git a/databus-mods/databus-mods-void/src/main/scala/org/dbpedia/databus_mods/void/RDFVoidWorker.scala b/databus-mods/databus-mods-void/src/main/scala/org/dbpedia/databus_mods/void/RDFVoidWorker.scala index b6726f9..4ba5a13 100644 --- a/databus-mods/databus-mods-void/src/main/scala/org/dbpedia/databus_mods/void/RDFVoidWorker.scala +++ b/databus-mods/databus-mods-void/src/main/scala/org/dbpedia/databus_mods/void/RDFVoidWorker.scala @@ -25,7 +25,7 @@ class RDFVoidWorker { override def perform(request: ModActivityRequest, builder: ModActivityMetadataBuilder): ModActivityMetadata = { builder.withType("https://mods.tools.dbpedia.org/ns/rdf#VoidMod") - ModActivity +// ModActivity // val is = DataUtil.openStream(request.accessUri)) diff --git a/pom.xml b/pom.xml index 72c7aaf..e63ba05 100755 --- a/pom.xml +++ b/pom.xml @@ -68,6 +68,12 @@ 10.14.2.0 + + org.apache.derby + derbytools + 10.14.2.0 + +