Skip to content

Commit

Permalink
Version 0.2.5.
Browse files Browse the repository at this point in the history
  • Loading branch information
@khituras
khituras committed Dec 9, 2018
1 parent d71f24d commit 90a10dc
Show file tree
Hide file tree
Showing 11 changed files with 120 additions and 65 deletions.
2 changes: 1 addition & 1 deletion jcore-pipeline-builder-base/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
<parent>
<groupId>de.julielab</groupId>
<artifactId>jcore-pipeline-modules</artifactId>
<version>0.2.5-SNAPSHOT</version>
<version>0.2.5</version>
</parent>

<build>
Expand Down
23 changes: 17 additions & 6 deletions ...er-base/src/main/java/de/julielab/jcore/pipeline/builder/base/main/JCoReUIMAPipeline.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,13 +21,15 @@
import org.apache.uima.collection.impl.metadata.cpe.CpeComponentDescriptorImpl;
import org.apache.uima.collection.impl.metadata.cpe.CpeIncludeImpl;
import org.apache.uima.collection.impl.metadata.cpe.CpeIntegratedCasProcessorImpl;
import org.apache.uima.collection.metadata.CpeComponentDescriptor;
import org.apache.uima.collection.metadata.CpeDescription;
import org.apache.uima.collection.metadata.CpeDescriptorException;
import org.apache.uima.fit.cpe.CpeBuilder;
import org.apache.uima.fit.factory.AnalysisEngineFactory;
import org.apache.uima.resource.ResourceCreationSpecifier;
import org.apache.uima.resource.ResourceInitializationException;
import org.apache.uima.resource.ResourceSpecifier;
import org.apache.uima.resource.metadata.Import;
import org.apache.uima.resource.metadata.MetaDataObject;
import org.apache.uima.resource.metadata.ResourceMetaData;
import org.apache.uima.resource.metadata.impl.Import_impl;
Expand Down Expand Up @@ -333,20 +335,29 @@ public void store(File directory, boolean clearLibDir) throws PipelineIOExceptio
cpeAAEFile));
builder.setAnalysisEngine(cpeAAE);
CpeDescription cpeDescription = builder.getCpeDescription();
if (crDescription != null)
cpeDescription.getAllCollectionCollectionReaders()[0].getCollectionIterator().
getDescriptor().getInclude().set(crFile.getName());
if (crDescription != null) {
Import_impl crImport = new Import_impl();
crImport.setLocation(crFile.getName());
final CpeComponentDescriptor crDescriptor = cpeDescription.getAllCollectionCollectionReaders()[0].getCollectionIterator().
getDescriptor();
// delete the automatically generated include; we don't want an include (Absolute URLs are used)
// but an import by location (a path relative to the CPE.xml descriptor is used)
crDescriptor.setInclude(null);
crDescriptor.setImport(crImport);
}

CpeIncludeImpl cpeInclude = new CpeIncludeImpl();
cpeInclude.set(cpeAAEFile.getName());
Import_impl cpeAaeImport = new Import_impl();
cpeAaeImport.setLocation(cpeAAEFile.getName());
CpeComponentDescriptorImpl cpeComponentDescriptor = new CpeComponentDescriptorImpl();
cpeComponentDescriptor.setInclude(cpeInclude);
cpeComponentDescriptor.setImport(cpeAaeImport);
CpeIntegratedCasProcessorImpl cpeIntegratedCasProcessor = new CpeIntegratedCasProcessorImpl();
cpeIntegratedCasProcessor.setCpeComponentDescriptor(cpeComponentDescriptor);
cpeIntegratedCasProcessor.setName("CPE AAE");
cpeIntegratedCasProcessor.setBatchSize(500);
CpeCasProcessorsImpl cpeCasProcessors = new CpeCasProcessorsImpl();
cpeCasProcessors.addCpeCasProcessor(cpeIntegratedCasProcessor);
cpeDescription.setCpeCasProcessors(cpeCasProcessors);
cpeDescription.getCpeCasProcessors().setPoolSize(24);
cpeDescription.toXML(FileUtilities.getWriterToFile(
cpeFile
));
Expand Down
2 changes: 1 addition & 1 deletion jcore-pipeline-builder-cli/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
<parent>
<groupId>de.julielab</groupId>
<artifactId>jcore-pipeline-modules</artifactId>
<version>0.2.5-SNAPSHOT</version>
<version>0.2.5</version>
</parent>
<dependencies>
<dependency>
Expand Down
2 changes: 1 addition & 1 deletion jcore-pipeline-runner/jcore-pipeline-runner-assembly/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
<parent>
<artifactId>jcore-pipeline-runner</artifactId>
<groupId>de.julielab</groupId>
<version>0.2.5-SNAPSHOT</version>
<version>0.2.5</version>
</parent>
<modelVersion>4.0.0</modelVersion>

Expand Down
2 changes: 1 addition & 1 deletion jcore-pipeline-runner/jcore-pipeline-runner-base/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
<parent>
<artifactId>jcore-pipeline-runner</artifactId>
<groupId>de.julielab</groupId>
<version>0.2.5-SNAPSHOT</version>
<version>0.2.5</version>
</parent>
<modelVersion>4.0.0</modelVersion>

Expand Down
48 changes: 41 additions & 7 deletions ...eline-runner-base/src/main/java/de/julielab/jcore/pipeline/runner/CPEBootstrapRunner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@
import org.slf4j.LoggerFactory;
import org.xml.sax.SAXException;

import java.io.File;
import java.io.IOException;
import java.io.*;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
Expand All @@ -46,17 +46,24 @@ public class CPEBootstrapRunner implements IPipelineRunner {
public void runPipeline(JCoReUIMAPipeline pipeline, HierarchicalConfiguration<ImmutableNode> runnerConfig) throws PipelineInstantiationException, PipelineRunningException, PipelineIOException {
try {

Stream<File> classpathElements = pipeline.getClasspathElements();
//classpathElements.forEach(JarLoader::addJarToClassPath);
String classpath = classpathElements.map(File::getAbsolutePath).collect(Collectors.joining(File.pathSeparator));
pipeline.load(false);
final String plp = pipeline.getLoadDirectory().getAbsolutePath();
int numThreads = runnerConfig.containsKey(NUMTHREADS) ? runnerConfig.getInt(NUMTHREADS) : 2;
final File cpeRunnerJar = findCpeRunnerJar();
Stream<File> classpathElements = pipeline.getClasspathElements();
classpathElements = Stream.concat(classpathElements, Stream.of(cpeRunnerJar, new File(plp + File.separator + JCoReUIMAPipeline.DIR_CONF), new File(plp + File.separator + "resources")));
String classpath = classpathElements.map(File::getAbsolutePath).collect(Collectors.joining(File.pathSeparator));

final String[] cmdarray = {"java", "-cp", classpath, "-jar", cpeRunnerJar.getAbsolutePath()};
final String[] cmdarray = {"java", "-cp", classpath, "de.julielab.jcore.pipeline.runner.cpe.CPERunner", "-d", plp + File.separator + JCoReUIMAPipeline.DIR_DESC + File.separator + "CPE.xml", "-t", String.valueOf(numThreads)};
log.info("Running the pipeline at {} with the following command line: {}", pipeline.getLoadDirectory(), Arrays.toString(cmdarray));
final Process exec = Runtime.getRuntime().exec(cmdarray);
exec.waitFor();
new InputStreamGobbler(exec.getInputStream(), "StdInGobbler", "std").start();
new InputStreamGobbler(exec.getErrorStream(), "ErrInGobbler", "err").start();

final int i = exec.waitFor();
if (i != 0)
throw new RuntimeException("Pipeline runner process exited with status " + i);

// // The CpePipeline.runPipeline() code was checked for the number of threads.
// log.info("Running pipeline with {} threads.", numThreads);
Expand Down Expand Up @@ -84,6 +91,33 @@ public void runPipeline(JCoReUIMAPipeline pipeline, HierarchicalConfiguration<Im
}
}

private class InputStreamGobbler extends Thread {
private InputStream is;
private String type;

public InputStreamGobbler(InputStream is, String threadName, String type) {
this.is = is;
this.type = type;
setName(threadName);
}

@Override
public void run() {
try (BufferedReader br = new BufferedReader(new InputStreamReader(is, StandardCharsets.UTF_8))) {
String line;
while ((line = br.readLine()) != null) {
if (type.equals("std")) {
System.out.println(line);
} else {
System.err.println(line);
}
}
} catch (IOException e) {
e.printStackTrace();
}
}
}

private File findCpeRunnerJar() {
String classpath = System.getProperty("java.class.path");
Stream<File> classpathDirs = Stream.of(classpath.split(File.pathSeparator)).map(File::new).map(f -> f.isDirectory() ? f : f.getParentFile()).distinct();
Expand All @@ -101,7 +135,7 @@ private File findCpeRunnerJar() {

@Override
public String getName() {
return "CPEBootstrapRunner";
return "CPERunner";
}

@Override
Expand Down
33 changes: 29 additions & 4 deletions jcore-pipeline-runner/jcore-pipeline-runner-cpe/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,38 @@
<parent>
<artifactId>jcore-pipeline-runner</artifactId>
<groupId>de.julielab</groupId>
<version>0.2.5-SNAPSHOT</version>
<version>0.2.5</version>
</parent>
<modelVersion>4.0.0</modelVersion>

<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>3.1.0</version>
<configuration>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
</configuration>
<executions>
<execution>
<id>assemble-all</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
<artifactId>jcore-pipeline-runner-cpe</artifactId>
<dependencies>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
</dependency>
<dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
Expand All @@ -25,12 +51,12 @@
<groupId>de.julielab</groupId>
<artifactId>jcore-types</artifactId>
<version>2.3.1</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>de.julielab</groupId>
<artifactId>jcore-pipeline-builder-base</artifactId>
<version>${project.parent.version}</version>
<scope>provided</scope>
<exclusions>
<exclusion>
<groupId>de.julielab</groupId>
Expand Down Expand Up @@ -63,5 +89,4 @@
</exclusions>
</dependency>
</dependencies>

</project>
46 changes: 13 additions & 33 deletions ...re-pipeline-runner-cpe/src/main/java/de/julielab/jcore/pipeline/runner/cpe/CPERunner.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/**
* DBStatusCallbackListener.java
* StatusCallbackListener.java
*
* Copyright (c) 2008, JULIE Lab.
* All rights reserved. This program and the accompanying materials
Expand All @@ -18,7 +18,6 @@
package de.julielab.jcore.pipeline.runner.cpe;

import de.julielab.jcore.pipeline.builder.base.exceptions.PipelineIOException;
import de.julielab.jcore.pipeline.builder.base.main.JCoReUIMAPipeline;
import org.apache.commons.cli.*;
import org.apache.uima.UIMAFramework;
import org.apache.uima.collection.CollectionProcessingEngine;
Expand All @@ -30,26 +29,18 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;

public class CPERunner {

/**
* the descriptor file for the CPE;
*/
protected String pipelinePath;
protected String descriptorPath;
protected CpeDescription cpeDescription;
protected Options options;
protected CollectionProcessingEngine cpe;
protected DBStatusCallbackListener statusCallbackListener;
/**
* the configuration taken from the Medline Reader and used to instantiate
* the Database Connector
*/
protected String configuration;
protected StatusCallbackListener statusCallbackListener;
protected Integer processingUnitThreadCount;
protected Integer casPoolSize;
protected Integer numbersOfDocuments;
Expand All @@ -65,7 +56,7 @@ public static void main(String[] args) {

public CPERunner() {
options = new Options();
options.addOption("p", true, "JCoRe Pipeline Path");
options.addOption("d", true, "JCoRe Pipeline Path");
options.addOption("n", true,
"numbers of documents to process (optional)");
options.addOption("t", true, "processing unit thread count (optional)");
Expand All @@ -84,9 +75,9 @@ public void parseArguments(String[] args) {
return;
}

pipelinePath = cmd.getOptionValue("p");
if (pipelinePath == null) {
System.err.println("-p option is missed");
descriptorPath = cmd.getOptionValue("d");
if (descriptorPath == null) {
System.err.println("-d option is missed");
error = true;
}

Expand Down Expand Up @@ -119,10 +110,10 @@ public void parseArguments(String[] args) {
*/
public void createCPEDescription() throws InvalidXMLException, IOException,
CpeDescriptorException, PipelineIOException {
LOGGER.info("Creating CPE description from " + pipelinePath + File.separator + "CPE.xml");
LOGGER.info("Creating CPE description from " + descriptorPath);

cpeDescription = UIMAFramework.getXMLParser().parseCpeDescription(
new XMLInputSource(pipelinePath));
new XMLInputSource(descriptorPath));

if (processingUnitThreadCount != null) {
LOGGER.info("Setting processing unit thread count to "
Expand Down Expand Up @@ -169,14 +160,7 @@ public void createCPE() throws InvalidXMLException, IOException,
LOGGER.info("Creating CPE... ");
cpe = UIMAFramework.produceCollectionProcessingEngine(cpeDescription);

InputStream is = null;
File dbcConfigFile = new File(configuration);
if (dbcConfigFile.exists())
is = new FileInputStream(dbcConfigFile);
else
is = getClass().getResourceAsStream("/" + configuration);

statusCallbackListener = new DBStatusCallbackListener(cpe,
statusCallbackListener = new StatusCallbackListener(cpe,
batchSize);
cpe.addStatusCallbackListener(statusCallbackListener);
}
Expand Down Expand Up @@ -240,8 +224,8 @@ protected void runCPE() {
}
}

public String getPipelinePath() {
return pipelinePath;
public String getDescriptorPath() {
return descriptorPath;
}

public CpeDescription getCpeDescription() {
Expand All @@ -256,14 +240,10 @@ public CollectionProcessingEngine getCpe() {
return cpe;
}

public DBStatusCallbackListener getStatusCallbackListener() {
public StatusCallbackListener getStatusCallbackListener() {
return statusCallbackListener;
}

public String getConfiguration() {
return configuration;
}

public Integer getProcessingUnitThreadCount() {
return processingUnitThreadCount;
}
Expand Down
Loading

0 comments on commit 90a10dc

Please sign in to comment.