Skip to content
This repository has been archived by the owner on Mar 24, 2022. It is now read-only.

Commit

Permalink
Move to outputfactor
Browse files Browse the repository at this point in the history
  • Loading branch information
wzorgdrager committed Feb 5, 2021
1 parent 4e7ef38 commit 885f945
Show file tree
Hide file tree
Showing 3 changed files with 24 additions and 18 deletions.
14 changes: 2 additions & 12 deletions src/main/java/eu/fasten/crawler/IncrementalMavenCrawler.java
Original file line number Diff line number Diff line change
@@ -1,9 +1,6 @@
package eu.fasten.crawler;

import eu.fasten.crawler.output.KafkaOutput;
import eu.fasten.crawler.output.Output;
import eu.fasten.crawler.output.RestOutput;
import eu.fasten.crawler.output.StdOutput;
import eu.fasten.crawler.output.*;
import org.apache.commons.cli.*;
import org.codehaus.plexus.util.FileUtils;
import org.slf4j.Logger;
Expand Down Expand Up @@ -97,18 +94,11 @@ public static void main(String[] args) {
}

// Setup arguments for crawler.
Output output = new StdOutput();
int batchSize = Integer.parseInt(properties.getProperty("batch_size"));
int startIndex = Integer.parseInt(properties.getProperty("index"));
int interval = Integer.parseInt(properties.getProperty("interval"));
String checkpointDir = properties.getProperty("checkpoint_dir");

// Setup Kafka.
if (properties.get("output").equals("kafka")) {
output = new KafkaOutput(properties.getProperty("kafka_topic"), properties.getProperty("kafka_brokers"), batchSize);
} else if (properties.get("output").equals("rest")) {
output = new RestOutput(properties.getProperty("rest_endpoint"));
}
Output output = OutputFactory.getOutput(properties.getProperty("output"), properties);

// Start cralwer and execute it with an interval.
IncrementalMavenCrawler crawler = new IncrementalMavenCrawler(startIndex, batchSize, output, checkpointDir);
Expand Down
17 changes: 17 additions & 0 deletions src/main/java/eu/fasten/crawler/output/OutputFactory.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
package eu.fasten.crawler.output;

import java.util.Properties;

public class OutputFactory {

public static Output getOutput(String outputName, Properties properties) {
switch (outputName) {
case "kafka":
return new KafkaOutput(properties.getProperty("kafka_topic"), properties.getProperty("kafka_brokers"), Integer.parseInt(properties.getProperty("batch_size")));
case "rest":
return new RestOutput(properties.getProperty("rest_endpoint"));
default:
return new StdOutput();
}
}
}
11 changes: 5 additions & 6 deletions src/main/java/eu/fasten/crawler/output/RestOutput.java
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.List;
import java.util.stream.Collectors;

public class RestOutput implements Output {

Expand All @@ -36,7 +35,7 @@ public boolean send(List<MavenArtifact> artifacts) {

try {
// Send batch.
StringEntity jsonList = new StringEntity(buildJsonList(artifacts.stream().map((x) -> x.toString()).collect(Collectors.toList())));
StringEntity jsonList = new StringEntity(buildJsonList(artifacts));
httpPost.setEntity(jsonList);
int responseCode = httpClient.execute(httpPost).getStatusLine().getStatusCode();

Expand All @@ -55,16 +54,16 @@ public boolean send(List<MavenArtifact> artifacts) {

/**
* Builds a json list of all artifacts.
* @param artifactStrings all artifacts.
* @param artifacts all artifacts.
* @return a stringified list of all artifacts.
*/
public String buildJsonList(List<String> artifactStrings) {
public String buildJsonList(List<MavenArtifact> artifacts) {
// Build array of JSON objects.
StringBuffer json = new StringBuffer();
json.append("[");

for (String af : artifactStrings) {
json.append(af + ",");
for (MavenArtifact af : artifacts) {
json.append(af.toString() + ",");
}

json.deleteCharAt(json.length() - 1);
Expand Down

0 comments on commit 885f945

Please sign in to comment.