diff --git a/README.md b/README.md index 8018752b3..ae0327661 100644 --- a/README.md +++ b/README.md @@ -538,13 +538,14 @@ mvn exec:java -Dexec.mainClass="benchmark.GraphBenchmark" -Dexec.args="aguments" mvn exec:java -Dexec.mainClass="benchmark.GraphBenchmark" -Dexec.args="-h" ``` ``` -usage: GraphBenchmark [-h] -d -gm -gp -gs -m -p -S -w +usage: GraphBenchmark [-h] [-mem] -d -gm -gp -gs -m -p -S -w -d,--dataset The name of the dataset, an important component of the file name with the results -gm,--grammar Path to JSON file contains context-free grammar -gp,--graph Path to directory contains files nodes.csv and edges.csv -gs,--graph_storage Graph storage type, allowed values: NEO4J, IN_MEMORY -h,--help Print help message -m,--measurement_iterations Number of measurement iterations + -mem,--memory_usage Whether to measure memory consumption -p,--problem Benchmarking algorithm, allowed values: REACHABILITY, ALL_PATHS -S,--scenario Benchmarking scenario and its argument, 's' property allowed values: ALL_PAIRS, MULTIPLE_SOURCES, 'a' property contains number of nodes if 's' equals ALL_PAIRS or path to file with vertices chunks if 's' equals @@ -559,7 +560,7 @@ Here is an example which can be run right after project is downloaded and result To run **all pairs reachability** algorithm on **Core** graph on **G1** grammar use the following command: ```bash -mvn exec:java -Dexec.mainClass="benchmark.GraphBenchmark" -Dexec.args="-d Core -gm test/resources/grammars/graph/g1/grammar.json -gp data/core -gs NEO4J -p REACHABILITY -S -s ALL_PAIRS -a $(( $(cat "data/core/nodes.csv" | wc -l)-1 )) -w 1 -m 10" +mvn exec:java -Dexec.mainClass="benchmark.GraphBenchmark" -Dexec.args="-d Core -gm test/resources/grammars/graph/g1/grammar.json -gp data/core -gs NEO4J -p REACHABILITY -S s=ALL_PAIRS -S a=$(( $(cat "data/core/nodes.csv" | wc -l)-1 )) -w 1 -m 10" ``` ### Data diff --git a/src/benchmark/BenchmarkProblem.java b/src/benchmark/BenchmarkProblem.java index 427eb1c8b..a5301b14d 100644 --- a/src/benchmark/BenchmarkProblem.java +++ b/src/benchmark/BenchmarkProblem.java @@ -15,4 +15,6 @@ public static BenchmarkProblem createBenchmarkProblem(Problem problem) { public abstract void runAlgo(IguanaParser parser, GraphInput input); public abstract long getResult(); + + public abstract void clearResult(); } diff --git a/src/benchmark/BenchmarkProblemAllPaths.java b/src/benchmark/BenchmarkProblemAllPaths.java index d195e8aa3..cdb81605d 100644 --- a/src/benchmark/BenchmarkProblemAllPaths.java +++ b/src/benchmark/BenchmarkProblemAllPaths.java @@ -22,6 +22,9 @@ public long getResult() { return parseResults.size(); } + @Override + public void clearResult() { parseResults = null; } + @Override public String toString() { return "SPPF"; diff --git a/src/benchmark/BenchmarkProblemReachability.java b/src/benchmark/BenchmarkProblemReachability.java index 38902ac61..e8ec4529e 100644 --- a/src/benchmark/BenchmarkProblemReachability.java +++ b/src/benchmark/BenchmarkProblemReachability.java @@ -25,6 +25,9 @@ public long getResult() { return parseResults.count(); } + @Override + public void clearResult() { parseResults = null; } + @Override public String toString() { return "REACHABILITY"; diff --git a/src/benchmark/BenchmarkScenarioAllPairs.java b/src/benchmark/BenchmarkScenarioAllPairs.java index 4f9c7bec4..7bd11bfbd 100644 --- a/src/benchmark/BenchmarkScenarioAllPairs.java +++ b/src/benchmark/BenchmarkScenarioAllPairs.java @@ -6,10 +6,10 @@ public class BenchmarkScenarioAllPairs extends BenchmarkScenario { - private final int nodesCount; + private final List> startVerticesChunks; public BenchmarkScenarioAllPairs(int nodesCount) { - this.nodesCount = nodesCount; + this.startVerticesChunks = List.of(Interval.zeroTo(nodesCount - 1)); } @Override @@ -24,7 +24,7 @@ public String getCsvRecord(int chunkIndex, double stepPrepareTime, double stepRu @Override public List> getStartVerticesChunks() { - return List.of(Interval.zeroTo(nodesCount - 1)); + return startVerticesChunks; } @Override diff --git a/src/benchmark/BenchmarkSettings.java b/src/benchmark/BenchmarkSettings.java index a8a8708d0..0d70e3dc7 100644 --- a/src/benchmark/BenchmarkSettings.java +++ b/src/benchmark/BenchmarkSettings.java @@ -19,8 +19,8 @@ public static BenchmarkSettings parseCli(CommandLine cmd) throws ParseException cmd.getOptionValue(CliParser.GRAPH_OPT), cmd.getOptionValue(CliParser.DATASET_OPT), Integer.parseInt(cmd.getOptionValue(CliParser.WARMUP_ITERATIONS_OPT)), - Integer.parseInt(cmd.getOptionValue(CliParser.MEASUREMENT_ITERATIONS_OPT))); - + Integer.parseInt(cmd.getOptionValue(CliParser.MEASUREMENT_ITERATIONS_OPT)), + cmd.hasOption(CliParser.MEMORY_USAGE)); } private final GraphStorage storageType; @@ -31,6 +31,7 @@ public static BenchmarkSettings parseCli(CommandLine cmd) throws ParseException private final String graphPath; private final int warmupIterations; private final int measurementIterations; + private final boolean withMemoryUsage; private BenchmarkSettings(GraphStorage storageType, Problem problem, @@ -39,7 +40,8 @@ private BenchmarkSettings(GraphStorage storageType, String graphPath, String datasetName, int warmupIterations, - int measurementIterations) { + int measurementIterations, + boolean withMemoryUsage) { this.storageType = storageType; this.problem = problem; this.scenario = scenario; @@ -48,6 +50,7 @@ private BenchmarkSettings(GraphStorage storageType, this.graphPath = graphPath; this.warmupIterations = warmupIterations; this.measurementIterations = measurementIterations; + this.withMemoryUsage = withMemoryUsage; } public GraphStorage getStorageType() { @@ -82,6 +85,8 @@ public int getMeasurementIterations() { return measurementIterations; } + public boolean isWithMemoryUsage() { return withMemoryUsage; } + public static class ScenarioSettings { private final Scenario scenario; diff --git a/src/benchmark/CliParser.java b/src/benchmark/CliParser.java index 1130be3fc..35ca4bb4b 100644 --- a/src/benchmark/CliParser.java +++ b/src/benchmark/CliParser.java @@ -17,6 +17,7 @@ public class CliParser { public static final String GRAPH_OPT = "gp"; public static final String WARMUP_ITERATIONS_OPT = "w"; public static final String MEASUREMENT_ITERATIONS_OPT = "m"; + public static final String MEMORY_USAGE = "mem"; private final Option helpOption; private final Options allOptions; @@ -105,6 +106,12 @@ public CliParser() { .optionalArg(false) .required(true) .build()); + allOptions.addOption(Option.builder(MEMORY_USAGE) + .longOpt("memory_usage") + .hasArg(false) + .desc("Whether to measure memory consumption") + .required(false) + .build()); } public boolean hasHelp(String[] args) { diff --git a/src/benchmark/CsvGenerator.java b/src/benchmark/CsvGenerator.java new file mode 100644 index 000000000..a31ce330b --- /dev/null +++ b/src/benchmark/CsvGenerator.java @@ -0,0 +1,45 @@ +package benchmark; + +import java.io.File; + +public class CsvGenerator { + + private final String header; + private final BenchmarkScenario scenario; + private final String filename; + + public CsvGenerator(BenchmarkScenario scenario, + String resultDir, + String datasetName, + String problem, + String graphStorage) { + this.scenario = scenario; + this.header = scenario.getCsvHeader(); + + this.filename = resultDir + + File.separator + + datasetName + + '_' + + scenario + + '_' + + problem + + '_' + + graphStorage; + } + + protected String getFilenamePart() { + return filename; + } + + public String getFilename() { + return filename + ".csv"; + } + + public String getHeader() { + return header; + } + + public String getRecord(int chunkIndex, double stepPrepareTime, double stepRunTime, long result) { + return scenario.getCsvRecord(chunkIndex, stepPrepareTime, stepRunTime, result); + } +} diff --git a/src/benchmark/CsvWithMemoryGenerator.java b/src/benchmark/CsvWithMemoryGenerator.java new file mode 100644 index 000000000..c81341e4d --- /dev/null +++ b/src/benchmark/CsvWithMemoryGenerator.java @@ -0,0 +1,36 @@ +package benchmark; + +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryType; + +public class CsvWithMemoryGenerator extends CsvGenerator { + public CsvWithMemoryGenerator(BenchmarkScenario scenario, String resultDir, String datasetName, String problem, String graphStorage) { + super(scenario, resultDir, datasetName, problem, graphStorage); + } + + @Override + public String getFilename() { + return super.getFilenamePart() + "_MEMORY.csv"; + } + + @Override + public String getHeader() { + return super.getHeader() + ",heap_mem,non_heap_mem"; + } + + @Override + public String getRecord(int chunkIndex, double stepPrepareTime, double stepRunTime, long result) { + var heapMem = 0L; + var nonHeapMem = 0L; + var pools = ManagementFactory.getMemoryPoolMXBeans(); + for (var memoryPoolMXBean: pools) { + var peakUsed = memoryPoolMXBean.getPeakUsage().getUsed(); + if (memoryPoolMXBean.getType() == MemoryType.HEAP) { + heapMem += peakUsed; + } else { + nonHeapMem += peakUsed; + } + } + return super.getRecord(chunkIndex, stepPrepareTime, stepRunTime, result) + "," + heapMem + "," + nonHeapMem; + } +} diff --git a/src/benchmark/GraphBenchmark.java b/src/benchmark/GraphBenchmark.java index e6ea7ec9f..1ea57fca6 100644 --- a/src/benchmark/GraphBenchmark.java +++ b/src/benchmark/GraphBenchmark.java @@ -14,7 +14,7 @@ public class GraphBenchmark { private static final String RESULTS_DIR = "results"; - public static void main(String[] args) throws IOException { + public static void main(String[] args) throws IOException, InterruptedException { final var cliParser = new CliParser(); if (cliParser.hasHelp(args)) { cliParser.printHelp(); @@ -40,6 +40,7 @@ public static void main(String[] args) throws IOException { private final String graphPath; private final int warmupIterations; private final int measurementIterations; + private final CsvGenerator csvGenerator; private GraphBenchmark(BenchmarkSettings settings) { graphStorage = BenchmarkGraphStorage.createBenchmarkStorage(settings.getStorageType()); @@ -50,24 +51,21 @@ private GraphBenchmark(BenchmarkSettings settings) { graphPath = settings.getGraphPath(); warmupIterations = settings.getWarmupIterations(); measurementIterations = settings.getMeasurementIterations(); + csvGenerator = settings.isWithMemoryUsage() + ? new CsvWithMemoryGenerator(scenario, RESULTS_DIR, datasetName, problem.toString(), graphStorage.toString()) + : new CsvGenerator(scenario, RESULTS_DIR, datasetName, problem.toString(), graphStorage.toString()); } - void benchmark() throws IOException { + void benchmark() throws IOException, InterruptedException { + MemoryCleaner memoryCleaner = new MemoryCleaner(); Grammar grammar = Grammar.load(grammarPath, "json"); graphStorage.loadGraph(graphPath); - File outFile = new File("%s%s%s_%s_%s_%s.csv".formatted( - RESULTS_DIR, - File.separator, - datasetName, - scenario.toString(), - problem.toString(), - graphStorage.toString()) - ); + File outFile = new File(csvGenerator.getFilename()); boolean fileExists = !outFile.createNewFile(); System.out.printf("Benchmarking %s for %s graph%n", problem, datasetName); try (PrintWriter outStatsTime = new PrintWriter(new FileOutputStream(outFile, true), true)) { if (!fileExists) { - outStatsTime.println(scenario.getCsvHeader()); + outStatsTime.println(csvGenerator.getHeader()); } final int maxIters = warmupIterations + measurementIterations; @@ -88,13 +86,20 @@ void benchmark() throws IOException { final double stepRunTime = (double) (stepStopTime - stepStartTime) / 1_000_000_000.; if (iter >= warmupIterations) { - outStatsTime.println(scenario.getCsvRecord(chunkIndex, stepPrepareTime, stepRunTime, problem.getResult())); + outStatsTime.println(csvGenerator.getRecord(chunkIndex, stepPrepareTime, stepRunTime, problem.getResult())); + outStatsTime.flush(); } + problem.clearResult(); graphStorage.onIterationFinish(); + parser = null; + input = null; + chunk = null; + memoryCleaner.freeMemory(); } } } graphStorage.close(); } + } diff --git a/src/benchmark/MemoryCleaner.java b/src/benchmark/MemoryCleaner.java new file mode 100644 index 000000000..9b18d8921 --- /dev/null +++ b/src/benchmark/MemoryCleaner.java @@ -0,0 +1,37 @@ +package benchmark; + +import java.lang.management.GarbageCollectorMXBean; +import java.lang.management.ManagementFactory; + +public class MemoryCleaner { + + public void freeMemory() throws InterruptedException { + long m; + long m2 = getReallyUsedMemory(); + do { + Thread.sleep(567); + m = m2; + m2 = getReallyUsedMemory(); + } while (m2 < m); + } + + private long getReallyUsedMemory() { + long before = getGcCount(); + System.gc(); + while (getGcCount() == before); + return getCurrentlyUsedMemory(); + } + private long getGcCount() { + long sum = 0; + for (GarbageCollectorMXBean b : ManagementFactory.getGarbageCollectorMXBeans()) { + long count = b.getCollectionCount(); + if (count != -1) { sum += count; } + } + return sum; + } + + private long getCurrentlyUsedMemory() { + return ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getUsed() + + ManagementFactory.getMemoryMXBean().getNonHeapMemoryUsage().getUsed(); + } +}