Add memory usage measurement in benchmarks

README.md

@@ -538,13 +538,14 @@ mvn exec:java -Dexec.mainClass="benchmark.GraphBenchmark" -Dexec.args="aguments"
 mvn exec:java -Dexec.mainClass="benchmark.GraphBenchmark" -Dexec.args="-h"
 ```
 ```
-usage: GraphBenchmark [-h] -d <dataset name> -gm <path> -gp <path> -gs <storage type> -m <number> -p <problem type> -S <s=value1 a=value2> -w <number>
+usage: GraphBenchmark [-h] [-mem] -d <dataset name> -gm <path> -gp <path> -gs <storage type> -m <number> -p <problem type> -S <s=value1 a=value2> -w <number>
  -d,--dataset <dataset name>            The name of the dataset, an important component of the file name with the results
  -gm,--grammar <path>                   Path to JSON file contains context-free grammar
  -gp,--graph <path>                     Path to directory contains files nodes.csv and edges.csv
  -gs,--graph_storage <storage type>     Graph storage type, allowed values: NEO4J, IN_MEMORY
  -h,--help                              Print help message
  -m,--measurement_iterations <number>   Number of measurement iterations
+ -mem,--memory_usage                    Whether to measure memory consumption
  -p,--problem <problem type>            Benchmarking algorithm, allowed values: REACHABILITY, ALL_PATHS
  -S,--scenario <s=value1 a=value2>      Benchmarking scenario and its argument, 's' property allowed values: ALL_PAIRS, MULTIPLE_SOURCES, 'a' property
                                         contains number of nodes if 's' equals ALL_PAIRS or path to file with vertices chunks if 's' equals
@@ -559,7 +560,7 @@ Here is an example which can be run right after project is downloaded and result
 To run **all pairs reachability** algorithm on **Core** graph on **G<sub>1</sub>** grammar use the following command:
 
 ```bash
-mvn exec:java -Dexec.mainClass="benchmark.GraphBenchmark" -Dexec.args="-d Core -gm test/resources/grammars/graph/g1/grammar.json -gp data/core -gs NEO4J -p REACHABILITY -S -s ALL_PAIRS -a $(( $(cat "data/core/nodes.csv" | wc -l)-1 )) -w 1 -m 10"
+mvn exec:java -Dexec.mainClass="benchmark.GraphBenchmark" -Dexec.args="-d Core -gm test/resources/grammars/graph/g1/grammar.json -gp data/core -gs NEO4J -p REACHABILITY -S s=ALL_PAIRS -S a=$(( $(cat "data/core/nodes.csv" | wc -l)-1 )) -w 1 -m 10"
 ```
 
 ### Data

src/benchmark/BenchmarkProblem.java

@@ -15,4 +15,6 @@ public static BenchmarkProblem createBenchmarkProblem(Problem problem) {
     public abstract void runAlgo(IguanaParser parser, GraphInput input);
 
     public abstract long getResult();
+
+    public abstract void clearResult();
 }

src/benchmark/BenchmarkProblemAllPaths.java

@@ -22,6 +22,9 @@ public long getResult() {
         return parseResults.size();
     }
 
+    @Override
+    public void clearResult() { parseResults = null; }
+
     @Override
     public String toString() {
         return "SPPF";

src/benchmark/BenchmarkProblemReachability.java

@@ -25,6 +25,9 @@ public long getResult() {
         return parseResults.count();
     }
 
+    @Override
+    public void clearResult() { parseResults = null; }
+
     @Override
     public String toString() {
         return "REACHABILITY";

src/benchmark/BenchmarkScenarioAllPairs.java

@@ -6,10 +6,10 @@
 
 public class BenchmarkScenarioAllPairs extends BenchmarkScenario {
 
-    private final int nodesCount;
+    private final List<List<Integer>> startVerticesChunks;
 
     public BenchmarkScenarioAllPairs(int nodesCount) {
-        this.nodesCount = nodesCount;
+        this.startVerticesChunks = List.of(Interval.zeroTo(nodesCount - 1));
     }
 
     @Override
@@ -24,7 +24,7 @@ public String getCsvRecord(int chunkIndex, double stepPrepareTime, double stepRu
 
     @Override
     public List<List<Integer>> getStartVerticesChunks() {
-        return List.of(Interval.zeroTo(nodesCount - 1));
+        return startVerticesChunks;
     }
 
     @Override

src/benchmark/BenchmarkSettings.java

@@ -19,8 +19,8 @@ public static BenchmarkSettings parseCli(CommandLine cmd) throws ParseException
                 cmd.getOptionValue(CliParser.GRAPH_OPT),
                 cmd.getOptionValue(CliParser.DATASET_OPT),
                 Integer.parseInt(cmd.getOptionValue(CliParser.WARMUP_ITERATIONS_OPT)),
-                Integer.parseInt(cmd.getOptionValue(CliParser.MEASUREMENT_ITERATIONS_OPT)));
-
+                Integer.parseInt(cmd.getOptionValue(CliParser.MEASUREMENT_ITERATIONS_OPT)),
+                cmd.hasOption(CliParser.MEMORY_USAGE));
     }
 
     private final GraphStorage storageType;
@@ -31,6 +31,7 @@ public static BenchmarkSettings parseCli(CommandLine cmd) throws ParseException
     private final String graphPath;
     private final int warmupIterations;
     private final int measurementIterations;
+    private final boolean withMemoryUsage;
 
     private BenchmarkSettings(GraphStorage storageType,
                               Problem problem,
@@ -39,7 +40,8 @@ private BenchmarkSettings(GraphStorage storageType,
                               String graphPath,
                               String datasetName,
                               int warmupIterations,
-                              int measurementIterations) {
+                              int measurementIterations,
+                              boolean withMemoryUsage) {
         this.storageType = storageType;
         this.problem = problem;
         this.scenario = scenario;
@@ -48,6 +50,7 @@ private BenchmarkSettings(GraphStorage storageType,
         this.graphPath = graphPath;
         this.warmupIterations = warmupIterations;
         this.measurementIterations = measurementIterations;
+        this.withMemoryUsage = withMemoryUsage;
     }
 
     public GraphStorage getStorageType() {
@@ -82,6 +85,8 @@ public int getMeasurementIterations() {
         return measurementIterations;
     }
 
+    public boolean isWithMemoryUsage() { return withMemoryUsage; }
+
     public static class ScenarioSettings {
 
         private final Scenario scenario;

src/benchmark/CliParser.java

@@ -17,6 +17,7 @@ public class CliParser {
     public static final String GRAPH_OPT = "gp";
     public static final String WARMUP_ITERATIONS_OPT = "w";
     public static final String MEASUREMENT_ITERATIONS_OPT = "m";
+    public static final String MEMORY_USAGE = "mem";
     private final Option helpOption;
     private final Options allOptions;
 
@@ -105,6 +106,12 @@ public CliParser() {
                 .optionalArg(false)
                 .required(true)
                 .build());
+        allOptions.addOption(Option.builder(MEMORY_USAGE)
+                .longOpt("memory_usage")
+                .hasArg(false)
+                .desc("Whether to measure memory consumption")
+                .required(false)
+                .build());
     }
 
     public boolean hasHelp(String[] args) {

src/benchmark/CsvGenerator.java

@@ -0,0 +1,45 @@
+package benchmark;
+
+import java.io.File;
+
+public class CsvGenerator {
+
+    private final String header;
+    private final BenchmarkScenario scenario;
+    private final String filename;
+
+    public CsvGenerator(BenchmarkScenario scenario,
+                        String resultDir,
+                        String datasetName,
+                        String problem,
+                        String graphStorage) {
+        this.scenario = scenario;
+        this.header = scenario.getCsvHeader();
+
+        this.filename = resultDir +
+                File.separator +
+                datasetName +
+                '_' +
+                scenario +
+                '_' +
+                problem +
+                '_' +
+                graphStorage;
+    }
+
+    protected String getFilenamePart() {
+        return filename;
+    }
+
+    public String getFilename() {
+        return filename + ".csv";
+    }
+
+    public String getHeader() {
+        return header;
+    }
+
+    public String getRecord(int chunkIndex, double stepPrepareTime, double stepRunTime, long result) {
+        return scenario.getCsvRecord(chunkIndex, stepPrepareTime, stepRunTime, result);
+    }
+}

src/benchmark/CsvWithMemoryGenerator.java

@@ -0,0 +1,36 @@
+package benchmark;
+
+import java.lang.management.ManagementFactory;
+import java.lang.management.MemoryType;
+
+public class CsvWithMemoryGenerator extends CsvGenerator {
+    public CsvWithMemoryGenerator(BenchmarkScenario scenario, String resultDir, String datasetName, String problem, String graphStorage) {
+        super(scenario, resultDir, datasetName, problem, graphStorage);
+    }
+
+    @Override
+    public String getFilename() {
+        return super.getFilenamePart() + "_MEMORY.csv";
+    }
+
+    @Override
+    public String getHeader() {
+        return super.getHeader() + ",heap_mem,non_heap_mem";
+    }
+
+    @Override
+    public String getRecord(int chunkIndex, double stepPrepareTime, double stepRunTime, long result) {
+        var heapMem = 0L;
+        var nonHeapMem = 0L;
+        var pools = ManagementFactory.getMemoryPoolMXBeans();
+        for (var memoryPoolMXBean: pools) {
+            var peakUsed = memoryPoolMXBean.getPeakUsage().getUsed();
+            if (memoryPoolMXBean.getType() == MemoryType.HEAP) {
+                heapMem += peakUsed;
+            } else {
+                nonHeapMem += peakUsed;
+            }
+        }
+        return super.getRecord(chunkIndex, stepPrepareTime, stepRunTime, result) + "," + heapMem + "," + nonHeapMem;
+    }
+}

src/benchmark/GraphBenchmark.java

@@ -14,7 +14,7 @@
 public class GraphBenchmark {
     private static final String RESULTS_DIR = "results";
 
-    public static void main(String[] args) throws IOException {
+    public static void main(String[] args) throws IOException, InterruptedException {
         final var cliParser = new CliParser();
         if (cliParser.hasHelp(args)) {
             cliParser.printHelp();
@@ -40,6 +40,7 @@ public static void main(String[] args) throws IOException {
     private final String graphPath;
     private final int warmupIterations;
     private final int measurementIterations;
+    private final CsvGenerator csvGenerator;
 
     private GraphBenchmark(BenchmarkSettings settings) {
         graphStorage = BenchmarkGraphStorage.createBenchmarkStorage(settings.getStorageType());
@@ -50,24 +51,21 @@ private GraphBenchmark(BenchmarkSettings settings) {
         graphPath = settings.getGraphPath();
         warmupIterations = settings.getWarmupIterations();
         measurementIterations = settings.getMeasurementIterations();
+        csvGenerator = settings.isWithMemoryUsage()
+                ? new CsvWithMemoryGenerator(scenario, RESULTS_DIR, datasetName, problem.toString(), graphStorage.toString())
+                : new CsvGenerator(scenario, RESULTS_DIR, datasetName, problem.toString(), graphStorage.toString());
     }
 
-    void benchmark() throws IOException {
+    void benchmark() throws IOException, InterruptedException {
+        MemoryCleaner memoryCleaner = new MemoryCleaner();
         Grammar grammar = Grammar.load(grammarPath, "json");
         graphStorage.loadGraph(graphPath);
-        File outFile = new File("%s%s%s_%s_%s_%s.csv".formatted(
-                RESULTS_DIR,
-                File.separator,
-                datasetName,
-                scenario.toString(),
-                problem.toString(),
-                graphStorage.toString())
-        );
+        File outFile = new File(csvGenerator.getFilename());
         boolean fileExists = !outFile.createNewFile();
         System.out.printf("Benchmarking %s for %s graph%n", problem, datasetName);
         try (PrintWriter outStatsTime = new PrintWriter(new FileOutputStream(outFile, true), true)) {
             if (!fileExists) {
-                outStatsTime.println(scenario.getCsvHeader());
+                outStatsTime.println(csvGenerator.getHeader());
             }
 
             final int maxIters = warmupIterations + measurementIterations;
@@ -88,13 +86,20 @@ void benchmark() throws IOException {
                     final double stepRunTime = (double) (stepStopTime - stepStartTime) / 1_000_000_000.;
 
                     if (iter >= warmupIterations) {
-                        outStatsTime.println(scenario.getCsvRecord(chunkIndex, stepPrepareTime, stepRunTime, problem.getResult()));
+                        outStatsTime.println(csvGenerator.getRecord(chunkIndex, stepPrepareTime, stepRunTime, problem.getResult()));
+                        outStatsTime.flush();
                     }
+                    problem.clearResult();
                     graphStorage.onIterationFinish();
+                    parser = null;
+                    input = null;
+                    chunk = null;
+                    memoryCleaner.freeMemory();
                 }
 
             }
         }
         graphStorage.close();
     }
+
 }

src/benchmark/MemoryCleaner.java

@@ -0,0 +1,37 @@
+package benchmark;
+
+import java.lang.management.GarbageCollectorMXBean;
+import java.lang.management.ManagementFactory;
+
+public class MemoryCleaner {
+
+    public void freeMemory() throws InterruptedException {
+        long m;
+        long m2 = getReallyUsedMemory();
+        do {
+            Thread.sleep(567);
+            m = m2;
+            m2 = getReallyUsedMemory();
+        } while (m2 < m);
+    }
+
+    private long getReallyUsedMemory() {
+        long before = getGcCount();
+        System.gc();
+        while (getGcCount() == before);
+        return getCurrentlyUsedMemory();
+    }
+    private long getGcCount() {
+        long sum = 0;
+        for (GarbageCollectorMXBean b : ManagementFactory.getGarbageCollectorMXBeans()) {
+            long count = b.getCollectionCount();
+            if (count != -1) { sum +=  count; }
+        }
+        return sum;
+    }
+
+    private long getCurrentlyUsedMemory() {
+        return ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getUsed()
+                + ManagementFactory.getMemoryMXBean().getNonHeapMemoryUsage().getUsed();
+    }
+}