From e0ec47d147c371929df9cd013726ed73cddc19ae Mon Sep 17 00:00:00 2001 From: Pierre Laporte Date: Fri, 26 Feb 2021 10:51:49 +0100 Subject: [PATCH 1/7] Read data from InputStream instead of File --- src/main/java/HdrToCsv.java | 50 ++++---- src/main/java/SplitHistogramLogs.java | 64 +++++----- src/main/java/SummarizeHistogramLogs.java | 113 +++++++++--------- src/main/java/UnionHistogramLogs.java | 15 +-- .../lob/saw/OrderedHistogramLogReader.java | 20 ++-- 5 files changed, 138 insertions(+), 124 deletions(-) diff --git a/src/main/java/HdrToCsv.java b/src/main/java/HdrToCsv.java index 9b77399..f5c1bba 100644 --- a/src/main/java/HdrToCsv.java +++ b/src/main/java/HdrToCsv.java @@ -3,7 +3,9 @@ import psy.lob.saw.OrderedHistogramLogReader; import java.io.File; -import java.io.FileNotFoundException; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; import java.nio.file.Paths; import java.util.Locale; @@ -34,34 +36,32 @@ public void setInputFile(String fileName) @Override public void run() { - OrderedHistogramLogReader reader = null; - try + try(InputStream inputStream = new FileInputStream(inputFile)) { - reader = new OrderedHistogramLogReader(inputFile); + OrderedHistogramLogReader reader = new OrderedHistogramLogReader(inputStream); + System.out.println( + "#Absolute timestamp,Relative timestamp,Throughput,Min,Avg,p50,p90,p95,p99,p999,p9999,Max"); + while (reader.hasNext()) + { + Histogram interval = (Histogram) reader.nextIntervalHistogram(); + System.out.printf(Locale.US, + "%.3f,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d%n", + interval.getStartTimeStamp() / 1000.0, + interval.getStartTimeStamp() / 1000 - (long) reader.getStartTimeSec(), + interval.getTotalCount(), interval.getMinValue(), + (long) interval.getMean(), + interval.getValueAtPercentile(50), + interval.getValueAtPercentile(90), + interval.getValueAtPercentile(95), + interval.getValueAtPercentile(99), + interval.getValueAtPercentile(99.9), + interval.getValueAtPercentile(99.99), + interval.getMaxValue()); + } } - catch (FileNotFoundException e) + catch (IOException e) { throw new RuntimeException(e); } - - System.out.println( - "#Absolute timestamp,Relative timestamp,Throughput,Min,Avg,p50,p90,p95,p99,p999,p9999,Max"); - while (reader.hasNext()) - { - Histogram interval = (Histogram) reader.nextIntervalHistogram(); - System.out.printf(Locale.US, - "%.3f,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d%n", - interval.getStartTimeStamp() / 1000.0, - interval.getStartTimeStamp() / 1000 - (long) reader.getStartTimeSec(), - interval.getTotalCount(), interval.getMinValue(), - (long) interval.getMean(), - interval.getValueAtPercentile(50), - interval.getValueAtPercentile(90), - interval.getValueAtPercentile(95), - interval.getValueAtPercentile(99), - interval.getValueAtPercentile(99.9), - interval.getValueAtPercentile(99.99), - interval.getMaxValue()); - } } } \ No newline at end of file diff --git a/src/main/java/SplitHistogramLogs.java b/src/main/java/SplitHistogramLogs.java index b7b1e8c..eb24622 100644 --- a/src/main/java/SplitHistogramLogs.java +++ b/src/main/java/SplitHistogramLogs.java @@ -5,7 +5,9 @@ import psy.lob.saw.OrderedHistogramLogReader; import java.io.File; -import java.io.FileNotFoundException; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; import java.util.HashMap; import java.util.HashSet; import java.util.Map; @@ -101,36 +103,42 @@ public void run() } } - private void split() throws FileNotFoundException + private void split() throws IOException { - OrderedHistogramLogReader reader = new OrderedHistogramLogReader( - inputFile, - start, - end, - tag -> shouldSkipTag(tag)); - Map writerByTag = new HashMap<>(); - Histogram interval; - int i = 0; - while (reader.hasNext()) + try(InputStream inputStream = new FileInputStream(inputFile)) { - interval = (Histogram) reader.nextIntervalHistogram(); - if (interval == null) + OrderedHistogramLogReader reader = new OrderedHistogramLogReader( + inputStream, + start, + end, + tag -> shouldSkipTag(tag)); + Map writerByTag = new HashMap<>(); + Histogram interval; + int i = 0; + while (reader.hasNext()) { - continue; + interval = (Histogram) reader.nextIntervalHistogram(); + if (interval == null) + { + continue; + } + String ntag = interval.getTag(); + if (shouldSkipTag(ntag)) + { + throw new IllegalStateException("Should be filtered upfront by the reader"); + } + if (verbose) + { + logHistogramForVerbose(System.out, interval, i++); + } + interval.setTag(null); + HistogramLogWriter writer = writerByTag.computeIfAbsent(ntag, k -> createWriterForTag(reader, k)); + writer.outputIntervalHistogram(interval); } - String ntag = interval.getTag(); - if (shouldSkipTag(ntag)) - { - throw new IllegalStateException("Should be filtered upfront by the reader"); - } - if (verbose) - { - logHistogramForVerbose(System.out, interval, i++); - } - interval.setTag(null); - HistogramLogWriter writer = writerByTag.computeIfAbsent(ntag, k -> createWriterForTag(reader, k)); - writer.outputIntervalHistogram(interval); - + } + catch (IOException e) + { + throw new RuntimeException(e); } } @@ -148,4 +156,4 @@ private HistogramLogWriter createWriterForTag(OrderedHistogramLogReader reader, HistogramLogWriter writer = HdrHistogramUtil.createLogWriter(outputFile, comment, reader.getStartTimeSec()); return writer; } -} +} \ No newline at end of file diff --git a/src/main/java/SummarizeHistogramLogs.java b/src/main/java/SummarizeHistogramLogs.java index a62f9c7..97b9bad 100644 --- a/src/main/java/SummarizeHistogramLogs.java +++ b/src/main/java/SummarizeHistogramLogs.java @@ -3,10 +3,7 @@ import org.kohsuke.args4j.Option; import psy.lob.saw.OrderedHistogramLogReader; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.PrintStream; +import java.io.*; import java.util.*; import java.util.function.Predicate; import java.util.regex.Pattern; @@ -134,69 +131,77 @@ private void summarizeAndPrint() throws FileNotFoundException long intervalLengthSum = 0; for (File inputFile : inputFiles) { - if (verbose) + try(InputStream inputStream = new FileInputStream(inputFile)) { - System.out.println("Summarizing file: " + inputFile.getName()); - } - OrderedHistogramLogReader reader = new OrderedHistogramLogReader( - inputFile, - start, - end, - tag -> shouldSkipTag(tag)); - Histogram interval; - int i = 0; - boolean first = true; - long startTime = 0; - - while (reader.hasNext()) - { - interval = (Histogram) reader.nextIntervalHistogram(); - if (interval == null) + if (verbose) { - continue; + System.out.println("Summarizing file: " + inputFile.getName()); } - if (first) + OrderedHistogramLogReader reader = new OrderedHistogramLogReader( + inputStream, + start, + end, + tag -> shouldSkipTag(tag)); + Histogram interval; + int i = 0; + boolean first = true; + long startTime = 0; + + while (reader.hasNext()) { - first = false; - startTime = interval.getStartTimeStamp(); - if (verbose) + interval = (Histogram) reader.nextIntervalHistogram(); + if (interval == null) { - System.out.println("StartTime: " + new Date(startTime)); + continue; } + if (first) + { + first = false; + startTime = interval.getStartTimeStamp(); + if (verbose) + { + System.out.println("StartTime: " + new Date(startTime)); + } + } + String ntag = ignoreTag ? null : interval.getTag(); + final int numberOfSignificantValueDigits = interval.getNumberOfSignificantValueDigits(); + Histogram sum = sumByTag.computeIfAbsent(ntag, k -> + { + Histogram h = new Histogram(numberOfSignificantValueDigits); + h.setTag(k); + return h; + }); + final long intervalLength = interval.getEndTimeStamp() - interval.getStartTimeStamp(); + intervalLengthSum += intervalLength; + sum.add(interval); + if (verbose) + { + logHistogramForVerbose(System.out, interval, i++, outputValueUnitRatio); + } } - String ntag = ignoreTag ? null : interval.getTag(); - final int numberOfSignificantValueDigits = interval.getNumberOfSignificantValueDigits(); - Histogram sum = sumByTag.computeIfAbsent(ntag, k -> - { - Histogram h = new Histogram(numberOfSignificantValueDigits); - h.setTag(k); - return h; - }); - final long intervalLength = interval.getEndTimeStamp() - interval.getStartTimeStamp(); - intervalLengthSum += intervalLength; - sum.add(interval); - if (verbose) + // calculate period + long maxPeriod = 0; + for (Histogram sum : sumByTag.values()) { - logHistogramForVerbose(System.out, interval, i++, outputValueUnitRatio); + long sumPeriod = (sum.getEndTimeStamp() - sum.getStartTimeStamp()); + if (verbose) + { + System.out.print(inputFile.getName()); + System.out.print(", "); + logHistogramForVerbose(System.out, sum, i++, outputValueUnitRatio); + } + sum.setEndTimeStamp(0); + sum.setStartTimeStamp(Long.MAX_VALUE); + maxPeriod = Math.max(maxPeriod, sumPeriod); } + period += maxPeriod; } - // calculate period - long maxPeriod = 0; - for (Histogram sum : sumByTag.values()) + catch (IOException e) { - long sumPeriod = (sum.getEndTimeStamp() - sum.getStartTimeStamp()); - if (verbose) - { - System.out.print(inputFile.getName()); - System.out.print(", "); - logHistogramForVerbose(System.out, sum, i++, outputValueUnitRatio); - } - sum.setEndTimeStamp(0); - sum.setStartTimeStamp(Long.MAX_VALUE); - maxPeriod = Math.max(maxPeriod, sumPeriod); + throw new RuntimeException(e); } - period += maxPeriod; + } if (ignoreTimeStamps) { diff --git a/src/main/java/UnionHistogramLogs.java b/src/main/java/UnionHistogramLogs.java index 46f5cc6..5966163 100644 --- a/src/main/java/UnionHistogramLogs.java +++ b/src/main/java/UnionHistogramLogs.java @@ -6,9 +6,7 @@ import psy.lob.saw.OrderedHistogramLogReader; import psy.lob.saw.UnionHistograms; -import java.io.File; -import java.io.FileOutputStream; -import java.io.PrintStream; +import java.io.*; import java.util.*; import java.util.function.Predicate; import java.util.regex.Pattern; @@ -152,10 +150,13 @@ public void run() List ins = new ArrayList<>(); for (File inputFile : inputFiles) { - ins.add(new HistogramIterator( - new OrderedHistogramLogReader(inputFile, start, end), - inputFilesTags.get(inputFile), - relative)); + try(InputStream inputStream = new FileInputStream(inputFile)) + { + ins.add(new HistogramIterator( + new OrderedHistogramLogReader(inputStream, start, end), + inputFilesTags.get(inputFile), + relative)); + } } UnionHistograms unionHistograms = new UnionHistograms(verbose, System.out, ins, new HistogramSink() { diff --git a/src/main/java/psy/lob/saw/OrderedHistogramLogReader.java b/src/main/java/psy/lob/saw/OrderedHistogramLogReader.java index f3c02fc..17df9aa 100644 --- a/src/main/java/psy/lob/saw/OrderedHistogramLogReader.java +++ b/src/main/java/psy/lob/saw/OrderedHistogramLogReader.java @@ -10,8 +10,8 @@ import org.HdrHistogram.EncodableHistogram; import java.io.Closeable; -import java.io.File; import java.io.FileNotFoundException; +import java.io.InputStream; import java.util.function.Predicate; import java.util.zip.DataFormatException; @@ -143,26 +143,26 @@ public boolean onException(Throwable t) private EncodableHistogram nextHistogram; private boolean inRange = true; - public OrderedHistogramLogReader(final File inputFile) throws FileNotFoundException + public OrderedHistogramLogReader(final InputStream inputStream) throws FileNotFoundException { - this(inputFile, 0.0, Long.MAX_VALUE * 1.0, s -> false, false); + this(inputStream, 0.0, Long.MAX_VALUE * 1.0, s -> false, false); } - public OrderedHistogramLogReader(File inputFile, double start, double end) throws FileNotFoundException + public OrderedHistogramLogReader(InputStream inputStream, double start, double end) throws FileNotFoundException { - this(inputFile, start, end, s -> false, false); + this(inputStream, start, end, s -> false, false); } - public OrderedHistogramLogReader(File inputFile, double start, double end, Predicate shouldExcludeTag) + public OrderedHistogramLogReader(InputStream inputStream, double start, double end, Predicate shouldExcludeTag) throws FileNotFoundException { - this(inputFile, start, end, shouldExcludeTag, false); + this(inputStream, start, end, shouldExcludeTag, false); } /** * Constructs a new OrderedHistogramLogReader that produces intervals read from the specified file. * - * @param inputFile The File to read from + * @param inputStream The data stream to read from * @param rangeStartTimeSec * @param rangeEndTimeSec * @param shouldExcludeTag predicate returns true is tag should be skipped @@ -170,12 +170,12 @@ public OrderedHistogramLogReader(File inputFile, double start, double end, Predi * @throws FileNotFoundException when unable to find inputFile */ public OrderedHistogramLogReader( - final File inputFile, + final InputStream inputStream, double rangeStartTimeSec, double rangeEndTimeSec, Predicate shouldExcludeTag, boolean absolute) throws FileNotFoundException { - scanner = new HistogramLogScanner(inputFile); + scanner = new HistogramLogScanner(inputStream); this.rangeStartTimeSec = rangeStartTimeSec; this.rangeEndTimeSec = rangeEndTimeSec; this.absolute = absolute; From 8039662525270b5805417e84e972299687b5b727 Mon Sep 17 00:00:00 2001 From: Pierre Laporte Date: Fri, 26 Feb 2021 14:37:54 +0100 Subject: [PATCH 2/7] Extract the split logic into a class that consumes an InputStream --- src/main/java/SplitHistogramLogs.java | 67 ++++++++++++++++----------- 1 file changed, 41 insertions(+), 26 deletions(-) diff --git a/src/main/java/SplitHistogramLogs.java b/src/main/java/SplitHistogramLogs.java index eb24622..57cb06f 100644 --- a/src/main/java/SplitHistogramLogs.java +++ b/src/main/java/SplitHistogramLogs.java @@ -6,12 +6,13 @@ import java.io.File; import java.io.FileInputStream; -import java.io.IOException; +import java.io.FileNotFoundException; import java.io.InputStream; import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Set; +import java.util.function.Predicate; import static psy.lob.saw.HdrHistogramUtil.logHistogramForVerbose; @@ -93,9 +94,9 @@ public void run() System.out.printf("start:%.2f end: MAX path:%s file:%s \n", start, absolutePath, name); } } - try + try(InputStream inputStream = new FileInputStream(inputFile)) { - split(); + new Splitter(inputFile.getName(), inputStream, start, end, verbose, this::shouldSkipTag).split(); } catch (Exception e) { @@ -103,15 +104,40 @@ public void run() } } - private void split() throws IOException + private boolean shouldSkipTag(String ntag) { - try(InputStream inputStream = new FileInputStream(inputFile)) + ntag = (ntag == null) ? "default" : ntag; + return excludeTags.contains(ntag) || (!includeTags.isEmpty() && !includeTags.contains(ntag)); + } + + public static class Splitter + { + private final String inputFileName; + private final InputStream inputStream; + private final double start; + private final double end; + private final boolean verbose; + private final Predicate tagExclusionPredicate; + + public Splitter(String inputFileName, InputStream inputStream, + double start, double end, + boolean verbose, Predicate tagExclusionPredicate) + { + this.inputFileName = inputFileName; + this.inputStream = inputStream; + this.start = start; + this.end = end; + this.verbose = verbose; + this.tagExclusionPredicate = tagExclusionPredicate; + } + + public void split() throws FileNotFoundException { OrderedHistogramLogReader reader = new OrderedHistogramLogReader( inputStream, start, end, - tag -> shouldSkipTag(tag)); + tagExclusionPredicate); Map writerByTag = new HashMap<>(); Histogram interval; int i = 0; @@ -123,37 +149,26 @@ private void split() throws IOException continue; } String ntag = interval.getTag(); - if (shouldSkipTag(ntag)) + if (tagExclusionPredicate.test(ntag)) { throw new IllegalStateException("Should be filtered upfront by the reader"); } - if (verbose) + if (this.verbose) { logHistogramForVerbose(System.out, interval, i++); } interval.setTag(null); - HistogramLogWriter writer = writerByTag.computeIfAbsent(ntag, k -> createWriterForTag(reader, k)); + HistogramLogWriter writer = writerByTag.computeIfAbsent(ntag, k -> createWriterForTag(reader, k, inputFileName)); writer.outputIntervalHistogram(interval); } } - catch (IOException e) + + private HistogramLogWriter createWriterForTag(OrderedHistogramLogReader reader, String tag, String inputFileName) { - throw new RuntimeException(e); + tag = (tag == null) ? "default" : tag; + File outputFile = new File(tag + "." + inputFileName); + String comment = "Splitting of:" + inputFileName + " start:" + start + " end:" + end; + return HdrHistogramUtil.createLogWriter(outputFile, comment, reader.getStartTimeSec()); } } - - private boolean shouldSkipTag(String ntag) - { - ntag = (ntag == null) ? "default" : ntag; - return excludeTags.contains(ntag) || (!includeTags.isEmpty() && !includeTags.contains(ntag)); - } - - private HistogramLogWriter createWriterForTag(OrderedHistogramLogReader reader, String tag) - { - tag = (tag == null) ? "default" : tag; - File outputFile = new File(tag + "." + inputFile.getName()); - String comment = "Splitting of:" + inputFile.getName() + " start:" + start + " end:" + end; - HistogramLogWriter writer = HdrHistogramUtil.createLogWriter(outputFile, comment, reader.getStartTimeSec()); - return writer; - } } \ No newline at end of file From ef0571925f631077b35aaf59a31e5d5831f2c2d1 Mon Sep 17 00:00:00 2001 From: Pierre Laporte Date: Fri, 26 Feb 2021 14:48:39 +0100 Subject: [PATCH 3/7] Extract the CSV conversion logic into a class that consumes an InputStream --- src/main/java/HdrToCsv.java | 54 ++++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 21 deletions(-) diff --git a/src/main/java/HdrToCsv.java b/src/main/java/HdrToCsv.java index f5c1bba..7fa4845 100644 --- a/src/main/java/HdrToCsv.java +++ b/src/main/java/HdrToCsv.java @@ -2,10 +2,7 @@ import org.kohsuke.args4j.Option; import psy.lob.saw.OrderedHistogramLogReader; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; +import java.io.*; import java.nio.file.Paths; import java.util.Locale; @@ -37,31 +34,46 @@ public void setInputFile(String fileName) public void run() { try(InputStream inputStream = new FileInputStream(inputFile)) + { + new Converter(inputStream).convert(); + } + catch (IOException e) + { + throw new RuntimeException(e); + } + } + + public static class Converter + { + private final InputStream inputStream; + + public Converter(InputStream inputStream) + { + this.inputStream = inputStream; + } + + public void convert() throws FileNotFoundException { OrderedHistogramLogReader reader = new OrderedHistogramLogReader(inputStream); System.out.println( - "#Absolute timestamp,Relative timestamp,Throughput,Min,Avg,p50,p90,p95,p99,p999,p9999,Max"); + "#Absolute timestamp,Relative timestamp,Throughput,Min,Avg,p50,p90,p95,p99,p999,p9999,Max"); while (reader.hasNext()) { Histogram interval = (Histogram) reader.nextIntervalHistogram(); System.out.printf(Locale.US, - "%.3f,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d%n", - interval.getStartTimeStamp() / 1000.0, - interval.getStartTimeStamp() / 1000 - (long) reader.getStartTimeSec(), - interval.getTotalCount(), interval.getMinValue(), - (long) interval.getMean(), - interval.getValueAtPercentile(50), - interval.getValueAtPercentile(90), - interval.getValueAtPercentile(95), - interval.getValueAtPercentile(99), - interval.getValueAtPercentile(99.9), - interval.getValueAtPercentile(99.99), - interval.getMaxValue()); + "%.3f,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d%n", + interval.getStartTimeStamp() / 1000.0, + interval.getStartTimeStamp() / 1000 - (long) reader.getStartTimeSec(), + interval.getTotalCount(), interval.getMinValue(), + (long) interval.getMean(), + interval.getValueAtPercentile(50), + interval.getValueAtPercentile(90), + interval.getValueAtPercentile(95), + interval.getValueAtPercentile(99), + interval.getValueAtPercentile(99.9), + interval.getValueAtPercentile(99.99), + interval.getMaxValue()); } } - catch (IOException e) - { - throw new RuntimeException(e); - } } } \ No newline at end of file From 38f67f9c9101bd029eccbaf7d72c11ff459f6d6f Mon Sep 17 00:00:00 2001 From: Pierre Laporte Date: Fri, 26 Feb 2021 18:19:43 +0100 Subject: [PATCH 4/7] Move tasks to package so that they can be imported --- src/main/java/HdrToCsv.java | 40 +--------- src/main/java/SplitHistogramLogs.java | 73 +----------------- src/main/java/psy/lob/saw/CsvConverter.java | 41 ++++++++++ .../java/psy/lob/saw/HistogramsSplitter.java | 75 +++++++++++++++++++ 4 files changed, 120 insertions(+), 109 deletions(-) create mode 100644 src/main/java/psy/lob/saw/CsvConverter.java create mode 100644 src/main/java/psy/lob/saw/HistogramsSplitter.java diff --git a/src/main/java/HdrToCsv.java b/src/main/java/HdrToCsv.java index 7fa4845..a073704 100644 --- a/src/main/java/HdrToCsv.java +++ b/src/main/java/HdrToCsv.java @@ -1,10 +1,8 @@ -import org.HdrHistogram.Histogram; import org.kohsuke.args4j.Option; -import psy.lob.saw.OrderedHistogramLogReader; +import psy.lob.saw.CsvConverter; import java.io.*; import java.nio.file.Paths; -import java.util.Locale; public class HdrToCsv implements Runnable { @@ -35,45 +33,11 @@ public void run() { try(InputStream inputStream = new FileInputStream(inputFile)) { - new Converter(inputStream).convert(); + new CsvConverter(inputStream).convert(); } catch (IOException e) { throw new RuntimeException(e); } } - - public static class Converter - { - private final InputStream inputStream; - - public Converter(InputStream inputStream) - { - this.inputStream = inputStream; - } - - public void convert() throws FileNotFoundException - { - OrderedHistogramLogReader reader = new OrderedHistogramLogReader(inputStream); - System.out.println( - "#Absolute timestamp,Relative timestamp,Throughput,Min,Avg,p50,p90,p95,p99,p999,p9999,Max"); - while (reader.hasNext()) - { - Histogram interval = (Histogram) reader.nextIntervalHistogram(); - System.out.printf(Locale.US, - "%.3f,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d%n", - interval.getStartTimeStamp() / 1000.0, - interval.getStartTimeStamp() / 1000 - (long) reader.getStartTimeSec(), - interval.getTotalCount(), interval.getMinValue(), - (long) interval.getMean(), - interval.getValueAtPercentile(50), - interval.getValueAtPercentile(90), - interval.getValueAtPercentile(95), - interval.getValueAtPercentile(99), - interval.getValueAtPercentile(99.9), - interval.getValueAtPercentile(99.99), - interval.getMaxValue()); - } - } - } } \ No newline at end of file diff --git a/src/main/java/SplitHistogramLogs.java b/src/main/java/SplitHistogramLogs.java index 57cb06f..abcb5fd 100644 --- a/src/main/java/SplitHistogramLogs.java +++ b/src/main/java/SplitHistogramLogs.java @@ -1,18 +1,11 @@ -import org.HdrHistogram.Histogram; -import org.HdrHistogram.HistogramLogWriter; import org.kohsuke.args4j.Option; -import psy.lob.saw.HdrHistogramUtil; -import psy.lob.saw.OrderedHistogramLogReader; +import psy.lob.saw.HistogramsSplitter; import java.io.File; import java.io.FileInputStream; -import java.io.FileNotFoundException; import java.io.InputStream; -import java.util.HashMap; import java.util.HashSet; -import java.util.Map; import java.util.Set; -import java.util.function.Predicate; import static psy.lob.saw.HdrHistogramUtil.logHistogramForVerbose; @@ -96,7 +89,7 @@ public void run() } try(InputStream inputStream = new FileInputStream(inputFile)) { - new Splitter(inputFile.getName(), inputStream, start, end, verbose, this::shouldSkipTag).split(); + new HistogramsSplitter(inputFile.getName(), inputStream, start, end, verbose, this::shouldSkipTag).split(); } catch (Exception e) { @@ -109,66 +102,4 @@ private boolean shouldSkipTag(String ntag) ntag = (ntag == null) ? "default" : ntag; return excludeTags.contains(ntag) || (!includeTags.isEmpty() && !includeTags.contains(ntag)); } - - public static class Splitter - { - private final String inputFileName; - private final InputStream inputStream; - private final double start; - private final double end; - private final boolean verbose; - private final Predicate tagExclusionPredicate; - - public Splitter(String inputFileName, InputStream inputStream, - double start, double end, - boolean verbose, Predicate tagExclusionPredicate) - { - this.inputFileName = inputFileName; - this.inputStream = inputStream; - this.start = start; - this.end = end; - this.verbose = verbose; - this.tagExclusionPredicate = tagExclusionPredicate; - } - - public void split() throws FileNotFoundException - { - OrderedHistogramLogReader reader = new OrderedHistogramLogReader( - inputStream, - start, - end, - tagExclusionPredicate); - Map writerByTag = new HashMap<>(); - Histogram interval; - int i = 0; - while (reader.hasNext()) - { - interval = (Histogram) reader.nextIntervalHistogram(); - if (interval == null) - { - continue; - } - String ntag = interval.getTag(); - if (tagExclusionPredicate.test(ntag)) - { - throw new IllegalStateException("Should be filtered upfront by the reader"); - } - if (this.verbose) - { - logHistogramForVerbose(System.out, interval, i++); - } - interval.setTag(null); - HistogramLogWriter writer = writerByTag.computeIfAbsent(ntag, k -> createWriterForTag(reader, k, inputFileName)); - writer.outputIntervalHistogram(interval); - } - } - - private HistogramLogWriter createWriterForTag(OrderedHistogramLogReader reader, String tag, String inputFileName) - { - tag = (tag == null) ? "default" : tag; - File outputFile = new File(tag + "." + inputFileName); - String comment = "Splitting of:" + inputFileName + " start:" + start + " end:" + end; - return HdrHistogramUtil.createLogWriter(outputFile, comment, reader.getStartTimeSec()); - } - } } \ No newline at end of file diff --git a/src/main/java/psy/lob/saw/CsvConverter.java b/src/main/java/psy/lob/saw/CsvConverter.java new file mode 100644 index 0000000..8394cd3 --- /dev/null +++ b/src/main/java/psy/lob/saw/CsvConverter.java @@ -0,0 +1,41 @@ +package psy.lob.saw; + +import org.HdrHistogram.Histogram; + +import java.io.FileNotFoundException; +import java.io.InputStream; +import java.util.Locale; + +public class CsvConverter +{ + private final InputStream inputStream; + + public CsvConverter(InputStream inputStream) + { + this.inputStream = inputStream; + } + + public void convert() throws FileNotFoundException + { + OrderedHistogramLogReader reader = new OrderedHistogramLogReader(inputStream); + System.out.println( + "#Absolute timestamp,Relative timestamp,Throughput,Min,Avg,p50,p90,p95,p99,p999,p9999,Max"); + while (reader.hasNext()) + { + Histogram interval = (Histogram) reader.nextIntervalHistogram(); + System.out.printf(Locale.US, + "%.3f,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d%n", + interval.getStartTimeStamp() / 1000.0, + interval.getStartTimeStamp() / 1000 - (long) reader.getStartTimeSec(), + interval.getTotalCount(), interval.getMinValue(), + (long) interval.getMean(), + interval.getValueAtPercentile(50), + interval.getValueAtPercentile(90), + interval.getValueAtPercentile(95), + interval.getValueAtPercentile(99), + interval.getValueAtPercentile(99.9), + interval.getValueAtPercentile(99.99), + interval.getMaxValue()); + } + } +} diff --git a/src/main/java/psy/lob/saw/HistogramsSplitter.java b/src/main/java/psy/lob/saw/HistogramsSplitter.java new file mode 100644 index 0000000..746660f --- /dev/null +++ b/src/main/java/psy/lob/saw/HistogramsSplitter.java @@ -0,0 +1,75 @@ +package psy.lob.saw; + +import org.HdrHistogram.Histogram; +import org.HdrHistogram.HistogramLogWriter; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.InputStream; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Predicate; + +import static psy.lob.saw.HdrHistogramUtil.logHistogramForVerbose; + +public class HistogramsSplitter +{ + private final String inputFileName; + private final InputStream inputStream; + private final double start; + private final double end; + private final boolean verbose; + private final Predicate tagExclusionPredicate; + + public HistogramsSplitter(String inputFileName, InputStream inputStream, + double start, double end, + boolean verbose, Predicate tagExclusionPredicate) + { + this.inputFileName = inputFileName; + this.inputStream = inputStream; + this.start = start; + this.end = end; + this.verbose = verbose; + this.tagExclusionPredicate = tagExclusionPredicate; + } + + public void split() throws FileNotFoundException + { + OrderedHistogramLogReader reader = new OrderedHistogramLogReader( + inputStream, + start, + end, + tagExclusionPredicate); + Map writerByTag = new HashMap<>(); + Histogram interval; + int i = 0; + while (reader.hasNext()) + { + interval = (Histogram) reader.nextIntervalHistogram(); + if (interval == null) + { + continue; + } + String ntag = interval.getTag(); + if (tagExclusionPredicate.test(ntag)) + { + throw new IllegalStateException("Should be filtered upfront by the reader"); + } + if (this.verbose) + { + logHistogramForVerbose(System.out, interval, i++); + } + interval.setTag(null); + HistogramLogWriter writer = writerByTag.computeIfAbsent(ntag, k -> createWriterForTag(reader, k, inputFileName)); + writer.outputIntervalHistogram(interval); + } + } + + private HistogramLogWriter createWriterForTag(OrderedHistogramLogReader reader, String tag, String inputFileName) + { + tag = (tag == null) ? "default" : tag; + File outputFile = new File(tag + "." + inputFileName); + String comment = "Splitting of:" + inputFileName + " start:" + start + " end:" + end; + return HdrHistogramUtil.createLogWriter(outputFile, comment, reader.getStartTimeSec()); + } +} From 2095efaf4020a82690831931dcb21dc891a49cbf Mon Sep 17 00:00:00 2001 From: Pierre Laporte Date: Fri, 26 Feb 2021 19:07:18 +0100 Subject: [PATCH 5/7] Pass output directory to splitter --- src/main/java/SplitHistogramLogs.java | 6 +++++- src/main/java/psy/lob/saw/HistogramsSplitter.java | 13 ++++++++----- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/main/java/SplitHistogramLogs.java b/src/main/java/SplitHistogramLogs.java index abcb5fd..d3adf06 100644 --- a/src/main/java/SplitHistogramLogs.java +++ b/src/main/java/SplitHistogramLogs.java @@ -4,6 +4,7 @@ import java.io.File; import java.io.FileInputStream; import java.io.InputStream; +import java.nio.file.Path; import java.util.HashSet; import java.util.Set; @@ -23,6 +24,7 @@ public class SplitHistogramLogs implements Runnable private File inputFile; private Set excludeTags = new HashSet<>(); private Set includeTags = new HashSet<>(); + private String fileName; public static void main(String[] args) throws Exception { @@ -89,7 +91,9 @@ public void run() } try(InputStream inputStream = new FileInputStream(inputFile)) { - new HistogramsSplitter(inputFile.getName(), inputStream, start, end, verbose, this::shouldSkipTag).split(); + Path outputDir = inputFile.toPath().getParent(); + fileName = inputFile.getName(); + new HistogramsSplitter(fileName, inputStream, start, end, verbose, this::shouldSkipTag, outputDir).split(); } catch (Exception e) { diff --git a/src/main/java/psy/lob/saw/HistogramsSplitter.java b/src/main/java/psy/lob/saw/HistogramsSplitter.java index 746660f..052a7ef 100644 --- a/src/main/java/psy/lob/saw/HistogramsSplitter.java +++ b/src/main/java/psy/lob/saw/HistogramsSplitter.java @@ -6,6 +6,7 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.InputStream; +import java.nio.file.Path; import java.util.HashMap; import java.util.Map; import java.util.function.Predicate; @@ -20,10 +21,11 @@ public class HistogramsSplitter private final double end; private final boolean verbose; private final Predicate tagExclusionPredicate; + private final Path outputDirectory; public HistogramsSplitter(String inputFileName, InputStream inputStream, double start, double end, - boolean verbose, Predicate tagExclusionPredicate) + boolean verbose, Predicate tagExclusionPredicate, Path outputDirectory) { this.inputFileName = inputFileName; this.inputStream = inputStream; @@ -31,6 +33,7 @@ public HistogramsSplitter(String inputFileName, InputStream inputStream, this.end = end; this.verbose = verbose; this.tagExclusionPredicate = tagExclusionPredicate; + this.outputDirectory = outputDirectory; } public void split() throws FileNotFoundException @@ -60,16 +63,16 @@ public void split() throws FileNotFoundException logHistogramForVerbose(System.out, interval, i++); } interval.setTag(null); - HistogramLogWriter writer = writerByTag.computeIfAbsent(ntag, k -> createWriterForTag(reader, k, inputFileName)); + HistogramLogWriter writer = writerByTag.computeIfAbsent(ntag, k -> createWriterForTag(reader, k)); writer.outputIntervalHistogram(interval); } } - private HistogramLogWriter createWriterForTag(OrderedHistogramLogReader reader, String tag, String inputFileName) + private HistogramLogWriter createWriterForTag(OrderedHistogramLogReader reader, String tag) { tag = (tag == null) ? "default" : tag; - File outputFile = new File(tag + "." + inputFileName); + Path outputPath = outputDirectory.resolve(tag + "." + inputFileName); String comment = "Splitting of:" + inputFileName + " start:" + start + " end:" + end; - return HdrHistogramUtil.createLogWriter(outputFile, comment, reader.getStartTimeSec()); + return HdrHistogramUtil.createLogWriter(outputPath.toFile(), comment, reader.getStartTimeSec()); } } From cfa91eb05bb54f7da046cdf368c0d4bda72421da Mon Sep 17 00:00:00 2001 From: Pierre Laporte Date: Fri, 26 Feb 2021 19:09:22 +0100 Subject: [PATCH 6/7] Pass output stream to CSV converted instead of assuming stdout --- src/main/java/HdrToCsv.java | 2 +- src/main/java/psy/lob/saw/CsvConverter.java | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/main/java/HdrToCsv.java b/src/main/java/HdrToCsv.java index a073704..5a19293 100644 --- a/src/main/java/HdrToCsv.java +++ b/src/main/java/HdrToCsv.java @@ -33,7 +33,7 @@ public void run() { try(InputStream inputStream = new FileInputStream(inputFile)) { - new CsvConverter(inputStream).convert(); + new CsvConverter(inputStream, System.out).convert(); } catch (IOException e) { diff --git a/src/main/java/psy/lob/saw/CsvConverter.java b/src/main/java/psy/lob/saw/CsvConverter.java index 8394cd3..d905146 100644 --- a/src/main/java/psy/lob/saw/CsvConverter.java +++ b/src/main/java/psy/lob/saw/CsvConverter.java @@ -4,26 +4,29 @@ import java.io.FileNotFoundException; import java.io.InputStream; +import java.io.PrintStream; import java.util.Locale; public class CsvConverter { private final InputStream inputStream; + private final PrintStream outputStream; - public CsvConverter(InputStream inputStream) + public CsvConverter(InputStream inputStream, PrintStream outputStream) { this.inputStream = inputStream; + this.outputStream = outputStream; } public void convert() throws FileNotFoundException { OrderedHistogramLogReader reader = new OrderedHistogramLogReader(inputStream); - System.out.println( + outputStream.println( "#Absolute timestamp,Relative timestamp,Throughput,Min,Avg,p50,p90,p95,p99,p999,p9999,Max"); while (reader.hasNext()) { Histogram interval = (Histogram) reader.nextIntervalHistogram(); - System.out.printf(Locale.US, + outputStream.printf(Locale.US, "%.3f,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d%n", interval.getStartTimeStamp() / 1000.0, interval.getStartTimeStamp() / 1000 - (long) reader.getStartTimeSec(), From 274e279f25e89eab58ce0fb1b8544e5a0b72baa2 Mon Sep 17 00:00:00 2001 From: Pierre Laporte Date: Mon, 1 Mar 2021 10:21:21 +0100 Subject: [PATCH 7/7] Return the list of files that were created during split --- .../java/psy/lob/saw/HistogramsSplitter.java | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/main/java/psy/lob/saw/HistogramsSplitter.java b/src/main/java/psy/lob/saw/HistogramsSplitter.java index 052a7ef..a2650a3 100644 --- a/src/main/java/psy/lob/saw/HistogramsSplitter.java +++ b/src/main/java/psy/lob/saw/HistogramsSplitter.java @@ -7,6 +7,7 @@ import java.io.FileNotFoundException; import java.io.InputStream; import java.nio.file.Path; +import java.util.Collection; import java.util.HashMap; import java.util.Map; import java.util.function.Predicate; @@ -36,14 +37,15 @@ public HistogramsSplitter(String inputFileName, InputStream inputStream, this.outputDirectory = outputDirectory; } - public void split() throws FileNotFoundException + public Collection split() throws FileNotFoundException { OrderedHistogramLogReader reader = new OrderedHistogramLogReader( inputStream, start, end, tagExclusionPredicate); - Map writerByTag = new HashMap<>(); + Map outputFileByTag = new HashMap<>(); + Map writerByPath = new HashMap<>(); Histogram interval; int i = 0; while (reader.hasNext()) @@ -62,17 +64,18 @@ public void split() throws FileNotFoundException { logHistogramForVerbose(System.out, interval, i++); } - interval.setTag(null); - HistogramLogWriter writer = writerByTag.computeIfAbsent(ntag, k -> createWriterForTag(reader, k)); + File outputFile = outputFileByTag.computeIfAbsent( + (ntag == null) ? "default" : ntag, + tag -> outputDirectory.resolve(tag + "." + inputFileName).toFile()); + HistogramLogWriter writer = writerByPath.computeIfAbsent(outputFile, o -> createWriterForTag(reader, o)); writer.outputIntervalHistogram(interval); } + return outputFileByTag.values(); } - private HistogramLogWriter createWriterForTag(OrderedHistogramLogReader reader, String tag) + private HistogramLogWriter createWriterForTag(OrderedHistogramLogReader reader, File outputFile) { - tag = (tag == null) ? "default" : tag; - Path outputPath = outputDirectory.resolve(tag + "." + inputFileName); String comment = "Splitting of:" + inputFileName + " start:" + start + " end:" + end; - return HdrHistogramUtil.createLogWriter(outputPath.toFile(), comment, reader.getStartTimeSec()); + return HdrHistogramUtil.createLogWriter(outputFile, comment, reader.getStartTimeSec()); } -} +} \ No newline at end of file