Index: jmh-core/src/main/java/org/openjdk/jmh/util/Statistics.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- jmh-core/src/main/java/org/openjdk/jmh/util/Statistics.java (revision 1325:49e2fb0c9f3bbbe9ffb6935aa25d0113d9d42f7c) +++ jmh-core/src/main/java/org/openjdk/jmh/util/Statistics.java (revision 1325+:49e2fb0c9f3b+) @@ -27,7 +27,10 @@ import org.apache.commons.math3.stat.descriptive.StatisticalSummary; import java.io.Serializable; +import java.util.Iterator; +import java.util.Map.Entry; + public interface Statistics extends Serializable, StatisticalSummary, Comparable { /** @@ -133,4 +136,14 @@ * @return histogram data */ int[] getHistogram(double[] levels); + + /** + * Returns the raw data for this statistics. This data can be useful for + * custom postprocessing and statistics computations. Note, that values of + * multiple calls may not be unique. Ordering of the values is not specified. + * + * @return iterator to raw data. Each item is pair of actual value and + * number of occurrences of this value. + */ + Iterator> getRawData(); } Index: jmh-core/src/main/java/org/openjdk/jmh/util/SingletonStatistics.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- jmh-core/src/main/java/org/openjdk/jmh/util/SingletonStatistics.java (revision 1325:49e2fb0c9f3bbbe9ffb6935aa25d0113d9d42f7c) +++ jmh-core/src/main/java/org/openjdk/jmh/util/SingletonStatistics.java (revision 1325+:49e2fb0c9f3b+) @@ -24,6 +24,10 @@ */ package org.openjdk.jmh.util; +import java.util.AbstractMap; +import java.util.Iterator; +import java.util.Map; + /** * Calculate statistics with just a single value. */ @@ -77,5 +81,30 @@ } } return result; + } + + @Override + public Iterator> getRawData() { + return new SingletonStatisticsIterator(); + } + + private class SingletonStatisticsIterator implements Iterator> { + private boolean entryReturned = false; + + @Override + public boolean hasNext() { + return !entryReturned; + } + + @Override + public Map.Entry next() { + entryReturned = true; + return new AbstractMap.SimpleImmutableEntry(value, 1L); + } + + @Override + public void remove() { + throw new UnsupportedOperationException("Element cannot be removed."); + } } } Index: jmh-core/src/main/java/org/openjdk/jmh/util/Utils.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- jmh-core/src/main/java/org/openjdk/jmh/util/Utils.java (revision 1325:49e2fb0c9f3bbbe9ffb6935aa25d0113d9d42f7c) +++ jmh-core/src/main/java/org/openjdk/jmh/util/Utils.java (revision 1325+:49e2fb0c9f3b+) @@ -443,4 +443,19 @@ return messages; } + /** + * Allow using for-each loop over raw data from Statistics implementations. + * + * @param it iterator to be looped over + * @return iterable for given iterator + */ + public static Iterable> loopOver(final Iterator> it) { + return new Iterable>() { + @Override + public Iterator> iterator() { + return it; + } + }; + } + } Index: jmh-core/src/test/java/org/openjdk/jmh/util/TestMultisetStatistics.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- jmh-core/src/test/java/org/openjdk/jmh/util/TestMultisetStatistics.java (revision 1325:49e2fb0c9f3bbbe9ffb6935aa25d0113d9d42f7c) +++ jmh-core/src/test/java/org/openjdk/jmh/util/TestMultisetStatistics.java (revision 1325+:49e2fb0c9f3b+) @@ -28,6 +28,8 @@ import org.junit.BeforeClass; import org.junit.Test; +import java.util.Map; + import static org.junit.Assert.assertEquals; /** @@ -383,6 +385,52 @@ new double[] {0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100}, new int[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9} ); + } + + /** + * Test of iterator which make accessible raw data. + * Iterate over default instance with no duplicates. + */ + @Test + public strictfp void testRawDataIterator_no_duplicates() { + int itemCount = 0; + for (Map.Entry entry : Utils.loopOver(instance.getRawData())) { + Assert.assertEquals(entry.getValue().longValue(), 1L); + + // Check if key (the actual data) is in the VALUES collection, + // else fail the test (the Multiset was constructed with values + // from VALUES collection, so it should be there). + boolean keyIsPresent = false; + double key = entry.getKey(); + for (double value : VALUES) { + if (Double.compare(value, key) == 0) { + keyIsPresent = true; + } + } + Assert.assertTrue("Value from iterator is not present in source collection", keyIsPresent); + + itemCount++; + } + Assert.assertEquals(itemCount, VALUES.length); + } + + /** + * Test of iterator which make accessible raw data. + * Iterate over new instance with duplicates. + */ + @Test + public strictfp void testRawDataIterator_duplicates() { + MultisetStatistics s = new MultisetStatistics(); + for (int c = 0; c <= 10; c++) { + s.addValue(c * 10, c); + } + + int itemCount = 0; + for (Map.Entry entry : Utils.loopOver(s.getRawData())) { + Assert.assertEquals(entry.getKey(), (double)(entry.getValue() * 10)); + itemCount++; + } + Assert.assertEquals(itemCount, 10); } } Index: jmh-core/src/main/java/org/openjdk/jmh/results/format/JSONResultFormat.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- jmh-core/src/main/java/org/openjdk/jmh/results/format/JSONResultFormat.java (revision 1325:49e2fb0c9f3bbbe9ffb6935aa25d0113d9d42f7c) +++ jmh-core/src/main/java/org/openjdk/jmh/results/format/JSONResultFormat.java (revision 1325+:49e2fb0c9f3b+) @@ -24,12 +24,14 @@ */ package org.openjdk.jmh.results.format; +import org.openjdk.jmh.annotations.Mode; import org.openjdk.jmh.infra.BenchmarkParams; import org.openjdk.jmh.results.BenchmarkResult; import org.openjdk.jmh.results.IterationResult; import org.openjdk.jmh.results.Result; import org.openjdk.jmh.results.RunResult; import org.openjdk.jmh.util.Statistics; +import org.openjdk.jmh.util.Utils; import java.io.PrintStream; import java.io.PrintWriter; @@ -59,6 +61,7 @@ if (first) { first = false; + pw.println(); } else { pw.println(","); } @@ -88,20 +91,15 @@ pw.println("\"scoreConfidence\" : " + emit(primaryResult.getScoreConfidence()) + ","); pw.println(emitPercentiles(primaryResult.getStatistics())); pw.println("\"scoreUnit\" : \"" + primaryResult.getScoreUnit() + "\","); + + boolean printRawData = Boolean.parseBoolean(System.getProperty("jmh.json.rawData", "true")); + boolean printHistogram = params.getMode() == Mode.SampleTime; pw.println("\"rawData\" :"); + pw.println(getRawData(runResult, printRawData && !printHistogram, printHistogram) + ","); + pw.println("\"rawDataHistogram\" :"); + pw.println(getRawData(runResult, printRawData && printHistogram, printHistogram)); - { - Collection l1 = new ArrayList(); - for (BenchmarkResult benchmarkResult : runResult.getBenchmarkResults()) { - Collection scores = new ArrayList(); - for (IterationResult r : benchmarkResult.getIterationResults()) { - scores.add(emit(r.getPrimaryResult().getScore())); - } - l1.add(printMultiple(scores, "[", "]")); - } - pw.println(printMultiple(l1, "[", "]")); - pw.println("},"); - } + pw.println("},"); // primaryMetric end Collection secondaries = new ArrayList(); for (Map.Entry e : runResult.getSecondaryResults().entrySet()) { @@ -130,6 +128,7 @@ } sb.append(printMultiple(l2, "[", "]")); + sb.append(", \"rawDataHistogram\" : []"); sb.append("}"); secondaries.add(sb.toString()); } @@ -137,14 +136,38 @@ pw.println(printMultiple(secondaries, "", "")); pw.println("}"); - pw.println("}"); - + pw.print("}"); // benchmark end } pw.println("]"); out.println(tidy(sw.toString())); } + private String getRawData(RunResult runResult, boolean printData, boolean histogram) { + StringBuilder sb = new StringBuilder(); + Collection runs = new ArrayList(); + + if (printData) { + for (BenchmarkResult benchmarkResult : runResult.getBenchmarkResults()) { + Collection iterations = new ArrayList(); + for (IterationResult r : benchmarkResult.getIterationResults()) { + for (Map.Entry item : + Utils.loopOver(r.getPrimaryResult().getStatistics().getRawData())) { + if (histogram) { + iterations.add("< " + emit(item.getKey()) + "; " + item.getValue() + " >"); + } else { + iterations.add(emit(item.getKey())); + } + } + } + runs.add(printMultiple(iterations, "[", "]")); + } + } + sb.append(printMultiple(runs, "[", "]")); + + return sb.toString(); + } + private String emitParams(BenchmarkParams params) { StringBuilder sb = new StringBuilder(); boolean isFirst = true; @@ -217,6 +240,9 @@ s = s.replaceAll("\\]\n,\n", "],\n"); s = s.replaceAll("\\}\n,\n", "},\n"); s = s.replaceAll("\n( *)\n", "\n"); + s = s.replaceAll(";", ","); + s = s.replaceAll("\\<", "["); + s = s.replaceAll("\\>", "]"); String[] lines = s.split("\n"); @@ -228,7 +254,7 @@ if (prevL != null && (prevL.endsWith("{") || prevL.endsWith("["))) { ident++; } - if (l.endsWith("}") || l.endsWith("]") || l.endsWith("},") || l.endsWith("],")) { + if (l.equals("}") || l.equals("]") || l.equals("},") || l.equals("],")) { ident--; } Index: jmh-core/src/test/java/org/openjdk/jmh/util/TestListStatistics.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- jmh-core/src/test/java/org/openjdk/jmh/util/TestListStatistics.java (revision 1325:49e2fb0c9f3bbbe9ffb6935aa25d0113d9d42f7c) +++ jmh-core/src/test/java/org/openjdk/jmh/util/TestListStatistics.java (revision 1325+:49e2fb0c9f3b+) @@ -28,6 +28,9 @@ import org.junit.BeforeClass; import org.junit.Test; +import java.util.Iterator; +import java.util.Map; + import static org.junit.Assert.assertEquals; /** @@ -390,6 +393,21 @@ new double[] {0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100}, new int[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9} ); + } + + /** + * Test of iterator which make accessible raw data. + */ + @Test + public strictfp void testRawDataIterator() { + Iterator> listIter = instance.getRawData(); + for (double item : VALUES) { + Assert.assertTrue(listIter.hasNext()); + Map.Entry entry = listIter.next(); + Assert.assertEquals(entry.getKey(), item); + Assert.assertEquals(entry.getValue().longValue(), 1L); + } + Assert.assertFalse(listIter.hasNext()); } } Index: jmh-core/src/test/resources/org/openjdk/jmh/results/format/output-golden.json IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- jmh-core/src/test/resources/org/openjdk/jmh/results/format/output-golden.json (revision 1325:49e2fb0c9f3bbbe9ffb6935aa25d0113d9d42f7c) +++ jmh-core/src/test/resources/org/openjdk/jmh/results/format/output-golden.json (revision 1325+:49e2fb0c9f3b+) @@ -66,6 +66,8 @@ 121.0, 826.0 ] + ], + "rawDataHistogram" : [ ] }, "secondaryMetrics" : { @@ -118,6 +120,8 @@ 918.0, 849.0 ] + ], + "rawDataHistogram" : [ ] }, "secondary2" : { @@ -169,6 +173,8 @@ 56.0, 992.0 ] + ], + "rawDataHistogram" : [ ] }, "secondary3" : { @@ -211,11 +217,12 @@ [ 532.0 ] + ], + "rawDataHistogram" : [ ] } } - } - , + }, { "benchmark" : "benchmark_1", "mode" : "thrpt", @@ -258,6 +265,8 @@ [ 439.0 ] + ], + "rawDataHistogram" : [ ] }, "secondaryMetrics" : { @@ -285,6 +294,8 @@ [ 953.0 ] + ], + "rawDataHistogram" : [ ] }, "secondary2" : { @@ -311,11 +322,12 @@ [ 367.0 ] + ], + "rawDataHistogram" : [ ] } } - } - , + }, { "benchmark" : "benchmark_2", "mode" : "thrpt", @@ -368,6 +380,8 @@ 379.0, 986.0 ] + ], + "rawDataHistogram" : [ ] }, "secondaryMetrics" : { @@ -405,6 +419,8 @@ 499.0, 333.0 ] + ], + "rawDataHistogram" : [ ] }, "secondary2" : { @@ -441,6 +457,8 @@ 339.0, 820.0 ] + ], + "rawDataHistogram" : [ ] }, "secondary3" : { @@ -473,11 +491,12 @@ 397.0, 825.0 ] + ], + "rawDataHistogram" : [ ] } } - } - , + }, { "benchmark" : "benchmark_3", "mode" : "thrpt", @@ -539,6 +558,8 @@ 125.0, 979.0 ] + ], + "rawDataHistogram" : [ ] }, "secondaryMetrics" : { @@ -585,6 +606,8 @@ 983.0, 232.0 ] + ], + "rawDataHistogram" : [ ] }, "secondary2" : { @@ -630,6 +653,8 @@ 392.0, 710.0 ] + ], + "rawDataHistogram" : [ ] }, "secondary3" : { @@ -667,11 +692,12 @@ ], [ ] + ], + "rawDataHistogram" : [ ] } } - } - , + }, { "benchmark" : "benchmark_4", "mode" : "thrpt", @@ -714,6 +740,8 @@ [ 956.0 ] + ], + "rawDataHistogram" : [ ] }, "secondaryMetrics" : { @@ -741,6 +769,8 @@ [ 688.0 ] + ], + "rawDataHistogram" : [ ] }, "secondary2" : { @@ -767,6 +797,8 @@ [ 237.0 ] + ], + "rawDataHistogram" : [ ] }, "secondary3" : { @@ -793,6 +825,8 @@ [ 599.0 ] + ], + "rawDataHistogram" : [ ] } } Index: jmh-core/src/main/java/org/openjdk/jmh/util/MultisetStatistics.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- jmh-core/src/main/java/org/openjdk/jmh/util/MultisetStatistics.java (revision 1325:49e2fb0c9f3bbbe9ffb6935aa25d0113d9d42f7c) +++ jmh-core/src/main/java/org/openjdk/jmh/util/MultisetStatistics.java (revision 1325+:49e2fb0c9f3b+) @@ -24,9 +24,7 @@ */ package org.openjdk.jmh.util; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; +import java.util.*; public class MultisetStatistics extends AbstractStatistics { private static final long serialVersionUID = -4401871054963903938L; @@ -135,5 +133,10 @@ } return result; + } + + @Override + public Iterator> getRawData() { + return values.entrySet().iterator(); } } Index: jmh-core/src/test/java/org/openjdk/jmh/util/TestSingletonStatistics.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- jmh-core/src/test/java/org/openjdk/jmh/util/TestSingletonStatistics.java (revision 1325:49e2fb0c9f3bbbe9ffb6935aa25d0113d9d42f7c) +++ jmh-core/src/test/java/org/openjdk/jmh/util/TestSingletonStatistics.java (revision 1325+:49e2fb0c9f3b+) @@ -28,6 +28,9 @@ import org.junit.BeforeClass; import org.junit.Test; +import java.util.Iterator; +import java.util.Map; + import static org.junit.Assert.assertEquals; /** @@ -243,6 +246,19 @@ new double[] {0, 2, 4, 8, 10}, new int[] {0, 0, 1, 0} ); + } + + /** + * Test of iterator which make accessible raw data. + */ + @Test + public strictfp void testRawDataIterator() { + Iterator> singIter = singStats.getRawData(); + Assert.assertTrue(singIter.hasNext()); + Map.Entry entry = singIter.next(); + Assert.assertEquals(entry.getKey(), VALUE); + Assert.assertEquals(entry.getValue().longValue(), 1L); + Assert.assertFalse(singIter.hasNext()); } } Index: jmh-core/src/main/java/org/openjdk/jmh/util/ListStatistics.java IDEA additional info: Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP <+>UTF-8 =================================================================== --- jmh-core/src/main/java/org/openjdk/jmh/util/ListStatistics.java (revision 1325:49e2fb0c9f3bbbe9ffb6935aa25d0113d9d42f7c) +++ jmh-core/src/main/java/org/openjdk/jmh/util/ListStatistics.java (revision 1325+:49e2fb0c9f3b+) @@ -26,7 +26,10 @@ import org.apache.commons.math3.stat.descriptive.rank.Percentile; +import java.util.AbstractMap; import java.util.Arrays; +import java.util.Iterator; +import java.util.Map; /** * Calculate statistics over a list of doubles. @@ -150,6 +153,11 @@ } @Override + public Iterator> getRawData() { + return new ListStatisticsIterator(); + } + + @Override public double getVariance() { if (count > 1) { double v = 0; @@ -160,6 +168,25 @@ return v / (count - 1); } else { return Double.NaN; + } + } + + private class ListStatisticsIterator implements Iterator> { + private int currentIndex = 0; + + @Override + public boolean hasNext() { + return currentIndex < count; + } + + @Override + public Map.Entry next() { + return new AbstractMap.SimpleImmutableEntry(values[currentIndex++], 1L); + } + + @Override + public void remove() { + throw new UnsupportedOperationException("Element cannot be removed."); } }