diff --git a/test/sdk-benchmarks/pom.xml b/test/sdk-benchmarks/pom.xml
index d87e62b42060..4e71d4e28a24 100644
--- a/test/sdk-benchmarks/pom.xml
+++ b/test/sdk-benchmarks/pom.xml
@@ -205,6 +205,11 @@
${awsjavasdk.version}
compile
+
+ commons-cli
+ commons-cli
+ compile
+
@@ -368,6 +373,8 @@
-classpath
software.amazon.awssdk.benchmark.BenchmarkRunner
+
+ -c
diff --git a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/BenchmarkResultProcessor.java b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/BenchmarkResultProcessor.java
index 580471fa1a3b..938aa0de3c08 100644
--- a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/BenchmarkResultProcessor.java
+++ b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/BenchmarkResultProcessor.java
@@ -24,6 +24,7 @@
import java.net.URL;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
@@ -34,6 +35,7 @@
import software.amazon.awssdk.benchmark.stats.SdkBenchmarkParams;
import software.amazon.awssdk.benchmark.stats.SdkBenchmarkResult;
import software.amazon.awssdk.benchmark.stats.SdkBenchmarkStatistics;
+import software.amazon.awssdk.benchmark.utils.BenchmarkProcessorOutput;
import software.amazon.awssdk.utils.Logger;
@@ -66,15 +68,18 @@ class BenchmarkResultProcessor {
* Process benchmark results
*
* @param results the results of the benchmark
- * @return the benchmark Id that failed the regression
+ * @return the benchmark results
*/
- List processBenchmarkResult(Collection results) {
- List currentData = new ArrayList<>();
+ BenchmarkProcessorOutput processBenchmarkResult(Collection results) {
+ Map benchmarkResults = new HashMap<>();
+
for (RunResult result : results) {
String benchmarkId = getBenchmarkId(result.getParams());
+ SdkBenchmarkResult sdkBenchmarkData = constructSdkBenchmarkResult(result);
+
+ benchmarkResults.put(benchmarkId, sdkBenchmarkData);
SdkBenchmarkResult baselineResult = baseline.get(benchmarkId);
- SdkBenchmarkResult sdkBenchmarkData = constructSdkBenchmarkResult(result);
if (baselineResult == null) {
log.warn(() -> {
@@ -90,15 +95,14 @@ List processBenchmarkResult(Collection results) {
continue;
}
- currentData.add(sdkBenchmarkData);
-
if (!validateBenchmarkResult(sdkBenchmarkData, baselineResult)) {
failedBenchmarkIds.add(benchmarkId);
}
}
- log.info(() -> "Current result: " + serializeResult(currentData));
- return failedBenchmarkIds;
+ BenchmarkProcessorOutput output = new BenchmarkProcessorOutput(benchmarkResults, failedBenchmarkIds);
+ log.info(() -> "Current result: " + serializeResult(output));
+ return output;
}
private SdkBenchmarkResult constructSdkBenchmarkResult(RunResult runResult) {
@@ -169,9 +173,9 @@ private boolean validateBenchmarkParams(SdkBenchmarkParams current, SdkBenchmark
return current.getMode() == baseline.getMode();
}
- private String serializeResult(List currentData) {
+ private String serializeResult(BenchmarkProcessorOutput processorOutput) {
try {
- return OBJECT_MAPPER.writeValueAsString(currentData);
+ return OBJECT_MAPPER.writeValueAsString(processorOutput);
} catch (JsonProcessingException e) {
log.error(() -> "Failed to serialize current result", e);
}
diff --git a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/BenchmarkRunner.java b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/BenchmarkRunner.java
index 92ca28d12acc..4c49f0270a87 100644
--- a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/BenchmarkRunner.java
+++ b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/BenchmarkRunner.java
@@ -15,11 +15,23 @@
package software.amazon.awssdk.benchmark;
-import com.fasterxml.jackson.core.JsonProcessingException;
+import static software.amazon.awssdk.benchmark.utils.BenchmarkConstant.OBJECT_MAPPER;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
+import java.util.stream.Collectors;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.DefaultParser;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
import org.openjdk.jmh.results.RunResult;
import org.openjdk.jmh.runner.Runner;
import org.openjdk.jmh.runner.RunnerException;
@@ -45,6 +57,8 @@
import software.amazon.awssdk.benchmark.enhanced.dynamodb.EnhancedClientQueryV1MapperComparisonBenchmark;
import software.amazon.awssdk.benchmark.enhanced.dynamodb.EnhancedClientScanV1MapperComparisonBenchmark;
import software.amazon.awssdk.benchmark.enhanced.dynamodb.EnhancedClientUpdateV1MapperComparisonBenchmark;
+import software.amazon.awssdk.benchmark.stats.SdkBenchmarkResult;
+import software.amazon.awssdk.benchmark.utils.BenchmarkProcessorOutput;
import software.amazon.awssdk.utils.Logger;
@@ -84,13 +98,15 @@ public class BenchmarkRunner {
private final List benchmarksToRun;
private final BenchmarkResultProcessor resultProcessor;
+ private final BenchmarkRunnerOptions options;
- private BenchmarkRunner(List benchmarksToRun) {
+ private BenchmarkRunner(List benchmarksToRun, BenchmarkRunnerOptions options) {
this.benchmarksToRun = benchmarksToRun;
this.resultProcessor = new BenchmarkResultProcessor();
+ this.options = options;
}
- public static void main(String... args) throws RunnerException, JsonProcessingException {
+ public static void main(String... args) throws Exception {
List benchmarksToRun = new ArrayList<>();
benchmarksToRun.addAll(SYNC_BENCHMARKS);
benchmarksToRun.addAll(ASYNC_BENCHMARKS);
@@ -99,13 +115,14 @@ public static void main(String... args) throws RunnerException, JsonProcessingEx
log.info(() -> "Skipping tests, to reduce benchmark times: \n" + MAPPER_BENCHMARKS + "\n" + METRIC_BENCHMARKS);
-
- BenchmarkRunner runner = new BenchmarkRunner(benchmarksToRun);
+ BenchmarkRunner runner = new BenchmarkRunner(benchmarksToRun, parseOptions(args));
runner.runBenchmark();
}
private void runBenchmark() throws RunnerException {
+ log.info(() -> "Running with options: " + options);
+
ChainedOptionsBuilder optionsBuilder = new OptionsBuilder();
benchmarksToRun.forEach(optionsBuilder::include);
@@ -114,11 +131,70 @@ private void runBenchmark() throws RunnerException {
Collection results = new Runner(optionsBuilder.build()).run();
- List failedResult = resultProcessor.processBenchmarkResult(results);
+ BenchmarkProcessorOutput processedResults = resultProcessor.processBenchmarkResult(results);
+ List failedResults = processedResults.getFailedBenchmarks();
+
+ if (options.outputPath != null) {
+ log.info(() -> "Writing results to " + options.outputPath);
+ writeResults(processedResults, options.outputPath);
+ }
+
+ if (options.check && !failedResults.isEmpty()) {
+ log.info(() -> "Failed perf regression tests: " + failedResults);
+ throw new RuntimeException("Perf regression tests failed: " + failedResults);
+ }
+ }
+
+ private static BenchmarkRunnerOptions parseOptions(String[] args) throws ParseException {
+ Options cliOptions = new Options();
+ cliOptions.addOption("o", "output", true,
+ "The path to write the benchmark results to.");
+ cliOptions.addOption("c", "check", false,
+ "If specified, exit with error code 1 if the results are not within the baseline.");
+
+ CommandLineParser parser = new DefaultParser();
+ CommandLine cmdLine = parser.parse(cliOptions, args);
+
+ BenchmarkRunnerOptions options = new BenchmarkRunnerOptions()
+ .check(cmdLine.hasOption("c"));
+
+ if (cmdLine.hasOption("o")) {
+ options.outputPath(Paths.get(cmdLine.getOptionValue("o")));
+ }
+
+ return options;
+ }
+
+ private static void writeResults(BenchmarkProcessorOutput output, Path outputPath) {
+ List results = output.getBenchmarkResults().values().stream().collect(Collectors.toList());
+ try (OutputStream os = Files.newOutputStream(outputPath)) {
+ OBJECT_MAPPER.writeValue(os, results);
+ } catch (IOException e) {
+ log.error(() -> "Failed to write the results to " + outputPath, e);
+ throw new RuntimeException(e);
+ }
+ }
+
+ private static class BenchmarkRunnerOptions {
+ private Path outputPath;
+ private boolean check;
+
+ public BenchmarkRunnerOptions outputPath(Path outputPath) {
+ this.outputPath = outputPath;
+ return this;
+ }
+
+ public BenchmarkRunnerOptions check(boolean check) {
+ this.check = check;
+ return this;
+ }
- if (!failedResult.isEmpty()) {
- log.info(() -> "Failed perf regression tests: " + failedResult);
- throw new RuntimeException("Perf regression tests failed: " + failedResult);
+ @Override
+ public String toString() {
+ return "BenchmarkRunnerOptions{" +
+ "outputPath=" + outputPath +
+ ", check=" + check +
+ '}';
}
}
}
diff --git a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/utils/BenchmarkProcessorOutput.java b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/utils/BenchmarkProcessorOutput.java
new file mode 100644
index 000000000000..902ac3034730
--- /dev/null
+++ b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/utils/BenchmarkProcessorOutput.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ * http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package software.amazon.awssdk.benchmark.utils;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import java.util.List;
+import java.util.Map;
+import software.amazon.awssdk.benchmark.stats.SdkBenchmarkResult;
+
+/**
+ * The output object of the benchmark processor. This contains the results of the all the benchmarks that were run, and the
+ * list of benchmarks that failed.
+ */
+public final class BenchmarkProcessorOutput {
+ private final Map benchmarkResults;
+ private final List failedBenchmarks;
+
+ @JsonCreator
+ public BenchmarkProcessorOutput(Map benchmarkResults, List failedBenchmarks) {
+ this.benchmarkResults = benchmarkResults;
+ this.failedBenchmarks = failedBenchmarks;
+ }
+
+ public Map getBenchmarkResults() {
+ return benchmarkResults;
+ }
+
+ public List getFailedBenchmarks() {
+ return failedBenchmarks;
+ }
+}