From 6fa1153da0e6149111f34e288cf01b05e62c7b5e Mon Sep 17 00:00:00 2001 From: Dongie Agnir Date: Thu, 24 Aug 2023 14:38:38 -0700 Subject: [PATCH] Add option to output all results to disk --- test/sdk-benchmarks/pom.xml | 7 ++ .../benchmark/BenchmarkResultProcessor.java | 24 +++-- .../awssdk/benchmark/BenchmarkRunner.java | 94 +++++++++++++++++-- .../utils/BenchmarkProcessorOutput.java | 44 +++++++++ 4 files changed, 150 insertions(+), 19 deletions(-) create mode 100644 test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/utils/BenchmarkProcessorOutput.java diff --git a/test/sdk-benchmarks/pom.xml b/test/sdk-benchmarks/pom.xml index d87e62b42060..4e71d4e28a24 100644 --- a/test/sdk-benchmarks/pom.xml +++ b/test/sdk-benchmarks/pom.xml @@ -205,6 +205,11 @@ ${awsjavasdk.version} compile + + commons-cli + commons-cli + compile + @@ -368,6 +373,8 @@ -classpath software.amazon.awssdk.benchmark.BenchmarkRunner + + -c diff --git a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/BenchmarkResultProcessor.java b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/BenchmarkResultProcessor.java index 580471fa1a3b..938aa0de3c08 100644 --- a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/BenchmarkResultProcessor.java +++ b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/BenchmarkResultProcessor.java @@ -24,6 +24,7 @@ import java.net.URL; import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -34,6 +35,7 @@ import software.amazon.awssdk.benchmark.stats.SdkBenchmarkParams; import software.amazon.awssdk.benchmark.stats.SdkBenchmarkResult; import software.amazon.awssdk.benchmark.stats.SdkBenchmarkStatistics; +import software.amazon.awssdk.benchmark.utils.BenchmarkProcessorOutput; import software.amazon.awssdk.utils.Logger; @@ -66,15 +68,18 @@ class BenchmarkResultProcessor { * Process benchmark results * * @param results the results of the benchmark - * @return the benchmark Id that failed the regression + * @return the benchmark results */ - List processBenchmarkResult(Collection results) { - List currentData = new ArrayList<>(); + BenchmarkProcessorOutput processBenchmarkResult(Collection results) { + Map benchmarkResults = new HashMap<>(); + for (RunResult result : results) { String benchmarkId = getBenchmarkId(result.getParams()); + SdkBenchmarkResult sdkBenchmarkData = constructSdkBenchmarkResult(result); + + benchmarkResults.put(benchmarkId, sdkBenchmarkData); SdkBenchmarkResult baselineResult = baseline.get(benchmarkId); - SdkBenchmarkResult sdkBenchmarkData = constructSdkBenchmarkResult(result); if (baselineResult == null) { log.warn(() -> { @@ -90,15 +95,14 @@ List processBenchmarkResult(Collection results) { continue; } - currentData.add(sdkBenchmarkData); - if (!validateBenchmarkResult(sdkBenchmarkData, baselineResult)) { failedBenchmarkIds.add(benchmarkId); } } - log.info(() -> "Current result: " + serializeResult(currentData)); - return failedBenchmarkIds; + BenchmarkProcessorOutput output = new BenchmarkProcessorOutput(benchmarkResults, failedBenchmarkIds); + log.info(() -> "Current result: " + serializeResult(output)); + return output; } private SdkBenchmarkResult constructSdkBenchmarkResult(RunResult runResult) { @@ -169,9 +173,9 @@ private boolean validateBenchmarkParams(SdkBenchmarkParams current, SdkBenchmark return current.getMode() == baseline.getMode(); } - private String serializeResult(List currentData) { + private String serializeResult(BenchmarkProcessorOutput processorOutput) { try { - return OBJECT_MAPPER.writeValueAsString(currentData); + return OBJECT_MAPPER.writeValueAsString(processorOutput); } catch (JsonProcessingException e) { log.error(() -> "Failed to serialize current result", e); } diff --git a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/BenchmarkRunner.java b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/BenchmarkRunner.java index 92ca28d12acc..4c49f0270a87 100644 --- a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/BenchmarkRunner.java +++ b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/BenchmarkRunner.java @@ -15,11 +15,23 @@ package software.amazon.awssdk.benchmark; -import com.fasterxml.jackson.core.JsonProcessingException; +import static software.amazon.awssdk.benchmark.utils.BenchmarkConstant.OBJECT_MAPPER; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.List; +import java.util.stream.Collectors; +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.Options; +import org.apache.commons.cli.ParseException; import org.openjdk.jmh.results.RunResult; import org.openjdk.jmh.runner.Runner; import org.openjdk.jmh.runner.RunnerException; @@ -45,6 +57,8 @@ import software.amazon.awssdk.benchmark.enhanced.dynamodb.EnhancedClientQueryV1MapperComparisonBenchmark; import software.amazon.awssdk.benchmark.enhanced.dynamodb.EnhancedClientScanV1MapperComparisonBenchmark; import software.amazon.awssdk.benchmark.enhanced.dynamodb.EnhancedClientUpdateV1MapperComparisonBenchmark; +import software.amazon.awssdk.benchmark.stats.SdkBenchmarkResult; +import software.amazon.awssdk.benchmark.utils.BenchmarkProcessorOutput; import software.amazon.awssdk.utils.Logger; @@ -84,13 +98,15 @@ public class BenchmarkRunner { private final List benchmarksToRun; private final BenchmarkResultProcessor resultProcessor; + private final BenchmarkRunnerOptions options; - private BenchmarkRunner(List benchmarksToRun) { + private BenchmarkRunner(List benchmarksToRun, BenchmarkRunnerOptions options) { this.benchmarksToRun = benchmarksToRun; this.resultProcessor = new BenchmarkResultProcessor(); + this.options = options; } - public static void main(String... args) throws RunnerException, JsonProcessingException { + public static void main(String... args) throws Exception { List benchmarksToRun = new ArrayList<>(); benchmarksToRun.addAll(SYNC_BENCHMARKS); benchmarksToRun.addAll(ASYNC_BENCHMARKS); @@ -99,13 +115,14 @@ public static void main(String... args) throws RunnerException, JsonProcessingEx log.info(() -> "Skipping tests, to reduce benchmark times: \n" + MAPPER_BENCHMARKS + "\n" + METRIC_BENCHMARKS); - - BenchmarkRunner runner = new BenchmarkRunner(benchmarksToRun); + BenchmarkRunner runner = new BenchmarkRunner(benchmarksToRun, parseOptions(args)); runner.runBenchmark(); } private void runBenchmark() throws RunnerException { + log.info(() -> "Running with options: " + options); + ChainedOptionsBuilder optionsBuilder = new OptionsBuilder(); benchmarksToRun.forEach(optionsBuilder::include); @@ -114,11 +131,70 @@ private void runBenchmark() throws RunnerException { Collection results = new Runner(optionsBuilder.build()).run(); - List failedResult = resultProcessor.processBenchmarkResult(results); + BenchmarkProcessorOutput processedResults = resultProcessor.processBenchmarkResult(results); + List failedResults = processedResults.getFailedBenchmarks(); + + if (options.outputPath != null) { + log.info(() -> "Writing results to " + options.outputPath); + writeResults(processedResults, options.outputPath); + } + + if (options.check && !failedResults.isEmpty()) { + log.info(() -> "Failed perf regression tests: " + failedResults); + throw new RuntimeException("Perf regression tests failed: " + failedResults); + } + } + + private static BenchmarkRunnerOptions parseOptions(String[] args) throws ParseException { + Options cliOptions = new Options(); + cliOptions.addOption("o", "output", true, + "The path to write the benchmark results to."); + cliOptions.addOption("c", "check", false, + "If specified, exit with error code 1 if the results are not within the baseline."); + + CommandLineParser parser = new DefaultParser(); + CommandLine cmdLine = parser.parse(cliOptions, args); + + BenchmarkRunnerOptions options = new BenchmarkRunnerOptions() + .check(cmdLine.hasOption("c")); + + if (cmdLine.hasOption("o")) { + options.outputPath(Paths.get(cmdLine.getOptionValue("o"))); + } + + return options; + } + + private static void writeResults(BenchmarkProcessorOutput output, Path outputPath) { + List results = output.getBenchmarkResults().values().stream().collect(Collectors.toList()); + try (OutputStream os = Files.newOutputStream(outputPath)) { + OBJECT_MAPPER.writeValue(os, results); + } catch (IOException e) { + log.error(() -> "Failed to write the results to " + outputPath, e); + throw new RuntimeException(e); + } + } + + private static class BenchmarkRunnerOptions { + private Path outputPath; + private boolean check; + + public BenchmarkRunnerOptions outputPath(Path outputPath) { + this.outputPath = outputPath; + return this; + } + + public BenchmarkRunnerOptions check(boolean check) { + this.check = check; + return this; + } - if (!failedResult.isEmpty()) { - log.info(() -> "Failed perf regression tests: " + failedResult); - throw new RuntimeException("Perf regression tests failed: " + failedResult); + @Override + public String toString() { + return "BenchmarkRunnerOptions{" + + "outputPath=" + outputPath + + ", check=" + check + + '}'; } } } diff --git a/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/utils/BenchmarkProcessorOutput.java b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/utils/BenchmarkProcessorOutput.java new file mode 100644 index 000000000000..902ac3034730 --- /dev/null +++ b/test/sdk-benchmarks/src/main/java/software/amazon/awssdk/benchmark/utils/BenchmarkProcessorOutput.java @@ -0,0 +1,44 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.awssdk.benchmark.utils; + +import com.fasterxml.jackson.annotation.JsonCreator; +import java.util.List; +import java.util.Map; +import software.amazon.awssdk.benchmark.stats.SdkBenchmarkResult; + +/** + * The output object of the benchmark processor. This contains the results of the all the benchmarks that were run, and the + * list of benchmarks that failed. + */ +public final class BenchmarkProcessorOutput { + private final Map benchmarkResults; + private final List failedBenchmarks; + + @JsonCreator + public BenchmarkProcessorOutput(Map benchmarkResults, List failedBenchmarks) { + this.benchmarkResults = benchmarkResults; + this.failedBenchmarks = failedBenchmarks; + } + + public Map getBenchmarkResults() { + return benchmarkResults; + } + + public List getFailedBenchmarks() { + return failedBenchmarks; + } +}