Skip to content

Commit f8154ed

Browse files
ci: Design and implement a comprehensive performance regression presubmit/continuous job
1 parent 5658c83 commit f8154ed

File tree

6 files changed

+470
-1
lines changed

6 files changed

+470
-1
lines changed

google-cloud-spanner/pom.xml

Lines changed: 26 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -276,6 +276,12 @@
276276
<artifactId>proto-google-cloud-monitoring-v3</artifactId>
277277
<version>3.76.0</version>
278278
</dependency>
279+
<dependency>
280+
<groupId>com.google.api.grpc</groupId>
281+
<artifactId>grpc-google-cloud-monitoring-v3</artifactId>
282+
<version>3.63.0</version>
283+
<scope>test</scope>
284+
</dependency>
279285
<dependency>
280286
<groupId>com.google.auth</groupId>
281287
<artifactId>google-auth-library-oauth2-http</artifactId>
@@ -522,7 +528,11 @@
522528
<argument>-classpath</argument>
523529
<classpath/>
524530
<argument>org.openjdk.jmh.Main</argument>
525-
<argument>${benchmark.name}</argument>
531+
<argument>${benchmark.name}</argument>
532+
<argument>-rf</argument>
533+
<argument>JSON</argument>
534+
<argument>-rff</argument>
535+
<argument>jmh-results.json</argument>
526536
</arguments>
527537
</configuration>
528538
</execution>
@@ -544,6 +554,21 @@
544554
</plugins>
545555
</build>
546556
</profile>
557+
<profile>
558+
<id>validate-benchmark</id>
559+
<build>
560+
<plugins>
561+
<plugin>
562+
<groupId>org.codehaus.mojo</groupId>
563+
<artifactId>exec-maven-plugin</artifactId>
564+
<configuration>
565+
<mainClass>com.google.cloud.spanner.benchmarking.BenchmarkValidator</mainClass>
566+
<classpathScope>test</classpathScope>
567+
</configuration>
568+
</plugin>
569+
</plugins>
570+
</build>
571+
</profile>
547572
<profile>
548573
<id>slow-tests</id>
549574
<build>

google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerCloudMonitoringExporter.java

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import com.google.api.gax.core.CredentialsProvider;
2323
import com.google.api.gax.core.FixedCredentialsProvider;
2424
import com.google.api.gax.core.NoCredentialsProvider;
25+
import com.google.api.gax.grpc.InstantiatingGrpcChannelProvider;
2526
import com.google.api.gax.rpc.PermissionDeniedException;
2627
import com.google.auth.Credentials;
2728
import com.google.cloud.monitoring.v3.MetricServiceClient;
@@ -34,6 +35,7 @@
3435
import com.google.monitoring.v3.ProjectName;
3536
import com.google.monitoring.v3.TimeSeries;
3637
import com.google.protobuf.Empty;
38+
import io.grpc.inprocess.InProcessChannelBuilder;
3739
import io.opentelemetry.sdk.common.CompletableResultCode;
3840
import io.opentelemetry.sdk.metrics.InstrumentType;
3941
import io.opentelemetry.sdk.metrics.data.AggregationTemporality;
@@ -92,6 +94,15 @@ static SpannerCloudMonitoringExporter create(
9294
settingsBuilder.setUniverseDomain(universeDomain);
9395
}
9496

97+
if (System.getProperty("jmh.monitoring-server") != null) {
98+
settingsBuilder.setTransportChannelProvider(
99+
InstantiatingGrpcChannelProvider.newBuilder()
100+
.setChannelConfigurator(
101+
managedChannelBuilder ->
102+
InProcessChannelBuilder.forName(System.getProperty("jmh.monitoring-server")))
103+
.build());
104+
}
105+
95106
Duration timeout = Duration.ofMinutes(1);
96107
// TODO: createServiceTimeSeries needs special handling if the request failed. Leaving
97108
// it as not retried for now.
Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
/*
2+
* Copyright 2025 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.cloud.spanner.benchmarking;
18+
19+
import com.google.cloud.spanner.benchmarking.BenchmarkValidator.BaselineResult.BenchmarkResult;
20+
import com.google.cloud.spanner.benchmarking.BenchmarkValidator.BaselineResult.BenchmarkResult.Percentile;
21+
import com.google.gson.Gson;
22+
import com.google.gson.reflect.TypeToken;
23+
import java.io.File;
24+
import java.io.IOException;
25+
import java.net.URL;
26+
import java.nio.file.Files;
27+
import java.nio.file.Paths;
28+
import java.util.ArrayList;
29+
import java.util.List;
30+
import java.util.Map;
31+
32+
public class BenchmarkValidator {
33+
34+
private final BaselineResult expectedResults;
35+
private final List<ActualBenchmarkResult> actualResults;
36+
37+
public BenchmarkValidator(String baselineFile, String actualFile) {
38+
Gson gson = new Gson();
39+
// Load expected result JSON from resource folder
40+
this.expectedResults = gson.fromJson(loadJsonFromResources(baselineFile), BaselineResult.class);
41+
// Load the actual result from current benchmarking run
42+
this.actualResults =
43+
gson.fromJson(
44+
loadJsonFromFile(actualFile),
45+
new TypeToken<ArrayList<ActualBenchmarkResult>>() {}.getType());
46+
}
47+
48+
void validate() {
49+
// Validating the resultant percentile against expected percentile with allowed threshold
50+
for (ActualBenchmarkResult actualResult : actualResults) {
51+
BenchmarkResult expectResult = expectedResults.benchmarkResultMap.get(actualResult.benchmark);
52+
if (expectResult == null) {
53+
throw new ValidationException(
54+
"Missing expected benchmark configuration for actual benchmarking");
55+
}
56+
Map<String, Double> actualPercentilesMap = actualResult.primaryMetric.scorePercentiles;
57+
// We will only be comparing the percentiles(p50, p90, p90) which are configured in the
58+
// expected
59+
// percentiles. This allows some checks to be disabled if required.
60+
for (Percentile expectedPercentile : expectResult.scorePercentiles) {
61+
String percentile = expectedPercentile.percentile;
62+
double difference =
63+
calculatePercentageDifference(
64+
expectedPercentile.baseline, actualPercentilesMap.get(percentile));
65+
// if an absolute different in percentage is greater than allowed difference
66+
// Then we are throwing validation error
67+
if (Math.abs(Math.ceil(difference)) > expectedPercentile.difference) {
68+
throw new ValidationException(
69+
String.format(
70+
"[%s][%s] Expected percentile %s[+/-%s] but got %s",
71+
actualResult.benchmark,
72+
percentile,
73+
expectedPercentile.baseline,
74+
expectedPercentile.difference,
75+
actualPercentilesMap.get(percentile)));
76+
}
77+
}
78+
}
79+
}
80+
81+
public static double calculatePercentageDifference(double base, double compareWith) {
82+
if (base == 0) {
83+
return 0.0;
84+
}
85+
return ((compareWith - base) / base) * 100;
86+
}
87+
88+
private String loadJsonFromFile(String file) {
89+
try {
90+
return new String(Files.readAllBytes(Paths.get(file)));
91+
} catch (IOException e) {
92+
throw new ValidationException("Failed to read file: " + file, e);
93+
}
94+
}
95+
96+
private String loadJsonFromResources(String baselineFile) {
97+
URL resourceUrl = getClass().getClassLoader().getResource(baselineFile);
98+
if (resourceUrl == null) {
99+
throw new ValidationException("File not found: " + baselineFile);
100+
}
101+
File file = new File(resourceUrl.getFile());
102+
return loadJsonFromFile(file.getAbsolutePath());
103+
}
104+
105+
static class ActualBenchmarkResult {
106+
String benchmark;
107+
PrimaryMetric primaryMetric;
108+
109+
static class PrimaryMetric {
110+
Map<String, Double> scorePercentiles;
111+
}
112+
}
113+
114+
static class BaselineResult {
115+
Map<String, BenchmarkResult> benchmarkResultMap;
116+
117+
static class BenchmarkResult {
118+
List<Percentile> scorePercentiles;
119+
120+
static class Percentile {
121+
String percentile;
122+
Double baseline;
123+
Double difference;
124+
}
125+
}
126+
}
127+
128+
static class ValidationException extends RuntimeException {
129+
ValidationException(String message) {
130+
super(message);
131+
}
132+
133+
ValidationException(String message, Throwable cause) {
134+
super(message, cause);
135+
}
136+
}
137+
138+
private static String parseCommandLineArg(String arg) {
139+
if (arg == null || arg.isEmpty()) {
140+
return "";
141+
}
142+
String[] args = arg.split("=");
143+
if (args.length != 2) {
144+
return "";
145+
}
146+
return args[1];
147+
}
148+
149+
public static void main(String[] args) {
150+
String actualFile = parseCommandLineArg(args[0]);
151+
new BenchmarkValidator("com/google/cloud/spanner/jmh/jmh-baseline.json", actualFile).validate();
152+
}
153+
}
Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
/*
2+
* Copyright 2025 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* https://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.cloud.spanner.benchmarking;
18+
19+
import com.google.monitoring.v3.CreateTimeSeriesRequest;
20+
import com.google.monitoring.v3.MetricServiceGrpc.MetricServiceImplBase;
21+
import com.google.protobuf.Empty;
22+
import io.grpc.stub.StreamObserver;
23+
24+
class MonitoringServiceImpl extends MetricServiceImplBase {
25+
26+
@Override
27+
public void createServiceTimeSeries(
28+
CreateTimeSeriesRequest request, StreamObserver<Empty> responseObserver) {
29+
try {
30+
Thread.sleep(100);
31+
responseObserver.onNext(Empty.getDefaultInstance());
32+
responseObserver.onCompleted();
33+
} catch (InterruptedException e) {
34+
responseObserver.onError(e);
35+
}
36+
}
37+
}

0 commit comments

Comments
 (0)