Skip to content

Commit 7383362

Browse files
surbhigarg92surbhigarg92
committed
Adding gfe_latencies metric to built-in metrics
1 parent ea1ebad commit 7383362

File tree

8 files changed

+205
-36
lines changed

8 files changed

+205
-36
lines changed

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInMetricsConstant.java‎

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -34,9 +34,9 @@
3434
public class BuiltInMetricsConstant {
3535

3636
public static final String METER_NAME = "spanner.googleapis.com/internal/client";
37-
3837
public static final String GAX_METER_NAME = OpenTelemetryMetricsRecorder.GAX_METER_NAME;
39-
38+
static final String SPANNER_METER_NAME = "spanner-java";
39+
static final String GFE_LATENCIES_NAME = "gfe_latencies";
4040
static final String OPERATION_LATENCIES_NAME = "operation_latencies";
4141
static final String ATTEMPT_LATENCIES_NAME = "attempt_latencies";
4242
static final String OPERATION_LATENCY_NAME = "operation_latency";
@@ -114,27 +114,39 @@ static Map<InstrumentSelector, View> getAllViews() {
114114
ImmutableMap.Builder<InstrumentSelector, View> views = ImmutableMap.builder();
115115
defineView(
116116
views,
117+
BuiltInMetricsConstant.GAX_METER_NAME,
117118
BuiltInMetricsConstant.OPERATION_LATENCY_NAME,
118119
BuiltInMetricsConstant.OPERATION_LATENCIES_NAME,
119120
BuiltInMetricsConstant.AGGREGATION_WITH_MILLIS_HISTOGRAM,
120121
InstrumentType.HISTOGRAM,
121122
"ms");
122123
defineView(
123124
views,
125+
BuiltInMetricsConstant.GAX_METER_NAME,
124126
BuiltInMetricsConstant.ATTEMPT_LATENCY_NAME,
125127
BuiltInMetricsConstant.ATTEMPT_LATENCIES_NAME,
126128
BuiltInMetricsConstant.AGGREGATION_WITH_MILLIS_HISTOGRAM,
127129
InstrumentType.HISTOGRAM,
128130
"ms");
129131
defineView(
130132
views,
133+
BuiltInMetricsConstant.SPANNER_METER_NAME,
134+
BuiltInMetricsConstant.GFE_LATENCIES_NAME,
135+
BuiltInMetricsConstant.GFE_LATENCIES_NAME,
136+
BuiltInMetricsConstant.AGGREGATION_WITH_MILLIS_HISTOGRAM,
137+
InstrumentType.HISTOGRAM,
138+
"ms");
139+
defineView(
140+
views,
141+
BuiltInMetricsConstant.GAX_METER_NAME,
131142
BuiltInMetricsConstant.OPERATION_COUNT_NAME,
132143
BuiltInMetricsConstant.OPERATION_COUNT_NAME,
133144
Aggregation.sum(),
134145
InstrumentType.COUNTER,
135146
"1");
136147
defineView(
137148
views,
149+
BuiltInMetricsConstant.GAX_METER_NAME,
138150
BuiltInMetricsConstant.ATTEMPT_COUNT_NAME,
139151
BuiltInMetricsConstant.ATTEMPT_COUNT_NAME,
140152
Aggregation.sum(),
@@ -145,6 +157,7 @@ static Map<InstrumentSelector, View> getAllViews() {
145157

146158
private static void defineView(
147159
ImmutableMap.Builder<InstrumentSelector, View> viewMap,
160+
String meterName,
148161
String metricName,
149162
String metricViewName,
150163
Aggregation aggregation,
@@ -153,7 +166,7 @@ private static void defineView(
153166
InstrumentSelector selector =
154167
InstrumentSelector.builder()
155168
.setName(BuiltInMetricsConstant.METER_NAME + '/' + metricName)
156-
.setMeterName(BuiltInMetricsConstant.GAX_METER_NAME)
169+
.setMeterName(meterName)
157170
.setType(type)
158171
.setUnit(unit)
159172
.build();

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInOpenTelemetryMetricsProvider.java‎

Lines changed: 29 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,8 @@
2828
import com.google.cloud.opentelemetry.detection.AttributeKeys;
2929
import com.google.cloud.opentelemetry.detection.DetectedPlatform;
3030
import com.google.cloud.opentelemetry.detection.GCPPlatformDetector;
31+
import com.google.common.cache.Cache;
32+
import com.google.common.cache.CacheBuilder;
3133
import com.google.common.hash.HashFunction;
3234
import com.google.common.hash.Hashing;
3335
import io.opentelemetry.api.OpenTelemetry;
@@ -42,6 +44,7 @@
4244
import java.util.HashMap;
4345
import java.util.Map;
4446
import java.util.UUID;
47+
import java.util.concurrent.ExecutionException;
4548
import java.util.logging.Level;
4649
import java.util.logging.Logger;
4750
import javax.annotation.Nullable;
@@ -57,6 +60,9 @@ final class BuiltInOpenTelemetryMetricsProvider {
5760

5861
private OpenTelemetry openTelemetry;
5962

63+
private final Cache<String, Map<String, String>> clientAttributesCache =
64+
CacheBuilder.newBuilder().maximumSize(1000).build();
65+
6066
private BuiltInOpenTelemetryMetricsProvider() {}
6167

6268
OpenTelemetry getOrCreateOpenTelemetry(String projectId, @Nullable Credentials credentials) {
@@ -78,16 +84,29 @@ OpenTelemetry getOrCreateOpenTelemetry(String projectId, @Nullable Credentials c
7884
}
7985
}
8086

81-
Map<String, String> createClientAttributes(String projectId, String client_name) {
82-
Map<String, String> clientAttributes = new HashMap<>();
83-
clientAttributes.put(LOCATION_ID_KEY.getKey(), detectClientLocation());
84-
clientAttributes.put(PROJECT_ID_KEY.getKey(), projectId);
85-
clientAttributes.put(INSTANCE_CONFIG_ID_KEY.getKey(), "unknown");
86-
clientAttributes.put(CLIENT_NAME_KEY.getKey(), client_name);
87-
String clientUid = getDefaultTaskValue();
88-
clientAttributes.put(CLIENT_UID_KEY.getKey(), clientUid);
89-
clientAttributes.put(CLIENT_HASH_KEY.getKey(), generateClientHash(clientUid));
90-
return clientAttributes;
87+
Map<String, String> createOrGetClientAttributes(String projectId, String client_name) {
88+
try {
89+
String key = projectId + client_name;
90+
return clientAttributesCache.get(
91+
key,
92+
() -> {
93+
Map<String, String> clientAttributes = new HashMap<>();
94+
clientAttributes.put(LOCATION_ID_KEY.getKey(), detectClientLocation());
95+
clientAttributes.put(PROJECT_ID_KEY.getKey(), projectId);
96+
clientAttributes.put(INSTANCE_CONFIG_ID_KEY.getKey(), "unknown");
97+
clientAttributes.put(CLIENT_NAME_KEY.getKey(), client_name);
98+
String clientUid = getDefaultTaskValue();
99+
clientAttributes.put(CLIENT_UID_KEY.getKey(), clientUid);
100+
clientAttributes.put(CLIENT_HASH_KEY.getKey(), generateClientHash(clientUid));
101+
return clientAttributes;
102+
});
103+
} catch (ExecutionException executionException) {
104+
logger.log(
105+
Level.WARNING,
106+
"Unable to get Client Attributes for client side metrics, will skip exporting client side metrics",
107+
executionException);
108+
return null;
109+
}
91110
}
92111

93112
/**

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/BuiltInOpenTelemetryMetricsRecorder.java‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
/*
2+
* Copyright 2024 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.cloud.spanner;
18+
19+
import com.google.api.gax.core.GaxProperties;
20+
import com.google.common.annotations.VisibleForTesting;
21+
import com.google.common.base.Preconditions;
22+
import io.opentelemetry.api.OpenTelemetry;
23+
import io.opentelemetry.api.common.Attributes;
24+
import io.opentelemetry.api.common.AttributesBuilder;
25+
import io.opentelemetry.api.metrics.DoubleHistogram;
26+
import io.opentelemetry.api.metrics.Meter;
27+
import java.util.HashMap;
28+
import java.util.Map;
29+
30+
/** OpenTelemetry implementation of recording built in metrics. */
31+
public class BuiltInOpenTelemetryMetricsRecorder {
32+
33+
private final DoubleHistogram gfeLatencyRecorder;
34+
private final Map<String, String> attributes = new HashMap<>();
35+
36+
/**
37+
* Creates the following instruments for the following metrics:
38+
*
39+
* <ul>
40+
* <li>GFE Latency: Histogram
41+
* </ul>
42+
*
43+
* @param openTelemetry OpenTelemetry instance
44+
*/
45+
public BuiltInOpenTelemetryMetricsRecorder(
46+
OpenTelemetry openTelemetry, Map<String, String> clientAttributes) {
47+
if (openTelemetry != null && clientAttributes != null) {
48+
gfeLatencyRecorder = null;
49+
return;
50+
}
51+
Meter meter =
52+
openTelemetry
53+
.meterBuilder(BuiltInMetricsConstant.SPANNER_METER_NAME)
54+
.setInstrumentationVersion(GaxProperties.getLibraryVersion(getClass()))
55+
.build();
56+
this.gfeLatencyRecorder =
57+
meter
58+
.histogramBuilder(
59+
BuiltInMetricsConstant.METER_NAME + '/' + BuiltInMetricsConstant.GFE_LATENCIES_NAME)
60+
.setDescription(
61+
"Latency between Google's network receiving an RPC and reading back the first byte of the response")
62+
.setUnit("ms")
63+
.build();
64+
this.attributes.putAll(clientAttributes);
65+
}
66+
67+
/**
68+
* Record the latency between Google's network receiving an RPC and reading back the first byte of
69+
* the response. Data is stored in a Histogram.
70+
*
71+
* @param gfeLatency Attempt Latency in ms
72+
* @param attributes Map of the attributes to store
73+
*/
74+
public void recordGFELatency(double gfeLatency, Map<String, String> attributes) {
75+
if (gfeLatencyRecorder != null) {
76+
this.attributes.putAll(attributes);
77+
gfeLatencyRecorder.record(gfeLatency, toOtelAttributes(this.attributes));
78+
}
79+
}
80+
81+
@VisibleForTesting
82+
Attributes toOtelAttributes(Map<String, String> attributes) {
83+
Preconditions.checkNotNull(attributes, "Attributes map cannot be null");
84+
AttributesBuilder attributesBuilder = Attributes.builder();
85+
attributes.forEach(attributesBuilder::put);
86+
return attributesBuilder.build();
87+
}
88+
}

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java‎

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1680,6 +1680,18 @@ public OpenTelemetry getOpenTelemetry() {
16801680
}
16811681
}
16821682

1683+
/** Returns an instance of OpenTelemetry object for Built-in Client metrics. */
1684+
public OpenTelemetry getBuiltInMetricsOpenTelemetry() {
1685+
return this.builtInOpenTelemetryMetricsProvider.getOrCreateOpenTelemetry(
1686+
this.getProjectId(), getCredentials());
1687+
}
1688+
1689+
/** Returns attributes for an instance of Built-in Client metrics. */
1690+
public Map<String, String> getBuiltInMetricsClientAttributes() {
1691+
return builtInOpenTelemetryMetricsProvider.createOrGetClientAttributes(
1692+
this.getProjectId(), "spanner-java/" + GaxProperties.getLibraryVersion(getClass()));
1693+
}
1694+
16831695
@Override
16841696
public ApiTracerFactory getApiTracerFactory() {
16851697
return createApiTracerFactory(false, false);
@@ -1729,11 +1741,13 @@ private ApiTracerFactory createMetricsApiTracerFactory() {
17291741
this.builtInOpenTelemetryMetricsProvider.getOrCreateOpenTelemetry(
17301742
this.getProjectId(), getCredentials());
17311743

1732-
return openTelemetry != null
1744+
Map<String, String> clientAttributes =
1745+
builtInOpenTelemetryMetricsProvider.createOrGetClientAttributes(
1746+
this.getProjectId(), "spanner-java/" + GaxProperties.getLibraryVersion(getClass()));
1747+
return openTelemetry != null && clientAttributes != null
17331748
? new MetricsTracerFactory(
17341749
new OpenTelemetryMetricsRecorder(openTelemetry, BuiltInMetricsConstant.METER_NAME),
1735-
builtInOpenTelemetryMetricsProvider.createClientAttributes(
1736-
this.getProjectId(), "spanner-java/" + GaxProperties.getLibraryVersion(getClass())))
1750+
clientAttributes)
17371751
: null;
17381752
}
17391753

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpc.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -357,6 +357,8 @@ public GapicSpannerRpc(final SpannerOptions options) {
357357
options.getInterceptorProvider(),
358358
SpannerInterceptorProvider.createDefault(
359359
options.getOpenTelemetry(),
360+
options.getBuiltInMetricsOpenTelemetry(),
361+
options.getBuiltInMetricsClientAttributes(),
360362
(() -> directPathEnabledSupplier.get()))))
361363
// This sets the trace context headers.
362364
.withTraceContext(endToEndTracingEnabled, options.getOpenTelemetry())

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/HeaderInterceptor.java‎

Lines changed: 35 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525

2626
import com.google.api.gax.tracing.ApiTracer;
2727
import com.google.cloud.spanner.BuiltInMetricsConstant;
28+
import com.google.cloud.spanner.BuiltInOpenTelemetryMetricsRecorder;
2829
import com.google.cloud.spanner.CompositeTracer;
2930
import com.google.cloud.spanner.SpannerExceptionFactory;
3031
import com.google.cloud.spanner.SpannerRpcMetrics;
@@ -94,12 +95,17 @@ class HeaderInterceptor implements ClientInterceptor {
9495
private static final Level LEVEL = Level.INFO;
9596
private final SpannerRpcMetrics spannerRpcMetrics;
9697

98+
private final BuiltInOpenTelemetryMetricsRecorder builtInOpenTelemetryMetricsRecorder;
99+
97100
private final Supplier<Boolean> directPathEnabledSupplier;
98101

99102
HeaderInterceptor(
100-
SpannerRpcMetrics spannerRpcMetrics, Supplier<Boolean> directPathEnabledSupplier) {
103+
SpannerRpcMetrics spannerRpcMetrics,
104+
BuiltInOpenTelemetryMetricsRecorder builtInOpenTelemetryMetricsRecorder,
105+
Supplier<Boolean> directPathEnabledSupplier) {
101106
this.spannerRpcMetrics = spannerRpcMetrics;
102107
this.directPathEnabledSupplier = directPathEnabledSupplier;
108+
this.builtInOpenTelemetryMetricsRecorder = builtInOpenTelemetryMetricsRecorder;
103109
}
104110

105111
@Override
@@ -118,17 +124,22 @@ public void start(Listener<RespT> responseListener, Metadata headers) {
118124
TagContext tagContext = getTagContext(key, method.getFullMethodName(), databaseName);
119125
Attributes attributes =
120126
getMetricAttributes(key, method.getFullMethodName(), databaseName);
121-
Map<String, String> builtInMetricsAttributes =
122-
getBuiltInMetricAttributes(key, databaseName);
127+
Map<String, String> commonBuiltInMetricAttributes =
128+
getCommonBuiltInMetricAttributes(key, databaseName);
123129
super.start(
124130
new SimpleForwardingClientCallListener<RespT>(responseListener) {
125131
@Override
126132
public void onHeaders(Metadata metadata) {
127133
Boolean isDirectPathUsed =
128134
isDirectPathUsed(getAttributes().get(Grpc.TRANSPORT_ATTR_REMOTE_ADDR));
129135
addBuiltInMetricAttributes(
130-
compositeTracer, builtInMetricsAttributes, isDirectPathUsed);
131-
processHeader(metadata, tagContext, attributes, span);
136+
compositeTracer, commonBuiltInMetricAttributes, isDirectPathUsed);
137+
processHeader(
138+
metadata,
139+
tagContext,
140+
attributes,
141+
span,
142+
getBuiltInMetricAttributes(commonBuiltInMetricAttributes, isDirectPathUsed));
132143
super.onHeaders(metadata);
133144
}
134145
},
@@ -142,7 +153,11 @@ public void onHeaders(Metadata metadata) {
142153
}
143154

144155
private void processHeader(
145-
Metadata metadata, TagContext tagContext, Attributes attributes, Span span) {
156+
Metadata metadata,
157+
TagContext tagContext,
158+
Attributes attributes,
159+
Span span,
160+
Map<String, String> builtInMetricsAttributes) {
146161
MeasureMap measureMap = STATS_RECORDER.newMeasureMap();
147162
String serverTiming = metadata.get(SERVER_TIMING_HEADER_KEY);
148163
if (serverTiming != null && serverTiming.startsWith(SERVER_TIMING_HEADER_PREFIX)) {
@@ -154,6 +169,7 @@ private void processHeader(
154169

155170
spannerRpcMetrics.recordGfeLatency(latency, attributes);
156171
spannerRpcMetrics.recordGfeHeaderMissingCount(0L, attributes);
172+
builtInOpenTelemetryMetricsRecorder.recordGFELatency(latency, builtInMetricsAttributes);
157173

158174
if (span != null) {
159175
span.setAttribute("gfe_latency", String.valueOf(latency));
@@ -224,8 +240,8 @@ private Attributes getMetricAttributes(String key, String method, DatabaseName d
224240
});
225241
}
226242

227-
private Map<String, String> getBuiltInMetricAttributes(String key, DatabaseName databaseName)
228-
throws ExecutionException {
243+
private Map<String, String> getCommonBuiltInMetricAttributes(
244+
String key, DatabaseName databaseName) throws ExecutionException {
229245
return builtInAttributesCache.get(
230246
key,
231247
() -> {
@@ -240,17 +256,21 @@ private Map<String, String> getBuiltInMetricAttributes(String key, DatabaseName
240256
});
241257
}
242258

259+
private Map<String, String> getBuiltInMetricAttributes(
260+
Map<String, String> commonBuiltInMetricsAttributes, Boolean isDirectPathUsed) {
261+
Map<String, String> builtInMetricAttributes = new HashMap<>(commonBuiltInMetricsAttributes);
262+
builtInMetricAttributes.put(
263+
BuiltInMetricsConstant.DIRECT_PATH_USED_KEY.getKey(), Boolean.toString(isDirectPathUsed));
264+
return builtInMetricAttributes;
265+
}
266+
243267
private void addBuiltInMetricAttributes(
244268
CompositeTracer compositeTracer,
245-
Map<String, String> builtInMetricsAttributes,
269+
Map<String, String> commonBuiltInMetricsAttributes,
246270
Boolean isDirectPathUsed) {
247271
if (compositeTracer != null) {
248-
// Direct Path used attribute
249-
Map<String, String> attributes = new HashMap<>(builtInMetricsAttributes);
250-
attributes.put(
251-
BuiltInMetricsConstant.DIRECT_PATH_USED_KEY.getKey(), Boolean.toString(isDirectPathUsed));
252-
253-
compositeTracer.addAttributes(attributes);
272+
compositeTracer.addAttributes(
273+
getBuiltInMetricAttributes(commonBuiltInMetricsAttributes, isDirectPathUsed));
254274
}
255275
}
256276

0 commit comments

Comments
 (0)