Skip to content

Commit bd75326

Browse files
zoewanggJonathanHenson
zoewangg
and
JonathanHenson
authored
Add client metrics for the CRT HTTP SPI implementation and wrap CRT HttpException with IOException (#3531)
* Added client metrics for the CRT HTTP SPI implementation. (#3482) * Added client metrics for the CRT HTTP SPI implementation. * fix build error in crt request executor with correct import statement. * Added concurrency acquire duration metrics to crt http SPI implementation. * simplify conditional. * Made http exceptions io exceptions so they can be retried. * Updated the exception mapping to be in the correct adapter. * Fix checkstyle errors and add tests Co-authored-by: Jonathan M. Henson <[email protected]>
1 parent db52448 commit bd75326

File tree

8 files changed

+108
-32
lines changed

8 files changed

+108
-32
lines changed

‎http-clients/aws-crt-client/pom.xml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,11 @@
6666
<artifactId>http-client-spi</artifactId>
6767
<version>${awsjavasdk.version}</version>
6868
</dependency>
69+
<dependency>
70+
<groupId>software.amazon.awssdk</groupId>
71+
<artifactId>metrics-spi</artifactId>
72+
<version>${awsjavasdk.version}</version>
73+
</dependency>
6974
<dependency>
7075
<groupId>software.amazon.awssdk</groupId>
7176
<artifactId>utils</artifactId>

‎http-clients/aws-crt-client/src/main/java/software/amazon/awssdk/http/crt/AwsCrtAsyncHttpClient.java

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515

1616
package software.amazon.awssdk.http.crt;
1717

18+
import static software.amazon.awssdk.http.HttpMetric.HTTP_CLIENT_NAME;
1819
import static software.amazon.awssdk.utils.FunctionalUtils.invokeSafely;
1920
import static software.amazon.awssdk.utils.Validate.paramNotNull;
2021

@@ -43,6 +44,8 @@
4344
import software.amazon.awssdk.http.async.SdkAsyncHttpClient;
4445
import software.amazon.awssdk.http.crt.internal.CrtRequestContext;
4546
import software.amazon.awssdk.http.crt.internal.CrtRequestExecutor;
47+
import software.amazon.awssdk.metrics.MetricCollector;
48+
import software.amazon.awssdk.metrics.NoOpMetricCollector;
4649
import software.amazon.awssdk.utils.AttributeMap;
4750
import software.amazon.awssdk.utils.IoUtils;
4851
import software.amazon.awssdk.utils.Logger;
@@ -253,6 +256,14 @@ public CompletableFuture<Void> execute(AsyncExecuteRequest asyncRequest) {
253256
paramNotNull(asyncRequest.requestContentPublisher(), "RequestContentPublisher");
254257
paramNotNull(asyncRequest.responseHandler(), "ResponseHandler");
255258

259+
if (asyncRequest.metricCollector().isPresent()) {
260+
MetricCollector metricCollector = asyncRequest.metricCollector().get();
261+
262+
if (metricCollector != null && !(metricCollector instanceof NoOpMetricCollector)) {
263+
metricCollector.reportMetric(HTTP_CLIENT_NAME, clientName());
264+
}
265+
}
266+
256267
/*
257268
* See the note on getOrCreateConnectionPool()
258269
*

‎http-clients/aws-crt-client/src/main/java/software/amazon/awssdk/http/crt/internal/CrtRequestContext.java

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,17 +18,20 @@
1818
import software.amazon.awssdk.annotations.SdkInternalApi;
1919
import software.amazon.awssdk.crt.http.HttpClientConnectionManager;
2020
import software.amazon.awssdk.http.async.AsyncExecuteRequest;
21+
import software.amazon.awssdk.metrics.MetricCollector;
2122

2223
@SdkInternalApi
2324
public final class CrtRequestContext {
2425
private final AsyncExecuteRequest request;
2526
private final int readBufferSize;
2627
private final HttpClientConnectionManager crtConnPool;
28+
private final MetricCollector metricCollector;
2729

2830
private CrtRequestContext(Builder builder) {
2931
this.request = builder.request;
3032
this.readBufferSize = builder.readBufferSize;
3133
this.crtConnPool = builder.crtConnPool;
34+
this.metricCollector = request.metricCollector().orElse(null);
3235
}
3336

3437
public static Builder builder() {
@@ -47,6 +50,10 @@ public HttpClientConnectionManager crtConnPool() {
4750
return crtConnPool;
4851
}
4952

53+
public MetricCollector metricCollector() {
54+
return metricCollector;
55+
}
56+
5057
public static class Builder {
5158
private AsyncExecuteRequest request;
5259
private int readBufferSize;

‎http-clients/aws-crt-client/src/main/java/software/amazon/awssdk/http/crt/internal/CrtRequestExecutor.java

Lines changed: 47 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,33 +15,61 @@
1515

1616
package software.amazon.awssdk.http.crt.internal;
1717

18+
import static software.amazon.awssdk.http.HttpMetric.AVAILABLE_CONCURRENCY;
19+
import static software.amazon.awssdk.http.HttpMetric.CONCURRENCY_ACQUIRE_DURATION;
20+
import static software.amazon.awssdk.http.HttpMetric.LEASED_CONCURRENCY;
21+
import static software.amazon.awssdk.http.HttpMetric.MAX_CONCURRENCY;
22+
import static software.amazon.awssdk.http.HttpMetric.PENDING_CONCURRENCY_ACQUIRES;
23+
1824
import java.io.IOException;
25+
import java.time.Duration;
1926
import java.util.concurrent.CompletableFuture;
2027
import software.amazon.awssdk.annotations.SdkInternalApi;
2128
import software.amazon.awssdk.crt.CrtRuntimeException;
2229
import software.amazon.awssdk.crt.http.HttpClientConnection;
30+
import software.amazon.awssdk.crt.http.HttpClientConnectionManager;
31+
import software.amazon.awssdk.crt.http.HttpManagerMetrics;
2332
import software.amazon.awssdk.crt.http.HttpRequest;
2433
import software.amazon.awssdk.crt.http.HttpStreamResponseHandler;
2534
import software.amazon.awssdk.http.SdkCancellationException;
2635
import software.amazon.awssdk.http.async.AsyncExecuteRequest;
2736
import software.amazon.awssdk.http.async.SdkAsyncHttpResponseHandler;
2837
import software.amazon.awssdk.http.crt.internal.request.CrtRequestAdapter;
2938
import software.amazon.awssdk.http.crt.internal.response.CrtResponseAdapter;
39+
import software.amazon.awssdk.metrics.MetricCollector;
40+
import software.amazon.awssdk.metrics.NoOpMetricCollector;
3041
import software.amazon.awssdk.utils.Logger;
3142

3243
@SdkInternalApi
3344
public final class CrtRequestExecutor {
3445
private static final Logger log = Logger.loggerFor(CrtRequestExecutor.class);
3546

3647
public CompletableFuture<Void> execute(CrtRequestContext executionContext) {
48+
// go ahead and get a reference to the metricCollector since multiple futures will
49+
// need it regardless.
50+
MetricCollector metricCollector = executionContext.metricCollector();
51+
boolean shouldPublishMetrics = metricCollector != null && !(metricCollector instanceof NoOpMetricCollector);
52+
53+
long acquireStartTime = 0;
54+
55+
if (shouldPublishMetrics) {
56+
// go ahead and get acquireStartTime for the concurrency timer as early as possible,
57+
// so it's as accurate as possible, but only do it in a branch since clock_gettime()
58+
// results in a full sys call barrier (multiple mutexes and a hw interrupt).
59+
acquireStartTime = System.nanoTime();
60+
}
61+
3762
CompletableFuture<Void> requestFuture = createExecutionFuture(executionContext.sdkRequest());
3863

3964
// When a Connection is ready from the Connection Pool, schedule the Request on the connection
4065
CompletableFuture<HttpClientConnection> httpClientConnectionCompletableFuture =
4166
executionContext.crtConnPool().acquireConnection();
4267

68+
long finalAcquireStartTime = acquireStartTime;
69+
4370
httpClientConnectionCompletableFuture.whenComplete((crtConn, throwable) -> {
4471
AsyncExecuteRequest asyncRequest = executionContext.sdkRequest();
72+
4573
// If we didn't get a connection for some reason, fail the request
4674
if (throwable != null) {
4775
reportFailure(new IOException("An exception occurred when acquiring a connection", throwable),
@@ -50,6 +78,12 @@ public CompletableFuture<Void> execute(CrtRequestContext executionContext) {
5078
return;
5179
}
5280

81+
if (shouldPublishMetrics) {
82+
long acquireCompletionTime = System.nanoTime();
83+
Duration acquireTimeTaken = Duration.ofNanos(acquireCompletionTime - finalAcquireStartTime);
84+
metricCollector.reportMetric(CONCURRENCY_ACQUIRE_DURATION, acquireTimeTaken);
85+
}
86+
5387
HttpRequest crtRequest = CrtRequestAdapter.toCrtRequest(executionContext);
5488
HttpStreamResponseHandler crtResponseHandler =
5589
CrtResponseAdapter.toCrtResponseHandler(crtConn, requestFuture, asyncRequest.responseHandler());
@@ -64,6 +98,18 @@ public CompletableFuture<Void> execute(CrtRequestContext executionContext) {
6498
}
6599
});
66100

101+
requestFuture.whenComplete((obj, err) -> {
102+
if (shouldPublishMetrics) {
103+
HttpClientConnectionManager connManager = executionContext.crtConnPool();
104+
HttpManagerMetrics managerMetrics = connManager.getManagerMetrics();
105+
// currently this executor only handles HTTP 1.1. Until H2 is added, the max concurrency settings are 1:1 with TCP
106+
// connections. When H2 is added, this code needs to be updated to handle stream multiplexing
107+
metricCollector.reportMetric(MAX_CONCURRENCY, connManager.getMaxConnections());
108+
metricCollector.reportMetric(AVAILABLE_CONCURRENCY, (int) managerMetrics.getAvailableConcurrency());
109+
metricCollector.reportMetric(LEASED_CONCURRENCY, (int) managerMetrics.getLeasedConcurrency());
110+
metricCollector.reportMetric(PENDING_CONCURRENCY_ACQUIRES, (int) managerMetrics.getPendingConcurrencyAcquires());
111+
}
112+
});
67113
return requestFuture;
68114
}
69115

@@ -91,7 +137,7 @@ private CompletableFuture<Void> createExecutionFuture(AsyncExecuteRequest reques
91137
/**
92138
* Notify the provided response handler and future of the failure.
93139
*/
94-
private void reportFailure(Throwable cause,
140+
private void reportFailure(IOException cause,
95141
CompletableFuture<Void> executeFuture,
96142
SdkAsyncHttpResponseHandler responseHandler) {
97143
try {

‎http-clients/aws-crt-client/src/main/java/software/amazon/awssdk/http/crt/internal/response/CrtResponseAdapter.java

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515

1616
package software.amazon.awssdk.http.crt.internal.response;
1717

18+
import java.io.IOException;
1819
import java.nio.ByteBuffer;
1920
import java.util.concurrent.CompletableFuture;
2021
import software.amazon.awssdk.annotations.SdkInternalApi;
@@ -126,9 +127,11 @@ private void onSuccessfulResponseComplete(HttpStream stream) {
126127
}
127128

128129
private void onFailedResponseComplete(HttpStream stream, HttpException error) {
129-
log.error(() -> "HTTP response encountered an error.", error);
130-
responsePublisher.error(error);
131-
failResponseHandlerAndFuture(stream, error);
130+
log.debug(() -> "HTTP response encountered an error.", error);
131+
132+
IOException wrappedError = new IOException(error);
133+
responsePublisher.error(wrappedError);
134+
failResponseHandlerAndFuture(stream, wrappedError);
132135
}
133136

134137
private void failResponseHandlerAndFuture(HttpStream stream, Throwable error) {

‎http-clients/aws-crt-client/src/test/java/software/amazon/awssdk/http/crt/AwsCrtHttpClientSpiVerificationTest.java

Lines changed: 27 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -20,31 +20,37 @@
2020
import static com.github.tomakehurst.wiremock.client.WireMock.binaryEqualTo;
2121
import static com.github.tomakehurst.wiremock.client.WireMock.stubFor;
2222
import static com.github.tomakehurst.wiremock.client.WireMock.urlEqualTo;
23+
import static com.github.tomakehurst.wiremock.client.WireMock.urlPathEqualTo;
2324
import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig;
2425
import static java.util.Collections.emptyMap;
2526
import static org.apache.commons.codec.digest.DigestUtils.sha256Hex;
2627
import static org.assertj.core.api.Assertions.assertThat;
2728
import static org.assertj.core.api.Assertions.assertThatThrownBy;
29+
import static software.amazon.awssdk.http.HttpTestUtils.createProvider;
30+
import static software.amazon.awssdk.http.crt.CrtHttpClientTestUtils.createRequest;
2831

2932
import com.github.tomakehurst.wiremock.http.Fault;
3033
import com.github.tomakehurst.wiremock.junit.WireMockRule;
34+
import java.io.IOException;
3135
import java.net.URI;
3236
import java.nio.ByteBuffer;
3337
import java.time.Duration;
3438
import java.util.Random;
3539
import java.util.concurrent.CompletableFuture;
3640
import java.util.concurrent.TimeUnit;
3741
import java.util.concurrent.atomic.AtomicReference;
38-
import org.junit.After;
39-
import org.junit.Before;
42+
import org.junit.AfterClass;
43+
import org.junit.BeforeClass;
4044
import org.junit.Rule;
4145
import org.junit.Test;
4246
import org.reactivestreams.Publisher;
4347
import org.reactivestreams.Subscriber;
4448
import org.reactivestreams.Subscription;
4549
import software.amazon.awssdk.crt.CrtResource;
50+
import software.amazon.awssdk.crt.http.HttpException;
4651
import software.amazon.awssdk.crt.io.EventLoopGroup;
4752
import software.amazon.awssdk.crt.io.HostResolver;
53+
import software.amazon.awssdk.http.RecordingResponseHandler;
4854
import software.amazon.awssdk.http.SdkHttpMethod;
4955
import software.amazon.awssdk.http.SdkHttpRequest;
5056
import software.amazon.awssdk.http.SdkHttpResponse;
@@ -62,10 +68,10 @@ public class AwsCrtHttpClientSpiVerificationTest {
6268
.dynamicPort()
6369
.dynamicHttpsPort());
6470

65-
private SdkAsyncHttpClient client;
71+
private static SdkAsyncHttpClient client;
6672

67-
@Before
68-
public void setup() throws Exception {
73+
@BeforeClass
74+
public static void setup() throws Exception {
6975
CrtResource.waitForNoResources();
7076

7177
client = AwsCrtAsyncHttpClient.builder()
@@ -74,8 +80,8 @@ public void setup() throws Exception {
7480
.build();
7581
}
7682

77-
@After
78-
public void tearDown() {
83+
@AfterClass
84+
public static void tearDown() {
7985
client.close();
8086
EventLoopGroup.closeStaticDefault();
8187
HostResolver.closeStaticDefault();
@@ -110,8 +116,20 @@ public void onError(Throwable error) {
110116
.build());
111117

112118
assertThat(errorSignaled.get(1, TimeUnit.SECONDS)).isTrue();
113-
assertThatThrownBy(executeFuture::join).hasCauseInstanceOf(Exception.class);
119+
assertThatThrownBy(executeFuture::join).hasCauseInstanceOf(IOException.class).hasRootCauseInstanceOf(HttpException.class);
120+
}
114121

122+
@Test
123+
public void requestFailed_connectionTimeout_shouldWrapException() {
124+
try (SdkAsyncHttpClient client = AwsCrtAsyncHttpClient.builder().connectionTimeout(Duration.ofNanos(1)).build()) {
125+
URI uri = URI.create("http://localhost:" + mockServer.port());
126+
stubFor(any(urlPathEqualTo("/")).willReturn(aResponse().withFault(Fault.RANDOM_DATA_THEN_CLOSE)));
127+
SdkHttpRequest request = createRequest(uri);
128+
RecordingResponseHandler recorder = new RecordingResponseHandler();
129+
client.execute(AsyncExecuteRequest.builder().request(request).requestContentPublisher(createProvider("")).responseHandler(recorder).build());
130+
assertThatThrownBy(() -> recorder.completeFuture().get(5, TimeUnit.SECONDS)).hasCauseInstanceOf(IOException.class)
131+
.hasRootCauseInstanceOf(HttpException.class);
132+
}
115133
}
116134

117135
@Test
@@ -135,7 +153,7 @@ public void onStream(Publisher<ByteBuffer> stream) {
135153

136154
SdkHttpRequest request = CrtHttpClientTestUtils.createRequest(URI.create("http://localhost:" + mockServer.port()));
137155

138-
CompletableFuture future = client.execute(AsyncExecuteRequest.builder()
156+
CompletableFuture<Void> future = client.execute(AsyncExecuteRequest.builder()
139157
.request(request)
140158
.responseHandler(handler)
141159
.requestContentPublisher(new EmptyPublisher())

‎http-clients/aws-crt-client/src/test/java/software/amazon/awssdk/http/crt/AwsCrtHttpClientWireMockTest.java

Lines changed: 4 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -28,15 +28,11 @@
2828
import com.github.tomakehurst.wiremock.junit.WireMockRule;
2929
import java.net.URI;
3030
import java.util.concurrent.TimeUnit;
31-
import org.junit.After;
32-
import org.junit.Before;
31+
import org.junit.AfterClass;
3332
import org.junit.BeforeClass;
3433
import org.junit.Rule;
3534
import org.junit.Test;
3635
import software.amazon.awssdk.crt.CrtResource;
37-
import software.amazon.awssdk.crt.io.EventLoopGroup;
38-
import software.amazon.awssdk.crt.io.HostResolver;
39-
import software.amazon.awssdk.http.RecordingNetworkTrafficListener;
4036
import software.amazon.awssdk.http.RecordingResponseHandler;
4137
import software.amazon.awssdk.http.SdkHttpRequest;
4238
import software.amazon.awssdk.http.async.AsyncExecuteRequest;
@@ -45,29 +41,19 @@
4541

4642
public class AwsCrtHttpClientWireMockTest {
4743
private static final Logger log = Logger.loggerFor(AwsCrtHttpClientWireMockTest.class);
48-
private final RecordingNetworkTrafficListener wiremockTrafficListener = new RecordingNetworkTrafficListener();
4944

5045
@Rule
5146
public WireMockRule mockServer = new WireMockRule(wireMockConfig()
52-
.dynamicPort()
53-
.dynamicHttpsPort()
54-
.networkTrafficListener(wiremockTrafficListener));
47+
.dynamicPort());
5548

5649
@BeforeClass
5750
public static void setup() {
5851
System.setProperty("aws.crt.debugnative", "true");
5952
}
6053

61-
@Before
62-
public void methodSetup() {
63-
wiremockTrafficListener.reset();
64-
}
65-
66-
@After
67-
public void tearDown() {
54+
@AfterClass
55+
public static void tearDown() {
6856
// Verify there is no resource leak.
69-
EventLoopGroup.closeStaticDefault();
70-
HostResolver.closeStaticDefault();
7157
CrtResource.waitForNoResources();
7258
}
7359

‎pom.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,7 @@
114114
<rxjava.version>2.2.21</rxjava.version>
115115
<commons-codec.verion>1.10</commons-codec.verion>
116116
<jmh.version>1.29</jmh.version>
117-
<awscrt.version>0.19.2</awscrt.version>
117+
<awscrt.version>0.19.6</awscrt.version>
118118

119119
<!--Test dependencies -->
120120
<junit5.version>5.8.1</junit5.version>

0 commit comments

Comments
 (0)