diff --git a/.brazil.json b/.brazil.json index 5a1d830abd..dbb2135d1b 100644 --- a/.brazil.json +++ b/.brazil.json @@ -8,6 +8,7 @@ "com.squareup.okio:okio-jvm:3.*": "OkioJvm-3.x", "io.opentelemetry:opentelemetry-api:1.*": "Maven-io-opentelemetry_opentelemetry-api-1.x", "org.slf4j:slf4j-api:2.*": "Maven-org-slf4j_slf4j-api-2.x", + "aws.sdk.kotlin.crt:aws-crt-kotlin:0.9.*": "AwsCrtKotlin-0.9.x", "aws.sdk.kotlin.crt:aws-crt-kotlin:0.8.*": "AwsCrtKotlin-0.8.x", "com.squareup.okhttp3:okhttp:4.*": "OkHttp3-4.x" }, diff --git a/.changes/d3ce7511-6fb2-4435-8f46-db724551b384.json b/.changes/d3ce7511-6fb2-4435-8f46-db724551b384.json new file mode 100644 index 0000000000..aa89d9167e --- /dev/null +++ b/.changes/d3ce7511-6fb2-4435-8f46-db724551b384.json @@ -0,0 +1,5 @@ +{ + "id": "d3ce7511-6fb2-4435-8f46-db724551b384", + "type": "misc", + "description": "Add telemetry provider configuration to `DefaultAwsSigner`" +} \ No newline at end of file diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index 599293cc06..d47020ea5d 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -62,13 +62,12 @@ jobs: - name: Test shell: bash run: | - # FIXME K2. Re-enable warnings as errors after this warning is removed: https://youtrack.jetbrains.com/issue/KT-68532 - # echo "kotlinWarningsAsErrors=true" >> $GITHUB_WORKSPACE/local.properties + echo "kotlinWarningsAsErrors=true" >> $GITHUB_WORKSPACE/local.properties ./gradlew apiCheck ./gradlew test jvmTest - name: Save Test Reports if: failure() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: test-reports-${{ matrix.os }} path: '**/build/reports' diff --git a/CHANGELOG.md b/CHANGELOG.md index c324c5e681..2e925d0943 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [1.4.2] - 01/28/2025 + +### Fixes +* Ignore hop-by-hop headers when signing requests + +## [1.4.1] - 01/16/2025 + ## [1.4.0] - 01/15/2025 ### Features diff --git a/codegen/protocol-tests/model/error-correction-tests.smithy b/codegen/protocol-tests/model/error-correction-tests.smithy index 201ffc877f..4f246e1b99 100644 --- a/codegen/protocol-tests/model/error-correction-tests.smithy +++ b/codegen/protocol-tests/model/error-correction-tests.smithy @@ -39,8 +39,9 @@ operation SayHelloXml { output: TestOutput, errors: [Error] } structure TestOutputDocument with [TestStruct] { innerField: Nested, - // FIXME: This trait fails smithy validator - // @required + + // Note: This shape _should_ be @required, but causes Smithy httpResponseTests validation to fail. + // We expect `document` to be deserialized as `null` and enforce @required using a runtime check, but Smithy validator doesn't recognize / allow this. document: Document } structure TestOutput with [TestStruct] { innerField: Nested } @@ -65,8 +66,8 @@ structure TestStruct { @required nestedListValue: NestedList - // FIXME: This trait fails smithy validator - // @required + // Note: This shape _should_ be @required, but causes Smithy httpResponseTests validation to fail. + // We expect `nested` to be deserialized as `null` and enforce @required using a runtime check, but Smithy validator doesn't recognize / allow this. nested: Nested @required @@ -97,8 +98,7 @@ union MyUnion { } structure Nested { - // FIXME: This trait fails smithy validator - // @required + @required a: String } diff --git a/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/aws/protocols/core/AwsHttpBindingProtocolGenerator.kt b/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/aws/protocols/core/AwsHttpBindingProtocolGenerator.kt index 6e5834f7d1..bd41afcade 100644 --- a/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/aws/protocols/core/AwsHttpBindingProtocolGenerator.kt +++ b/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/aws/protocols/core/AwsHttpBindingProtocolGenerator.kt @@ -40,13 +40,7 @@ abstract class AwsHttpBindingProtocolGenerator : HttpBindingProtocolGenerator() // val targetedTest = TestMemberDelta(setOf("RestJsonComplexErrorWithNoMessage"), TestContainmentMode.RUN_TESTS) val ignoredTests = TestMemberDelta( - setOf( - "AwsJson10ClientErrorCorrectsWithDefaultValuesWhenServerFailsToSerializeRequiredValues", - "RestJsonNullAndEmptyHeaders", - "NullAndEmptyHeaders", - "RpcV2CborClientPopulatesDefaultsValuesWhenMissingInResponse", - "RpcV2CborClientPopulatesDefaultValuesInInput", - ), + setOf(), ) val requestTestBuilder = HttpProtocolUnitTestRequestGenerator.Builder() diff --git a/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/core/KotlinSymbolProvider.kt b/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/core/KotlinSymbolProvider.kt index e6dbd0dca1..f35570dfeb 100644 --- a/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/core/KotlinSymbolProvider.kt +++ b/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/core/KotlinSymbolProvider.kt @@ -193,7 +193,7 @@ class KotlinSymbolProvider(private val model: Model, private val settings: Kotli } else { // only use @default if type is `T` shape.getTrait()?.let { - defaultValue(it.getDefaultValue(targetShape)) + setDefaultValue(it, targetShape) } } } @@ -219,9 +219,10 @@ class KotlinSymbolProvider(private val model: Model, private val settings: Kotli } } - private fun DefaultTrait.getDefaultValue(targetShape: Shape): String? { - val node = toNode() - return when { + private fun Symbol.Builder.setDefaultValue(defaultTrait: DefaultTrait, targetShape: Shape) { + val node = defaultTrait.toNode() + + val defaultValue = when { node.toString() == "null" -> null // Check if target is an enum before treating the default like a regular number/string @@ -235,13 +236,20 @@ class KotlinSymbolProvider(private val model: Model, private val settings: Kotli "${enumSymbol.fullName}.fromValue($arg)" } - targetShape.isBlobShape && targetShape.isStreaming -> - node - .toString() - .takeUnless { it.isEmpty() } - ?.let { "ByteStream.fromString(${it.dq()})" } + targetShape.isBlobShape -> { + addReferences(RuntimeTypes.Core.Text.Encoding.decodeBase64) - targetShape.isBlobShape -> "${node.toString().dq()}.encodeToByteArray()" + if (targetShape.isStreaming) { + node.toString() + .takeUnless { it.isEmpty() } + ?.let { + addReferences(RuntimeTypes.Core.Content.ByteStream) + "ByteStream.fromString(${it.dq()}.decodeBase64())" + } + } else { + "${node.toString().dq()}.decodeBase64().encodeToByteArray()" + } + } targetShape.isDocumentShape -> getDefaultValueForDocument(node) targetShape.isTimestampShape -> getDefaultValueForTimestamp(node.asNumberNode().get()) @@ -252,6 +260,8 @@ class KotlinSymbolProvider(private val model: Model, private val settings: Kotli node.isStringNode -> node.toString().dq() else -> node.toString() } + + defaultValue(defaultValue) } private fun getDefaultValueForTimestamp(node: NumberNode): String { diff --git a/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/StructureGenerator.kt b/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/StructureGenerator.kt index c34eb5ab9d..5d90f376d2 100644 --- a/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/StructureGenerator.kt +++ b/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/StructureGenerator.kt @@ -248,6 +248,7 @@ class StructureGenerator( } else { memberSymbol } + write("public var #L: #E", memberName, builderMemberSymbol) } write("") diff --git a/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/protocol/HttpStringValuesMapSerializer.kt b/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/protocol/HttpStringValuesMapSerializer.kt index 932b2a796a..5dfc519962 100644 --- a/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/protocol/HttpStringValuesMapSerializer.kt +++ b/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/protocol/HttpStringValuesMapSerializer.kt @@ -157,9 +157,8 @@ class HttpStringValuesMapSerializer( val paramName = binding.locationName // addAll collection parameter 2 val param2 = if (mapFnContents.isEmpty()) "input.$memberName" else "input.$memberName.map { $mapFnContents }" - val nullCheck = if (memberSymbol.isNullable) "?" else "" writer.write( - "if (input.#L$nullCheck.isNotEmpty() == true) #L(#S, #L)", + "if (input.#L != null) #L(#S, #L)", memberName, binding.location.addAllFnName, paramName, @@ -174,8 +173,7 @@ class HttpStringValuesMapSerializer( val paramName = binding.locationName val memberSymbol = symbolProvider.toSymbol(binding.member) - // NOTE: query parameters are allowed to be empty, whereas headers should omit empty string - // values from serde + // NOTE: query parameters are allowed to be empty if ((location == HttpBinding.Location.QUERY || location == HttpBinding.Location.HEADER) && binding.member.hasTrait()) { // Call the idempotency token function if no supplied value. writer.addImport(RuntimeTypes.SmithyClient.IdempotencyTokenProviderExt) @@ -185,18 +183,7 @@ class HttpStringValuesMapSerializer( paramName, ) } else { - val nullCheck = - if (location == HttpBinding.Location.QUERY || - memberTarget.hasTrait< - @Suppress("DEPRECATION") - software.amazon.smithy.model.traits.EnumTrait, - >() - ) { - if (memberSymbol.isNullable) "input.$memberName != null" else "" - } else { - val nullCheck = if (memberSymbol.isNullable) "?" else "" - "input.$memberName$nullCheck.isNotEmpty() == true" - } + val nullCheck = if (memberSymbol.isNullable) "input.$memberName != null" else "" val cond = defaultCheck(binding.member) ?: nullCheck diff --git a/codegen/smithy-kotlin-codegen/src/test/kotlin/software/amazon/smithy/kotlin/codegen/core/SymbolProviderTest.kt b/codegen/smithy-kotlin-codegen/src/test/kotlin/software/amazon/smithy/kotlin/codegen/core/SymbolProviderTest.kt index 0cc4d2cb14..bb90fbc0fb 100644 --- a/codegen/smithy-kotlin-codegen/src/test/kotlin/software/amazon/smithy/kotlin/codegen/core/SymbolProviderTest.kt +++ b/codegen/smithy-kotlin-codegen/src/test/kotlin/software/amazon/smithy/kotlin/codegen/core/SymbolProviderTest.kt @@ -182,7 +182,7 @@ class SymbolProviderTest { "double,2.71828,2.71828", "byte,10,10.toByte()", "string,\"hello\",\"hello\"", - "blob,\"abcdefg\",\"abcdefg\".encodeToByteArray()", + "blob,\"abcdefg\",\"abcdefg\".decodeBase64().encodeToByteArray()", "boolean,true,true", "bigInteger,5,5", "bigDecimal,9.0123456789,9.0123456789", diff --git a/codegen/smithy-kotlin-codegen/src/test/kotlin/software/amazon/smithy/kotlin/codegen/rendering/protocol/HttpBindingProtocolGeneratorTest.kt b/codegen/smithy-kotlin-codegen/src/test/kotlin/software/amazon/smithy/kotlin/codegen/rendering/protocol/HttpBindingProtocolGeneratorTest.kt index 3c15579dab..6f8040e7dc 100644 --- a/codegen/smithy-kotlin-codegen/src/test/kotlin/software/amazon/smithy/kotlin/codegen/rendering/protocol/HttpBindingProtocolGeneratorTest.kt +++ b/codegen/smithy-kotlin-codegen/src/test/kotlin/software/amazon/smithy/kotlin/codegen/rendering/protocol/HttpBindingProtocolGeneratorTest.kt @@ -57,8 +57,8 @@ internal class SmokeTestOperationSerializer: HttpSerializer.NonStreaming(builder.build(), "signature".encodeToByteArray()) diff --git a/runtime/auth/aws-signing-default/api/aws-signing-default.api b/runtime/auth/aws-signing-default/api/aws-signing-default.api index 3b9535c403..e00e6d829a 100644 --- a/runtime/auth/aws-signing-default/api/aws-signing-default.api +++ b/runtime/auth/aws-signing-default/api/aws-signing-default.api @@ -1,4 +1,12 @@ +public final class aws/smithy/kotlin/runtime/auth/awssigning/DefaultAwsSignerBuilder { + public fun ()V + public final fun build ()Laws/smithy/kotlin/runtime/auth/awssigning/AwsSigner; + public final fun getTelemetryProvider ()Laws/smithy/kotlin/runtime/telemetry/TelemetryProvider; + public final fun setTelemetryProvider (Laws/smithy/kotlin/runtime/telemetry/TelemetryProvider;)V +} + public final class aws/smithy/kotlin/runtime/auth/awssigning/DefaultAwsSignerKt { + public static final fun DefaultAwsSigner (Lkotlin/jvm/functions/Function1;)Laws/smithy/kotlin/runtime/auth/awssigning/AwsSigner; public static final fun getDefaultAwsSigner ()Laws/smithy/kotlin/runtime/auth/awssigning/AwsSigner; } diff --git a/runtime/auth/aws-signing-default/common/src/aws/smithy/kotlin/runtime/auth/awssigning/Canonicalizer.kt b/runtime/auth/aws-signing-default/common/src/aws/smithy/kotlin/runtime/auth/awssigning/Canonicalizer.kt index 69d742de79..c7c47fe5a2 100644 --- a/runtime/auth/aws-signing-default/common/src/aws/smithy/kotlin/runtime/auth/awssigning/Canonicalizer.kt +++ b/runtime/auth/aws-signing-default/common/src/aws/smithy/kotlin/runtime/auth/awssigning/Canonicalizer.kt @@ -58,16 +58,27 @@ internal interface Canonicalizer { ): CanonicalRequest } -// Taken from https://github.com/awslabs/aws-c-auth/blob/dd505b55fd46222834f35c6e54165d8cbebbfaaa/source/aws_signing.c#L118-L156 private val skipHeaders = setOf( - "connection", "expect", // https://github.com/awslabs/aws-sdk-kotlin/issues/862 + + // Taken from https://github.com/awslabs/aws-c-auth/blob/274a1d21330731cc51bb742794adc70ada5f4380/source/aws_signing.c#L121-L164 "sec-websocket-key", "sec-websocket-protocol", "sec-websocket-version", - "upgrade", "user-agent", "x-amzn-trace-id", + + // Taken from https://datatracker.ietf.org/doc/html/rfc2616#section-13.5.1. These are "hop-by-hop" headers which may + // be modified/removed by intervening proxies or caches. These are unsafe to sign because if they change they render + // the signature invalid. + "connection", + "keep-alive", + "proxy-authenticate", + "proxy-authorization", + "te", + "trailers", + "transfer-encoding", + "upgrade", ) internal class DefaultCanonicalizer(private val sha256Supplier: HashSupplier = ::Sha256) : Canonicalizer { diff --git a/runtime/auth/aws-signing-default/common/src/aws/smithy/kotlin/runtime/auth/awssigning/DefaultAwsSigner.kt b/runtime/auth/aws-signing-default/common/src/aws/smithy/kotlin/runtime/auth/awssigning/DefaultAwsSigner.kt index df613d31b0..791d8fce0a 100644 --- a/runtime/auth/aws-signing-default/common/src/aws/smithy/kotlin/runtime/auth/awssigning/DefaultAwsSigner.kt +++ b/runtime/auth/aws-signing-default/common/src/aws/smithy/kotlin/runtime/auth/awssigning/DefaultAwsSigner.kt @@ -4,8 +4,10 @@ */ package aws.smithy.kotlin.runtime.auth.awssigning +import aws.smithy.kotlin.runtime.ExperimentalApi import aws.smithy.kotlin.runtime.http.Headers import aws.smithy.kotlin.runtime.http.request.HttpRequest +import aws.smithy.kotlin.runtime.telemetry.TelemetryProvider import aws.smithy.kotlin.runtime.telemetry.logging.logger import aws.smithy.kotlin.runtime.time.TimestampFormat import kotlin.coroutines.coroutineContext @@ -13,13 +15,30 @@ import kotlin.coroutines.coroutineContext /** The default implementation of [AwsSigner] */ public val DefaultAwsSigner: AwsSigner = DefaultAwsSignerImpl() +/** Creates a customized instance of [AwsSigner] */ +@Suppress("ktlint:standard:function-naming") +public fun DefaultAwsSigner(block: DefaultAwsSignerBuilder.() -> Unit): AwsSigner = + DefaultAwsSignerBuilder().apply(block).build() + +/** A builder class for creating instances of [AwsSigner] using the default implementation */ +public class DefaultAwsSignerBuilder { + public var telemetryProvider: TelemetryProvider? = null + + public fun build(): AwsSigner = DefaultAwsSignerImpl( + telemetryProvider = telemetryProvider, + ) +} + +@OptIn(ExperimentalApi::class) internal class DefaultAwsSignerImpl( private val canonicalizer: Canonicalizer = Canonicalizer.Default, private val signatureCalculator: SignatureCalculator = SignatureCalculator.Default, private val requestMutator: RequestMutator = RequestMutator.Default, + private val telemetryProvider: TelemetryProvider? = null, ) : AwsSigner { override suspend fun sign(request: HttpRequest, config: AwsSigningConfig): AwsSigningResult { - val logger = coroutineContext.logger() + val logger = telemetryProvider?.loggerProvider?.getOrCreateLogger("DefaultAwsSigner") + ?: coroutineContext.logger() // TODO: implement SigV4a if (config.algorithm != AwsSigningAlgorithm.SIGV4) { @@ -52,7 +71,8 @@ internal class DefaultAwsSignerImpl( prevSignature: ByteArray, config: AwsSigningConfig, ): AwsSigningResult { - val logger = coroutineContext.logger() + val logger = telemetryProvider?.loggerProvider?.getOrCreateLogger("DefaultAwsSigner") + ?: coroutineContext.logger() val stringToSign = signatureCalculator.chunkStringToSign(chunkBody, prevSignature, config) logger.trace { "Chunk string to sign:\n$stringToSign" } @@ -70,7 +90,8 @@ internal class DefaultAwsSignerImpl( prevSignature: ByteArray, config: AwsSigningConfig, ): AwsSigningResult { - val logger = coroutineContext.logger() + val logger = telemetryProvider?.loggerProvider?.getOrCreateLogger("DefaultAwsSigner") + ?: coroutineContext.logger() // FIXME - can we share canonicalization code more than we are..., also this reduce is inefficient. // canonicalize the headers diff --git a/runtime/auth/aws-signing-default/common/test/aws/smithy/kotlin/runtime/auth/awssigning/DefaultCanonicalizerTest.kt b/runtime/auth/aws-signing-default/common/test/aws/smithy/kotlin/runtime/auth/awssigning/DefaultCanonicalizerTest.kt index 7fc20ddb83..2dfbdf48e1 100644 --- a/runtime/auth/aws-signing-default/common/test/aws/smithy/kotlin/runtime/auth/awssigning/DefaultCanonicalizerTest.kt +++ b/runtime/auth/aws-signing-default/common/test/aws/smithy/kotlin/runtime/auth/awssigning/DefaultCanonicalizerTest.kt @@ -6,8 +6,12 @@ package aws.smithy.kotlin.runtime.auth.awssigning import aws.smithy.kotlin.runtime.auth.awscredentials.Credentials import aws.smithy.kotlin.runtime.auth.awssigning.tests.DEFAULT_TEST_CREDENTIALS -import aws.smithy.kotlin.runtime.http.* -import aws.smithy.kotlin.runtime.http.request.* +import aws.smithy.kotlin.runtime.http.Headers +import aws.smithy.kotlin.runtime.http.HttpBody +import aws.smithy.kotlin.runtime.http.HttpMethod +import aws.smithy.kotlin.runtime.http.request.HttpRequest +import aws.smithy.kotlin.runtime.http.request.headers +import aws.smithy.kotlin.runtime.http.request.url import aws.smithy.kotlin.runtime.net.Host import aws.smithy.kotlin.runtime.net.url.Url import aws.smithy.kotlin.runtime.time.Instant @@ -136,6 +140,7 @@ class DefaultCanonicalizerTest { // These should not be signed set("Expect", "100-continue") set("X-Amzn-Trace-Id", "qux") + set("Transfer-Encoding", "chunked") } body = HttpBody.Empty } diff --git a/runtime/auth/http-auth-aws/common/test/aws/smithy/kotlin/runtime/http/auth/AwsHttpSignerTestBase.kt b/runtime/auth/http-auth-aws/common/test/aws/smithy/kotlin/runtime/http/auth/AwsHttpSignerTestBase.kt index 2ce308df8f..7d10b35e14 100644 --- a/runtime/auth/http-auth-aws/common/test/aws/smithy/kotlin/runtime/http/auth/AwsHttpSignerTestBase.kt +++ b/runtime/auth/http-auth-aws/common/test/aws/smithy/kotlin/runtime/http/auth/AwsHttpSignerTestBase.kt @@ -12,7 +12,9 @@ import aws.smithy.kotlin.runtime.auth.awssigning.DefaultAwsSigner import aws.smithy.kotlin.runtime.auth.awssigning.internal.AWS_CHUNKED_THRESHOLD import aws.smithy.kotlin.runtime.collections.Attributes import aws.smithy.kotlin.runtime.collections.get -import aws.smithy.kotlin.runtime.http.* +import aws.smithy.kotlin.runtime.http.HttpBody +import aws.smithy.kotlin.runtime.http.HttpMethod +import aws.smithy.kotlin.runtime.http.SdkHttpClient import aws.smithy.kotlin.runtime.http.operation.* import aws.smithy.kotlin.runtime.http.request.HttpRequest import aws.smithy.kotlin.runtime.http.request.HttpRequestBuilder @@ -149,8 +151,8 @@ public abstract class AwsHttpSignerTestBase( val op = buildOperation(streaming = true, replayable = false, requestBody = "a".repeat(AWS_CHUNKED_THRESHOLD + 1)) val expectedDate = "20201016T195600Z" val expectedSig = "AWS4-HMAC-SHA256 Credential=AKID/20201016/us-east-1/demo/aws4_request, " + - "SignedHeaders=content-encoding;host;transfer-encoding;x-amz-archive-description;x-amz-date;x-amz-decoded-content-length;x-amz-security-token, " + - "Signature=ac341b9b248a0b23d2fcd9f7e805f4eb0b8a1b789bb23a8ec6adc6c48dd084ad" + "SignedHeaders=content-encoding;host;x-amz-archive-description;x-amz-date;x-amz-decoded-content-length;x-amz-security-token, " + + "Signature=ef06c95647c4d2daa6c89ac90274f1c780777cba8eaab772df6d8009def3eb8f" val signed = getSignedRequest(op) assertEquals(expectedDate, signed.headers["X-Amz-Date"]) @@ -162,8 +164,8 @@ public abstract class AwsHttpSignerTestBase( val op = buildOperation(streaming = true, replayable = true, requestBody = "a".repeat(AWS_CHUNKED_THRESHOLD + 1)) val expectedDate = "20201016T195600Z" val expectedSig = "AWS4-HMAC-SHA256 Credential=AKID/20201016/us-east-1/demo/aws4_request, " + - "SignedHeaders=content-encoding;host;transfer-encoding;x-amz-archive-description;x-amz-date;x-amz-decoded-content-length;x-amz-security-token, " + - "Signature=ac341b9b248a0b23d2fcd9f7e805f4eb0b8a1b789bb23a8ec6adc6c48dd084ad" + "SignedHeaders=content-encoding;host;x-amz-archive-description;x-amz-date;x-amz-decoded-content-length;x-amz-security-token, " + + "Signature=ef06c95647c4d2daa6c89ac90274f1c780777cba8eaab772df6d8009def3eb8f" val signed = getSignedRequest(op) assertEquals(expectedDate, signed.headers["X-Amz-Date"]) @@ -176,8 +178,8 @@ public abstract class AwsHttpSignerTestBase( val expectedDate = "20201016T195600Z" // should have same signature as testSignAwsChunkedStreamNonReplayable(), except for the hash, since the body is different val expectedSig = "AWS4-HMAC-SHA256 Credential=AKID/20201016/us-east-1/demo/aws4_request, " + - "SignedHeaders=content-encoding;host;transfer-encoding;x-amz-archive-description;x-amz-date;x-amz-decoded-content-length;x-amz-security-token, " + - "Signature=3f0277123c9ed8a8858f793886a0ac0fcb457bc54401ffc22d470f373397cff0" + "SignedHeaders=content-encoding;host;x-amz-archive-description;x-amz-date;x-amz-decoded-content-length;x-amz-security-token, " + + "Signature=a902702b57057a864bf41cc22ee846a1b7bd047e22784367ec6a459f6791330e" val signed = getSignedRequest(op) assertEquals(expectedDate, signed.headers["X-Amz-Date"]) diff --git a/runtime/protocol/http-client-engines/http-client-engine-okhttp/jvm/src/aws/smithy/kotlin/runtime/http/engine/okhttp/MetricsInterceptor.kt b/runtime/protocol/http-client-engines/http-client-engine-okhttp/jvm/src/aws/smithy/kotlin/runtime/http/engine/okhttp/MetricsInterceptor.kt index 0ea8ae5e66..8047b870c3 100644 --- a/runtime/protocol/http-client-engines/http-client-engine-okhttp/jvm/src/aws/smithy/kotlin/runtime/http/engine/okhttp/MetricsInterceptor.kt +++ b/runtime/protocol/http-client-engines/http-client-engine-okhttp/jvm/src/aws/smithy/kotlin/runtime/http/engine/okhttp/MetricsInterceptor.kt @@ -30,7 +30,7 @@ public object MetricsInterceptor : Interceptor { } val originalResponse = chain.proceed(request) - val response = if (originalResponse.body == null || originalResponse.body?.contentLength() == 0L) { + val response = if (originalResponse.body.contentLength() == 0L) { originalResponse } else { originalResponse.newBuilder() diff --git a/runtime/protocol/http-client-engines/http-client-engine-okhttp/jvm/src/aws/smithy/kotlin/runtime/http/engine/okhttp/OkHttpEngine.kt b/runtime/protocol/http-client-engines/http-client-engine-okhttp/jvm/src/aws/smithy/kotlin/runtime/http/engine/okhttp/OkHttpEngine.kt index 061c22a13d..a20387d0a0 100644 --- a/runtime/protocol/http-client-engines/http-client-engine-okhttp/jvm/src/aws/smithy/kotlin/runtime/http/engine/okhttp/OkHttpEngine.kt +++ b/runtime/protocol/http-client-engines/http-client-engine-okhttp/jvm/src/aws/smithy/kotlin/runtime/http/engine/okhttp/OkHttpEngine.kt @@ -15,6 +15,7 @@ import aws.smithy.kotlin.runtime.net.TlsVersion import aws.smithy.kotlin.runtime.operation.ExecutionContext import aws.smithy.kotlin.runtime.time.Instant import aws.smithy.kotlin.runtime.time.fromEpochMilliseconds +import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.job import okhttp3.* import okhttp3.ConnectionPool @@ -47,6 +48,7 @@ public class OkHttpEngine( private val connectionIdleMonitor = config.connectionIdlePollingInterval?.let { ConnectionIdleMonitor(it) } private val client = config.buildClientWithConnectionListener(metrics, connectionIdleMonitor) + @OptIn(ExperimentalCoroutinesApi::class) override suspend fun roundTrip(context: ExecutionContext, request: HttpRequest): HttpCall { val callContext = callContext() diff --git a/runtime/protocol/http-client-engines/http-client-engine-okhttp/jvm/src/aws/smithy/kotlin/runtime/http/engine/okhttp/OkHttpUtils.kt b/runtime/protocol/http-client-engines/http-client-engine-okhttp/jvm/src/aws/smithy/kotlin/runtime/http/engine/okhttp/OkHttpUtils.kt index c287228fdf..181ed400c3 100644 --- a/runtime/protocol/http-client-engines/http-client-engine-okhttp/jvm/src/aws/smithy/kotlin/runtime/http/engine/okhttp/OkHttpUtils.kt +++ b/runtime/protocol/http-client-engines/http-client-engine-okhttp/jvm/src/aws/smithy/kotlin/runtime/http/engine/okhttp/OkHttpUtils.kt @@ -104,7 +104,7 @@ public fun Headers.toOkHttpHeaders(): OkHttpHeaders = OkHttpHeaders.Builder().al @InternalApi public fun OkHttpResponse.toSdkResponse(): HttpResponse { val sdkHeaders = OkHttpHeadersAdapter(headers) - val httpBody = if (body == null || body!!.contentLength() == 0L) { + val httpBody = if (body.contentLength() == 0L) { HttpBody.Empty } else { object : HttpBody.SourceContent() { diff --git a/runtime/protocol/http-client-engines/http-client-engine-okhttp4/jvm/src/aws/smithy/kotlin/runtime/http/engine/okhttp4/OkHttp4Engine.kt b/runtime/protocol/http-client-engines/http-client-engine-okhttp4/jvm/src/aws/smithy/kotlin/runtime/http/engine/okhttp4/OkHttp4Engine.kt index ff59823201..026bbd942c 100644 --- a/runtime/protocol/http-client-engines/http-client-engine-okhttp4/jvm/src/aws/smithy/kotlin/runtime/http/engine/okhttp4/OkHttp4Engine.kt +++ b/runtime/protocol/http-client-engines/http-client-engine-okhttp4/jvm/src/aws/smithy/kotlin/runtime/http/engine/okhttp4/OkHttp4Engine.kt @@ -99,7 +99,7 @@ private suspend fun Call.executeAsync(): Response = call: Call, response: Response, ) { - continuation.resume(response) { + continuation.resume(response) { cause, _, _ -> response.closeQuietly() } } diff --git a/runtime/runtime-core/common/src/aws/smithy/kotlin/runtime/text/encoding/Base64.kt b/runtime/runtime-core/common/src/aws/smithy/kotlin/runtime/text/encoding/Base64.kt index 7a3a569899..9af6977d29 100644 --- a/runtime/runtime-core/common/src/aws/smithy/kotlin/runtime/text/encoding/Base64.kt +++ b/runtime/runtime-core/common/src/aws/smithy/kotlin/runtime/text/encoding/Base64.kt @@ -99,10 +99,17 @@ public fun String.decodeBase64Bytes(): ByteArray = encodeToByteArray().decodeBas * Decode [ByteArray] from base64 format */ public fun ByteArray.decodeBase64(): ByteArray { - val encoded = this + // Calculate the padding needed to make the length a multiple of 4 + val remainder = size % 4 + val encoded: ByteArray = if (remainder == 0) { + this + } else { + this + ByteArray(4 - remainder) { BASE64_PAD.code.toByte() } + } + val decodedLen = base64DecodedLen(encoded) val decoded = ByteArray(decodedLen) - val blockCnt = size / 4 + val blockCnt = encoded.size / 4 var readIdx = 0 var writeIdx = 0 diff --git a/runtime/runtime-core/common/src/aws/smithy/kotlin/runtime/util/JMESPath.kt b/runtime/runtime-core/common/src/aws/smithy/kotlin/runtime/util/JMESPath.kt index 9b691a79a2..c8437b2b33 100644 --- a/runtime/runtime-core/common/src/aws/smithy/kotlin/runtime/util/JMESPath.kt +++ b/runtime/runtime-core/common/src/aws/smithy/kotlin/runtime/util/JMESPath.kt @@ -62,8 +62,7 @@ public fun Any?.type(): String = when (this) { is List<*>, is Array<*> -> "array" is Number -> "number" is Any -> "object" - null -> "null" - else -> throw Exception("Undetected type for: $this") + else -> "null" } // Collection `flattenIfPossible` functions diff --git a/runtime/runtime-core/common/test/aws/smithy/kotlin/runtime/text/encoding/Base64Test.kt b/runtime/runtime-core/common/test/aws/smithy/kotlin/runtime/text/encoding/Base64Test.kt index 4c2ada2010..956fabee2e 100644 --- a/runtime/runtime-core/common/test/aws/smithy/kotlin/runtime/text/encoding/Base64Test.kt +++ b/runtime/runtime-core/common/test/aws/smithy/kotlin/runtime/text/encoding/Base64Test.kt @@ -76,14 +76,6 @@ class Base64Test { ex.message!!.shouldContain("decode base64: invalid input byte: 45") } - @Test - fun decodeNonMultipleOf4() { - val ex = assertFails { - "Zm9vY=".decodeBase64() - } - ex.message!!.shouldContain("invalid base64 string of length 6; not a multiple of 4") - } - @Test fun decodeInvalidPadding() { val ex = assertFails { @@ -116,4 +108,24 @@ class Base64Test { assertEquals(encoded, decoded.encodeBase64()) assertEquals(decoded, encoded.decodeBase64()) } + + @Test + fun testUnpaddedInputs() { + // from https://github.com/smithy-lang/smithy/pull/2502 + val input = "v2hkZWZhdWx0c79tZGVmYXVsdFN0cmluZ2JoaW5kZWZhdWx0Qm9vbGVhbvVrZGVmYXVsdExpc3Sf/3BkZWZhdWx0VGltZXN0YW1wwQBrZGVmYXVsdEJsb2JDYWJja2RlZmF1bHRCeXRlAWxkZWZhdWx0U2hvcnQBbmRlZmF1bHRJbnRlZ2VyCmtkZWZhdWx0TG9uZxhkbGRlZmF1bHRGbG9hdPo/gAAAbWRlZmF1bHREb3VibGX6P4AAAGpkZWZhdWx0TWFwv/9rZGVmYXVsdEVudW1jRk9PbmRlZmF1bHRJbnRFbnVtAWtlbXB0eVN0cmluZ2BsZmFsc2VCb29sZWFu9GllbXB0eUJsb2JAaHplcm9CeXRlAGl6ZXJvU2hvcnQAa3plcm9JbnRlZ2VyAGh6ZXJvTG9uZwBpemVyb0Zsb2F0+gAAAABqemVyb0RvdWJsZfoAAAAA//8" + input.decodeBase64() + + val inputs = mapOf( + "YQ" to "a", + "Yg" to "b", + "YWI" to "ab", + "YWJj" to "abc", + "SGVsbG8gd29ybGQ" to "Hello world", + ) + + inputs.forEach { (encoded, expected) -> + val actual = encoded.decodeBase64() + assertEquals(expected, actual) + } + } }