diff --git a/.changes/e9fa30cd-1aa7-4c5d-b01a-384078b8863a.json b/.changes/e9fa30cd-1aa7-4c5d-b01a-384078b8863a.json new file mode 100644 index 000000000..ad9fa3d72 --- /dev/null +++ b/.changes/e9fa30cd-1aa7-4c5d-b01a-384078b8863a.json @@ -0,0 +1,5 @@ +{ + "id": "e9fa30cd-1aa7-4c5d-b01a-384078b8863a", + "type": "feature", + "description": "Added a protocol tests runner, a new mechanism to run the tests and report the results in JSON format detached from the test phase of a regular build." +} diff --git a/build.gradle.kts b/build.gradle.kts index ab17214cd..b2b38f4f6 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -137,8 +137,11 @@ apiValidation { "testing", "smithy-kotlin-codegen", "smithy-kotlin-codegen-testutils", + "smithy-kotlin-protocol-tests-codegen", "smithy-aws-kotlin-codegen", "protocol-tests", + "protocol-tests-runner", + "protocol-tests-utils", "aws-signing-benchmarks", "channel-benchmarks", "http-benchmarks", diff --git a/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/CodegenVisitor.kt b/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/CodegenVisitor.kt index 4adf42b91..e98524f3b 100644 --- a/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/CodegenVisitor.kt +++ b/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/CodegenVisitor.kt @@ -114,6 +114,12 @@ class CodegenVisitor(context: PluginContext) : ShapeVisitor.Default() { } fun execute() { + generateShapes() + generateNonShapes() + writers.flushWriters() + } + + fun generateShapes() { logger.info("Generating Kotlin client for service ${settings.service}") logger.info("Walking shapes from ${settings.service} to find shapes to generate") @@ -122,15 +128,7 @@ class CodegenVisitor(context: PluginContext) : ShapeVisitor.Default() { serviceShapes.forEach { it.accept(this) } protocolGenerator?.apply { - val ctx = ProtocolGenerator.GenerationContext( - settings, - model, - service, - symbolProvider, - integrations, - protocol, - writers, - ) + val ctx = generationContext() logger.info("[${service.id}] Generating unit tests for protocol $protocol") generateProtocolUnitTests(ctx) @@ -144,7 +142,9 @@ class CodegenVisitor(context: PluginContext) : ShapeVisitor.Default() { logger.info("[${service.id}] Generating auth scheme provider for protocol $protocol") generateAuthSchemeProvider(ctx) } + } + fun generateNonShapes() { writers.finalize() if (settings.build.generateDefaultBuildFiles) { @@ -156,10 +156,18 @@ class CodegenVisitor(context: PluginContext) : ShapeVisitor.Default() { // write files defined by integrations integrations.forEach { it.writeAdditionalFiles(baseGenerationContext, writers) } - - writers.flushWriters() } + fun generationContext(): ProtocolGenerator.GenerationContext = ProtocolGenerator.GenerationContext( + settings, + model, + service, + symbolProvider, + integrations, + protocolGenerator!!.protocol, + writers, + ) + override fun getDefault(shape: Shape?) { } override fun structureShape(shape: StructureShape) { diff --git a/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/core/KotlinDependency.kt b/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/core/KotlinDependency.kt index b575efae6..21332274e 100644 --- a/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/core/KotlinDependency.kt +++ b/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/core/KotlinDependency.kt @@ -38,6 +38,7 @@ private fun getDefaultRuntimeVersion(): String { const val RUNTIME_GROUP: String = "aws.smithy.kotlin" val RUNTIME_VERSION: String = System.getProperty("smithy.kotlin.codegen.clientRuntimeVersion", getDefaultRuntimeVersion()) val KOTLIN_COMPILER_VERSION: String = System.getProperty("smithy.kotlin.codegen.kotlinCompilerVersion", "2.1.0") +val SHADOW_JAR_VERSION: String = System.getProperty("com.github.johnrengelman.shadowVersion", "8.1.1") enum class SourceSet { CommonMain, diff --git a/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/GradleGenerator.kt b/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/GradleGenerator.kt index 07f7541a0..8b9c17706 100644 --- a/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/GradleGenerator.kt +++ b/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/GradleGenerator.kt @@ -22,6 +22,7 @@ fun writeGradleBuild( settings: KotlinSettings, manifest: FileManifest, dependencies: List, + enableApplication: Boolean = false, ) { val writer = GradleWriter() @@ -53,6 +54,12 @@ fun writeGradleBuild( } }, ) + if (enableApplication) { + indent() + write("application") + write("id(\"com.github.johnrengelman.shadow\") version #S", SHADOW_JAR_VERSION) + dedent() + } } when { @@ -74,6 +81,13 @@ fun writeGradleBuild( ) } + if (enableApplication) { + writer.write("") + writer.openBlock("application {") + writer.write("mainClass.set(\"${settings.pkg.name}.pt.RunnerKt\")") + writer.closeBlock("}") + } + val contents = writer.toString() manifest.writeFile("build.gradle.kts", contents) if (settings.build.generateFullProject) { diff --git a/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/protocol/HttpProtocolUnitTestGenerator.kt b/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/protocol/HttpProtocolUnitTestGenerator.kt index d5640735d..6ca9c8155 100644 --- a/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/protocol/HttpProtocolUnitTestGenerator.kt +++ b/codegen/smithy-kotlin-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/rendering/protocol/HttpProtocolUnitTestGenerator.kt @@ -5,7 +5,6 @@ package software.amazon.smithy.kotlin.codegen.rendering.protocol import software.amazon.smithy.codegen.core.SymbolProvider -import software.amazon.smithy.kotlin.codegen.core.KotlinDependency import software.amazon.smithy.kotlin.codegen.core.KotlinWriter import software.amazon.smithy.model.Model import software.amazon.smithy.model.shapes.OperationShape @@ -40,9 +39,7 @@ protected constructor( /** * Render a test class and unit tests for the specified [testCases] */ - fun renderTestClass(testClassName: String) { - writer.addImport(KotlinDependency.KOTLIN_TEST.namespace, "Test") - + open fun renderTestClass(testClassName: String) { writer.write("") .openBlock("class $testClassName {") .call { @@ -58,7 +55,7 @@ protected constructor( /** * Write a single unit test function using the given [writer] */ - private fun renderTestFunction(test: T) { + protected open fun renderTestFunction(test: T) { test.documentation.ifPresent { writer.dokka(it) } diff --git a/codegen/smithy-kotlin-protocol-tests-codegen/build.gradle.kts b/codegen/smithy-kotlin-protocol-tests-codegen/build.gradle.kts new file mode 100644 index 000000000..e28a25aeb --- /dev/null +++ b/codegen/smithy-kotlin-protocol-tests-codegen/build.gradle.kts @@ -0,0 +1,103 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +import aws.sdk.kotlin.gradle.dsl.configurePublishing +import org.jetbrains.kotlin.gradle.dsl.JvmTarget +import org.jetbrains.kotlin.gradle.tasks.KotlinCompile + +plugins { + alias(libs.plugins.kotlin.jvm) + jacoco + `maven-publish` +} + +val codegenVersion: String by project +description = "Smithy Kotlin Codegen for Protocol tests" +group = "software.amazon.smithy.kotlin" +version = codegenVersion + +dependencies { + api(project(":codegen:smithy-kotlin-codegen")) + + api(libs.smithy.aws.traits) + api(libs.smithy.protocol.test.traits) + api(libs.smithy.protocol.traits) + + testImplementation(libs.junit.jupiter) + testImplementation(libs.junit.jupiter.params) + testImplementation(libs.kotest.assertions.core.jvm) + testImplementation(libs.kotlin.test.junit5) + testImplementation(project(":codegen:smithy-kotlin-codegen-testutils")) + + testImplementation(libs.slf4j.api) + testImplementation(libs.slf4j.simple) + testImplementation(libs.kotlinx.serialization.json) +} + +tasks.withType { + compilerOptions { + jvmTarget.set(JvmTarget.JVM_17) + } +} + +tasks.withType { + sourceCompatibility = JavaVersion.VERSION_17.toString() + targetCompatibility = JavaVersion.VERSION_17.toString() +} + +// Reusable license copySpec +val licenseSpec = copySpec { + from("${project.rootDir}/LICENSE") + from("${project.rootDir}/NOTICE") +} + +// Configure jars to include license related info +tasks.jar { + metaInf.with(licenseSpec) + inputs.property("moduleName", project.name) + manifest { + attributes["Automatic-Module-Name"] = project.name + } +} + +tasks.test { + useJUnitPlatform() + testLogging { + events("passed", "skipped", "failed") + showStandardStreams = true + showStackTraces = true + showExceptions = true + exceptionFormat = org.gradle.api.tasks.testing.logging.TestExceptionFormat.FULL + } +} + +// Configure jacoco (code coverage) to generate an HTML report +tasks.jacocoTestReport { + reports { + xml.required.set(false) + csv.required.set(false) + html.outputLocation.set(layout.buildDirectory.dir("reports/jacoco")) + } +} + +// Always run the jacoco test report after testing. +tasks["test"].finalizedBy(tasks["jacocoTestReport"]) + +val sourcesJar by tasks.creating(Jar::class) { + group = "publishing" + description = "Assembles Kotlin sources jar" + archiveClassifier.set("sources") + from(sourceSets.getByName("main").allSource) +} + +publishing { + publications { + create("codegen") { + from(components["java"]) + artifact(sourcesJar) + } + } +} + +configurePublishing("smithy-kotlin", "smithy-lang") diff --git a/codegen/smithy-kotlin-protocol-tests-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/pt/KotlinProtocolTestCodegenPlugin.kt b/codegen/smithy-kotlin-protocol-tests-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/pt/KotlinProtocolTestCodegenPlugin.kt new file mode 100644 index 000000000..d372f058f --- /dev/null +++ b/codegen/smithy-kotlin-protocol-tests-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/pt/KotlinProtocolTestCodegenPlugin.kt @@ -0,0 +1,79 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package software.amazon.smithy.kotlin.codegen.pt + +import software.amazon.smithy.build.PluginContext +import software.amazon.smithy.build.SmithyBuildPlugin +import software.amazon.smithy.kotlin.codegen.CodegenVisitor +import software.amazon.smithy.kotlin.codegen.core.GradleConfiguration +import software.amazon.smithy.kotlin.codegen.core.KOTLIN_COMPILER_VERSION +import software.amazon.smithy.kotlin.codegen.core.KotlinDependency +import software.amazon.smithy.kotlin.codegen.rendering.protocol.TestMemberDelta +import software.amazon.smithy.kotlin.codegen.rendering.protocol.pt.ProtocolTestGenerator +import software.amazon.smithy.kotlin.codegen.rendering.writeGradleBuild + +// We redefine the kotlin-test and smithy-tests dependencies since for this use case +// we need them in the implementation scope instead of just in the test scope. +val KOTLIN_TEST_RT = KotlinDependency( + GradleConfiguration.Implementation, + "kotlin.test", + "org.jetbrains.kotlin", + "kotlin-test", + KOTLIN_COMPILER_VERSION, +) +val SMITHY_TEST_RT = KotlinDependency( + GradleConfiguration.Implementation, + KotlinDependency.SMITHY_TEST.namespace, + KotlinDependency.SMITHY_TEST.group, + KotlinDependency.SMITHY_TEST.artifact, + KotlinDependency.SMITHY_TEST.version, +) + +/** + * Plugin to trigger Kotlin protocol tests code generation. This plugin also generates the client and the + * request/response shapes. + */ +public class KotlinProtocolTestCodegenPlugin : SmithyBuildPlugin { + + override fun getName(): String = "kotlin-protocol-tests-codegen" + + override fun execute(context: PluginContext?) { + // Run the regular codegen + var codegen = CodegenVisitor(context ?: error("context was null")) + codegen.generateShapes() + + // Generate the protocol tests + val ctx = codegen.generationContext() + val requestTestBuilder = ProtocolTestRequestGenerator.Builder() + val responseTestBuilder = ProtocolTestResponseGenerator.Builder() + val errorTestBuilder = ProtocolTestErrorGenerator.Builder() + val ignoredTests = TestMemberDelta( + setOf(), + ) + ProtocolTestGenerator( + ctx, + requestTestBuilder, + responseTestBuilder, + errorTestBuilder, + ignoredTests, + ).generateProtocolTests() + + val writers = ctx.delegator + writers.finalize() + val settings = ctx.settings + + if (settings.build.generateDefaultBuildFiles) { + val dependencies = writers.dependencies + .mapNotNull { it.properties["dependency"] as? KotlinDependency } + .distinct() + val newDependencies = ArrayList() + newDependencies.addAll(dependencies) + newDependencies.add(KOTLIN_TEST_RT) + newDependencies.add(SMITHY_TEST_RT) + writeGradleBuild(settings, writers.fileManifest, newDependencies, enableApplication = true) + } + writers.flushWriters() + } +} diff --git a/codegen/smithy-kotlin-protocol-tests-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/pt/ProtocolTestErrorGenerator.kt b/codegen/smithy-kotlin-protocol-tests-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/pt/ProtocolTestErrorGenerator.kt new file mode 100644 index 000000000..e9c94b486 --- /dev/null +++ b/codegen/smithy-kotlin-protocol-tests-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/pt/ProtocolTestErrorGenerator.kt @@ -0,0 +1,72 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package software.amazon.smithy.kotlin.codegen.pt + +import software.amazon.smithy.kotlin.codegen.rendering.protocol.HttpProtocolUnitTestErrorGenerator +import software.amazon.smithy.kotlin.codegen.rendering.protocol.HttpProtocolUnitTestGenerator +import software.amazon.smithy.kotlin.codegen.rendering.protocol.pt.renderTestCaseEpilogue +import software.amazon.smithy.kotlin.codegen.rendering.protocol.pt.renderTestCasePrelude +import software.amazon.smithy.protocoltests.traits.HttpResponseTestCase + +/** + * Renders a class with all the defined protocol tests for responses with errors. + */ +open class ProtocolTestErrorGenerator protected constructor(builder: HttpProtocolUnitTestErrorGenerator.Builder) : HttpProtocolUnitTestErrorGenerator(builder) { + + /** + * Render a test class and unit tests for the specified [testCases] + */ + override fun renderTestClass(testClassName: String) { + writer.addImport("${ctx.settings.pkg.name}", "*") + + writer.write("") + .openBlock("internal class $testClassName(val results: MutableList) {") + .openBlock("public fun runAll() {") + .call { + for (test in testCases) { + renderTestFunctionCall(test) + } + } + .closeBlock("}") + .call { + for (test in testCases) { + renderTestFunction(test) + } + } + .closeBlock("}") + } + + /** + * Write a single unit test function using the given [writer] + */ + override fun renderTestFunction(test: HttpResponseTestCase) { + test.documentation.ifPresent { + writer.dokka(it) + } + + writer.write("") + .openBlock("private fun `${test.id}`() {") + .call { renderTestCasePrelude(writer, test.id, "RESPONSE") } + .openBlock(openTestBlock()) + .call { renderTestBody(test) } + .closeBlock("}") + .call { renderTestCaseEpilogue(writer) } + .closeBlock("}") + } + + fun renderTestFunctionCall(test: HttpResponseTestCase) { + writer.write("`${test.id}`()") + } + + fun openTestBlock(): String { + val respType = responseSymbol?.name ?: "Unit" + return "httpResponseTest<$respType> {" + } + + open class Builder : HttpProtocolUnitTestErrorGenerator.Builder() { + override fun build(): HttpProtocolUnitTestGenerator = + ProtocolTestErrorGenerator(this) + } +} diff --git a/codegen/smithy-kotlin-protocol-tests-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/pt/ProtocolTestRequestGenerator.kt b/codegen/smithy-kotlin-protocol-tests-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/pt/ProtocolTestRequestGenerator.kt new file mode 100644 index 000000000..8877b1658 --- /dev/null +++ b/codegen/smithy-kotlin-protocol-tests-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/pt/ProtocolTestRequestGenerator.kt @@ -0,0 +1,66 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package software.amazon.smithy.kotlin.codegen.pt + +import software.amazon.smithy.kotlin.codegen.rendering.protocol.HttpProtocolUnitTestGenerator +import software.amazon.smithy.kotlin.codegen.rendering.protocol.HttpProtocolUnitTestRequestGenerator +import software.amazon.smithy.kotlin.codegen.rendering.protocol.pt.renderTestCaseEpilogue +import software.amazon.smithy.kotlin.codegen.rendering.protocol.pt.renderTestCasePrelude +import software.amazon.smithy.protocoltests.traits.HttpRequestTestCase + +/** + * Renders a class with all the defined protocol tests for requests. + */ +open class ProtocolTestRequestGenerator protected constructor(builder: Builder) : HttpProtocolUnitTestRequestGenerator(builder) { + + /** + * Render a test class and unit tests for the specified [testCases] + */ + override fun renderTestClass(testClassName: String) { + writer.addImport("${ctx.settings.pkg.name}", "*") + + writer.write("") + .openBlock("internal class $testClassName(val results: MutableList) {") + .openBlock("public fun runAll() {") + .call { + for (test in testCases) { + renderTestFunctionCall(test) + } + } + .closeBlock("}") + .call { + for (test in testCases) { + renderTestFunction(test) + } + } + .closeBlock("}") + } + + /** + * Write a single unit test function using the given [writer] + */ + override fun renderTestFunction(test: HttpRequestTestCase) { + test.documentation.ifPresent { + writer.dokka(it) + } + + writer.openBlock("private fun `${test.id}`() {") + .call { renderTestCasePrelude(writer, test.id, "REQUEST") } + .openBlock("httpRequestTest {") + .call { renderTestBody(test) } + .closeBlock("}") + .call { renderTestCaseEpilogue(writer) } + .closeBlock("}") + } + + fun renderTestFunctionCall(test: HttpRequestTestCase) { + writer.write("`${test.id}`()") + } + + open class Builder : HttpProtocolUnitTestRequestGenerator.Builder() { + override fun build(): HttpProtocolUnitTestGenerator = + ProtocolTestRequestGenerator(this) + } +} diff --git a/codegen/smithy-kotlin-protocol-tests-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/pt/ProtocolTestResponseGenerator.kt b/codegen/smithy-kotlin-protocol-tests-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/pt/ProtocolTestResponseGenerator.kt new file mode 100644 index 000000000..a2201fd02 --- /dev/null +++ b/codegen/smithy-kotlin-protocol-tests-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/pt/ProtocolTestResponseGenerator.kt @@ -0,0 +1,71 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package software.amazon.smithy.kotlin.codegen.pt + +import software.amazon.smithy.kotlin.codegen.rendering.protocol.HttpProtocolUnitTestGenerator +import software.amazon.smithy.kotlin.codegen.rendering.protocol.HttpProtocolUnitTestResponseGenerator +import software.amazon.smithy.kotlin.codegen.rendering.protocol.pt.renderTestCaseEpilogue +import software.amazon.smithy.kotlin.codegen.rendering.protocol.pt.renderTestCasePrelude +import software.amazon.smithy.protocoltests.traits.HttpResponseTestCase + +/** + * Renders a class with all the defined protocol tests for responses. + */ +open class ProtocolTestResponseGenerator protected constructor(builder: Builder) : HttpProtocolUnitTestResponseGenerator(builder) { + + /** + * Render a test class and unit tests for the specified [testCases] + */ + override fun renderTestClass(testClassName: String) { + writer.addImport("${ctx.settings.pkg.name}", "*") + + writer.write("") + .openBlock("internal class $testClassName(val results: MutableList) {") + .openBlock("public fun runAll() {") + .call { + for (test in testCases) { + renderTestFunctionCall(test) + } + } + .closeBlock("}") + .call { + for (test in testCases) { + renderTestFunction(test) + } + } + .closeBlock("}") + } + + /** + * Write a single unit test function using the given [writer] + */ + override fun renderTestFunction(test: HttpResponseTestCase) { + test.documentation.ifPresent { + writer.dokka(it) + } + + writer.openBlock("private fun `${test.id}`() {") + .call { renderTestCasePrelude(writer, test.id, "RESPONSE") } + .openBlock(openTestBlock()) + .call { renderTestBody(test) } + .closeBlock("}") + .call { renderTestCaseEpilogue(writer) } + .closeBlock("}") + } + + fun renderTestFunctionCall(test: HttpResponseTestCase) { + writer.write("`${test.id}`()") + } + + fun openTestBlock(): String { + val respType = responseSymbol?.name ?: "Unit" + return "httpResponseTest<$respType> {" + } + + open class Builder : HttpProtocolUnitTestResponseGenerator.Builder() { + override fun build(): HttpProtocolUnitTestGenerator = + ProtocolTestResponseGenerator(this) + } +} diff --git a/codegen/smithy-kotlin-protocol-tests-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/pt/ProtocolTestsGenerator.kt b/codegen/smithy-kotlin-protocol-tests-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/pt/ProtocolTestsGenerator.kt new file mode 100644 index 000000000..f6b00caa7 --- /dev/null +++ b/codegen/smithy-kotlin-protocol-tests-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/pt/ProtocolTestsGenerator.kt @@ -0,0 +1,202 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package software.amazon.smithy.kotlin.codegen.rendering.protocol.pt + +import software.amazon.smithy.kotlin.codegen.core.KotlinWriter +import software.amazon.smithy.kotlin.codegen.core.closeAndOpenBlock +import software.amazon.smithy.kotlin.codegen.core.defaultName +import software.amazon.smithy.kotlin.codegen.model.getTrait +import software.amazon.smithy.kotlin.codegen.pt.ProtocolTestErrorGenerator +import software.amazon.smithy.kotlin.codegen.pt.ProtocolTestRequestGenerator +import software.amazon.smithy.kotlin.codegen.pt.ProtocolTestResponseGenerator +import software.amazon.smithy.kotlin.codegen.pt.ProtocolTestsUtils +import software.amazon.smithy.kotlin.codegen.rendering.protocol.ProtocolGenerator +import software.amazon.smithy.kotlin.codegen.rendering.protocol.TestContainmentMode +import software.amazon.smithy.kotlin.codegen.rendering.protocol.TestMemberDelta +import software.amazon.smithy.model.knowledge.OperationIndex +import software.amazon.smithy.model.knowledge.TopDownIndex +import software.amazon.smithy.model.shapes.OperationShape +import software.amazon.smithy.model.shapes.Shape +import software.amazon.smithy.protocoltests.traits.AppliesTo +import software.amazon.smithy.protocoltests.traits.HttpMessageTestCase +import software.amazon.smithy.protocoltests.traits.HttpRequestTestsTrait +import software.amazon.smithy.protocoltests.traits.HttpResponseTestsTrait +import java.util.* +import java.util.logging.Logger + +/** + * Generates protocol unit tests for the HTTP protocol from smithy models. + */ +class ProtocolTestGenerator( + private val ctx: ProtocolGenerator.GenerationContext, + private val requestTestBuilder: ProtocolTestRequestGenerator.Builder, + private val responseTestBuilder: ProtocolTestResponseGenerator.Builder, + private val errorTestBuilder: ProtocolTestErrorGenerator.Builder, + // list of test ID's to ignore/skip + private val testDelta: TestMemberDelta = TestMemberDelta(setOf()), +) { + private val logger = Logger.getLogger(javaClass.name) + + /** + * Generates the API HTTP protocol tests defined in the smithy model. + */ + fun generateProtocolTests() { + val operationIndex: OperationIndex = OperationIndex.of(ctx.model) + val topDownIndex: TopDownIndex = TopDownIndex.of(ctx.model) + + val classes = ArrayList() + for (operation in TreeSet(topDownIndex.getContainedOperations(ctx.service).filterNot(::serverOnly))) { + // 1. Generate test cases for each request. + generateRequestProtocolTests(operation, classes) + + // 2. Generate test cases for each response. + generateResponseProtocolTests(operation, classes) + + // 3. Generate test cases for each error on each operation. + generateErrorProtocolTests(operationIndex, operation, classes) + } + + generateProtocolTestsRunner(classes) + } + + private fun isTestCaseAllowedForRunMode(test: T): Boolean = when (testDelta.runMode) { + TestContainmentMode.EXCLUDE_TESTS -> test.protocol == ctx.protocol && test.id !in testDelta.members + TestContainmentMode.RUN_TESTS -> test.protocol == ctx.protocol && test.id in testDelta.members + } + + private fun generateRequestProtocolTests(operation: OperationShape, classes: MutableList) { + // 1. Generate test cases for each request. + val requestTests = operation.getTrait() + ?.getTestCasesFor(AppliesTo.CLIENT) + ?.filter(::isTestCaseAllowedForRunMode) + + if (requestTests?.isEmpty() != false) { + return + } + requestTests.let { testCases -> + val testOperationName = operation.id.name.replaceFirstChar { c -> c.uppercaseChar() } + val testClassName = "${testOperationName}RequestTest" + val testFilename = "$testClassName.kt" + classes.add(testClassName) + ctx.delegator.useFileWriter(testFilename, ctx.settings.pkg.name + ".pt") { writer -> + logger.fine("Generating request protocol test cases for ${operation.id}") + requestTestBuilder + .ctx(ctx) + .writer(writer) + .model(ctx.model) + .symbolProvider(ctx.symbolProvider) + .operation(operation) + .service(ctx.service) + .testCases(testCases) + .build() + .renderTestClass(testClassName) + } + } + } + + private fun generateResponseProtocolTests(operation: OperationShape, classes: MutableList) { + val responseTests = operation.getTrait() + ?.getTestCasesFor(AppliesTo.CLIENT) + ?.filter(::isTestCaseAllowedForRunMode) + + if (responseTests?.isEmpty() != false) { + return + } + responseTests.let { testCases -> + val testOperationName = operation.id.name.replaceFirstChar { c -> c.uppercaseChar() } + val testClassName = "${testOperationName}ResponseTest" + val testFilename = "$testClassName.kt" + classes.add(testClassName) + ctx.delegator.useFileWriter(testFilename, ctx.settings.pkg.name + ".pt") { writer -> + logger.fine("Generating response protocol test cases for ${operation.id}") + responseTestBuilder + .ctx(ctx) + .writer(writer) + .model(ctx.model) + .symbolProvider(ctx.symbolProvider) + .operation(operation) + .service(ctx.service) + .testCases(testCases) + .build() + .renderTestClass(testClassName) + } + } + } + + private fun generateErrorProtocolTests( + operationIndex: OperationIndex, + operation: OperationShape, + classes: MutableList, + ) { + for (error in operationIndex.getErrors(operation).filterNot(::serverOnly)) { + val errorTests = error.getTrait() + ?.getTestCasesFor(AppliesTo.CLIENT) + ?.filter(::isTestCaseAllowedForRunMode) + if (errorTests?.isEmpty() != false) { + return + } + errorTests.let { testCases -> + // use operation name as filename + val opName = operation.id.name.replaceFirstChar { c -> c.uppercaseChar() } + val testFilename = "${opName}ErrorTest.kt" + // multiple error (tests) may be associated with a single operation, + // use the operation name + error name as the class name + val testClassName = "${opName}${error.defaultName(ctx.service)}Test" + classes.add(testClassName) + ctx.delegator.useFileWriter(testFilename, ctx.settings.pkg.name + ".pt") { writer -> + logger.fine("Generating error protocol test cases for ${operation.id}") + errorTestBuilder + .error(error) + .ctx(ctx) + .writer(writer) + .model(ctx.model) + .symbolProvider(ctx.symbolProvider) + .operation(operation) + .service(ctx.service) + .testCases(testCases) + .build() + .renderTestClass(testClassName) + } + } + } + } + + private fun generateProtocolTestsRunner(classes: List) { + val testClassName = "Runner" + val testFilename = "$testClassName.kt" + ctx.delegator.useFileWriter(testFilename, ctx.settings.pkg.name + ".pt") { writer -> + logger.fine("Generating protocol test runner") + writer.openBlock("public fun main() {") + .write("val results = ArrayList<#T>()", ProtocolTestsUtils.TestResult) + for (testClass in classes) { + writer.write("#L(results).runAll()", testClass) + } + writer.write("#T(#S, #S, results)", ProtocolTestsUtils.writeResults, ctx.service.id, ctx.protocol) + writer.closeBlock("}") + } + } +} + +fun renderTestCasePrelude(writer: KotlinWriter, testId: String, type: String) { + writer.write("val res = #T(#S, #T.#L)", ProtocolTestsUtils.TestResult, testId, ProtocolTestsUtils.TestType, type) + .write("results.add(res)") + .openBlock("try {") +} + +fun renderTestCaseEpilogue(writer: KotlinWriter) { + writer.closeAndOpenBlock("} catch (ex: AssertionError) {") + .write("res.result = #T.FAILED", ProtocolTestsUtils.Result) + .write("val sw = java.io.StringWriter()") + .write("ex.printStackTrace(java.io.PrintWriter(sw))") + .write("res.log = sw.toString()") + .closeAndOpenBlock("} catch (ex: Exception) {") + .write("res.result = #T.ERRORED", ProtocolTestsUtils.Result) + .write("val sw = java.io.StringWriter()") + .write("ex.printStackTrace(java.io.PrintWriter(sw))") + .write("res.log = sw.toString()") + .closeBlock("}") +} + +private fun serverOnly(shape: Shape): Boolean = shape.hasTag("server-only") diff --git a/codegen/smithy-kotlin-protocol-tests-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/pt/ProtocolTestsRuntimeTypes.kt b/codegen/smithy-kotlin-protocol-tests-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/pt/ProtocolTestsRuntimeTypes.kt new file mode 100644 index 000000000..29da0111b --- /dev/null +++ b/codegen/smithy-kotlin-protocol-tests-codegen/src/main/kotlin/software/amazon/smithy/kotlin/codegen/pt/ProtocolTestsRuntimeTypes.kt @@ -0,0 +1,24 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package software.amazon.smithy.kotlin.codegen.pt + +import software.amazon.smithy.kotlin.codegen.core.GradleConfiguration +import software.amazon.smithy.kotlin.codegen.core.KotlinDependency +import software.amazon.smithy.kotlin.codegen.core.RuntimeTypePackage + +val PROTOCOL_TEST_RUNTIME_UTILS = KotlinDependency( + GradleConfiguration.Implementation, + "software.amazon.smithy.kotlin.protocolTests.utils", + "software.amazon.smithy.kotlin", + "protocol-tests-utils", + "0.34.17-SNAPSHOT", +) + +object ProtocolTestsUtils : RuntimeTypePackage(PROTOCOL_TEST_RUNTIME_UTILS) { + val TestType = symbol("TestType") + val Result = symbol("Result") + val TestResult = symbol("TestResult") + val writeResults = symbol("writeResults") +} diff --git a/codegen/smithy-kotlin-protocol-tests-codegen/src/main/resources/META-INF/services/software.amazon.smithy.build.SmithyBuildPlugin b/codegen/smithy-kotlin-protocol-tests-codegen/src/main/resources/META-INF/services/software.amazon.smithy.build.SmithyBuildPlugin new file mode 100644 index 000000000..03baef5f7 --- /dev/null +++ b/codegen/smithy-kotlin-protocol-tests-codegen/src/main/resources/META-INF/services/software.amazon.smithy.build.SmithyBuildPlugin @@ -0,0 +1 @@ +software.amazon.smithy.kotlin.codegen.pt.KotlinProtocolTestCodegenPlugin diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index fda89e3a2..7d41c1967 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -31,6 +31,7 @@ docker-java-version = "3.4.0" ktor-version = "3.1.1" kaml-version = "0.55.0" jsoup-version = "1.19.1" +shadow-version = "8.1.1" [libraries] aws-kotlin-repo-tools-build-support = { module="aws.sdk.kotlin.gradle:build-support", version.ref = "aws-kotlin-repo-tools-version" } @@ -67,11 +68,13 @@ micrometer-core = { module = "io.micrometer:micrometer-core", version.ref = "mic smithy-codegen-core = { module = "software.amazon.smithy:smithy-codegen-core", version.ref = "smithy-version" } smithy-cli = { module = "software.amazon.smithy:smithy-cli", version.ref = "smithy-version" } smithy-model = { module = "software.amazon.smithy:smithy-model", version.ref = "smithy-version" } +smithy-build = { module = "software.amazon.smithy:smithy-build", version.ref = "smithy-version" } smithy-waiters = { module = "software.amazon.smithy:smithy-waiters", version.ref = "smithy-version" } smithy-protocol-traits = { module = "software.amazon.smithy:smithy-protocol-traits", version.ref = "smithy-version" } smithy-protocol-tests = { module = "software.amazon.smithy:smithy-protocol-tests", version.ref = "smithy-version" } smithy-protocol-test-traits = { module = "software.amazon.smithy:smithy-protocol-test-traits", version.ref = "smithy-version" } smithy-aws-traits = { module = "software.amazon.smithy:smithy-aws-traits", version.ref = "smithy-version" } +smithy-validation-model = { module = "software.amazon.smithy:smithy-validation-model", version.ref = "smithy-version" } smithy-rules-engine = { module = "software.amazon.smithy:smithy-rules-engine", version.ref = "smithy-version" } smithy-aws-endpoints = { module = "software.amazon.smithy:smithy-aws-endpoints", version.ref = "smithy-version" } smithy-aws-protocol-tests = { module = "software.amazon.smithy:smithy-aws-protocol-tests", version.ref = "smithy-version" } @@ -111,3 +114,4 @@ kotlinx-serialization = { id = "org.jetbrains.kotlin.plugin.serialization", vers aws-kotlin-repo-tools-kmp = { id = "aws.sdk.kotlin.gradle.kmp", version.ref = "aws-kotlin-repo-tools-version" } aws-kotlin-repo-tools-smithybuild = { id = "aws.sdk.kotlin.gradle.smithybuild", version.ref = "aws-kotlin-repo-tools-version" } aws-kotlin-repo-tools-artifactsizemetrics = { id = "aws.sdk.kotlin.gradle.artifactsizemetrics", version.ref = "aws-kotlin-repo-tools-version" } +shadow-jar = { id = "com.github.johnrengelman.shadow", version.ref = "shadow-version"} diff --git a/settings.gradle.kts b/settings.gradle.kts index 2c8d3df11..e469a0fdf 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -107,6 +107,7 @@ include(":runtime:testing") include(":codegen:smithy-kotlin-codegen") include(":codegen:smithy-kotlin-codegen-testutils") include(":codegen:smithy-aws-kotlin-codegen") +include(":codegen:smithy-kotlin-protocol-tests-codegen") include(":codegen:protocol-tests") include(":tests") @@ -123,3 +124,5 @@ include(":tests:codegen:waiter-tests") include(":tests:integration:slf4j-1x-consumer") include(":tests:integration:slf4j-2x-consumer") include(":tests:integration:slf4j-hybrid-consumer") +include(":tests:protocol-tests-runner") +include(":tests:protocol-tests-utils") diff --git a/tests/protocol-tests-runner/build.gradle.kts b/tests/protocol-tests-runner/build.gradle.kts new file mode 100644 index 000000000..ce61e507e --- /dev/null +++ b/tests/protocol-tests-runner/build.gradle.kts @@ -0,0 +1,45 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +import aws.sdk.kotlin.gradle.dsl.skipPublishing + +plugins { + alias(libs.plugins.kotlin.jvm) + alias(libs.plugins.shadow.jar) + application +} + +repositories { + mavenLocal() + mavenCentral() +} + +skipPublishing() + +description = "Protocol test runner" + +dependencies { + implementation(kotlin("stdlib")) + implementation(libs.smithy.model) + implementation(libs.smithy.build) + implementation(libs.smithy.validation.model) + implementation(libs.smithy.aws.traits) + implementation(libs.smithy.protocol.test.traits) + implementation(libs.smithy.protocol.traits) + implementation(project(":tests:protocol-tests-utils")) + implementation(project(":codegen:smithy-kotlin-protocol-tests-codegen")) + implementation(project(":codegen:smithy-aws-kotlin-codegen")) + implementation(project(":codegen:smithy-kotlin-codegen")) +} + +application { + mainClass.set("software.amazon.smithy.kotlin.protocolTests.RunnerKt") +} + +tasks { + shadowJar { + append("META-INF/smithy/manifest") + mergeServiceFiles() + } +} diff --git a/tests/protocol-tests-runner/src/main/kotlin/software/amazon/smithy/kotlin/protocolTests/Generation.kt b/tests/protocol-tests-runner/src/main/kotlin/software/amazon/smithy/kotlin/protocolTests/Generation.kt new file mode 100644 index 000000000..717273ed4 --- /dev/null +++ b/tests/protocol-tests-runner/src/main/kotlin/software/amazon/smithy/kotlin/protocolTests/Generation.kt @@ -0,0 +1,67 @@ +package software.amazon.smithy.kotlin.protocolTests + +import software.amazon.smithy.build.FileManifest +import software.amazon.smithy.build.PluginContext +import software.amazon.smithy.kotlin.codegen.pt.KotlinProtocolTestCodegenPlugin +import software.amazon.smithy.model.Model +import software.amazon.smithy.model.node.ArrayNode +import software.amazon.smithy.model.node.ObjectNode +import software.amazon.smithy.model.node.StringNode +import software.amazon.smithy.model.shapes.ServiceShape +import java.nio.file.Path + +/** + * Runs the `KotlinProtocolTestCodegenPlugin` directly. Leaves the results in the directory + * that is given as an argument. + */ +fun generateProtocolTests(model: Model, service: ServiceShape, codegenPath: Path) { + val context = PluginContext.builder() + .model(model) + .fileManifest(FileManifest.create(codegenPath)) + .settings(settingsForService(service)) + .build() + val plugin = KotlinProtocolTestCodegenPlugin() + plugin.execute(context) +} + +/** + * Creates the `Node` instance that represents the configuration used to run the + * `KotlinProtocolTestCodegenPlugin` plugin. + */ +private fun settingsForService(service: ServiceShape): ObjectNode = ObjectNode.builder() + .withMember("service", service.id.toString()) + .withMember("package", packageSettings(service)) + .withMember("build", buildSettings()) + .withMember("api", apiSettings()) + .build() + +/** + * Crates the `Node` instance that defines the package settings. + */ +private fun packageSettings(service: ServiceShape): ObjectNode { + val version = service.version ?: "1.0" + return ObjectNode.builder() + .withMember("version", version) + .withMember("name", "smithy.protocolTests") + .build() +} + +/** + * Creates the `Node` instance that defines the build settings. + */ +private fun buildSettings(): ObjectNode = ObjectNode.builder() + .withMember("generateFullProject", true) + .withMember( + "optInAnnotations", + ArrayNode.arrayNode(StringNode.from("aws.smithy.kotlin.runtime.InternalApi")), + ) + .withMember("rootProject", true) + .build() + +/** + * Creates the `Node` instance that defines the api settings. + */ +private fun apiSettings(): ObjectNode = ObjectNode.builder() + // By default, this value is WHEN_DIFFERENT, which is incorrect for all protocols + .withMember("defaultValueSerializationMode", "always") + .build() diff --git a/tests/protocol-tests-runner/src/main/kotlin/software/amazon/smithy/kotlin/protocolTests/Report.kt b/tests/protocol-tests-runner/src/main/kotlin/software/amazon/smithy/kotlin/protocolTests/Report.kt new file mode 100644 index 000000000..c873dc9d1 --- /dev/null +++ b/tests/protocol-tests-runner/src/main/kotlin/software/amazon/smithy/kotlin/protocolTests/Report.kt @@ -0,0 +1,58 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package software.amazon.smithy.kotlin.protocolTests + +import software.amazon.smithy.kotlin.protocolTests.utils.JsonWriter +import java.time.Instant + +/** + * Writes the start of the protocol tests execution report. + */ +fun writeReportStart(writer: JsonWriter) { + writer.startObject() + writer.writeKvp("product", "AWS SDK for Kotlin") + writer.writeKvp("model", "smithy") + writer.writeKvp("sdkVersion", sdkVersion(writer)) + writer.writeKvp("date", Instant.now().toString()) + writer.writeKey("tags") + writeTags(writer) + writer.writeKey("suites") + writer.startArray() +} + +/** + * Writes the end of the protocol tests execution report. + */ +fun writeReportEnd(writer: JsonWriter) { + writer.endArray() + writer.endObject() +} + +/** + * Writes the tags for the protocol tests report. + */ +fun writeTags(writer: JsonWriter) { + writer.startObject() + System.getProperty("os.name").let { + writer.writeKvp("os.name", it) + } + System.getProperty("os.version").let { + writer.writeKvp("os.version", it) + } + System.getProperty("os.arch").let { + writer.writeKvp("os.arch", it) + } + System.getProperty("java.version").let { + writer.writeKvp("java.version", it) + } + writer.endObject() +} + +private fun sdkVersion(writer: JsonWriter): String { + val sdkVersion = writer.javaClass.classLoader + .getResourceAsStream("software/amazon/smithy/kotlin/codegen/core/sdk-version.txt")!! + .readBytes().toString(Charsets.UTF_8) + return sdkVersion +} diff --git a/tests/protocol-tests-runner/src/main/kotlin/software/amazon/smithy/kotlin/protocolTests/Runner.kt b/tests/protocol-tests-runner/src/main/kotlin/software/amazon/smithy/kotlin/protocolTests/Runner.kt new file mode 100644 index 000000000..590cce798 --- /dev/null +++ b/tests/protocol-tests-runner/src/main/kotlin/software/amazon/smithy/kotlin/protocolTests/Runner.kt @@ -0,0 +1,114 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package software.amazon.smithy.kotlin.protocolTests + +import software.amazon.smithy.kotlin.protocolTests.utils.JsonWriter +import software.amazon.smithy.model.Model +import software.amazon.smithy.model.knowledge.TopDownIndex +import software.amazon.smithy.model.shapes.ServiceShape +import software.amazon.smithy.model.traits.TagsTrait +import software.amazon.smithy.protocoltests.traits.HttpRequestTestsTrait +import software.amazon.smithy.protocoltests.traits.HttpResponseTestsTrait +import java.io.InputStreamReader +import java.io.PrintWriter +import java.io.StringWriter +import java.nio.charset.StandardCharsets +import java.nio.file.Files +import java.nio.file.Path + +/** + * Assumes each argument in args to be a JAR file defining protocol tests. Each argument is passed as-is to the + * Smithy model assembler, and then, for each valid service defining protocol tests we + * + * 1. Generate the protocol tests + * 2. Build the generated protocol tests + * 3. Execute the protocol tests + * 4. Collect the output of the service into the final result + * + * The final report that aggregates the results for each suite (client) found in the model is stored + * in a file `results.json`. + */ +fun main(args: Array) { + val assembler = Model + .assembler() + .discoverModels() + for (arg in args) { + assembler.addImport(arg) + } + val model = assembler.assemble().unwrap() + PrintWriter("result.json").use { out -> + val jsonWriter = JsonWriter(out) + writeReportStart(jsonWriter) + for (service in model.serviceShapes) { + if (hasStandardProtocolTests(service, model)) { + val codegenPath = Files.createTempDirectory("protocol-tests-${service.id.name}") + // Generate the client and protocol tests assertions + generateProtocolTests(model, service!!, codegenPath) + // Build the client and protocol tests + buildProtocolTests(codegenPath) + // Run the protocol tests and get the results + val results = runProtocolTests(codegenPath) + // Add the result to the final report + jsonWriter.writeEncodedValue(results) + } + } + writeReportEnd(jsonWriter) + } +} + +private fun hasStandardProtocolTests(service: ServiceShape, model: Model): Boolean { + val isAwsServiceTest = service.findTrait(TagsTrait.ID) + .map { trait -> TagsTrait::class.java.cast(trait) } + // We add a tag to protocol tests for specific AWS services + // that require customizations to run properly. Those + // not standard protocol tests are filtered out here. + .map { tags -> tags.values.contains("aws-service-test") } + .orElse(false) + + if (isAwsServiceTest) { + return false + } + // Check that at least one operation in the service has a test trait, either for + // requests or for responses. + TopDownIndex.of(model).getContainedOperations(service).forEach { operation -> + if (operation.hasTrait(HttpResponseTestsTrait.ID) || + operation.hasTrait(HttpRequestTestsTrait.ID) + ) { + return true + } + } + return false +} + +/** + * Executes `gradle build -x test` in the given directory to build the generated + * code that includes the client and the protocol tests. This build is expected + * to generate an uber JAR that we can the call directly without having to setup + * all the classpath elements. + */ +private fun buildProtocolTests(codegenPath: Path) { + ProcessBuilder("gradle", "build", "-x", "test") + .directory(codegenPath.toFile()) + .inheritIO() + .start() + .waitFor() +} + +/** + * After building the protocol tests and client source files the uber jar is found inside + * `build/libs/`. We now just run java with this jar as an argument. The report with the + * results will be printed out to the standard output, we capture it here and return it + * to fill up the final report. + */ +private fun runProtocolTests(codegenPath: Path): String { + val name = codegenPath.toFile().name + val process = ProcessBuilder("java", "-jar", "build/libs/$name-all.jar") + .directory(codegenPath.toFile()) + .start() + val writer = StringWriter() + InputStreamReader(process.inputStream, StandardCharsets.UTF_8).transferTo(writer) + process.waitFor() + return writer.toString() +} diff --git a/tests/protocol-tests-utils/build.gradle.kts b/tests/protocol-tests-utils/build.gradle.kts new file mode 100644 index 000000000..6c8df4332 --- /dev/null +++ b/tests/protocol-tests-utils/build.gradle.kts @@ -0,0 +1,116 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +import aws.sdk.kotlin.gradle.dsl.configurePublishing +import org.jetbrains.kotlin.gradle.dsl.JvmTarget +import org.jetbrains.kotlin.gradle.tasks.KotlinCompile + +plugins { + alias(libs.plugins.kotlin.jvm) + jacoco + `maven-publish` +} + +description = "Utils for running protocol tests" +extra["displayName"] = "Smithy :: Kotlin :: Protocol Tests :: Utils" +extra["moduleName"] = "software.amazon.smithy.kotlin.pt.util" + +val codegenVersion: String by project +group = "software.amazon.smithy.kotlin" +version = codegenVersion + +val sdkVersion: String by project +val runtimeVersion = sdkVersion + +dependencies { + + // Test dependencies + testImplementation(libs.junit.jupiter) + testImplementation(libs.kotest.assertions.core.jvm) + testImplementation(libs.kotlin.test) + testImplementation(libs.kotlin.test.junit5) +} + +val generateSdkRuntimeVersion by tasks.registering { + // generate the version of the runtime to use as a resource. + // this keeps us from having to manually change version numbers in multiple places + val resourcesDir = layout.buildDirectory.dir("resources/main/software/amazon/smithy/kotlin/codegen/core").get() + val versionFile = file("$resourcesDir/sdk-version.txt") + val gradlePropertiesFile = rootProject.file("gradle.properties") + inputs.file(gradlePropertiesFile) + outputs.file(versionFile) + sourceSets.main.get().output.dir(resourcesDir) + doLast { + versionFile.writeText(runtimeVersion) + } +} + +tasks.withType { + compilerOptions { + jvmTarget.set(JvmTarget.JVM_1_8) + freeCompilerArgs.add("-opt-in=kotlin.RequiresOptIn") + } + dependsOn(generateSdkRuntimeVersion) +} + +tasks.withType { + sourceCompatibility = JavaVersion.VERSION_1_8.toString() + targetCompatibility = JavaVersion.VERSION_1_8.toString() +} + +// Reusable license copySpec +val licenseSpec = copySpec { + from("${project.rootDir}/LICENSE") + from("${project.rootDir}/NOTICE") +} + +// Configure jars to include license related info +tasks.jar { + metaInf.with(licenseSpec) + inputs.property("moduleName", project.name) + manifest { + attributes["Automatic-Module-Name"] = project.name + } +} + +val sourcesJar by tasks.creating(Jar::class) { + group = "publishing" + description = "Assembles Kotlin sources jar" + archiveClassifier.set("sources") + from(sourceSets.getByName("main").allSource) +} + +tasks.test { + useJUnitPlatform() + testLogging { + events("passed", "skipped", "failed") + showStandardStreams = true + showStackTraces = true + showExceptions = true + exceptionFormat = org.gradle.api.tasks.testing.logging.TestExceptionFormat.FULL + } +} + +// Configure jacoco (code coverage) to generate an HTML report +tasks.jacocoTestReport { + reports { + xml.required.set(false) + csv.required.set(false) + html.outputLocation.set(layout.buildDirectory.dir("reports/jacoco")) + } +} + +// Always run the jacoco test report after testing. +tasks["test"].finalizedBy(tasks["jacocoTestReport"]) + +publishing { + publications { + create("codegen") { + from(components["java"]) + artifact(sourcesJar) + } + } +} + +configurePublishing("smithy-kotlin", "smithy-lang") diff --git a/tests/protocol-tests-utils/src/main/kotlin/software/amazon/smithy/kotlin/protocolTests/utils/JsonWriter.kt b/tests/protocol-tests-utils/src/main/kotlin/software/amazon/smithy/kotlin/protocolTests/utils/JsonWriter.kt new file mode 100644 index 000000000..639d238e3 --- /dev/null +++ b/tests/protocol-tests-utils/src/main/kotlin/software/amazon/smithy/kotlin/protocolTests/utils/JsonWriter.kt @@ -0,0 +1,192 @@ +package software.amazon.smithy.kotlin.protocolTests.utils + +import java.io.Closeable +import java.io.Writer + +/** + * A JSON simple, streaming, JSON writer. + */ +class JsonWriter(private val writer: Writer) : Closeable { + internal enum class State { + Start, + End, + ObjectStart, + ObjectFirstKey, + ObjectAfterFirstKey, + ObjectValue, + ArrayStart, + ArrayValue, + } + + private val stack = ArrayDeque() + + init { + stack.add(State.Start) + } + + /** + * Starts a JSON object. + */ + fun startObject(): JsonWriter { + pushTransition(State.ObjectStart) + writer.write("{") + return this + } + + /** + * Ends a JSON object. + */ + fun endObject(): JsonWriter { + val current = stack.removeLast() + if (current != State.ObjectStart && current != State.ObjectValue) { + throw Exception("Cannot end object while state is $current") + } + writer.write("}") + return this + } + + /** + * Writes a key for a JSON object. + */ + fun writeKey(key: String): JsonWriter { + val current = stack.removeLast() + if (current == State.ObjectStart) { + stack.addLast(State.ObjectFirstKey) + } else if (current == State.ObjectValue) { + writer.write(",") + stack.addLast(State.ObjectAfterFirstKey) + } else { + throw Exception("Cannot write key $key while state is $current") + } + writeString(key) + writer.write(":") + return this + } + + /** + * Writes a key value pair for a JSON object. + */ + fun writeKvp(key: String, value: Any?): JsonWriter { + writeKey(key) + writeValue(value) + return this + } + + /** + * Starts a JSON array. + */ + fun startArray(): JsonWriter { + pushTransition(State.ArrayStart) + writer.write("[") + return this + } + + /** + * Ends a JSON array. + */ + fun endArray(): JsonWriter { + val current = stack.removeLast() + if (current != State.ArrayStart && current != State.ArrayValue) { + throw Exception("Cannot end array while state is $current") + } + writer.write("]") + return this + } + + /** + * Writes any value. Possible values allowed are String, Int, Double, Float, + * Long, and, Boolean. + */ + fun writeValue(input: Any?): JsonWriter { + val current = stack.last() + transitionForValue(current) + + if (input == null) { + writer.write("null") + } + when (input) { + is String -> writeString(input) + is Int -> writer.write(input.toString()) + is Double -> writer.write(input.toString()) + is Float -> writer.write(input.toString()) + is Long -> writer.write(input.toString()) + is Boolean -> writer.write(input.toString()) + else -> throw Exception("Unsupported input type: $input") + } + return this + } + + /** + * Writes a literal String that is already JSON encoded. + */ + fun writeEncodedValue(input: String): JsonWriter { + val current = stack.last() + transitionForValue(current) + writer.write(input) + return this + } + + /** + * Transitions and pushes a new state onto the stack. + */ + private fun pushTransition(state: State) { + transitionForValue(state) + stack.addLast(state) + } + + /** + * Transitions to a new proper state to add a new value to the stream. + */ + private fun transitionForValue(state: State) { + when (val current = stack.removeLast()) { + State.Start -> { + stack.addLast(State.End) + } + + State.ObjectFirstKey, State.ObjectAfterFirstKey -> { + stack.addLast(State.ObjectValue) + } + + State.ArrayStart -> { + stack.addLast(State.ArrayValue) + } + + State.ArrayValue -> { + writer.write(",") + stack.addLast(State.ArrayValue) + } + + else -> { + throw Exception("The current state $current cannot precede an structured of type $state") + } + } + } + + /** + * Writes a JSON string encoding any special characters to output correct JSON. + */ + private fun writeString(input: String) { + writer.write('"'.code) + input.forEach { c -> + when (c) { + '\\' -> writer.write("\\\\") + '"' -> writer.write("\\\"") + '\b' -> writer.write("\\b") + '\u000C' -> writer.write("\\f") + '\n' -> writer.write("\\n") + '\r' -> writer.write("\\r") + '\t' -> writer.write("\\t") + in '\u0000'..'\u001F' -> writer.write(String.format("\\u%04X", c.code)) + else -> writer.write(c.code) + } + } + writer.write('"'.code) + } + + /** + * Closes the underlying writer. + */ + override fun close() { + writer.close() + } +} diff --git a/tests/protocol-tests-utils/src/main/kotlin/software/amazon/smithy/kotlin/protocolTests/utils/Utils.kt b/tests/protocol-tests-utils/src/main/kotlin/software/amazon/smithy/kotlin/protocolTests/utils/Utils.kt new file mode 100644 index 000000000..edac59e7f --- /dev/null +++ b/tests/protocol-tests-utils/src/main/kotlin/software/amazon/smithy/kotlin/protocolTests/utils/Utils.kt @@ -0,0 +1,61 @@ +package software.amazon.smithy.kotlin.protocolTests.utils + +import java.io.OutputStreamWriter + +enum class Result(val value: String) { + PASSED("passed"), + FAILED("failed"), + ERRORED("errored"), + SKIPPED("skipped"), +} + +enum class TestType(val value: String) { + REQUEST("request"), + RESPONSE("response"), +} + +data class TestResult( + val testId: String, + val testType: TestType, + var result: Result = Result.PASSED, + var log: String? = null, +) + +fun writeResults( + serviceId: String, + protocolId: String, + results: List, +) { + JsonWriter(OutputStreamWriter(System.out)).use { writer -> + writeResults(writer, serviceId, protocolId, results) + } +} + +fun writeResults( + writer: JsonWriter, + serviceId: String, + protocolId: String, + results: List, +) { + writer.startObject() + writer.writeKvp("service", serviceId) + writer.writeKvp("protocol", protocolId) + writer.writeKey("results") + writer.startArray() + for (result in results) { + writeTestResult(writer, result) + } + writer.endArray() + writer.endObject() +} + +internal fun writeTestResult(writer: JsonWriter, result: TestResult) { + writer.startObject() + .writeKvp("id", result.testId) + .writeKvp("type", result.testType.value) + .writeKvp("result", result.result.value) + result.log?.let { + writer.writeKvp("log", it) + } + writer.endObject() +} diff --git a/tests/protocol-tests-utils/src/test/kotlin/software/amazon/smithy/kotlin/protocolTests/utils/JsonWriterTest.kt b/tests/protocol-tests-utils/src/test/kotlin/software/amazon/smithy/kotlin/protocolTests/utils/JsonWriterTest.kt new file mode 100644 index 000000000..0b79fbf0e --- /dev/null +++ b/tests/protocol-tests-utils/src/test/kotlin/software/amazon/smithy/kotlin/protocolTests/utils/JsonWriterTest.kt @@ -0,0 +1,178 @@ +package software.amazon.smithy.kotlin.protocolTests.utils + +import org.junit.jupiter.api.Assertions.* +import java.io.StringWriter +import kotlin.test.Test + +class JsonWriterTest { + + @Test + fun `write empty array`() { + var result = setup { + startArray() + endArray() + } + + assertEquals("[]", result) + } + + @Test + fun `write one element array`() { + var result = setup { + startArray() + writeValue(123) + endArray() + } + + assertEquals("[123]", result) + } + + @Test + fun `write two elements array`() { + var result = setup { + startArray() + writeValue(123) + writeValue(456) + endArray() + } + + assertEquals("[123,456]", result) + } + + @Test + fun `write nested empty array`() { + var result = setup { + startArray() + startArray() + endArray() + endArray() + } + + assertEquals("[[]]", result) + } + + @Test + fun `write nested one element array`() { + var result = setup { + startArray() + startArray() + writeValue("foo bar") + endArray() + endArray() + } + + assertEquals("[[\"foo bar\"]]", result) + } + + @Test + fun `write nested two element array`() { + var result = setup { + startArray() + startArray() + writeValue("foo bar") + writeValue(123) + endArray() + endArray() + } + assertEquals("[[\"foo bar\",123]]", result) + } + + @Test + fun `write nested empty object array`() { + var result = setup { + startArray() + startObject() + endObject() + endArray() + } + assertEquals("[{}]", result) + } + + @Test + fun `write nested objects array`() { + var result = setup { + startArray() + startObject() + endObject() + startObject() + writeKey("name") + writeValue("Joe Dow") + endObject() + endArray() + } + assertEquals("[{},{\"name\":\"Joe Dow\"}]", result) + } + + @Test + fun `write empty object`() { + var result = setup { + startObject() + endObject() + } + assertEquals("{}", result) + } + + @Test + fun `write one pair object`() { + var result = setup { + startObject() + writeKey("name") + writeValue("Joe Dow") + endObject() + } + assertEquals("{\"name\":\"Joe Dow\"}", result) + } + + @Test + fun `write one pair with nested object`() { + var result = setup { + startObject() + writeKey("personal") + startObject() + writeKey("name") + writeValue("Joe Doe") + endObject() + endObject() + } + assertEquals("{\"personal\":{\"name\":\"Joe Doe\"}}", result) + } + + @Test + fun `writes two pairs object`() { + var result = setup { + startObject() + writeKey("name") + writeValue("Joe Doe") + writeKey("age") + writeValue(20.3) + endObject() + } + assertEquals("{\"name\":\"Joe Doe\",\"age\":20.3}", result) + } + + @Test + fun `escapes special chars`() { + var result = setup { + writeValue("Joe \n \t \\ \"foo\" Doe") + } + assertEquals("\"Joe \\n \\t \\\\ \\\"foo\\\" Doe\"", result) + } + + @Test + fun `handles literal json`() { + var result = setup { + startObject() + writeKey("foo") + writeEncodedValue("{\"bar\": \"baz\"}") + endObject() + } + assertEquals("{\"foo\":{\"bar\": \"baz\"}}", result) + } + + internal fun setup(block: JsonWriter.() -> Unit): String { + var writer = StringWriter() + var jsonWriter = JsonWriter(writer) + block(jsonWriter) + return writer.toString() + } +}