From a526150f919048084855886a5c599ea9610761cc Mon Sep 17 00:00:00 2001 From: IlyaMuravjov Date: Mon, 27 Nov 2023 15:04:16 +0300 Subject: [PATCH] Automatically remove uncompilable test methods --- .../codegen/renderer/CgAbstractRenderer.kt | 5 ++ .../main/kotlin/org/utbot/contest/Contest.kt | 10 +++ .../org/utbot/contest/ContestEstimator.kt | 61 +++++++++++++++++-- 3 files changed, 71 insertions(+), 5 deletions(-) diff --git a/utbot-framework/src/main/kotlin/org/utbot/framework/codegen/renderer/CgAbstractRenderer.kt b/utbot-framework/src/main/kotlin/org/utbot/framework/codegen/renderer/CgAbstractRenderer.kt index bb6863e063..ce811eefb1 100644 --- a/utbot-framework/src/main/kotlin/org/utbot/framework/codegen/renderer/CgAbstractRenderer.kt +++ b/utbot-framework/src/main/kotlin/org/utbot/framework/codegen/renderer/CgAbstractRenderer.kt @@ -251,12 +251,14 @@ abstract class CgAbstractRenderer( } override fun visit(element: CgTestMethod) { + visit(CgSingleLineComment(TEST_METHOD_START_MARKER)) renderMethodDocumentation(element) for (annotation in element.annotations) { annotation.accept(this) } renderMethodSignature(element) visit(element as CgMethod) + visit(CgSingleLineComment(TEST_METHOD_END_MARKER)) } override fun visit(element: CgErrorTestMethod) { @@ -953,6 +955,9 @@ abstract class CgAbstractRenderer( private fun ClassId.isAccessibleBySimpleName(): Boolean = isAccessibleBySimpleNameImpl(this) companion object { + const val TEST_METHOD_START_MARKER = "test method start marker" + const val TEST_METHOD_END_MARKER = "test method end marker" + fun makeRenderer( context: CgContext, printer: CgPrinter = CgPrinterImpl() diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Contest.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Contest.kt index 92a2700820..4e98919133 100644 --- a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Contest.kt +++ b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/Contest.kt @@ -148,6 +148,16 @@ fun main(args: Array) { expectedExceptions = ExpectedExceptionsForClass(), methodNameFilter = null ) + + val compiledClassFileDir = File(outputDir.absolutePath, "compiledClassFiles") + compiledClassFileDir.mkdirs() + compileClassAndRemoveUncompilableTests( + testDir = compiledClassFileDir.absolutePath, + classPath = classpathString, + testClass = cut.generatedTestFile.absolutePath + ) + compiledClassFileDir.deleteRecursively() + println("${ContestMessage.READY}") } } diff --git a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt index 1b94267218..d0b75c78f1 100644 --- a/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt +++ b/utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt @@ -32,6 +32,7 @@ import org.utbot.features.FeatureExtractorFactoryImpl import org.utbot.features.FeatureProcessorWithStatesRepetitionFactory import org.utbot.framework.PathSelectorType import org.utbot.framework.UtSettings +import org.utbot.framework.codegen.renderer.CgAbstractRenderer import org.utbot.framework.plugin.api.util.id import org.utbot.framework.plugin.api.util.withUtContext import org.utbot.framework.plugin.services.JdkInfoService @@ -49,7 +50,10 @@ private val javaHome = System.getenv("JAVA_HOME") private val javacCmd = "$javaHome/bin/javac" private val javaCmd = "$javaHome/bin/java" -private const val compileAttempts = 2 +// first attempt is for --add-opens +// second attempt is for removing test methods that still don't compile +// last attempt is for checking if final result compiles +private const val compileAttempts = 3 private data class UnnamedPackageInfo(val pack: String, val module: String) @@ -62,7 +66,16 @@ private fun findAllNotExportedPackages(report: String): List }.toList().distinct() } -private fun compileClass(testDir: String, classPath: String, testClass: String): Int { +private fun findErrorLines(report: String): List { + // (\d+) is line number + // (:(\d+)) is optional column number + val regex = """\.java:(\d+)(:(\d+))?: error""".toRegex() + return regex.findAll(report).map { + it.groupValues[1].toInt() - 1 + }.toList().distinct() +} + +fun compileClassAndRemoveUncompilableTests(testDir: String, classPath: String, testClass: String): Int = try { val exports = mutableSetOf() var exitCode = 0 @@ -93,10 +106,48 @@ private fun compileClass(testDir: String, classPath: String, testClass: String): if (errors.isNotEmpty()) logger.error { "Compilation errors: $errors" } exports += findAllNotExportedPackages(errors) + if (attemptNumber >= 1) { + val testFile = File(testClass) + val testClassLines = testFile.readLines() + val testStarts = testClassLines.withIndex() + .filter { it.value.contains(CgAbstractRenderer.TEST_METHOD_START_MARKER) } + .map { it.index } + val testEnds = testClassLines.withIndex() + .filter { it.value.contains(CgAbstractRenderer.TEST_METHOD_END_MARKER) } + .map { it.index } + val errorLines = findErrorLines(errors) + val errorRanges = errorLines.map { errorLine -> + // if error is outside test method, we can't fix that by removing test methods + val testStart = testStarts.filter { it <= errorLine }.maxOrNull() ?: return exitCode + val testEnd = testEnds.filter { it >= errorLine }.minOrNull() ?: return exitCode + testStart..testEnd + }.distinct() + if (errorRanges.size > testStarts.size / 5.0) { + logger.error { "Over 20% of test are uncompilable" } + logger.error { "Speculating that something is wrong with compilation settings, keeping all tests" } + return exitCode + } + val linesToRemove = mutableSetOf() + errorRanges.forEach { linesToRemove.addAll(it) } + val removedText = testClassLines.withIndex() + .filter { it.index in linesToRemove } + .joinToString("\n") { "${it.index}: ${it.value}" } + logger.info { "Removed uncompilable tests:\n$removedText" } + testFile.writeText(testClassLines.filterIndexed { i, _ -> i !in linesToRemove }.joinToString("\n")) + } } } - return exitCode + exitCode +} catch (e: Throwable) { + logger.error(e) { "compileClass failed" } + 1 +} finally { + val testFile = File(testClass) + testFile.writeText(testFile.readLines().filter { + !it.contains(CgAbstractRenderer.TEST_METHOD_START_MARKER) + && !it.contains(CgAbstractRenderer.TEST_METHOD_END_MARKER) + }.joinToString("\n")) } fun Array.toText() = joinToString(separator = ",") @@ -181,7 +232,7 @@ interface Tool { classStats.testClassFile = testClass logger.info().measureTime({ "Compiling class ${testClass.absolutePath}" }) { - val exitCode = compileClass( + val exitCode = compileClassAndRemoveUncompilableTests( compiledTestDir.absolutePath, project.compileClasspathString, testClass.absolutePath @@ -456,7 +507,7 @@ fun runEstimator( if (UtSettings.pathSelectorType == PathSelectorType.ML_SELECTOR || UtSettings.pathSelectorType == PathSelectorType.TORCH_SELECTOR) { Predictors.stateRewardPredictor = EngineAnalyticsContext.mlPredictorFactory() } - + logger.info { "PathSelectorType: ${UtSettings.pathSelectorType}" } if (UtSettings.pathSelectorType == PathSelectorType.ML_SELECTOR || UtSettings.pathSelectorType == PathSelectorType.TORCH_SELECTOR) { logger.info { "RewardModelPath: ${UtSettings.modelPath}" }