Skip to content

Out of process bug fixes #1109

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Oct 7, 2022
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions utbot-core/src/main/kotlin/org/utbot/common/JvmUtil.kt
Original file line number Diff line number Diff line change
@@ -3,3 +3,5 @@ package org.utbot.common
private val javaSpecificationVersion = System.getProperty("java.specification.version")
val isJvm8 = javaSpecificationVersion.equals("1.8")
val isJvm9Plus = !javaSpecificationVersion.contains(".") && javaSpecificationVersion.toInt() >= 9

fun osSpecificJavaExecutable() = if (isWindows) "javaw" else "java"
3 changes: 3 additions & 0 deletions utbot-framework-api/build.gradle.kts
Original file line number Diff line number Diff line change
@@ -12,6 +12,9 @@ plugins {
dependencies {
api(project(":utbot-core"))
api(project(":utbot-api"))
api(project(":utbot-rd"))
implementation(group ="com.jetbrains.rd", name = "rd-framework", version = "2022.3.1")
implementation(group ="com.jetbrains.rd", name = "rd-core", version = "2022.3.1")
implementation("com.github.UnitTestBot:soot:${sootCommitHash}")
implementation(group = "io.github.microutils", name = "kotlin-logging", version = kotlinLoggingVersion)
// TODO do we really need apache commons?
Original file line number Diff line number Diff line change
@@ -1,10 +1,8 @@
package org.utbot.framework

import com.jetbrains.rd.util.LogLevel
import mu.KotlinLogging
import org.utbot.common.AbstractSettings
import org.utbot.common.PropertiesSettingsContainer
import kotlin.reflect.KProperty

private val logger = KotlinLogging.logger {}

/**
@@ -266,7 +264,17 @@ object UtSettings : AbstractSettings(
)

/**
* Determines whether should errors from a child process and idea engine process be written to a log file or suppressed.
* Log level for engine process, which started in idea on generate tests action.
*/
var engineProcessLogLevel by getEnumProperty(LogLevel.Info)

/**
* Log level for concrete executor process.
*/
var childProcessLogLevel by getEnumProperty(LogLevel.Info)

/**
* Determines whether should errors from a child process be written to a log file or suppressed.
* Note: being enabled, this option can highly increase disk usage when using ContestEstimator.
*
* False by default (for saving disk space).
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
package org.utbot.framework.process

import org.utbot.framework.plugin.services.JdkInfoService

object OpenModulesContainer {
private val modulesContainer: List<String>
val javaVersionSpecificArguments: List<String>
get() = modulesContainer
.takeIf { JdkInfoService.provide().version > 8 }
?: emptyList()

init {
modulesContainer = buildList {
openPackage("java.base", "jdk.internal.misc")
openPackage("java.base", "java.lang")
openPackage("java.base", "java.lang.reflect")
openPackage("java.base", "sun.security.provider")
add("--illegal-access=warn")
}
}

private fun MutableList<String>.openPackage(module: String, pakage: String) {
add("--add-opens")
add("$module/$pakage=ALL-UNNAMED")
}
}
Original file line number Diff line number Diff line change
@@ -53,10 +53,20 @@ internal object HandlerClassesLoader : URLClassLoader(emptyArray()) {
* Command-line option to disable the sandbox
*/
const val DISABLE_SANDBOX_OPTION = "--disable-sandbox"
private val defaultLogLevel = LogLevel.Info
const val ENABLE_LOGS_OPTION = "--enable-logs"
private val logger = getLogger("ChildProcess")
private val messageFromMainTimeout: Duration = 120.seconds

fun logLevelArgument(level: LogLevel): String {
return "$ENABLE_LOGS_OPTION=$level"
}

private fun findLogLevel(args: Array<String>): LogLevel {
val logArgument = args.find{ it.contains(ENABLE_LOGS_OPTION) } ?: return LogLevel.Fatal

return enumValueOf(logArgument.split("=").last())
}

/**
* It should be compiled into separate jar file (child_process.jar) and be run with an agent (agent.jar) option.
*/
@@ -76,7 +86,9 @@ fun main(args: Array<String>) = runBlocking {
}
}

Logger.set(Lifetime.Eternal, UtRdConsoleLoggerFactory(defaultLogLevel, System.err))
val logLevel: LogLevel = findLogLevel(args)
Logger.set(Lifetime.Eternal, UtRdConsoleLoggerFactory(logLevel, System.err))

val port = findRdPort(args)

try {
Original file line number Diff line number Diff line change
@@ -7,6 +7,7 @@ import org.utbot.common.utBotTempDirectory
import org.utbot.framework.plugin.services.JdkInfoService
import org.utbot.framework.UtSettings
import org.utbot.framework.plugin.services.WorkingDirService
import org.utbot.framework.process.OpenModulesContainer
import org.utbot.instrumentation.Settings
import org.utbot.instrumentation.agent.DynamicClassTransformer
import org.utbot.rd.rdPortArgument
@@ -19,26 +20,20 @@ class ChildProcessRunner {
private val id = Random.nextLong()
private var processSeqN = 0
private val cmds: List<String> by lazy {
val debugCmd =
listOfNotNull(DEBUG_RUN_CMD.takeIf { Settings.runChildProcessWithDebug} )

val javaVersionSpecificArguments =
listOf("--add-opens", "java.base/jdk.internal.misc=ALL-UNNAMED", "--illegal-access=warn")
.takeIf { JdkInfoService.provide().version > 8 }
?: emptyList()

val debugCmd = listOfNotNull(DEBUG_RUN_CMD.takeIf { Settings.runChildProcessWithDebug })
val javaVersionSpecificArguments = OpenModulesContainer.javaVersionSpecificArguments
val pathToJava = JdkInfoService.provide().path

listOf(pathToJava.resolve("bin${File.separatorChar}java").toString()) +
listOf(pathToJava.resolve("bin${File.separatorChar}${osSpecificJavaExecutable()}").toString()) +
debugCmd +
javaVersionSpecificArguments +
listOf("-javaagent:$jarFile", "-ea", "-jar", "$jarFile")
}

var errorLogFile: File = NULL_FILE

fun start(port: Int): Process {
val portArgument = rdPortArgument(port)
fun start(rdPort: Int): Process {
val portArgument = rdPortArgument(rdPort)

logger.debug { "Starting child process: ${cmds.joinToString(" ")} $portArgument" }
processSeqN++
@@ -54,6 +49,9 @@ class ChildProcessRunner {
if (!UtSettings.useSandbox) {
add(DISABLE_SANDBOX_OPTION)
}
if (UtSettings.logConcreteExecutionErrors) {
add(logLevelArgument(UtSettings.childProcessLogLevel))
}
add(portArgument)
}

@@ -62,10 +60,10 @@ class ChildProcessRunner {
.directory(directory)

return processBuilder.start().also {
logger.debug { "Process started with PID=${it.getPid}" }
logger.info { "Process started with PID=${it.getPid}" }

if (UtSettings.logConcreteExecutionErrors) {
logger.debug { "Child process error log: ${errorLogFile.absolutePath}" }
logger.info { "Child process error log: ${errorLogFile.absolutePath}" }
}
}
}
@@ -75,7 +73,7 @@ class ChildProcessRunner {
private const val ERRORS_FILE_PREFIX = "utbot-childprocess-errors"
private const val INSTRUMENTATION_LIB = "lib"

private const val DEBUG_RUN_CMD = "-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,quiet=y,address=5005"
private const val DEBUG_RUN_CMD = "-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,quiet=y,address=5006"

private val UT_BOT_TEMP_DIR: File = File(utBotTempDirectory.toFile(), ERRORS_FILE_PREFIX)

Original file line number Diff line number Diff line change
@@ -54,7 +54,7 @@ import org.utbot.framework.plugin.api.CodegenLanguage
import org.utbot.intellij.plugin.models.GenerateTestsModel
import org.utbot.intellij.plugin.models.packageName
import org.utbot.intellij.plugin.process.EngineProcess
import org.utbot.intellij.plugin.process.RdGTestenerationResult
import org.utbot.intellij.plugin.process.RdTestGenerationResult
import org.utbot.intellij.plugin.sarif.SarifReportIdea
import org.utbot.intellij.plugin.sarif.SourceFindingStrategyIdea
import org.utbot.intellij.plugin.ui.*
@@ -83,7 +83,7 @@ object CodeGenerationController {

fun generateTests(
model: GenerateTestsModel,
classesWithTests: Map<PsiClass, RdGTestenerationResult>,
classesWithTests: Map<PsiClass, RdTestGenerationResult>,
psi2KClass: Map<PsiClass, ClassId>,
proc: EngineProcess
) {
Original file line number Diff line number Diff line change
@@ -36,7 +36,7 @@ import org.utbot.intellij.plugin.generator.CodeGenerationController.generateTest
import org.utbot.intellij.plugin.models.GenerateTestsModel
import org.utbot.intellij.plugin.models.packageName
import org.utbot.intellij.plugin.process.EngineProcess
import org.utbot.intellij.plugin.process.RdGTestenerationResult
import org.utbot.intellij.plugin.process.RdTestGenerationResult
import org.utbot.intellij.plugin.settings.Settings
import org.utbot.intellij.plugin.ui.GenerateTestsDialogWindow
import org.utbot.intellij.plugin.ui.utils.isBuildWithGradle
@@ -46,7 +46,6 @@ import org.utbot.intellij.plugin.ui.utils.testModules
import org.utbot.intellij.plugin.util.*
import org.utbot.rd.terminateOnException
import java.io.File
import java.net.URLClassLoader
import java.nio.file.Path
import java.nio.file.Paths
import java.util.concurrent.TimeUnit
@@ -136,12 +135,12 @@ object UtTestsDialogProcessor {

val (buildDirs, classpath, classpathList, pluginJarsPath) = buildPaths

val testSetsByClass = mutableMapOf<PsiClass, RdGTestenerationResult>()
val testSetsByClass = mutableMapOf<PsiClass, RdTestGenerationResult>()
val psi2KClass = mutableMapOf<PsiClass, ClassId>()
var processedClasses = 0
val totalClasses = model.srcClasses.size

val proc = EngineProcess(lifetime)
val proc = EngineProcess(lifetime, project)

proc.setupUtContext(buildDirs + classpathList)
proc.createTestGenerator(
@@ -283,10 +282,6 @@ object UtTestsDialogProcessor {
appendLine("Alternatively, you could try to increase current timeout $timeout sec for generating tests in generation dialog.")
}


private fun urlClassLoader(classpath: List<String>) =
URLClassLoader(classpath.map { File(it).toURI().toURL() }.toTypedArray())

private fun findSrcModule(srcClasses: Set<PsiClass>): Module {
val srcModules = srcClasses.mapNotNull { it.module }.distinct()
return when (srcModules.size) {
Original file line number Diff line number Diff line change
@@ -1,29 +1,29 @@
package org.utbot.intellij.plugin.process

import com.intellij.ide.plugins.cl.PluginClassLoader
import com.intellij.openapi.application.ApplicationManager
import com.intellij.openapi.project.DumbService
import com.intellij.openapi.project.Project
import com.intellij.psi.PsiMethod
import com.intellij.refactoring.util.classMembers.MemberInfo
import com.jetbrains.rd.util.lifetime.Lifetime
import com.jetbrains.rd.util.lifetime.onTermination
import com.jetbrains.rd.util.lifetime.throwIfNotAlive
import kotlinx.coroutines.runBlocking
import kotlinx.coroutines.sync.Mutex
import kotlinx.coroutines.sync.withLock
import mu.KotlinLogging
import org.jetbrains.kotlin.scripting.resolve.classId
import org.utbot.common.AbstractSettings
import org.utbot.common.getPid
import org.utbot.common.osSpecificJavaExecutable
import org.utbot.common.utBotTempDirectory
import org.utbot.framework.UtSettings
import org.utbot.framework.codegen.*
import org.utbot.framework.codegen.model.UtilClassKind
import org.utbot.framework.codegen.model.constructor.tree.TestsGenerationReport
import org.utbot.framework.plugin.api.*
import org.utbot.framework.plugin.services.JdkInfo
import org.utbot.framework.plugin.services.JdkInfoDefaultProvider
import org.utbot.framework.plugin.services.JdkInfoService
import org.utbot.framework.plugin.services.WorkingDirService
import org.utbot.framework.process.OpenModulesContainer
import org.utbot.framework.process.generated.*
import org.utbot.framework.process.generated.Signature
import org.utbot.framework.util.Conflict
@@ -39,85 +39,77 @@ import org.utbot.rd.startUtProcessWithRdServer
import org.utbot.sarif.SourceFindingStrategy
import java.io.File
import java.nio.file.Path
import java.util.*
import kotlin.io.path.deleteIfExists
import kotlin.io.path.pathString
import kotlin.random.Random
import kotlin.reflect.KProperty1
import kotlin.reflect.full.memberProperties

private val engineProcessLogConfigurations = utBotTempDirectory.toFile().resolve("rdEngineProcessLogConfigurations")
private val logger = KotlinLogging.logger {}
private val engineProcessLogDirectory = utBotTempDirectory.toFile().resolve("rdEngineProcessLogs")

data class RdGTestenerationResult(val notEmptyCases: Int, val testSetsId: Long)
data class RdTestGenerationResult(val notEmptyCases: Int, val testSetsId: Long)

class EngineProcess(val lifetime: Lifetime) {
class EngineProcess(parent: Lifetime, val project: Project) {
private val ldef = parent.createNested()
private val id = Random.nextLong()
private var count = 0
private var configPath: Path? = null

companion object {
private var configPath: Path? = null
private fun getOrCreateLogConfig(): String {
var realPath = configPath
if (realPath == null) {
synchronized(this) {
realPath = configPath
if (realPath == null) {
utBotTempDirectory.toFile().mkdirs()
configPath = utBotTempDirectory.toFile().resolve("EngineProcess_log4j2.xml").apply {
writeText(
"""<?xml version="1.0" encoding="UTF-8"?>
private fun getOrCreateLogConfig(): String {
var realPath = configPath
if (realPath == null) {
engineProcessLogConfigurations.mkdirs()
configPath = File.createTempFile("epl", ".xml", engineProcessLogConfigurations).apply {
val onMatch = if (UtSettings.logConcreteExecutionErrors) "NEUTRAL" else "DENY"
writeText(
"""<?xml version="1.0" encoding="UTF-8"?>
<Configuration>
<Appenders>
<Console name="Console" target="SYSTEM_OUT">
<ThresholdFilter level="ERROR" onMatch="DENY" onMismatch="DENY"/>
<ThresholdFilter level="${UtSettings.engineProcessLogLevel.name.uppercase()}" onMatch="$onMatch" onMismatch="DENY"/>
<PatternLayout pattern="%d{HH:mm:ss.SSS} | %-5level | %c{1} | %msg%n"/>
</Console>
</Appenders>
<Loggers>
<Root level="error">
<Root level="${UtSettings.engineProcessLogLevel.name.lowercase()}">
<AppenderRef ref="Console"/>
</Root>
</Loggers>
</Configuration>"""
)
}.toPath()
realPath = configPath
}
}
}
return realPath!!.pathString
)
}.toPath()
realPath = configPath
}
return realPath!!.pathString
}
// because we cannot load idea bundled lifetime or it will break everything

private fun debugArgument(): String {
return "-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,quiet=y,address=5005".takeIf { Settings.runIdeaProcessWithDebug }
?: ""
}

private val kryoHelper = KryoHelper(lifetime)
private val kryoHelper = KryoHelper(ldef)

private suspend fun engineModel(): EngineProcessModel {
lifetime.throwIfNotAlive()
ldef.throwIfNotAlive()
return lock.withLock {
var proc = current

if (proc == null) {
proc = startUtProcessWithRdServer(lifetime) { port ->
proc = startUtProcessWithRdServer(ldef) { port ->
val current = JdkInfoDefaultProvider().info
val required = JdkInfoService.jdkInfoProvider.info
val java =
JdkInfoService.jdkInfoProvider.info.path.resolve("bin${File.separatorChar}javaw").toString()
JdkInfoService.jdkInfoProvider.info.path.resolve("bin${File.separatorChar}${osSpecificJavaExecutable()}").toString()
val cp = (this.javaClass.classLoader as PluginClassLoader).classPath.baseUrls.joinToString(
separator = ";",
prefix = "\"",
postfix = "\""
)
val classname = "org.utbot.framework.process.EngineMainKt"
val javaVersionSpecificArguments =
listOf("--add-opens", "java.base/jdk.internal.misc=ALL-UNNAMED", "--illegal-access=warn")
.takeIf { JdkInfoService.provide().version > 8 }
?: emptyList()
val javaVersionSpecificArguments = OpenModulesContainer.javaVersionSpecificArguments
val directory = WorkingDirService.provide().toFile()
val log4j2ConfigFile = "\"-Dlog4j2.configurationFile=${getOrCreateLogConfig()}\""
val debugArg = debugArgument()
@@ -172,7 +164,7 @@ class EngineProcess(val lifetime: Lifetime) {
private var current: ProcessWithRdServer? = null

fun setupUtContext(classpathForUrlsClassloader: List<String>) = runBlocking {
engineModel().setupUtContext.startSuspending(lifetime, SetupContextParams(classpathForUrlsClassloader))
engineModel().setupUtContext.startSuspending(ldef, SetupContextParams(classpathForUrlsClassloader))
}

// suppose that only 1 simultaneous test generator process can be executed in idea
@@ -186,7 +178,7 @@ class EngineProcess(val lifetime: Lifetime) {
) = runBlocking {
engineModel().isCancelled.set(handler = isCancelled)
engineModel().createTestGenerator.startSuspending(
lifetime,
ldef,
TestGeneratorParams(buildDir.toTypedArray(), classPath, dependencyPaths, JdkInfo(jdkInfo.path.pathString, jdkInfo.version))
)
}
@@ -238,9 +230,9 @@ class EngineProcess(val lifetime: Lifetime) {
isFuzzingEnabled: Boolean,
fuzzingValue: Double,
searchDirectory: String
): RdGTestenerationResult = runBlocking {
): RdTestGenerationResult = runBlocking {
val result = engineModel().generate.startSuspending(
lifetime,
ldef,
GenerateParams(
mockInstalled,
staticsMockingIsConfigured,
@@ -257,7 +249,7 @@ class EngineProcess(val lifetime: Lifetime) {
)
)

return@runBlocking RdGTestenerationResult(result.notEmptyCases, result.testSetsId)
return@runBlocking RdTestGenerationResult(result.notEmptyCases, result.testSetsId)
}

fun render(
@@ -278,7 +270,7 @@ class EngineProcess(val lifetime: Lifetime) {
testClassPackageName: String
): Pair<String, UtilClassKind?> = runBlocking {
val result = engineModel().render.startSuspending(
lifetime, RenderParams(
ldef, RenderParams(
testSetsId,
kryoHelper.writeObject(classUnderTest),
kryoHelper.writeObject(paramNames),
@@ -300,9 +292,9 @@ class EngineProcess(val lifetime: Lifetime) {
}

fun forceTermination() = runBlocking {
configPath?.deleteIfExists()
engineModel().stopProcess.start(Unit)
current?.terminate()
engineModel().writeSarifReport
}

fun writeSarif(reportFilePath: Path,
@@ -312,23 +304,23 @@ class EngineProcess(val lifetime: Lifetime) {
) = runBlocking {
current!!.protocol.rdSourceFindingStrategy.let {
it.getSourceFile.set { params ->
ApplicationManager.getApplication().runReadAction<String?> {
DumbService.getInstance(project).runReadActionInSmartMode<String?> {
sourceFindingStrategy.getSourceFile(
params.classFqn,
params.extension
)?.canonicalPath
}
}
it.getSourceRelativePath.set { params ->
ApplicationManager.getApplication().runReadAction<String> {
DumbService.getInstance(project).runReadActionInSmartMode<String> {
sourceFindingStrategy.getSourceRelativePath(
params.classFqn,
params.extension
)
}
}
it.testsRelativePath.set { _ ->
ApplicationManager.getApplication().runReadAction<String> {
DumbService.getInstance(project).runReadActionInSmartMode<String> {
sourceFindingStrategy.testsRelativePath
}
}
@@ -374,8 +366,8 @@ class EngineProcess(val lifetime: Lifetime) {
}

init {
lifetime.onTermination {
current?.terminate()
ldef.onTermination {
forceTermination()
}
}
}
1 change: 1 addition & 0 deletions utbot-rd/src/main/kotlin/org/utbot/rd/ClientProcessUtil.kt
Original file line number Diff line number Diff line change
@@ -113,6 +113,7 @@ class ClientProtocolBuilder {
private var timeout = Duration.INFINITE

suspend fun start(port: Int, parent: Lifetime? = null, bindables: Protocol.(CallsSynchronizer) -> Unit) {
UtRdCoroutineScope.current // coroutine scope initialization
val pid = currentProcessPid.toInt()
val ldef = parent?.createNested() ?: LifetimeDefinition()
ldef.terminateOnException { _ ->
4 changes: 3 additions & 1 deletion utbot-rd/src/main/kotlin/org/utbot/rd/UtRdCoroutineScope.kt
Original file line number Diff line number Diff line change
@@ -4,6 +4,8 @@ import com.jetbrains.rd.framework.util.RdCoroutineScope
import com.jetbrains.rd.framework.util.asCoroutineDispatcher
import com.jetbrains.rd.util.lifetime.Lifetime

private val coroutineDispatcher = UtSingleThreadScheduler("UtCoroutineScheduler").asCoroutineDispatcher

class UtRdCoroutineScope(lifetime: Lifetime) : RdCoroutineScope(lifetime) {
companion object {
val current = UtRdCoroutineScope(Lifetime.Eternal)
@@ -13,5 +15,5 @@ class UtRdCoroutineScope(lifetime: Lifetime) : RdCoroutineScope(lifetime) {
override(lifetime, this)
}

override val defaultDispatcher = UtSingleThreadScheduler("UtCoroutineScheduler").asCoroutineDispatcher
override val defaultDispatcher = coroutineDispatcher
}