diff --git a/gradle.properties b/gradle.properties index d9a4e5ce5..a1b20769d 100644 --- a/gradle.properties +++ b/gradle.properties @@ -2,7 +2,7 @@ projectVersion=0.3.0 teamCityUrl=https://teamcity.jetbrains.com teamCityUsername=guest teamCityPassword=guest -kotlinVersion=1.3.40 -kotlinBuildType=Kotlin_1340_CompilerAllPlugins -kotlinBuild=1.3.41-release-150 -kotlinPluginBuild=1.3.41-release-IJ2019.2-1 +kotlinVersion=1.3.50 +kotlinBuildType=Kotlin_1350_CompilerAllPlugins +kotlinBuild=1.3.50-release-127 +kotlinPluginBuild= diff --git a/scripts/update_kt_version.py b/scripts/update_kt_version.py index 554e32b26..5d5029c09 100644 --- a/scripts/update_kt_version.py +++ b/scripts/update_kt_version.py @@ -18,6 +18,10 @@ def is_plugin_artifact(art_name): def to_plugin_build(art_name): return art_name.lstrip("kotlin-plugin-").rstrip(".zip") +class Unnamed: + def name(self): + return "" + def main(): props_file = "gradle.properties" @@ -39,7 +43,7 @@ def main(): build = prompt_by("build", builds, TeamCityNode.number).follow() artifacts = [art for art in build.follow("artifacts").findall("file") if is_plugin_artifact(art.name())] - artifact = prompt_by("plugin build", artifacts, lambda art: to_plugin_build(art.name())) + artifact = prompt_by("plugin build", artifacts, lambda art: to_plugin_build(art.name()), Unnamed()) changes = { "kotlinVersion": version.name(), diff --git a/scripts/utils/cli.py b/scripts/utils/cli.py index edf87dbd5..0fadc5d44 100644 --- a/scripts/utils/cli.py +++ b/scripts/utils/cli.py @@ -21,7 +21,7 @@ def require_not_none(description, x): if x == None: sys.exit(description + " not present") -def prompt_by(what, nodes, describer): +def prompt_by(what, nodes, describer, default=None): node_dict = {describer(node): node for node in nodes} sorted_described = sorted(node_dict.keys(), key=alphanum_sort_key) @@ -30,12 +30,12 @@ def prompt_by(what, nodes, describer): print() last_entry = sorted_described[-1] if len(sorted_described) > 0 else None - choice = input("Enter a " + what + " to choose [default: " + last_entry + "]: ").strip() + choice = input(f"Enter a {what} to choose [default: {last_entry}]: ").strip() print() - if len(choice) == 0: + if len(choice) == 0 and last_entry: return node_dict[last_entry] elif choice not in node_dict.keys(): - sys.exit("Invalid " + what + "!") + return default else: return node_dict[choice] diff --git a/server/build.gradle b/server/build.gradle index b7fe19870..da5af46bd 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -47,6 +47,7 @@ dependencies { implementation 'org.jetbrains.kotlin:kotlin-compiler-embeddable' implementation 'org.jetbrains.kotlin:kotlin-scripting-compiler-embeddable' implementation 'org.jetbrains.kotlin:kotlin-scripting-compiler-impl-embeddable' + implementation 'org.jetbrains.kotlin:kotlin-scripting-jvm-host-embeddable' implementation 'org.jetbrains.kotlin:kotlin-reflect' implementation 'org.jetbrains:fernflower:1.0' implementation 'com.pinterest.ktlint:ktlint-core:0.34.2' @@ -62,9 +63,9 @@ dependencies { testImplementation 'org.hamcrest:hamcrest-all:1.3' testImplementation 'junit:junit:4.11' testImplementation 'org.openjdk.jmh:jmh-core:1.20' - testImplementation 'org.jetbrains.kotlin:kotlin-scripting-jvm-host-embeddable' // See https://github.com/JetBrains/kotlin/blob/65b0a5f90328f4b9addd3a10c6f24f3037482276/libraries/examples/scripting/jvm-embeddable-host/build.gradle.kts#L8 + compileOnly 'org.jetbrains.kotlin:kotlin-scripting-jvm-host' testCompileOnly 'org.jetbrains.kotlin:kotlin-scripting-jvm-host' annotationProcessor 'org.openjdk.jmh:jmh-generator-annprocess:1.20' @@ -96,6 +97,11 @@ task copyPropertiesToTestWorkspace(type: Copy) { into file('src/test/resources/additionalWorkspace') } +task copyPropertiesToDSLTestWorkspace(type: Copy) { + from "$rootDir/gradle.properties" + into file('src/test/resources/kotlinDSLWorkspace') +} + task fixFilePermissions(type: Exec) { // When running on macOS or Linux the start script // needs executable permissions to run. @@ -132,7 +138,7 @@ run { } test { - dependsOn copyPropertiesToTestWorkspace + dependsOn copyPropertiesToTestWorkspace, copyPropertiesToDSLTestWorkspace testLogging { events 'failed' diff --git a/server/src/main/kotlin/org/javacs/kt/CompiledFile.kt b/server/src/main/kotlin/org/javacs/kt/CompiledFile.kt index e42c5a069..c1d2e22a1 100644 --- a/server/src/main/kotlin/org/javacs/kt/CompiledFile.kt +++ b/server/src/main/kotlin/org/javacs/kt/CompiledFile.kt @@ -15,15 +15,18 @@ import org.jetbrains.kotlin.psi.psiUtil.parentsWithSelf import org.jetbrains.kotlin.resolve.BindingContext import org.jetbrains.kotlin.resolve.scopes.LexicalScope import org.jetbrains.kotlin.types.KotlinType +import java.nio.file.Paths class CompiledFile( - val content: String, - val parse: KtFile, - val compile: BindingContext, - val container: ComponentProvider, - val sourcePath: Collection, - val classPath: CompilerClassPath) { - + val content: String, + val parse: KtFile, + val compile: BindingContext, + val container: ComponentProvider, + val sourcePath: Collection, + val classPath: CompilerClassPath, + val isScript: Boolean = false, + val kind: CompilationKind = CompilationKind.DEFAULT +) { /** * Find the type of the expression at `cursor` */ @@ -38,7 +41,7 @@ class CompiledFile( bindingContextOf(expression, scopeWithImports).getType(expression) fun bindingContextOf(expression: KtExpression, scopeWithImports: LexicalScope): BindingContext = - classPath.compiler.compileExpression(expression, scopeWithImports, sourcePath).first + classPath.compiler.compileExpression(expression, scopeWithImports, sourcePath, kind).first private fun expandForType(cursor: Int, surroundingExpr: KtExpression): KtExpression { val dotParent = surroundingExpr.parent as? KtDotQualifiedExpression @@ -120,7 +123,7 @@ class CompiledFile( } val padOffset = " ".repeat(offset) - val recompile = classPath.compiler.createFile(padOffset + surroundingContent) + val recompile = classPath.compiler.createFile(padOffset + surroundingContent, Paths.get("dummy.virtual" + if (isScript) ".kts" else ".kt"), kind) return recompile.findElementAt(cursor)?.findParent() } diff --git a/server/src/main/kotlin/org/javacs/kt/Compiler.kt b/server/src/main/kotlin/org/javacs/kt/Compiler.kt index 9c7fcdd57..364de25e4 100644 --- a/server/src/main/kotlin/org/javacs/kt/Compiler.kt +++ b/server/src/main/kotlin/org/javacs/kt/Compiler.kt @@ -15,6 +15,7 @@ import org.jetbrains.kotlin.cli.jvm.compiler.EnvironmentConfigFiles import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment import org.jetbrains.kotlin.cli.jvm.compiler.TopDownAnalyzerFacadeForJVM import org.jetbrains.kotlin.cli.jvm.config.addJvmClasspathRoots +import org.jetbrains.kotlin.cli.jvm.plugins.PluginCliParser import org.jetbrains.kotlin.config.CommonConfigurationKeys import org.jetbrains.kotlin.config.CompilerConfiguration as KotlinCompilerConfiguration import org.jetbrains.kotlin.config.JVMConfigurationKeys @@ -23,7 +24,7 @@ import org.jetbrains.kotlin.container.ComponentProvider import org.jetbrains.kotlin.container.get import org.jetbrains.kotlin.compiler.plugin.ComponentRegistrar import org.jetbrains.kotlin.idea.KotlinLanguage -import org.jetbrains.kotlin.load.java.JvmAbi +import org.jetbrains.kotlin.metadata.jvm.deserialization.JvmProtoBufUtil import org.jetbrains.kotlin.psi.* import org.jetbrains.kotlin.resolve.BindingContext import org.jetbrains.kotlin.resolve.BindingTraceContext @@ -31,28 +32,49 @@ import org.jetbrains.kotlin.resolve.LazyTopDownAnalyzer import org.jetbrains.kotlin.resolve.TopDownAnalysisMode.TopLevelDeclarations import org.jetbrains.kotlin.resolve.calls.components.InferenceSession import org.jetbrains.kotlin.resolve.calls.smartcasts.DataFlowInfo +import org.jetbrains.kotlin.resolve.extensions.ExtraImportsProviderExtension import org.jetbrains.kotlin.resolve.lazy.declarations.FileBasedDeclarationProviderFactory import org.jetbrains.kotlin.resolve.scopes.LexicalScope -import org.jetbrains.kotlin.scripting.configuration.ScriptingConfigurationKeys -import org.jetbrains.kotlin.scripting.compiler.plugin.definitions.CliScriptDefinitionProvider -import org.jetbrains.kotlin.scripting.compiler.plugin.definitions.CliScriptDependenciesProvider import org.jetbrains.kotlin.scripting.compiler.plugin.ScriptingCompilerConfigurationComponentRegistrar -import org.jetbrains.kotlin.scripting.definitions.KotlinScriptDefinition +import org.jetbrains.kotlin.scripting.compiler.plugin.definitions.CliScriptDefinitionProvider +import org.jetbrains.kotlin.scripting.configuration.ScriptingConfigurationKeys +import org.jetbrains.kotlin.scripting.definitions.ScriptCompilationConfigurationFromDefinition import org.jetbrains.kotlin.scripting.definitions.ScriptDefinitionProvider import org.jetbrains.kotlin.scripting.definitions.ScriptDependenciesProvider import org.jetbrains.kotlin.scripting.definitions.StandardScriptDefinition +import org.jetbrains.kotlin.scripting.definitions.ScriptDefinition +import org.jetbrains.kotlin.scripting.definitions.KotlinScriptDefinition // Legacy +import org.jetbrains.kotlin.scripting.definitions.findScriptDefinition +import org.jetbrains.kotlin.scripting.definitions.getEnvironment +import org.jetbrains.kotlin.scripting.extensions.ScriptExtraImportsProviderExtension +import org.jetbrains.kotlin.scripting.resolve.KotlinScriptDefinitionFromAnnotatedTemplate import org.jetbrains.kotlin.types.TypeUtils import org.jetbrains.kotlin.types.expressions.ExpressionTypingServices import org.jetbrains.kotlin.util.KotlinFrontEndException +import org.jetbrains.kotlin.utils.PathUtil import java.io.Closeable +import java.io.File import java.nio.file.Path import java.nio.file.Paths +import java.net.URLClassLoader import java.util.concurrent.locks.ReentrantLock import kotlin.concurrent.withLock +import kotlin.script.dependencies.Environment +import kotlin.script.dependencies.ScriptContents +import kotlin.script.experimental.dependencies.ScriptDependencies +import kotlin.script.experimental.api.ScriptCompilationConfiguration +import kotlin.script.experimental.dependencies.DependenciesResolver +import kotlin.script.experimental.dependencies.DependenciesResolver.ResolveResult +import kotlin.script.experimental.host.ScriptingHostConfiguration +import kotlin.script.experimental.host.configurationDependencies +import kotlin.script.experimental.jvm.defaultJvmScriptingHostConfiguration +import kotlin.script.experimental.jvm.JvmDependency import org.javacs.kt.util.KotlinLSException import org.javacs.kt.util.KotlinNullableNotNullManager import org.javacs.kt.util.LoggingMessageCollector +private val GRADLE_DSL_DEPENDENCY_PATTERN = Regex("^gradle-(?:kotlin-dsl|core).*\\.jar$") + /** * Kotlin compiler APIs used to parse, analyze and compile * files and expressions. @@ -71,11 +93,53 @@ private class CompilationEnvironment( parentDisposable = disposable, // Not to be confused with the CompilerConfiguration in the language server Configuration configuration = KotlinCompilerConfiguration().apply { - put(CommonConfigurationKeys.MODULE_NAME, JvmAbi.DEFAULT_MODULE_NAME) + put(CommonConfigurationKeys.MODULE_NAME, JvmProtoBufUtil.DEFAULT_MODULE_NAME) put(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY, LoggingMessageCollector) add(ComponentRegistrar.PLUGIN_COMPONENT_REGISTRARS, ScriptingCompilerConfigurationComponentRegistrar()) - add(ScriptingConfigurationKeys.SCRIPT_DEFINITIONS, StandardScriptDefinition) addJvmClasspathRoots(classPath.map { it.toFile() }) + + // Setup script templates (e.g. used by Gradle's Kotlin DSL) + val scriptDefinitions: MutableList = mutableListOf(ScriptDefinition.getDefault(defaultJvmScriptingHostConfiguration)) + + if (classPath.any { GRADLE_DSL_DEPENDENCY_PATTERN.matches(it.fileName.toString()) }) { + LOG.info("Configuring Kotlin DSL script templates...") + + val scriptTemplates = listOf( + "org.gradle.kotlin.dsl.KotlinInitScript", + "org.gradle.kotlin.dsl.KotlinSettingsScript", + "org.gradle.kotlin.dsl.KotlinBuildScript" + ) + + try { + // Load template classes + val scriptClassLoader = URLClassLoader(classPath.map { it.toUri().toURL() }.toTypedArray()) + val fileClassPath = classPath.map { it.toFile() } + val scriptHostConfig = ScriptingHostConfiguration(defaultJvmScriptingHostConfiguration) { + configurationDependencies(JvmDependency(fileClassPath)) + } + // TODO: KotlinScriptDefinition will soon be deprecated, use + // ScriptDefinition.compilationConfiguration and its defaultImports instead + // of KotlinScriptDefinition.dependencyResolver + // TODO: Use ScriptDefinition.FromLegacyTemplate directly if possible + // scriptDefinitions = scriptTemplates.map { ScriptDefinition.FromLegacyTemplate(scriptHostConfig, scriptClassLoader.loadClass(it).kotlin) } + scriptDefinitions.addAll(scriptTemplates.map { ScriptDefinition.FromLegacy(scriptHostConfig, object : KotlinScriptDefinitionFromAnnotatedTemplate( + scriptClassLoader.loadClass(it).kotlin, + scriptHostConfig[ScriptingHostConfiguration.getEnvironment]?.invoke() + ) { + override val dependencyResolver: DependenciesResolver = object : DependenciesResolver { + override fun resolve(scriptContents: ScriptContents, environment: Environment) = ResolveResult.Success(ScriptDependencies( + imports = listOf("org.gradle.kotlin.dsl.*") + )) + } + }) }) + } catch (e: Exception) { + LOG.error("Error while loading script template classes") + LOG.printStackTrace(e) + } + } + + LOG.info("Adding script definitions ${scriptDefinitions.map { it.asLegacyOrNull()?.template?.simpleName }}") + addAll(ScriptingConfigurationKeys.SCRIPT_DEFINITIONS, scriptDefinitions) }, configFiles = EnvironmentConfigFiles.JVM_CONFIG_FILES ) @@ -86,7 +150,7 @@ private class CompilationEnvironment( } parser = KtPsiFactory(project) - scripts = ScriptDefinitionProvider.getInstance(project)!! + scripts = ScriptDefinitionProvider.getInstance(project)!! as CliScriptDefinitionProvider } fun updateConfiguration(config: CompilerConfiguration) { @@ -124,18 +188,28 @@ private class CompilationEnvironment( } } +/** + * Determines the compilation environment used + * by the compiler (and thus the class path). + */ +enum class CompilationKind { + /** Uses the default class path. */ + DEFAULT, + /** Uses the Kotlin DSL class path if available. */ + BUILD_SCRIPT +} + /** * Incrementally compiles files and expressions. * The basic strategy for compiling one file at-a-time is outlined in OneFilePerformance. */ -class Compiler(classPath: Set) : Closeable { +class Compiler(classPath: Set, buildScriptClassPath: Set = emptySet()) : Closeable { private var closed = false private val localFileSystem: VirtualFileSystem - private val compileEnvironment = CompilationEnvironment(classPath) - private val compileLock = ReentrantLock() // TODO: Lock at file-level - val psiFileFactory - get() = PsiFileFactory.getInstance(compileEnvironment.environment.project) + private val defaultCompileEnvironment = CompilationEnvironment(classPath) + private val buildScriptCompileEnvironment = buildScriptClassPath.takeIf { it.isNotEmpty() }?.let(::CompilationEnvironment) + private val compileLock = ReentrantLock() // TODO: Lock at file-level companion object { init { @@ -152,29 +226,30 @@ class Compiler(classPath: Set) : Closeable { * configuration (which is a class from this project). */ fun updateConfiguration(config: CompilerConfiguration) { - compileEnvironment.updateConfiguration(config) + defaultCompileEnvironment.updateConfiguration(config) + buildScriptCompileEnvironment?.updateConfiguration(config) } - fun createFile(content: String, file: Path = Paths.get("dummy.virtual.kt")): KtFile { + fun createFile(content: String, file: Path = Paths.get("dummy.virtual.kt"), kind: CompilationKind = CompilationKind.DEFAULT): KtFile { assert(!content.contains('\r')) - val new = psiFileFactory.createFileFromText(file.toString(), KotlinLanguage.INSTANCE, content, true, false) as KtFile + val new = psiFileFactoryFor(kind).createFileFromText(file.toString(), KotlinLanguage.INSTANCE, content, true, false) as KtFile assert(new.virtualFile != null) return new } - fun createExpression(content: String, file: Path = Paths.get("dummy.virtual.kt")): KtExpression { - val property = parseDeclaration("val x = $content", file) as KtProperty + fun createExpression(content: String, file: Path = Paths.get("dummy.virtual.kt"), kind: CompilationKind = CompilationKind.DEFAULT): KtExpression { + val property = parseDeclaration("val x = $content", file, kind) as KtProperty return property.initializer!! } - fun createDeclaration(content: String, file: Path = Paths.get("dummy.virtual.kt")): KtDeclaration = - parseDeclaration(content, file) + fun createDeclaration(content: String, file: Path = Paths.get("dummy.virtual.kt"), kind: CompilationKind = CompilationKind.DEFAULT): KtDeclaration = + parseDeclaration(content, file, kind) - private fun parseDeclaration(content: String, file: Path): KtDeclaration { - val parse = createFile(content, file) + private fun parseDeclaration(content: String, file: Path, kind: CompilationKind = CompilationKind.DEFAULT): KtDeclaration { + val parse = createFile(content, file, kind) val declarations = parse.declarations assert(declarations.size == 1) { "${declarations.size} declarations in $content" } @@ -191,12 +266,25 @@ class Compiler(classPath: Set) : Closeable { else return onlyDeclaration } - fun compileFile(file: KtFile, sourcePath: Collection): Pair = - compileFiles(listOf(file), sourcePath) + private fun compileEnvironmentFor(kind: CompilationKind): CompilationEnvironment = when (kind) { + CompilationKind.DEFAULT -> defaultCompileEnvironment + CompilationKind.BUILD_SCRIPT -> buildScriptCompileEnvironment ?: defaultCompileEnvironment + } + + fun psiFileFactoryFor(kind: CompilationKind): PsiFileFactory = + PsiFileFactory.getInstance(compileEnvironmentFor(kind).environment.project) + + fun compileFile(file: KtFile, sourcePath: Collection, kind: CompilationKind = CompilationKind.DEFAULT): Pair = + compileFiles(listOf(file), sourcePath, kind) + + fun compileFiles(files: Collection, sourcePath: Collection, kind: CompilationKind = CompilationKind.DEFAULT): Pair { + if (kind == CompilationKind.BUILD_SCRIPT) { + // Print the (legacy) script template used by the compiled Kotlin DSL build file + files.forEach { LOG.debug { "$it -> ScriptDefinition: ${it.findScriptDefinition()?.asLegacyOrNull()?.template?.simpleName}" } } + } - fun compileFiles(files: Collection, sourcePath: Collection): Pair { compileLock.withLock { - val (container, trace) = compileEnvironment.createContainer(sourcePath) + val (container, trace) = compileEnvironmentFor(kind).createContainer(sourcePath) val topDownAnalyzer = container.get() topDownAnalyzer.analyzeDeclarations(TopLevelDeclarations, files) @@ -204,11 +292,11 @@ class Compiler(classPath: Set) : Closeable { } } - fun compileExpression(expression: KtExpression, scopeWithImports: LexicalScope, sourcePath: Collection): Pair { + fun compileExpression(expression: KtExpression, scopeWithImports: LexicalScope, sourcePath: Collection, kind: CompilationKind = CompilationKind.DEFAULT): Pair { try { // Use same lock as 'compileFile' to avoid concurrency issues such as #42 compileLock.withLock { - val (container, trace) = compileEnvironment.createContainer(sourcePath) + val (container, trace) = compileEnvironmentFor(kind).createContainer(sourcePath) val incrementalCompiler = container.get() incrementalCompiler.getTypeInfo( scopeWithImports, @@ -227,7 +315,8 @@ class Compiler(classPath: Set) : Closeable { override fun close() { if (!closed) { - compileEnvironment.close() + defaultCompileEnvironment.close() + buildScriptCompileEnvironment?.close() closed = true } else { LOG.warn("Compiler is already closed!") diff --git a/server/src/main/kotlin/org/javacs/kt/CompilerClassPath.kt b/server/src/main/kotlin/org/javacs/kt/CompilerClassPath.kt index 40d6f712d..83274a42b 100644 --- a/server/src/main/kotlin/org/javacs/kt/CompilerClassPath.kt +++ b/server/src/main/kotlin/org/javacs/kt/CompilerClassPath.kt @@ -7,31 +7,51 @@ import java.nio.file.Path class CompilerClassPath(private val config: CompilerConfiguration) : Closeable { private val workspaceRoots = mutableSetOf() private val classPath = mutableSetOf() - var compiler = Compiler(classPath) + private val buildScriptClassPath = mutableSetOf() + var compiler = Compiler(classPath, buildScriptClassPath) private set init { compiler.updateConfiguration(config) } - private fun refresh() { - val newClassPath = defaultClassPathResolver(workspaceRoots).classpathOrEmpty + private fun refresh(updateBuildScriptClassPath: Boolean = true) { + // TODO: Fetch class path and build script class path concurrently (and asynchronously) + val resolver = defaultClassPathResolver(workspaceRoots) + var refreshCompiler = false + val newClassPath = resolver.classpathOrEmpty if (newClassPath != classPath) { - val added = newClassPath - classPath - val removed = classPath - newClassPath + syncClassPath(classPath, newClassPath) + refreshCompiler = true + } - logAdded(added) - logRemoved(removed) + if (updateBuildScriptClassPath) { + val newBuildScriptClassPath = resolver.buildScriptClasspathOrEmpty + if (newBuildScriptClassPath != buildScriptClassPath) { + syncClassPath(buildScriptClassPath, newBuildScriptClassPath) + refreshCompiler = true + } + } - classPath.removeAll(removed) - classPath.addAll(added) + if (refreshCompiler) { compiler.close() - compiler = Compiler(classPath) + compiler = Compiler(classPath, buildScriptClassPath) updateCompilerConfiguration() } } + private fun syncClassPath(dest: MutableSet, new: Set) { + val added = new - dest + val removed = dest - new + + logAdded(added) + logRemoved(removed) + + dest.removeAll(removed) + dest.addAll(added) + } + fun updateCompilerConfiguration() { compiler.updateConfiguration(config) } @@ -61,8 +81,9 @@ class CompilerClassPath(private val config: CompilerConfiguration) : Closeable { } fun changedOnDisk(file: Path) { - if (file.fileName.toString() == "pom.xml") - refresh() + val name = file.fileName.toString() + if (name == "pom.xml" || name == "build.gradle" || name == "build.gradle.kts") + refresh(updateBuildScriptClassPath = false) } override fun close() { diff --git a/server/src/main/kotlin/org/javacs/kt/SourcePath.kt b/server/src/main/kotlin/org/javacs/kt/SourcePath.kt index 5a1279e4a..97057961e 100644 --- a/server/src/main/kotlin/org/javacs/kt/SourcePath.kt +++ b/server/src/main/kotlin/org/javacs/kt/SourcePath.kt @@ -17,22 +17,27 @@ class SourcePath( private val files = mutableMapOf() private inner class SourceFile( - val uri: URI, - var content: String, - val path: Path? = uri.filePath, - var parsed: KtFile? = null, - var compiledFile: KtFile? = null, - var compiledContext: BindingContext? = null, - var compiledContainer: ComponentProvider? = null, - val isTemporary: Boolean = false // A temporary source file will not be returned by .all() + val uri: URI, + var content: String, + val path: Path? = uri.filePath, + var parsed: KtFile? = null, + var compiledFile: KtFile? = null, + var compiledContext: BindingContext? = null, + var compiledContainer: ComponentProvider? = null, + val isTemporary: Boolean = false // A temporary source file will not be returned by .all() ) { + val isScript: Boolean = uri.toString().endsWith(".kts") + val kind: CompilationKind = + if (path?.fileName?.toString()?.endsWith(".gradle.kts") ?: false) CompilationKind.BUILD_SCRIPT + else CompilationKind.DEFAULT + fun put(newContent: String) { content = newContent } fun parseIfChanged(): SourceFile { if (content != parsed?.text) { - parsed = cp.compiler.createFile(content, path ?: Paths.get("sourceFile.virtual.kt")) + parsed = cp.compiler.createFile(content, path ?: Paths.get("sourceFile.virtual.kt"), kind) } return this @@ -54,7 +59,7 @@ class SourcePath( if (parsed?.text != compiledFile?.text) { LOG.debug("Compiling {}", path?.fileName) - val (context, container) = cp.compiler.compileFile(parsed!!, allIncludingThis()) + val (context, container) = cp.compiler.compileFile(parsed!!, allIncludingThis(), kind) compiledContext = context compiledContainer = container compiledFile = parsed @@ -67,7 +72,7 @@ class SourcePath( parseIfChanged().compileIfNull().doPrepareCompiledFile() private fun doPrepareCompiledFile(): CompiledFile = - CompiledFile(content, compiledFile!!, compiledContext!!, compiledContainer!!, allIncludingThis(), cp) + CompiledFile(content, compiledFile!!, compiledContext!!, compiledContainer!!, allIncludingThis(), cp, isScript, kind) private fun allIncludingThis(): Collection = parseIfChanged().let { if (isTemporary) (all().asSequence() + sequenceOf(parsed!!)).toList() @@ -137,23 +142,31 @@ class SourcePath( fun compileFiles(all: Collection): BindingContext { // Figure out what has changed val sources = all.map { files[it]!! } - val changed = sources.filter { it.content != it.compiledFile?.text } + val allChanged = sources.filter { it.content != it.compiledFile?.text } + val (changedBuildScripts, changedSources) = allChanged.partition { it.kind == CompilationKind.BUILD_SCRIPT } // Compile changed files - val parse = changed.map { it.parseIfChanged().parsed!! } - val (context, container) = cp.compiler.compileFiles(parse, all()) - - // Update cache - for (f in changed) { - f.compiledFile = f.parsed - f.compiledContext = context - f.compiledContainer = container + fun compileAndUpdate(changed: List, kind: CompilationKind): BindingContext? { + if (changed.isEmpty()) return null + val parse = changed.map { it.parseIfChanged().parsed!! } + val (context, container) = cp.compiler.compileFiles(parse, all(), kind) + + // Update cache + for (f in changed) { + f.compiledFile = f.parsed + f.compiledContext = context + f.compiledContainer = container + } + + return context } + val buildScriptsContext = compileAndUpdate(changedBuildScripts, CompilationKind.BUILD_SCRIPT) + val sourcesContext = compileAndUpdate(changedSources, CompilationKind.DEFAULT) + // Combine with past compilations - val combined = mutableListOf(context) - val same = sources - changed - combined.addAll(same.map { it.compiledContext!! }) + val same = sources - allChanged + val combined = listOf(buildScriptsContext, sourcesContext).filterNotNull() + same.map { it.compiledContext!! } return CompositeBindingContext.create(combined) } diff --git a/server/src/main/kotlin/org/javacs/kt/j2k/JavaToKotlinConverter.kt b/server/src/main/kotlin/org/javacs/kt/j2k/JavaToKotlinConverter.kt index c5bba1276..400c47ac0 100644 --- a/server/src/main/kotlin/org/javacs/kt/j2k/JavaToKotlinConverter.kt +++ b/server/src/main/kotlin/org/javacs/kt/j2k/JavaToKotlinConverter.kt @@ -6,10 +6,11 @@ import org.jetbrains.kotlin.com.intellij.openapi.project.Project // import org.jetbrains.kotlin.j2k.JavaToKotlinTranslator import org.javacs.kt.LOG import org.javacs.kt.Compiler +import org.javacs.kt.CompilationKind import org.javacs.kt.util.nonNull fun convertJavaToKotlin(javaCode: String, compiler: Compiler): String { - val psiFactory = compiler.psiFileFactory + val psiFactory = compiler.psiFileFactoryFor(CompilationKind.DEFAULT) val javaAST = psiFactory.createFileFromText("snippet.java", JavaLanguage.INSTANCE, javaCode) LOG.info("Parsed {} to {}", javaCode, javaAST) diff --git a/server/src/main/resources/kotlinDSLClassPathFinder.gradle b/server/src/main/resources/kotlinDSLClassPathFinder.gradle new file mode 100644 index 000000000..f1df027f6 --- /dev/null +++ b/server/src/main/resources/kotlinDSLClassPathFinder.gradle @@ -0,0 +1,19 @@ +import org.gradle.kotlin.dsl.accessors.AccessorsClassPathKt +import org.gradle.internal.classpath.ClassPath + +allprojects { project -> + task kotlinLSPKotlinDSLDeps { + // def pattern = ~/^(?:gradle-(?:kotlin-dsl|base-services|plugins|core)|kotlin-(?:compiler|stdlib)).*\.jar/ + def pattern = ~/^.*\.jar/ + doLast { + (fileTree("$gradle.gradleHomeDir/lib") + fileTree("$gradle.gradleUserHomeDir/caches/$gradle.gradleVersion/generated-gradle-jars")) + .findAll { it.name =~ pattern } + .forEach { System.out.println "kotlin-lsp-gradle $it" } + + // List dynamically generated Kotlin DSL accessors (e.g. the 'compile' configuration method) + def accessors = AccessorsClassPathKt.projectAccessorsClassPath(project, ClassPath.EMPTY) + accessors.bin.asFiles + .forEach { System.out.println "kotlin-lsp-gradle $it" } + } + } +} diff --git a/server/src/main/resources/classpathFinder.gradle b/server/src/main/resources/projectClassPathFinder.gradle similarity index 87% rename from server/src/main/resources/classpathFinder.gradle rename to server/src/main/resources/projectClassPathFinder.gradle index 1457c3420..6e4d9a0bc 100644 --- a/server/src/main/resources/classpathFinder.gradle +++ b/server/src/main/resources/projectClassPathFinder.gradle @@ -1,6 +1,6 @@ allprojects { project -> - task kotlinLSPDeps { - task -> doLast { + task kotlinLSPProjectDeps { task -> + doLast { System.out.println "" System.out.println "gradle-version $gradleVersion" System.out.println "kotlin-lsp-project ${project.name}" @@ -41,7 +41,7 @@ allprojects { project -> }.each { it.resolve().each { def inspected = it.inspect() - + if (inspected.endsWith("jar")) { if (!inspected.contains("zip!")) { System.out.println "kotlin-lsp-gradle $it" @@ -61,4 +61,12 @@ allprojects { project -> } } } + + task kotlinLSPAllGradleDeps { + doLast { + fileTree("$gradle.gradleHomeDir/lib") + .findAll { it.toString().endsWith '.jar' } + .forEach { System.out.println "kotlin-lsp-gradle $it" } + } + } } diff --git a/server/src/test/kotlin/org/javacs/kt/GradleDSLScriptTest.kt b/server/src/test/kotlin/org/javacs/kt/GradleDSLScriptTest.kt new file mode 100644 index 000000000..dd2b43acc --- /dev/null +++ b/server/src/test/kotlin/org/javacs/kt/GradleDSLScriptTest.kt @@ -0,0 +1,22 @@ +package org.javacs.kt + +import org.junit.Test +import org.junit.Assert.assertThat +import org.hamcrest.Matchers.* +import org.eclipse.lsp4j.MarkedString + +class GradleDSLScriptTest : SingleFileTestFixture("kotlinDSLWorkspace", "build.gradle.kts") { + @Test fun `edit repositories`() { + val completions = languageServer.textDocumentService.completion(completionParams(file, 7, 13)).get().right!! + val labels = completions.items.map { it.label } + + assertThat(labels, hasItem("repositories")) + } + + @Test fun `hover plugin`() { + val hover = languageServer.textDocumentService.hover(textDocumentPosition(file, 4, 8)).get()!! + val contents = hover.contents.left.first().right + + assertThat(contents, equalTo(MarkedString("kotlin", "fun PluginDependenciesSpec.kotlin(module: String): PluginDependencySpec"))) + } +} diff --git a/server/src/test/kotlin/org/javacs/kt/OneFilePerformance.kt b/server/src/test/kotlin/org/javacs/kt/OneFilePerformance.kt index 265c28746..e3351a9eb 100644 --- a/server/src/test/kotlin/org/javacs/kt/OneFilePerformance.kt +++ b/server/src/test/kotlin/org/javacs/kt/OneFilePerformance.kt @@ -17,7 +17,7 @@ import org.jetbrains.kotlin.config.CompilerConfiguration import org.jetbrains.kotlin.container.ComponentProvider import org.jetbrains.kotlin.container.ValueDescriptor import org.jetbrains.kotlin.descriptors.CallableDescriptor -import org.jetbrains.kotlin.load.java.JvmAbi +import org.jetbrains.kotlin.metadata.jvm.deserialization.JvmProtoBufUtil import org.jetbrains.kotlin.psi.KtElement import org.jetbrains.kotlin.psi.KtFile import org.jetbrains.kotlin.psi.KtPsiFactory @@ -48,7 +48,7 @@ class OneFilePerformance { class ReusableParts : Closeable { internal var config = CompilerConfiguration() init { - config.put(CommonConfigurationKeys.MODULE_NAME, JvmAbi.DEFAULT_MODULE_NAME) + config.put(CommonConfigurationKeys.MODULE_NAME, JvmProtoBufUtil.DEFAULT_MODULE_NAME) config.put(CLIConfigurationKeys.MESSAGE_COLLECTOR_KEY, LoggingMessageCollector) } internal val disposable = Disposer.newDisposable() diff --git a/server/src/test/kotlin/org/javacs/kt/SimpleScriptTest.kt b/server/src/test/kotlin/org/javacs/kt/SimpleScriptTest.kt index 5206f5de2..d13e70e52 100644 --- a/server/src/test/kotlin/org/javacs/kt/SimpleScriptTest.kt +++ b/server/src/test/kotlin/org/javacs/kt/SimpleScriptTest.kt @@ -32,6 +32,8 @@ private class SnippetRunner { } class SimpleScriptTest { + // TODO: Test a script using the language server instead + // of just experimenting with the API @Test fun basicScript() { val runner = SnippetRunner() diff --git a/server/src/test/resources/kotlinDSLWorkspace/.gitignore b/server/src/test/resources/kotlinDSLWorkspace/.gitignore new file mode 100644 index 000000000..1b915fd3b --- /dev/null +++ b/server/src/test/resources/kotlinDSLWorkspace/.gitignore @@ -0,0 +1,2 @@ +# Auto-generated by :server:copyPropertiesToDSLTestWorkspace +gradle.properties diff --git a/server/src/test/resources/kotlinDSLWorkspace/build.gradle.kts b/server/src/test/resources/kotlinDSLWorkspace/build.gradle.kts new file mode 100644 index 000000000..2f3017905 --- /dev/null +++ b/server/src/test/resources/kotlinDSLWorkspace/build.gradle.kts @@ -0,0 +1,9 @@ +plugins { + // TODO: Currently not possible, see https://github.com/gradle/gradle/issues/9830 + // kotlin("jvm") version "$kotlinVersion" + kotlin("jvm") version "1.3.50" +} + +repositories { + jcenter() +} diff --git a/server/src/test/resources/kotlinDSLWorkspace/settings.gradle.kts b/server/src/test/resources/kotlinDSLWorkspace/settings.gradle.kts new file mode 100644 index 000000000..1dbf00c9b --- /dev/null +++ b/server/src/test/resources/kotlinDSLWorkspace/settings.gradle.kts @@ -0,0 +1 @@ +rootProject.name = "Kotlin DSL Workspace" diff --git a/shared/src/main/kotlin/org/javacs/kt/classpath/ClassPathResolver.kt b/shared/src/main/kotlin/org/javacs/kt/classpath/ClassPathResolver.kt index f6c4f4409..2353f30c0 100644 --- a/shared/src/main/kotlin/org/javacs/kt/classpath/ClassPathResolver.kt +++ b/shared/src/main/kotlin/org/javacs/kt/classpath/ClassPathResolver.kt @@ -6,6 +6,7 @@ import java.nio.file.Path /** A source for creating class paths */ interface ClassPathResolver { val resolverType: String + val classpath: Set // may throw exceptions val classpathOrEmpty: Set // does not throw exceptions get() = try { @@ -15,6 +16,16 @@ interface ClassPathResolver { emptySet() } + val buildScriptClasspath: Set + get() = emptySet() + val buildScriptClasspathOrEmpty: Set + get() = try { + buildScriptClasspath + } catch (e: Exception) { + LOG.warn("Could not resolve buildscript classpath using {}: {}", resolverType, e.message) + emptySet() + } + companion object { /** A default empty classpath implementation */ val empty = object : ClassPathResolver { @@ -39,10 +50,14 @@ internal class UnionClassPathResolver(val lhs: ClassPathResolver, val rhs: Class override val resolverType: String get() = "(${lhs.resolverType} + ${rhs.resolverType})" override val classpath get() = lhs.classpath + rhs.classpath override val classpathOrEmpty get() = lhs.classpathOrEmpty + rhs.classpathOrEmpty + override val buildScriptClasspath get() = lhs.buildScriptClasspath + rhs.buildScriptClasspath + override val buildScriptClasspathOrEmpty get() = lhs.buildScriptClasspathOrEmpty + rhs.buildScriptClasspathOrEmpty } internal class FirstNonEmptyClassPathResolver(val lhs: ClassPathResolver, val rhs: ClassPathResolver) : ClassPathResolver { override val resolverType: String get() = "(${lhs.resolverType} or ${rhs.resolverType})" override val classpath get() = lhs.classpath.takeIf { it.isNotEmpty() } ?: rhs.classpath override val classpathOrEmpty get() = lhs.classpathOrEmpty.takeIf { it.isNotEmpty() } ?: rhs.classpathOrEmpty + override val buildScriptClasspath get() = lhs.buildScriptClasspath.takeIf { it.isNotEmpty() } ?: rhs.buildScriptClasspath + override val buildScriptClasspathOrEmpty get() = lhs.buildScriptClasspathOrEmpty.takeIf { it.isNotEmpty() } ?: rhs.buildScriptClasspathOrEmpty } diff --git a/shared/src/main/kotlin/org/javacs/kt/classpath/GradleClassPathResolver.kt b/shared/src/main/kotlin/org/javacs/kt/classpath/GradleClassPathResolver.kt index d176f4a1a..62508f376 100644 --- a/shared/src/main/kotlin/org/javacs/kt/classpath/GradleClassPathResolver.kt +++ b/shared/src/main/kotlin/org/javacs/kt/classpath/GradleClassPathResolver.kt @@ -3,33 +3,46 @@ package org.javacs.kt.classpath import org.javacs.kt.LOG import org.javacs.kt.util.firstNonNull import org.javacs.kt.util.tryResolving -import org.javacs.kt.util.execAndReadStdout +import org.javacs.kt.util.execAndReadStdoutAndStderr import org.javacs.kt.util.KotlinLSException import org.javacs.kt.util.isOSWindows import org.javacs.kt.util.findCommandOnPath import java.io.File -import java.nio.file.FileSystems import java.nio.file.Files import java.nio.file.Path +import java.nio.file.Paths -internal class GradleClassPathResolver(private val path: Path) : ClassPathResolver { +internal class GradleClassPathResolver(private val path: Path, private val includeKotlinDSL: Boolean): ClassPathResolver { override val resolverType: String = "Gradle" + private val projectDirectory: Path get() = path.getParent() override val classpath: Set get() { - val projectDirectory = path.getParent() - return readDependenciesViaGradleCLI(projectDirectory) - .orEmpty() + val scripts = listOf("projectClassPathFinder.gradle") + val tasks = listOf("kotlinLSPProjectDeps") + + return readDependenciesViaGradleCLI(projectDirectory, scripts, tasks) .apply { if (isNotEmpty()) LOG.info("Successfully resolved dependencies for '${projectDirectory.fileName}' using Gradle") } } + override val buildScriptClasspath: Set get() { + return if (includeKotlinDSL) { + val scripts = listOf("kotlinDSLClassPathFinder.gradle") + val tasks = listOf("kotlinLSPKotlinDSLDeps") + + return readDependenciesViaGradleCLI(projectDirectory, scripts, tasks) + .apply { if (isNotEmpty()) LOG.info("Successfully resolved build script dependencies for '${projectDirectory.fileName}' using Gradle") } + } else { + emptySet() + } + } companion object { /** Create a Gradle resolver if a file is a pom. */ fun maybeCreate(file: Path): GradleClassPathResolver? = file.takeIf { file.endsWith("build.gradle") || file.endsWith("build.gradle.kts") } - ?.let { GradleClassPathResolver(it) } + ?.let { GradleClassPathResolver(it, includeKotlinDSL = file.toString().endsWith(".kts")) } } } -private fun createTemporaryGradleFile(deleteOnExit: Boolean = false): File { +private fun gradleScriptToTempFile(scriptName: String, deleteOnExit: Boolean = false): File { val config = File.createTempFile("classpath", ".gradle") if (deleteOnExit) { config.deleteOnExit() @@ -38,7 +51,7 @@ private fun createTemporaryGradleFile(deleteOnExit: Boolean = false): File { LOG.debug("Creating temporary gradle file {}", config.absolutePath) config.bufferedWriter().use { configWriter -> - ClassLoader.getSystemResourceAsStream("classpathFinder.gradle").bufferedReader().use { configReader -> + ClassLoader.getSystemResourceAsStream(scriptName).bufferedReader().use { configReader -> configReader.copyTo(configWriter) } } @@ -58,28 +71,37 @@ private fun getGradleCommand(workspace: Path): Path { } } -private fun readDependenciesViaGradleCLI(projectDirectory: Path): Set? { - LOG.info("Resolving dependencies for '{}' through Gradle's CLI...", projectDirectory.fileName) - val config = createTemporaryGradleFile(deleteOnExit = false) +private fun readDependenciesViaGradleCLI(projectDirectory: Path, gradleScripts: List, gradleTasks: List): Set { + LOG.info("Resolving dependencies for '{}' through Gradle's CLI using tasks {}...", projectDirectory.fileName, gradleTasks) + + val tmpScripts = gradleScripts.map { gradleScriptToTempFile(it, deleteOnExit = false).toPath().toAbsolutePath() } val gradle = getGradleCommand(projectDirectory) - val cmd = "$gradle -I ${config.absolutePath} kotlinLSPDeps --console=plain" - LOG.debug(" -- executing {}", cmd) - val dependencies = findGradleCLIDependencies(cmd, projectDirectory) - config.delete() + + val command = "$gradle ${tmpScripts.map { "-I $it" }.joinToString(" ")} ${gradleTasks.joinToString(" ")} --console=plain" + val dependencies = findGradleCLIDependencies(command, projectDirectory) + ?.also { LOG.debug("Classpath for task {}", it) } + .orEmpty() + + tmpScripts.forEach(Files::delete) return dependencies } private fun findGradleCLIDependencies(command: String, projectDirectory: Path): Set? { - val result = execAndReadStdout(command, projectDirectory) + val (result, errors) = execAndReadStdoutAndStderr(command, projectDirectory) LOG.debug(result) + if ("FAILURE: Build failed" in errors) { + LOG.warn("Gradle task failed: {}", errors.lines().firstOrNull()) + } return parseGradleCLIDependencies(result) } -private val artifactPattern by lazy { "kotlin-lsp-gradle (.+)(\r?\n)".toRegex() } +private val artifactPattern by lazy { "kotlin-lsp-gradle (.+)(?:\r?\n)".toRegex() } +private val gradleErrorWherePattern by lazy { "\\*\\s+Where:[\r\n]+(\\S\\.*)".toRegex() } private fun parseGradleCLIDependencies(output: String): Set? { + LOG.debug(output) val artifacts = artifactPattern.findAll(output) - .mapNotNull { FileSystems.getDefault().getPath(it.groups[1]?.value) } + .mapNotNull { Paths.get(it.groups[1]?.value) } .filterNotNull() return artifacts.toSet() } diff --git a/shared/src/main/kotlin/org/javacs/kt/classpath/WithStdlibResolver.kt b/shared/src/main/kotlin/org/javacs/kt/classpath/WithStdlibResolver.kt index 42eb74254..0065544bc 100644 --- a/shared/src/main/kotlin/org/javacs/kt/classpath/WithStdlibResolver.kt +++ b/shared/src/main/kotlin/org/javacs/kt/classpath/WithStdlibResolver.kt @@ -7,6 +7,8 @@ internal class WithStdlibResolver(private val wrapped: ClassPathResolver) : Clas override val resolverType: String get() = "Stdlib + ${wrapped.resolverType}" override val classpath: Set get() = wrapWithStdlib(wrapped.classpath) override val classpathOrEmpty: Set get() = wrapWithStdlib(wrapped.classpathOrEmpty) + override val buildScriptClasspath: Set get() = wrapWithStdlib(wrapped.buildScriptClasspath) + override val buildScriptClasspathOrEmpty: Set get() = wrapWithStdlib(wrapped.buildScriptClasspathOrEmpty) } private fun wrapWithStdlib(paths: Set): Set { diff --git a/shared/src/main/kotlin/org/javacs/kt/util/Utils.kt b/shared/src/main/kotlin/org/javacs/kt/util/Utils.kt index 0b7a893fe..45cc3de19 100644 --- a/shared/src/main/kotlin/org/javacs/kt/util/Utils.kt +++ b/shared/src/main/kotlin/org/javacs/kt/util/Utils.kt @@ -10,11 +10,16 @@ import java.util.concurrent.CompletableFuture fun execAndReadStdout(shellCommand: String, directory: Path): String { val process = Runtime.getRuntime().exec(shellCommand, null, directory.toFile()) val stdout = process.inputStream - var result = "" - stdout.bufferedReader().use { - result = it.readText() - } - return result + return stdout.bufferedReader().use { it.readText() } +} + +fun execAndReadStdoutAndStderr(shellCommand: String, directory: Path): Pair { + val process = Runtime.getRuntime().exec(shellCommand, null, directory.toFile()) + val stdout = process.inputStream + val stderr = process.errorStream + val output = stdout.bufferedReader().use { it.readText() } + val errors = stderr.bufferedReader().use { it.readText() } + return Pair(output, errors) } inline fun withCustomStdout(delegateOut: PrintStream, task: () -> Unit) {