Backport "Add expression compiler" to 3.3 LTS

.gitignore

@@ -61,7 +61,6 @@ testlogs/
 
 # Put local stuff here
 local/
-compiler/test/debug/Gen.jar
 
 compiler/before-pickling.txt
 compiler/after-pickling.txt

compiler/src/dotty/tools/debug/ExpressionCompiler.scala

@@ -0,0 +1,31 @@
+package dotty.tools.debug
+
+import dotty.tools.dotc.Compiler
+import dotty.tools.dotc.core.Contexts.Context
+import dotty.tools.dotc.core.Phases.Phase
+import dotty.tools.dotc.transform.ElimByName
+
+/**
+  * The expression compiler powers the debug console in Metals and the IJ Scala plugin,
+  * enabling evaluation of arbitrary Scala expressions at runtime (even macros).
+  * It produces class files that can be loaded by the running Scala program,
+  * to compute the evaluation output.
+  * 
+  * To do so, it extends the Compiler with 3 phases:
+  *  - InsertExpression: parses and inserts the expression in the original source tree
+  *  - ExtractExpression: extract the typed expression and places it in the new expression class
+  *  - ResolveReflectEval: resolves local variables or inacessible members using reflection calls
+  */
+class ExpressionCompiler(config: ExpressionCompilerConfig) extends Compiler:
+
+  override protected def frontendPhases: List[List[Phase]] =
+    val parser :: others = super.frontendPhases: @unchecked
+    parser :: List(InsertExpression(config)) :: others
+
+  override protected def transformPhases: List[List[Phase]] =
+    val store = ExpressionStore()
+    // the ExtractExpression phase should be after ElimByName and ExtensionMethods, and before LambdaLift
+    val transformPhases = super.transformPhases
+    val index = transformPhases.indexWhere(_.exists(_.phaseName == ElimByName.name))
+    val (before, after) = transformPhases.splitAt(index + 1)
+    (before :+ List(ExtractExpression(config, store))) ++ (after :+ List(ResolveReflectEval(config, store)))

compiler/src/dotty/tools/debug/ExpressionCompilerBridge.scala

@@ -0,0 +1,38 @@
+package dotty.tools.debug
+
+import java.nio.file.Path
+import java.util.function.Consumer
+import java.{util => ju}
+import scala.jdk.CollectionConverters.*
+import scala.util.control.NonFatal
+import dotty.tools.dotc.reporting.StoreReporter
+import dotty.tools.dotc.core.Contexts.Context
+import dotty.tools.dotc.Driver
+
+class ExpressionCompilerBridge:
+  def run(
+      outputDir: Path,
+      classPath: String,
+      options: Array[String],
+      sourceFile: Path,
+      config: ExpressionCompilerConfig
+  ): Boolean =
+    val args = Array(
+      "-d",
+      outputDir.toString,
+      "-classpath",
+      classPath,
+      "-Yskip:pureStats"
+      // Debugging: Print the tree after phases of the debugger
+      // "-Vprint:insert-expression,resolve-reflect-eval",
+    ) ++ options :+ sourceFile.toString
+    val driver = new Driver:
+      protected override def newCompiler(using Context): ExpressionCompiler = ExpressionCompiler(config)
+    val reporter = ExpressionReporter(error => config.errorReporter.accept(error))
+    try
+      driver.process(args, reporter)
+      !reporter.hasErrors
+    catch
+      case NonFatal(cause) =>
+        cause.printStackTrace()
+        throw cause

compiler/src/dotty/tools/debug/ExpressionCompilerConfig.scala

@@ -0,0 +1,65 @@
+package dotty.tools.debug
+
+import dotty.tools.dotc.ast.tpd.*
+import dotty.tools.dotc.core.Symbols.*
+import dotty.tools.dotc.core.Types.*
+import dotty.tools.dotc.core.Names.*
+import dotty.tools.dotc.core.Flags.*
+import dotty.tools.dotc.core.Contexts.*
+import dotty.tools.dotc.core.SymUtils
+
+import java.{util => ju}
+import ju.function.Consumer
+
+class ExpressionCompilerConfig private[debug] (
+  packageName: String,
+  outputClassName: String,
+  private[debug] val breakpointLine: Int,
+  private[debug] val expression: String,
+  private[debug] val localVariables: ju.Set[String],
+  private[debug] val errorReporter: Consumer[String],
+  private[debug] val testMode: Boolean
+):
+  def this() = this(
+    packageName = "",
+    outputClassName = "",
+    breakpointLine = -1,
+    expression = "",
+    localVariables = ju.Collections.emptySet,
+    errorReporter = _ => (),
+    testMode = false,
+  )
+
+  def withPackageName(packageName: String): ExpressionCompilerConfig = copy(packageName = packageName)
+  def withOutputClassName(outputClassName: String): ExpressionCompilerConfig = copy(outputClassName = outputClassName)
+  def withBreakpointLine(breakpointLine: Int): ExpressionCompilerConfig = copy(breakpointLine = breakpointLine)
+  def withExpression(expression: String): ExpressionCompilerConfig = copy(expression = expression)
+  def withLocalVariables(localVariables: ju.Set[String]): ExpressionCompilerConfig = copy(localVariables = localVariables)
+  def withErrorReporter(errorReporter: Consumer[String]): ExpressionCompilerConfig = copy(errorReporter = errorReporter)
+
+  private[debug] val expressionTermName: TermName = termName(outputClassName.toLowerCase.toString)
+  private[debug] val expressionClassName: TypeName = typeName(outputClassName)
+
+  private[debug] def expressionClass(using Context): ClassSymbol =
+    if packageName.isEmpty then requiredClass(outputClassName)
+    else requiredClass(s"$packageName.$outputClassName")
+
+  private[debug] def evaluateMethod(using Context): Symbol =
+    expressionClass.info.decl(termName("evaluate")).symbol
+
+  private def copy(
+    packageName: String = packageName,
+    outputClassName: String = outputClassName,
+    breakpointLine: Int = breakpointLine,
+    expression: String = expression,
+    localVariables: ju.Set[String] = localVariables,
+    errorReporter: Consumer[String] = errorReporter,
+  ) = new ExpressionCompilerConfig(
+    packageName,
+    outputClassName,
+    breakpointLine,
+    expression,
+    localVariables,
+    errorReporter,
+    testMode
+  )

compiler/src/dotty/tools/debug/ExpressionReporter.scala

@@ -0,0 +1,17 @@
+package dotty.tools.debug
+
+import dotty.tools.dotc.core.Contexts.*
+import dotty.tools.dotc.reporting.AbstractReporter
+import dotty.tools.dotc.reporting.Diagnostic
+
+private class ExpressionReporter(reportError: String => Unit) extends AbstractReporter:
+  override def doReport(dia: Diagnostic)(using Context): Unit =
+    // Debugging: println(messageAndPos(dia))
+    dia match
+      case error: Diagnostic.Error =>
+        val newPos = error.pos.source.positionInUltimateSource(error.pos)
+        val errorWithNewPos = new Diagnostic.Error(error.msg, newPos)
+        reportError(stripColor(messageAndPos(errorWithNewPos)))
+      case _ =>
+        // TODO report the warnings in the expression
+        ()

compiler/src/dotty/tools/debug/ExpressionStore.scala

@@ -0,0 +1,24 @@
+package dotty.tools.debug
+
+import dotty.tools.dotc.ast.tpd.*
+import dotty.tools.dotc.core.Symbols.*
+import dotty.tools.dotc.core.Types.*
+import dotty.tools.dotc.core.Names.*
+import dotty.tools.dotc.core.Flags.*
+import dotty.tools.dotc.core.Contexts.*
+import dotty.tools.dotc.core.SymUtils
+
+private class ExpressionStore:
+  var symbol: TermSymbol | Null = null
+  // To resolve captured variables, we store:
+  // - All classes in the chain of owners of the expression
+  // - The first local method enclosing the expression
+  var classOwners: Seq[ClassSymbol] = Seq.empty
+  var capturingMethod: Option[TermSymbol] = None
+
+  def store(exprSym: Symbol)(using Context): Unit =
+    symbol = exprSym.asTerm
+    classOwners = exprSym.ownersIterator.collect { case cls: ClassSymbol => cls }.toSeq
+    capturingMethod = exprSym.ownersIterator
+      .find(sym => (sym.isClass || sym.is(Method)) && sym.enclosure.is(Method)) // the first local class or method
+      .collect { case sym if sym.is(Method) => sym.asTerm } // if it is a method