First step to pipelining support - enable reading Java symbols from TASTy (#19074)

This PR contains the minimal forward incompatible changes of pipelining
- reading Java and outline symbols from TASTy.
The other changes to implement pipelining have no impact on TASTy or the
standard library - so can come in a patch release.

To test reading TASTy produced from Java defined classes, we add two
private flags `-Yjava-tasty` and `-Yjava-tasty-output`, which are not
expected to be used by any build tool. The tests allow us to write just
the java signatures to TASTy, package them in a jar, and then read the
Java TASTy from the classpath.

- Keep Java compilation units up to Pickler phase if `-Yjava-tasty` is
set. Skip phases for Java when not needed.
- Add `JAVAattr` and `OUTLINEattr` TASTy attributes, `ELIDED` tree tag.
`ELIDED` trees are pickled as rhs of java term definitions. `ELIDED`
trees can only be unpickled if `OUTLINEattr` is present.
- Java units will set the `JAVAattr` TASTy attribute. As currently we
outline parse Java files we also set the `OUTLINEattr`.
- In the future we might expand `OUTLINEattr` to include outline Scala
typing.
- `OUTLINEattr` and `JAVAattr` do not need any special flags to _read_
from the classpath, however to read outline tasty in the `-from-tasty`
mode (so `tasty-inspector` and `scaladoc` are included) we do require an
explicit `-Yallow-outline-from-tasty`, as method bodies are required for
full functionality.
- write java tasty files to a special jar, set with
`-Yjava-tasty-output`
  this option is for testing purposes only.

Fix sealedDescendants method for Java Enums.
- Rename `JavaEnumTrait` flags to `JavaEnum` (reflecting the actual
flags set)
- test java enum in `SealedDescendantsTest`

fixes #15908

Author: bishabosha

compiler/src/dotty/tools/backend/jvm/BCodeHelpers.scala

@@ -292,7 +292,7 @@ trait BCodeHelpers extends BCodeIdiomatic {
           }
         case Ident(nme.WILDCARD) =>
           // An underscore argument indicates that we want to use the default value for this parameter, so do not emit anything
-        case t: tpd.RefTree if t.symbol.owner.linkedClass.isAllOf(JavaEnumTrait) =>
+        case t: tpd.RefTree if t.symbol.owner.linkedClass.isAllOf(JavaEnum) =>
           val edesc = innerClasesStore.typeDescriptor(t.tpe) // the class descriptor of the enumeration class.
           val evalue = t.symbol.javaSimpleName // value the actual enumeration value.
           av.visitEnum(name, edesc, evalue)

compiler/src/dotty/tools/backend/jvm/BTypesFromSymbols.scala

@@ -304,7 +304,7 @@ class BTypesFromSymbols[I <: DottyBackendInterface](val int: I, val frontendAcce
       .addFlagIf(sym.is(Bridge), ACC_BRIDGE | ACC_SYNTHETIC)
       .addFlagIf(sym.is(Artifact), ACC_SYNTHETIC)
       .addFlagIf(sym.isClass && !sym.isInterface, ACC_SUPER)
-      .addFlagIf(sym.isAllOf(JavaEnumTrait), ACC_ENUM)
+      .addFlagIf(sym.isAllOf(JavaEnum), ACC_ENUM)
       .addFlagIf(sym.is(JavaVarargs), ACC_VARARGS)
       .addFlagIf(sym.is(Synchronized), ACC_SYNCHRONIZED)
       .addFlagIf(sym.isDeprecated, ACC_DEPRECATED)

compiler/src/dotty/tools/dotc/CompilationUnit.scala

@@ -19,7 +19,7 @@ import scala.annotation.internal.sharable
 import scala.util.control.NoStackTrace
 import transform.MacroAnnotations
 
-class CompilationUnit protected (val source: SourceFile) {
+class CompilationUnit protected (val source: SourceFile, val info: CompilationUnitInfo | Null) {
 
   override def toString: String = source.toString
 
@@ -30,6 +30,13 @@ class CompilationUnit protected (val source: SourceFile) {
   /** Is this the compilation unit of a Java file */
   def isJava: Boolean = source.file.name.endsWith(".java")
 
+  /** Is this the compilation unit of a Java file, or TASTy derived from a Java file */
+  def typedAsJava = isJava || {
+    val infoNN = info
+    infoNN != null && infoNN.tastyInfo.exists(_.attributes.isJava)
+  }
+
+
   /** The source version for this unit, as determined by a language import */
   var sourceVersion: Option[SourceVersion] = None
 
@@ -106,7 +113,7 @@ class CompilationUnit protected (val source: SourceFile) {
     myAssignmentSpans.nn
 }
 
-@sharable object NoCompilationUnit extends CompilationUnit(NoSource) {
+@sharable object NoCompilationUnit extends CompilationUnit(NoSource, info = null) {
 
   override def isJava: Boolean = false
 
@@ -122,13 +129,14 @@ object CompilationUnit {
 
   /** Make a compilation unit for top class `clsd` with the contents of the `unpickled` tree */
   def apply(clsd: ClassDenotation, unpickled: Tree, forceTrees: Boolean)(using Context): CompilationUnit =
-    val file = clsd.symbol.associatedFile.nn
-    apply(SourceFile(file, Array.empty[Char]), unpickled, forceTrees)
+    val compilationUnitInfo = clsd.symbol.compilationUnitInfo.nn
+    val file = compilationUnitInfo.associatedFile
+    apply(SourceFile(file, Array.empty[Char]), unpickled, forceTrees, compilationUnitInfo)
 
   /** Make a compilation unit, given picked bytes and unpickled tree */
-  def apply(source: SourceFile, unpickled: Tree, forceTrees: Boolean)(using Context): CompilationUnit = {
+  def apply(source: SourceFile, unpickled: Tree, forceTrees: Boolean, info: CompilationUnitInfo)(using Context): CompilationUnit = {
     assert(!unpickled.isEmpty, unpickled)
-    val unit1 = new CompilationUnit(source)
+    val unit1 = new CompilationUnit(source, info)
     unit1.tpdTree = unpickled
     if (forceTrees) {
       val force = new Force
@@ -156,7 +164,8 @@ object CompilationUnit {
         NoSource
       }
       else source
-    new CompilationUnit(src)
+    val info = if src.exists then CompilationUnitInfo(src.file) else null
+    new CompilationUnit(src, info)
   }
 
   /** Force the tree to be loaded */

compiler/src/dotty/tools/dotc/ast/Desugar.scala

@@ -17,6 +17,7 @@ import reporting.*
 import annotation.constructorOnly
 import printing.Formatting.hl
 import config.Printers
+import parsing.Parsers
 
 import scala.annotation.internal.sharable
 import scala.annotation.threadUnsafe
@@ -143,8 +144,13 @@ object desugar {
 
   /** A value definition copied from `vdef` with a tpt typetree derived from it */
   def derivedTermParam(vdef: ValDef)(using Context): ValDef =
+    derivedTermParam(vdef, vdef.unforcedRhs)
+
+  def derivedTermParam(vdef: ValDef, rhs: LazyTree)(using Context): ValDef =
     cpy.ValDef(vdef)(
-      tpt = DerivedFromParamTree().withSpan(vdef.tpt.span).watching(vdef))
+      tpt = DerivedFromParamTree().withSpan(vdef.tpt.span).watching(vdef),
+      rhs = rhs
+    )
 
 // ----- Desugar methods -------------------------------------------------
 
@@ -544,8 +550,11 @@ object desugar {
       constrTparams.zipWithConserve(impliedTparams)((tparam, impliedParam) =>
         derivedTypeParam(tparam).withAnnotations(impliedParam.mods.annotations))
     val derivedVparamss =
-      constrVparamss.nestedMap(vparam =>
-        derivedTermParam(vparam).withAnnotations(Nil))
+      constrVparamss.nestedMap: vparam =>
+        val derived =
+          if ctx.compilationUnit.isJava then derivedTermParam(vparam, Parsers.unimplementedExpr)
+          else derivedTermParam(vparam)
+        derived.withAnnotations(Nil)
 
     val constr = cpy.DefDef(constr1)(paramss = joinParams(constrTparams, constrVparamss))
 

compiler/src/dotty/tools/dotc/config/ScalaSettings.scala

@@ -8,7 +8,7 @@ import dotty.tools.dotc.config.Settings.{Setting, SettingGroup}
 import dotty.tools.dotc.config.SourceVersion
 import dotty.tools.dotc.core.Contexts.*
 import dotty.tools.dotc.rewrites.Rewrites
-import dotty.tools.io.{AbstractFile, Directory, JDK9Reflectors, PlainDirectory}
+import dotty.tools.io.{AbstractFile, Directory, JDK9Reflectors, PlainDirectory, NoAbstractFile}
 import Setting.ChoiceWithHelp
 
 import scala.util.chaining.*
@@ -433,4 +433,9 @@ private sealed trait YSettings:
   val YforceInlineWhileTyping: Setting[Boolean] = BooleanSetting("-Yforce-inline-while-typing", "Make non-transparent inline methods inline when typing. Emulates the old inlining behavior of 3.0.0-M3.")
 
   val YdebugMacros: Setting[Boolean] = BooleanSetting("-Ydebug-macros", "Show debug info when quote pattern match fails")
+
+  // Pipeline compilation options
+  val YjavaTasty: Setting[Boolean] = BooleanSetting("-Yjava-tasty", "Pickler phase should compute pickles for .java defined symbols for use by build tools")
+  val YjavaTastyOutput: Setting[AbstractFile] = OutputSetting("-Yjava-tasty-output", "directory|jar", "(Internal use only!) destination for generated .tasty files containing Java type signatures.", NoAbstractFile)
+  val YallowOutlineFromTasty: Setting[Boolean] = BooleanSetting("-Yallow-outline-from-tasty", "Allow outline TASTy to be loaded with the -from-tasty option.")
 end YSettings

compiler/src/dotty/tools/dotc/core/Definitions.scala

@@ -996,6 +996,7 @@ class Definitions {
   @tu lazy val AnnotationClass: ClassSymbol = requiredClass("scala.annotation.Annotation")
   @tu lazy val StaticAnnotationClass: ClassSymbol = requiredClass("scala.annotation.StaticAnnotation")
   @tu lazy val RefiningAnnotationClass: ClassSymbol = requiredClass("scala.annotation.RefiningAnnotation")
+  @tu lazy val JavaAnnotationClass: ClassSymbol = requiredClass("java.lang.annotation.Annotation")
 
   // Annotation classes
   @tu lazy val AllowConversionsAnnot: ClassSymbol = requiredClass("scala.annotation.allowConversions")

compiler/src/dotty/tools/dotc/core/Flags.scala

@@ -576,7 +576,7 @@ object Flags {
   val InlineMethod: FlagSet                  = Inline | Method
   val InlineParam: FlagSet                   = Inline | Param
   val InlineByNameProxy: FlagSet             = InlineProxy | Method
-  val JavaEnumTrait: FlagSet                 = JavaDefined | Enum                             // A Java enum trait
+  val JavaEnum: FlagSet                      = JavaDefined | Enum                             // A Java enum trait
   val JavaEnumValue: FlagSet                 = JavaDefined | EnumValue                        // A Java enum value
   val StaticProtected: FlagSet               = JavaDefined | JavaStatic | Protected           // Java symbol which is `protected` and `static`
   val JavaModule: FlagSet                    = JavaDefined | Module                           // A Java companion object

compiler/src/dotty/tools/dotc/core/Phases.scala

@@ -333,16 +333,25 @@ object Phases {
     def subPhases: List[Run.SubPhase] = Nil
     final def traversals: Int = if subPhases.isEmpty then 1 else subPhases.length
 
+    /** skip the phase for a Java compilation unit, may depend on -Yjava-tasty */
+    def skipIfJava(using Context): Boolean = true
+
     /** @pre `isRunnable` returns true */
     def run(using Context): Unit
 
     /** @pre `isRunnable` returns true */
     def runOn(units: List[CompilationUnit])(using runCtx: Context): List[CompilationUnit] =
       val buf = List.newBuilder[CompilationUnit]
+      // factor out typedAsJava check when not needed
+      val doSkipJava = ctx.settings.YjavaTasty.value && this <= picklerPhase && skipIfJava
       for unit <- units do
         given unitCtx: Context = runCtx.fresh.setPhase(this.start).setCompilationUnit(unit).withRootImports
         if ctx.run.enterUnit(unit) then
-          try run
+          try
+            if doSkipJava && unit.typedAsJava then
+              ()
+            else
+              run
           catch case ex: Throwable if !ctx.run.enrichedErrorMessage =>
             println(ctx.run.enrichErrorMessage(s"unhandled exception while running $phaseName on $unit"))
             throw ex

compiler/src/dotty/tools/dotc/core/SymDenotations.scala

@@ -1695,7 +1695,7 @@ object SymDenotations {
             c.ensureCompleted()
       end completeChildrenIn
 
-      if is(Sealed) || isAllOf(JavaEnumTrait) then
+      if is(Sealed) || isAllOf(JavaEnum) && isClass then
         if !is(ChildrenQueried) then
           // Make sure all visible children are completed, so that
           // they show up in Child annotations. A possible child is visible if it

compiler/src/dotty/tools/dotc/core/tasty/Attributes.scala

@@ -11,6 +11,8 @@ class Attributes private[tasty](
   def explicitNulls: Boolean = booleanTags(EXPLICITNULLSattr)
   def captureChecked: Boolean = booleanTags(CAPTURECHECKEDattr)
   def withPureFuns: Boolean = booleanTags(WITHPUREFUNSattr)
+  def isJava: Boolean = booleanTags(JAVAattr)
+  def isOutline: Boolean = booleanTags(OUTLINEattr)
 }
 
 object Attributes:
@@ -19,12 +21,16 @@ object Attributes:
     explicitNulls: Boolean,
     captureChecked: Boolean,
     withPureFuns: Boolean,
+    isJava: Boolean,
+    isOutline: Boolean,
   ): Attributes =
     val booleanTags = BitSet.newBuilder
     if scala2StandardLibrary then booleanTags += SCALA2STANDARDLIBRARYattr
     if explicitNulls then booleanTags += EXPLICITNULLSattr
     if captureChecked then booleanTags += CAPTURECHECKEDattr
     if withPureFuns then booleanTags += WITHPUREFUNSattr
+    if isJava then booleanTags += JAVAattr
+    if isOutline then booleanTags += OUTLINEattr
     new Attributes(booleanTags.result())
   end apply
 

compiler/src/dotty/tools/dotc/core/tasty/DottyUnpickler.scala

@@ -36,6 +36,7 @@ object DottyUnpickler {
     def unpickle(reader: TastyReader, nameAtRef: NameTable): CommentUnpickler =
       new CommentUnpickler(reader)
   }
+
   class AttributesSectionUnpickler extends SectionUnpickler[AttributeUnpickler](AttributesSection) {
     def unpickle(reader: TastyReader, nameAtRef: NameTable): AttributeUnpickler =
       new AttributeUnpickler(reader)

compiler/src/dotty/tools/dotc/core/tasty/TreePickler.scala

@@ -23,7 +23,7 @@ import quoted.QuotePatterns
 object TreePickler:
   class StackSizeExceeded(val mdef: tpd.MemberDef) extends Exception
 
-class TreePickler(pickler: TastyPickler) {
+class TreePickler(pickler: TastyPickler, attributes: Attributes) {
   val buf: TreeBuffer = new TreeBuffer
   pickler.newSection(ASTsSection, buf)
   import buf.*
@@ -322,6 +322,11 @@ class TreePickler(pickler: TastyPickler) {
     if (!tree.isEmpty) pickleTree(tree)
   }
 
+  def pickleElidedUnlessEmpty(tree: Tree, tp: Type)(using Context): Unit =
+    if !tree.isEmpty then
+      writeByte(ELIDED)
+      pickleType(tp)
+
   def pickleDef(tag: Int, mdef: MemberDef, tpt: Tree, rhs: Tree = EmptyTree, pickleParams: => Unit = ())(using Context): Unit = {
     val sym = mdef.symbol
 
@@ -337,7 +342,12 @@ class TreePickler(pickler: TastyPickler) {
           case _: Template | _: Hole => pickleTree(tpt)
           case _ if tpt.isType => pickleTpt(tpt)
         }
-        pickleTreeUnlessEmpty(rhs)
+        if attributes.isOutline && sym.isTerm && attributes.isJava then
+          // TODO: if we introduce outline typing for Scala definitions
+          // then we will need to update the check here
+          pickleElidedUnlessEmpty(rhs, tpt.tpe)
+        else
+          pickleTreeUnlessEmpty(rhs)
         pickleModifiers(sym, mdef)
       }
     catch

compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala

@@ -104,6 +104,11 @@ class TreeUnpickler(reader: TastyReader,
   private val explicitNulls =
     attributeUnpicklerOpt.exists(_.attributes.explicitNulls)
 
+  private val unpicklingJava =
+    attributeUnpicklerOpt.exists(_.attributes.isJava)
+
+  private val isOutline = attributeUnpicklerOpt.exists(_.attributes.isOutline)
+
   private def registerSym(addr: Addr, sym: Symbol) =
     symAtAddr(addr) = sym
 
@@ -609,7 +614,10 @@ class TreeUnpickler(reader: TastyReader,
       val rhsIsEmpty = nothingButMods(end)
       if (!rhsIsEmpty) skipTree()
       val (givenFlags0, annotFns, privateWithin) = readModifiers(end)
-      val givenFlags = if isClass && unpicklingScala2Library then givenFlags0 | Scala2x | Scala2Tasty else givenFlags0
+      val givenFlags =
+        if isClass && unpicklingScala2Library then givenFlags0 | Scala2x | Scala2Tasty
+        else if unpicklingJava then givenFlags0 | JavaDefined
+        else givenFlags0
       pickling.println(i"creating symbol $name at $start with flags ${givenFlags.flagsString}, isAbsType = $isAbsType, $ttag")
       val flags = normalizeFlags(tag, givenFlags, name, isAbsType, rhsIsEmpty)
       def adjustIfModule(completer: LazyType) =
@@ -1037,6 +1045,8 @@ class TreeUnpickler(reader: TastyReader,
       val parentReader = fork
       val parents = readParents(withArgs = false)(using parentCtx)
       val parentTypes = parents.map(_.tpe.dealias)
+      if cls.is(JavaDefined) && parentTypes.exists(_.derivesFrom(defn.JavaAnnotationClass)) then
+        cls.setFlag(JavaAnnotation)
       val self =
         if (nextByte == SELFDEF) {
           readByte()
@@ -1197,7 +1207,12 @@ class TreeUnpickler(reader: TastyReader,
 
       def completeSelect(name: Name, sig: Signature, target: Name): Select =
         val qual = readTree()
-        val denot = accessibleDenot(qual.tpe.widenIfUnstable, name, sig, target)
+        val denot0 = accessibleDenot(qual.tpe.widenIfUnstable, name, sig, target)
+        val denot =
+          if unpicklingJava && name == tpnme.Object && denot0.symbol == defn.ObjectClass then
+            defn.FromJavaObjectType.denot
+          else
+            denot0
         makeSelect(qual, name, denot)
 
       def readQualId(): (untpd.Ident, TypeRef) =
@@ -1216,6 +1231,11 @@ class TreeUnpickler(reader: TastyReader,
           forkAt(readAddr()).readTree()
         case IDENT =>
           untpd.Ident(readName()).withType(readType())
+        case ELIDED =>
+          if !isOutline then
+            report.error(
+              s"Illegal elided tree in unpickler without ${attributeTagToString(OUTLINEattr)}, ${ctx.source}")
+          untpd.Ident(nme.WILDCARD).withType(readType())
         case IDENTtpt =>
           untpd.Ident(readName().toTypeName).withType(readType())
         case SELECT =>

compiler/src/dotty/tools/dotc/fromtasty/AlreadyLoadedCompilationUnit.scala

@@ -7,4 +7,4 @@ import dotty.tools.dotc.util.NoSource
  *  encountered, and attempted to inspect, something that has already been loaded, for example a Scala primitive or a
  *  library class like Option.
  */
-class AlreadyLoadedCompilationUnit(val className: String) extends CompilationUnit(NoSource)
+class AlreadyLoadedCompilationUnit(val className: String) extends CompilationUnit(NoSource, null)

compiler/src/dotty/tools/dotc/fromtasty/ReadTasty.scala

@@ -46,9 +46,16 @@ class ReadTasty extends Phase {
             case unpickler: tasty.DottyUnpickler =>
               if (cls.rootTree.isEmpty) None
               else {
-                val unit = CompilationUnit(cls, cls.rootTree, forceTrees = true)
-                unit.pickled += (cls -> (() => unpickler.unpickler.bytes))
-                Some(unit)
+                val attributes = unpickler.tastyAttributes
+                if attributes.isJava && !ctx.settings.YjavaTasty.value then
+                  // filter out Java compilation units if -Yjava-tasty is not set
+                  None
+                else if attributes.isOutline && !ctx.settings.YallowOutlineFromTasty.value then
+                  cannotUnpickle("it contains outline signatures and -Yallow-outline-from-tasty is not set.")
+                else
+                  val unit = CompilationUnit(cls, cls.rootTree, forceTrees = true)
+                  unit.pickled += (cls -> (() => unpickler.unpickler.bytes))
+                  Some(unit)
               }
             case tree: Tree[?] =>
               // TODO handle correctly this case correctly to get the tree or avoid it completely.

compiler/src/dotty/tools/dotc/fromtasty/TASTYCompilationUnit.scala

@@ -3,6 +3,6 @@ package dotty.tools.dotc.fromtasty
 import dotty.tools.dotc.CompilationUnit
 import dotty.tools.dotc.util.NoSource
 
-class TASTYCompilationUnit(val className: String) extends CompilationUnit(NoSource) {
+class TASTYCompilationUnit(val className: String) extends CompilationUnit(NoSource, null) {
   override def toString: String = s"class file $className"
 }

compiler/src/dotty/tools/dotc/parsing/JavaParsers.scala

@@ -136,7 +136,7 @@ object JavaParsers {
       ValDef(name, tpt, EmptyTree).withMods(Modifiers(Flags.JavaDefined | Flags.Param))
 
     def makeConstructor(formals: List[Tree], tparams: List[TypeDef], flags: FlagSet = Flags.JavaDefined): DefDef = {
-      val vparams = formals.zipWithIndex.map { case (p, i) => makeSyntheticParam(i + 1, p) }
+      val vparams = formals.zipWithIndex.map { case (p, i) => makeSyntheticParam(i + 1, p).withMods(Modifiers(flags)) }
       DefDef(nme.CONSTRUCTOR, joinParams(tparams, List(vparams)), TypeTree(), EmptyTree).withMods(Modifiers(flags))
     }
 
@@ -992,7 +992,7 @@ object JavaParsers {
         Select(New(javaLangDot(tpnme.Enum)), nme.CONSTRUCTOR), List(enumType)), Nil)
       val enumclazz = atSpan(start, nameOffset) {
         TypeDef(name,
-          makeTemplate(superclazz :: interfaces, body, List(), true)).withMods(mods | Flags.JavaEnumTrait)
+          makeTemplate(superclazz :: interfaces, body, List(), true)).withMods(mods | Flags.JavaEnum)
       }
       addCompanionObject(consts ::: statics ::: predefs, enumclazz)
     }
@@ -1011,7 +1011,7 @@ object JavaParsers {
           skipAhead()
           accept(RBRACE)
         }
-        ValDef(name.toTermName, enumType, unimplementedExpr).withMods(Modifiers(Flags.JavaEnumTrait | Flags.StableRealizable | Flags.JavaDefined | Flags.JavaStatic))
+        ValDef(name.toTermName, enumType, unimplementedExpr).withMods(Modifiers(Flags.JavaEnumValue | Flags.JavaStatic))
       }
     }
 

compiler/src/dotty/tools/dotc/parsing/Parsers.scala

@@ -97,6 +97,9 @@ object Parsers {
   private val InCond: Region => Region = Scanners.InParens(LPAREN, _)
   private val InFor : Region => Region = Scanners.InBraces(_)
 
+  def unimplementedExpr(using Context): Select =
+    Select(scalaDot(nme.Predef), nme.???)
+
   abstract class ParserCommon(val source: SourceFile)(using Context) {
 
     val in: ScannerCommon
@@ -164,9 +167,6 @@ object Parsers {
      */
     def syntaxError(msg: Message, span: Span): Unit =
       report.error(msg, source.atSpan(span))
-
-    def unimplementedExpr(using Context): Select =
-      Select(scalaDot(nme.Predef), nme.???)
   }
 
   trait OutlineParserCommon extends ParserCommon {