Test -print-tasty output of Java sources

bishabosha · bishabosha · commit c81113c49190 · 2024-03-12T18:24:04.000+01:00
diff --git a/compiler/src/dotty/tools/dotc/config/ScalaSettings.scala b/compiler/src/dotty/tools/dotc/config/ScalaSettings.scala
@@ -388,6 +388,7 @@ private sealed trait YSettings:
   val YshowPrintErrors: Setting[Boolean] = BooleanSetting("-Yshow-print-errors", "Don't suppress exceptions thrown during tree printing.")
   val YprintTasty: Setting[Boolean] = BooleanSetting("-Yprint-tasty", "Prints the generated TASTY to stdout.")
   val YtestPickler: Setting[Boolean] = BooleanSetting("-Ytest-pickler", "Self-test for pickling functionality; should be used with -Ystop-after:pickler.")
+  val YtestPicklerCheck: Setting[Boolean] = BooleanSetting("-Ytest-pickler-check", "Self-test for pickling -print-tasty output; should be used with -Ytest-pickler.")
   val YcheckReentrant: Setting[Boolean] = BooleanSetting("-Ycheck-reentrant", "Check that compiled program does not contain vars that can be accessed from a global root.")
   val YdropComments: Setting[Boolean] = BooleanSetting("-Ydrop-docs", "Drop documentation when scanning source files.", aliases = List("-Ydrop-comments"))
   val YcookComments: Setting[Boolean] = BooleanSetting("-Ycook-docs", "Cook the documentation (type check `@usecase`, etc.)", aliases = List("-Ycook-comments"))
diff --git a/compiler/src/dotty/tools/dotc/core/tasty/TastyAnsiiPrinter.scala b/compiler/src/dotty/tools/dotc/core/tasty/TastyAnsiiPrinter.scala
@@ -2,7 +2,10 @@ package dotty.tools.dotc
 package core
 package tasty
 
-class TastyAnsiiPrinter(bytes: Array[Byte]) extends TastyPrinter(bytes) {
+class TastyAnsiiPrinter(bytes: Array[Byte], testPickler: Boolean) extends TastyPrinter(bytes, testPickler) {
+
+  def this(bytes: Array[Byte]) = this(bytes, testPickler = false)
+
   override protected def nameStr(str: String): String = Console.MAGENTA + str + Console.RESET
   override protected def treeStr(str: String): String = Console.YELLOW + str + Console.RESET
   override protected def lengthStr(str: String): String = Console.CYAN + str + Console.RESET
diff --git a/compiler/src/dotty/tools/dotc/core/tasty/TastyPrinter.scala b/compiler/src/dotty/tools/dotc/core/tasty/TastyPrinter.scala
@@ -20,9 +20,12 @@ import dotty.tools.tasty.TastyBuffer.Addr
 object TastyPrinter:
 
   def showContents(bytes: Array[Byte], noColor: Boolean): String =
+    showContents(bytes, noColor, testPickler = false)
+
+  def showContents(bytes: Array[Byte], noColor: Boolean, testPickler: Boolean = false): String =
     val printer =
-      if noColor then new TastyPrinter(bytes)
-      else new TastyAnsiiPrinter(bytes)
+      if noColor then new TastyPrinter(bytes, testPickler)
+      else new TastyAnsiiPrinter(bytes, testPickler)
     printer.showContents()
 
   def main(args: Array[String]): Unit = {
@@ -62,7 +65,9 @@ object TastyPrinter:
       println(line)
   }
 
-class TastyPrinter(bytes: Array[Byte]) {
+class TastyPrinter(bytes: Array[Byte], val testPickler: Boolean) {
+
+  def this(bytes: Array[Byte]) = this(bytes, testPickler = false)
 
   class TastyPrinterUnpickler extends TastyUnpickler(bytes) {
     var namesStart: Addr = uninitialized
@@ -84,9 +89,16 @@ class TastyPrinter(bytes: Array[Byte]) {
   private def printHeader(sb: StringBuilder): Unit =
     val header = unpickler.header
     sb.append("Header:\n")
-    sb.append(s"  version: ${header.majorVersion}.${header.minorVersion}.${header.experimentalVersion}\n")
-    sb.append("  tooling: ").append(header.toolingVersion).append("\n")
-    sb.append("     UUID: ").append(header.uuid).append("\n")
+    if testPickler then
+      // these fields are not stable when the TASTy/compiler versions change, so not useful for testing
+      sb.append("  version: <elided>\n")
+      sb.append("  tooling: <elided>\n")
+      sb.append("     UUID: <elided>\n")
+    else
+      sb.append(s"  version: ${header.majorVersion}.${header.minorVersion}.${header.experimentalVersion}\n")
+      sb.append("  tooling: ").append(header.toolingVersion).append("\n")
+      sb.append("     UUID: ").append(header.uuid).append("\n")
+    end if
     sb.append("\n")
 
   private def printNames(sb: StringBuilder): Unit =
@@ -230,6 +242,8 @@ class TastyPrinter(bytes: Array[Byte]) {
       import reader.*
       sb.append(s"\n\nAttributes (${reader.endAddr.index - reader.startAddr.index} bytes, starting from $base):\n")
       while !isAtEnd do
+        // TODO: Should we elide attributes under testPickler? (i.e.
+        //   if we add new attributes many check files will need to be updated)
         val tag = readByte()
         sb.append("  ").append(attributeTagToString(tag))
         if isBooleanAttrTag(tag) then ()
diff --git a/compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala b/compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala
@@ -1085,10 +1085,10 @@ class TreeUnpickler(reader: TastyReader,
                   def complete(denot: SymDenotation)(using Context) =
                     denot.info = tdef.symbol.asType.info.subst(tparamRefs, derivedTparamRefs)
                 }
-                newSymbol(sym, tdef.name, Flags.JavaDefined | Flags.Param, completer, coord = cls.coord)
+                newSymbol(sym, tdef.name, Flags.JavaDefined | Flags.Param, completer, coord = coordAt(start))
               lazy val derivedTparamRefs: List[Type] = derivedTparamSyms.map(_.typeRef)
               val vparamSym =
-                newSymbol(sym, nme.syntheticParamName(1), pflags, defn.UnitType, coord = cls.coord)
+                newSymbol(sym, nme.syntheticParamName(1), pflags, defn.UnitType, coord = coordAt(start))
               val vparamSymss: List[List[Symbol]] = List(vparamSym) :: Nil
               val paramSymss =
                 if derivedTparamSyms.nonEmpty then derivedTparamSyms :: vparamSymss else vparamSymss
diff --git a/compiler/src/dotty/tools/dotc/parsing/JavaParsers.scala b/compiler/src/dotty/tools/dotc/parsing/JavaParsers.scala
@@ -894,6 +894,7 @@ object JavaParsers {
             parents = superclass :: interfaces,
             stats = canonicalConstructor :: accessors ::: body,
             tparams = tparams,
+            // ctorSpan = getSpan(start, nameOffset),
             needsDummyConstr = true
           )
         ).withMods(mods.withFlags(Flags.JavaDefined | Flags.Final))
diff --git a/compiler/src/dotty/tools/dotc/printing/RefinedPrinter.scala b/compiler/src/dotty/tools/dotc/printing/RefinedPrinter.scala
@@ -1010,7 +1010,10 @@ class RefinedPrinter(_ctx: Context) extends PlainPrinter(_ctx) {
         var modsText = modText(constr.mods, constr.symbol, "", isType = false)
         if (!modsText.isEmpty) modsText = " " ~ modsText
         if (constr.mods.hasAnnotations && !constr.mods.hasFlags) modsText = modsText ~~ " this"
-        withEnclosingDef(constr) { addParamssText(tparamsTxt ~~ modsText, constr.trailingParamss) }
+        val ctorParamss =
+          // for fake `(x$1: Unit): Foo` constructor, don't print the param (span is not reconstructed correctly)
+          if constr.symbol.isAllOf(JavaParsers.fakeFlags) then Nil else constr.trailingParamss
+        withEnclosingDef(constr) { addParamssText(tparamsTxt ~~ modsText, ctorParamss) }
       }
     val parentsText = Text(impl.parents.map(constrText), if (ofNew) keywordStr(" with ") else ", ")
     val derivedText = Text(impl.derived.map(toText(_)), ", ")
diff --git a/compiler/src/dotty/tools/dotc/transform/Pickler.scala b/compiler/src/dotty/tools/dotc/transform/Pickler.scala
@@ -69,6 +69,7 @@ class Pickler extends Phase {
 
   // Maps that keep a record if -Ytest-pickler is set.
   private val beforePickling = new mutable.HashMap[ClassSymbol, String]
+  private val beforePicklingPrinted = new mutable.HashMap[ClassSymbol, String]
   private val pickledBytes = new mutable.HashMap[ClassSymbol, (CompilationUnit, Array[Byte])]
 
   /** Drop any elements of this list that are linked module classes of other elements in the list */
@@ -184,7 +185,10 @@ class Pickler extends Phase {
         else
           val pickled = computePickled()
           reportPositionWarnings()
-          if ctx.settings.YtestPickler.value then pickledBytes(cls) = (unit, pickled)
+          if ctx.settings.YtestPickler.value then
+            pickledBytes(cls) = (unit, pickled)
+            if ctx.settings.YtestPicklerCheck.value then
+              beforePicklingPrinted(cls) = TastyPrinter.showContents(pickled, noColor = true, testPickler = true)
           () => pickled
 
       unit.pickled += (cls -> demandPickled)
@@ -251,15 +255,22 @@ class Pickler extends Phase {
   private def testUnpickler(using Context): Unit =
     pickling.println(i"testing unpickler at run ${ctx.runId}")
     ctx.initialize()
+    val resolveCheck = ctx.settings.YtestPicklerCheck.value
     val unpicklers =
       for ((cls, (unit, bytes)) <- pickledBytes) yield {
         val unpickler = new DottyUnpickler(unit.source.file, bytes)
         unpickler.enter(roots = Set.empty)
-        cls -> (unit, unpickler)
+        val optCheck =
+          if resolveCheck then
+            val resolved = unit.source.file.resolveSibling(s"${cls.name.mangledString}.tastycheck")
+            if resolved == null then None
+            else Some(resolved)
+          else None
+        cls -> (unit, unpickler, optCheck)
       }
     pickling.println("************* entered toplevel ***********")
     val rootCtx = ctx
-    for ((cls, (unit, unpickler)) <- unpicklers) do
+    for ((cls, (unit, unpickler, optCheck)) <- unpicklers) do
       val testJava = unit.typedAsJava
       if testJava then
         if unpickler.unpickler.nameAtRef.contents.exists(_ == nme.FromJavaObject) then
@@ -268,6 +279,15 @@ class Pickler extends Phase {
       val freshUnit = CompilationUnit(rootCtx.compilationUnit.source)
       freshUnit.needsCaptureChecking = unit.needsCaptureChecking
       freshUnit.knowsPureFuns = unit.knowsPureFuns
+      optCheck match
+        case Some(check) =>
+          import java.nio.charset.StandardCharsets.UTF_8
+          val checkContents = String(check.toByteArray, UTF_8)
+          inContext(rootCtx.fresh.setCompilationUnit(freshUnit)):
+            testSamePrinted(beforePicklingPrinted(cls), checkContents, cls, check)
+        case None =>
+          ()
+
       inContext(printerContext(testJava)(using rootCtx.fresh.setCompilationUnit(freshUnit))):
         testSame(i"$unpickled%\n%", beforePickling(cls), cls)
 
@@ -283,4 +303,15 @@ class Pickler extends Phase {
                     |
                     |  diff before-pickling.txt after-pickling.txt""")
   end testSame
+
+  private def testSamePrinted(printed: String, checkContents: String, cls: ClassSymbol, check: AbstractFile)(using Context) =
+    import java.nio.charset.StandardCharsets.UTF_8
+    def normal(s: String) = new String(s.getBytes(UTF_8), UTF_8)
+    val unequal = printed.length() != checkContents.length() || normal(printed) != normal(checkContents)
+    if unequal then
+      output("after-printing.txt", printed)
+      report.error(em"""TASTy printer difference for $cls in ${cls.source}, for details:
+                    |
+                    |  diff ${check.toString} after-printing.txt""")
+  end testSamePrinted
 }
diff --git a/tests/pos/i19806/J.tastycheck b/tests/pos/i19806/J.tastycheck
@@ -0,0 +1,161 @@
+Header:
+  version: <elided>
+  tooling: <elided>
+     UUID: <elided>
+
+Names (217 bytes, starting from 80):
+     0: ASTs
+     1: p
+     2: J
+     3: J[ModuleClass]
+     4: Object
+     5: java
+     6: lang
+     7: java[Qualified . lang]
+     8: _
+     9: <init>
+    10: Unit
+    11: scala
+    12: module2
+    13: Module
+    14: Module[ModuleClass]
+    15: module
+    16: innermodule2
+    17: InnerModule
+    18: InnerModule[ModuleClass]
+    19: innermodule
+    20: T
+    21: Nothing
+    22: Positions
+    23: tests/pos/i19806/J_SCALA_ONLY.java
+    24: Comments
+    25: Attributes
+
+
+Trees (145 bytes, starting from 300):
+     0: PACKAGE(142)
+     3:   TERMREFpkg 1 [p]
+     5:   VALDEF(11) 2 [J]
+     8:     IDENTtpt 3 [J[ModuleClass]]
+    10:       TYPEREFsymbol 18
+    12:         TERMREFpkg 1 [p]
+    14:     ELIDED
+    15:       SHAREDtype 10
+    17:     OBJECT
+    18:   TYPEDEF(86) 3 [J[ModuleClass]]
+    21:     TEMPLATE(82)
+    23:       TYPEREF 4 [Object]
+    25:         TERMREFpkg 7 [java[Qualified . lang]]
+    27:       SELFDEF 8 [_]
+    29:         SINGLETONtpt
+    30:           TERMREFsymbol 5
+    32:             SHAREDtype 12
+    34:       DEFDEF(7) 9 [<init>]
+    37:         EMPTYCLAUSE
+    38:         TYPEREF 10 [Unit]
+    40:           TERMREFpkg 11 [scala]
+    42:         STABLE
+    43:       DEFDEF(12) 12 [module2]
+    46:         EMPTYCLAUSE
+    47:         IDENTtpt 14 [Module[ModuleClass]]
+    49:           TYPEREF 14 [Module[ModuleClass]]
+    51:             SHAREDtype 12
+    53:         ELIDED
+    54:           SHAREDtype 49
+    56:         STATIC
+    57:       DEFDEF(12) 15 [module]
+    60:         EMPTYCLAUSE
+    61:         SELECTtpt 14 [Module[ModuleClass]]
+    63:           SHAREDtype 3
+    65:         ELIDED
+    66:           TYPEREF 14 [Module[ModuleClass]]
+    68:             SHAREDtype 3
+    70:         STATIC
+    71:       DEFDEF(14) 16 [innermodule2]
+    74:         EMPTYCLAUSE
+    75:         SELECTtpt 18 [InnerModule[ModuleClass]]
+    77:           TERMREF 13 [Module]
+    79:             SHAREDtype 12
+    81:         ELIDED
+    82:           TYPEREF 18 [InnerModule[ModuleClass]]
+    84:             SHAREDtype 77
+    86:         STATIC
+    87:       DEFDEF(16) 19 [innermodule]
+    90:         EMPTYCLAUSE
+    91:         SELECTtpt 18 [InnerModule[ModuleClass]]
+    93:           SELECT 13 [Module]
+    95:             SHAREDtype 3
+    97:         ELIDED
+    98:           TYPEREF 18 [InnerModule[ModuleClass]]
+   100:             TERMREF 13 [Module]
+   102:               SHAREDtype 3
+   104:         STATIC
+   105:     OBJECT
+   106:   TYPEDEF(37) 2 [J]
+   109:     TEMPLATE(34)
+   111:       TYPEPARAM(11) 20 [T]
+   114:         TYPEBOUNDS(6)
+   116:           TYPEREF 21 [Nothing]
+   118:             SHAREDtype 40
+   120:           SHAREDtype 23
+   122:         PRIVATE
+   123:         LOCAL
+   124:       SHAREDtype 120
+   126:       SPLITCLAUSE
+   127:       DEFDEF(16) 9 [<init>]
+   130:         TYPEPARAM(7) 20 [T]
+   133:           TYPEBOUNDStpt(4)
+   135:             SHAREDtype 116
+   137:             SHAREDtype 120
+   139:         EMPTYCLAUSE
+   140:         SHAREDtype 38
+   142:         ELIDED
+   143:           SHAREDtype 38
+   145:
+
+Positions (145 bytes, starting from 448):
+  lines: 23
+  line sizes:
+     10, 0, 19, 0, 15, 0, 35, 29, 3, 0, 36, 29, 3, 0, 52, 41, 3, 0, 53, 41
+     3, 1, 0
+  positions:
+     0: 0 .. 394
+     5: 12 .. 12
+     8: 12 .. 12
+    18: 12 .. 394
+    21: 52 .. 392
+    23: 25 .. 25
+    30: 52 .. 52
+    34: 52 .. 52
+    38: 52 .. 52
+    43: 52 .. 119
+    47: 66 .. 73
+    57: 123 .. 191
+    61: 137 .. 146
+    63: 137 .. 138
+    71: 195 .. 291
+    75: 209 .. 228
+    77: 209 .. 215
+    87: 295 .. 392
+    91: 309 .. 330
+    93: 309 .. 317
+    95: 309 .. 310
+   106: 12 .. 394
+   109: 27 .. 48
+   111: 27 .. 28
+   114: 27 .. 27
+   124: 28 .. 28
+   127: 35 .. 48
+   130: 27 .. 28
+   135: 27 .. 27
+   137: 27 .. 27
+   140: 46 .. 46
+
+  source paths:
+     0: 23 [tests/pos/i19806/J_SCALA_ONLY.java]
+
+
+Attributes (4 bytes, starting from 597):
+  JAVAattr
+  OUTLINEattr
+  SOURCEFILEattr 23 [tests/pos/i19806/J_SCALA_ONLY.java]
diff --git a/tests/pos/i19806/J_SCALA_ONLY.java b/tests/pos/i19806/J_SCALA_ONLY.java
@@ -0,0 +1,22 @@
+package p;
+
+public class J<T> {
+
+  public J() {}
+
+  public static Module$ module2() {
+    return p.Module$.MODULE$;
+  }
+
+  public static p.Module$ module() {
+    return p.Module$.MODULE$;
+  }
+
+  public static Module.InnerModule$ innermodule2() {
+    return p.Module.InnerModule$.MODULE$;
+  }
+
+  public static p.Module.InnerModule$ innermodule() {
+    return p.Module.InnerModule$.MODULE$;
+  }
+}
diff --git a/tests/pos/i19806/Module.scala b/tests/pos/i19806/Module.scala
@@ -0,0 +1,9 @@
+//> using options -Yjava-tasty -Ytest-pickler-check
+
+package p
+
+object Module:
+  object InnerModule
+
+class Outer:
+  object InnerModule
diff --git a/tests/run/i17255/Module.scala b/tests/run/i17255/Module.scala
@@ -1,3 +1,4 @@
+// scalajs: --skip
 package p {
   object Module {
     override def toString = "Module"

Original file line number	Diff line number	Diff line change
`@@ -894,6 +894,7 @@ object JavaParsers {`
`894`	`894`	`parents = superclass :: interfaces,`
`895`	`895`	`stats = canonicalConstructor :: accessors ::: body,`
`896`	`896`	`tparams = tparams,`
	`897`	`+ // ctorSpan = getSpan(start, nameOffset),`
`897`	`898`	`needsDummyConstr = true`
`898`	`899`	`)`
`899`	`900`	`).withMods(mods.withFlags(Flags.JavaDefined \| Flags.Final))`
Original file line number	Diff line number	Diff line change
`@@ -1,3 +1,4 @@`
	`1`	`+// scalajs: --skip`
`1`	`2`	`package p {`
`2`	`3`	`object Module {`
`3`	`4`	`override def toString = "Module"`