Merge pull request #5894 from dotty-staging/add-update-checkfiles

Add `updateCheckFiles` to override tests check files with output

Author: nicolasstucki

compiler/test/dotc/comptest.scala

@@ -11,6 +11,7 @@ object comptest extends ParallelTesting {
   def safeMode = false
   def isInteractive = true
   def testFilter = None
+  def updateCheckFiles: Boolean = false
 
   val posDir = "./tests/pos/"
   val negDir = "./tests/neg/"

compiler/test/dotty/Properties.scala

@@ -20,6 +20,9 @@ object Properties {
    */
   val testsFilter: Option[String] = sys.props.get("dotty.tests.filter")
 
+  /** Tests should override the checkfiles with the current output */
+  val testsUpdateCheckfile: Boolean = propIsNullOrTrue("dotty.tests.updateCheckfiles")
+
   /** When set, the run tests are only compiled - not run, a warning will be
    *  issued
    */

compiler/test/dotty/tools/dotc/BootstrappedOnlyCompilationTests.scala

@@ -7,13 +7,8 @@ import org.junit.Assert._
 import org.junit.Assume._
 import org.junit.experimental.categories.Category
 
-import java.nio.file._
-import java.util.stream.{ Stream => JStream }
-import scala.collection.JavaConverters._
-import scala.util.matching.Regex
 import scala.concurrent.duration._
 import vulpix._
-import dotty.tools.io.JFile
 
 @Category(Array(classOf[BootstrappedOnlyTests]))
 class BootstrappedOnlyCompilationTests extends ParallelTesting {
@@ -28,6 +23,7 @@ class BootstrappedOnlyCompilationTests extends ParallelTesting {
   def safeMode = Properties.testsSafeMode
   def isInteractive = SummaryReport.isInteractive
   def testFilter = Properties.testsFilter
+  def updateCheckFiles: Boolean = Properties.testsUpdateCheckfile
 
   // Positive tests ------------------------------------------------------------
 

compiler/test/dotty/tools/dotc/CompilationTests.scala

@@ -27,6 +27,7 @@ class CompilationTests extends ParallelTesting {
   def safeMode = Properties.testsSafeMode
   def isInteractive = SummaryReport.isInteractive
   def testFilter = Properties.testsFilter
+  def updateCheckFiles: Boolean = Properties.testsUpdateCheckfile
 
   // Positive tests ------------------------------------------------------------
 

compiler/test/dotty/tools/dotc/FromTastyTests.scala

@@ -20,7 +20,7 @@ class FromTastyTests extends ParallelTesting {
   def safeMode = Properties.testsSafeMode
   def isInteractive = SummaryReport.isInteractive
   def testFilter = Properties.testsFilter
-
+  def updateCheckFiles: Boolean = Properties.testsUpdateCheckfile
 
   @Test def posTestFromTasty: Unit = {
     // Can be reproduced with

compiler/test/dotty/tools/dotc/IdempotencyTests.scala

@@ -24,6 +24,7 @@ class IdempotencyTests extends ParallelTesting {
   def safeMode = Properties.testsSafeMode
   def isInteractive = SummaryReport.isInteractive
   def testFilter = Properties.testsFilter
+  def updateCheckFiles: Boolean = Properties.testsUpdateCheckfile
 
   @Category(Array(classOf[SlowTests]))
   @Test def idempotency: Unit = {

compiler/test/dotty/tools/vulpix/ParallelTesting.scala

@@ -46,6 +46,9 @@ trait ParallelTesting extends RunnerOrchestration { self =>
    */
   def testFilter: Option[String]
 
+  /** Tests should override the checkfiles with the current output */
+  def updateCheckFiles: Boolean
+
   /** A test source whose files or directory of files is to be compiled
    *  in a specific way defined by the `Test`
    */
@@ -505,6 +508,12 @@ trait ParallelTesting extends RunnerOrchestration { self =>
       this
     }
 
+    protected def updateCheckFile(checkFile: JFile, lines: Seq[String]): Unit = {
+      val outFile = dotty.tools.io.File(checkFile.toPath)
+      outFile.writeAll(lines.mkString("", EOL, EOL))
+      echo("Updated checkfile: " + checkFile.getPath)
+    }
+
     /** Returns all files in directory or the file if not a directory */
     private def flattenFiles(f: JFile): Array[JFile] =
       if (f.isDirectory) f.listFiles.flatMap(flattenFiles)
@@ -537,28 +546,32 @@ trait ParallelTesting extends RunnerOrchestration { self =>
                     val output = Source.fromFile(outDir.getParent + "_decompiled" + JFile.separator + outDir.getName
                       + JFile.separator + "decompiled.scala", "UTF-8").getLines().map {line =>
                       stripTrailingWhitespaces.unapplySeq(line).map(_.head).getOrElse(line)
-                    }.toList
+                    }.filter(!_.startsWith(ignoredFilePathLine)).toList
 
-                    val check: String = Source.fromFile(checkFile, "UTF-8").getLines().filter(!_.startsWith(ignoredFilePathLine))
+                    val check: String = Source.fromFile(checkFile, "UTF-8").getLines()
                       .mkString(EOL)
 
-                    if (output.filter(!_.startsWith(ignoredFilePathLine)).mkString(EOL) != check) {
+                    if (output.mkString(EOL) != check) {
                       val outFile = dotty.tools.io.File(checkFile.toPath).addExtension(".out")
-                      outFile.writeAll(output.mkString(EOL))
-                      val msg =
-                        s"""Output differed for test $name, use the following command to see the diff:
-                           |  > diff $checkFile $outFile
+                      if (updateCheckFiles) {
+                        updateCheckFile(checkFile, output)
+                      } else {
+                        outFile.writeAll(output.mkString("", EOL, ""))
+                        val msg =
+                          s"""Output differed for test $name, use the following command to see the diff:
+                             |  > diff $checkFile $outFile
                         """.stripMargin
 
-                      echo(msg)
-                      addFailureInstruction(msg)
+                        echo(msg)
+                        addFailureInstruction(msg)
 
-                      // Print build instructions to file and summary:
-                      val buildInstr = testSource.buildInstructions(0, rep.warningCount)
-                      addFailureInstruction(buildInstr)
+                        // Print build instructions to file and summary:
+                        val buildInstr = testSource.buildInstructions(0, rep.warningCount)
+                        addFailureInstruction(buildInstr)
 
-                      // Fail target:
-                      failTestSource(testSource)
+                        // Fail target:
+                        failTestSource(testSource)
+                      }
                     }
                   case _ =>
                 }
@@ -631,6 +644,9 @@ trait ParallelTesting extends RunnerOrchestration { self =>
 
             // Fail target:
             failTestSource(testSource)
+
+            if (updateCheckFiles)
+              updateCheckFile(checkFile.get, outputLines)
           }
         }
 
@@ -766,7 +782,11 @@ trait ParallelTesting extends RunnerOrchestration { self =>
         }
         def checkFileTest(sourceName: String, checkFile: JFile, actual: List[String]) = {
           val expexted = Source.fromFile(checkFile, "UTF-8").getLines().toList
-          diffMessage(sourceName, actual, expexted).foreach(fail)
+          for (msg <- diffMessage(sourceName, actual, expexted)) {
+            fail(msg)
+            if (updateCheckFiles)
+              updateCheckFile(checkFile, actual)
+          }
         }
 
         val (compilerCrashed, expectedErrors, actualErrors, hasMissingAnnotations, errorMap) = testSource match {
@@ -1371,7 +1391,7 @@ trait ParallelTesting extends RunnerOrchestration { self =>
             |Test '$title' compiled with $errors error(s) and $warnings warning(s),
             |the test can be reproduced by running:
             |
-            |  sbt "testFromTasty $file"
+            |  sbt "testCompilation --from-tasty $file"
             |
             |This tests can be disabled by adding `${file.getName}` to `compiler${JFile.separator}test${JFile.separator}dotc${JFile.separator}$runOrPos-$listName.blacklist`
             |

compiler/test/dotty/tools/vulpix/SummaryReport.scala

@@ -40,6 +40,7 @@ trait SummaryReporting {
 
   /** Echoes contents of `it` to file *immediately* then flushes */
   def echoToLog(it: Iterator[String]): Unit
+
 }
 
 /** A summary report that doesn't do anything */
@@ -53,6 +54,7 @@ final class NoSummaryReport extends SummaryReporting {
   def echoSummary(): Unit = ()
   def echoToLog(msg: String): Unit = ()
   def echoToLog(it: Iterator[String]): Unit = ()
+  def updateCheckFiles: Boolean = false
 }
 
 /** A summary report that logs to both stdout and the `TestReporter.logWriter`

compiler/test/dotty/tools/vulpix/VulpixMetaTests.scala

@@ -4,7 +4,6 @@ package vulpix
 import org.junit.Test
 import org.junit.experimental.categories.Category
 import scala.concurrent.duration._
-import dotty.Properties
 import TestConfiguration._
 
 /** Meta tests for the Vulpix test suite. This test follows the structure of
@@ -19,6 +18,7 @@ class VulpixMetaTests extends ParallelTesting {
   def safeMode = false // Don't fork a new VM after each run test
   def isInteractive = false // Don't beautify output for interactive use.
   def testFilter = None // Run all the tests.
+  def updateCheckFiles: Boolean = false
 
   implicit val summaryReport: SummaryReporting = new SummaryReport
   implicit def testGroup: TestGroup = TestGroup("VulpixMetaTests")

compiler/test/dotty/tools/vulpix/VulpixUnitTests.scala

@@ -21,6 +21,7 @@ class VulpixUnitTests extends ParallelTesting {
   def safeMode = sys.env.get("SAFEMODE").isDefined
   def isInteractive = !sys.env.contains("DRONE")
   def testFilter = None
+  def updateCheckFiles: Boolean = false
 
   // To fail with something else than an AssertionError
   def fail(): Unit = throw new Exception("didn't fail properly")

docs/docs/contributing/testing.md

@@ -80,9 +80,27 @@ This will run both the test `./tests/pos/companions.scala` and
 `./tests/neg/companions.scala` since both of these match the given string.
 This also means that you could run `testCompilation` with no arguments to run all integration tests.
 
+When complex checkfiles must be updated, `testCompilation` can run in a mode where it overrides the checkfiles with the test outputs.
+```bash
+$ sbt
+> testCompilation --update-checkfiles
+```
+
 ### Bootstrapped-only tests
 
 To run `testCompilation` on a bootstrapped Dotty compiler, use
 `dotty-compiler-bootstrapped/testCompilation` (with the same syntax as above).
 Some tests can only be run in bootstrapped compilers; that includes all tests
 with `with-compiler` in their name.
+
+### From TASTy tests
+
+`testCompilation` has a additional mode to run tests that compile code from a `.tasty` file, decompile a `.tasty` file and recompile the decompiled tasty.
+ Modify blacklist and whitelists in `compiler/test/dotc` to enable or disable tests from `.tasty` files.
+
+ ```bash
+ $ sbt
+ > testCompilation --from-tasty
+ ```
+ 
+ This mode can be combined with `--update-checkfiles` to update the `.decompiled` files or can be run under `dotty-compiler-bootstrapped/testCompilation` to test on a bootstrapped Dotty compiler. 

project/Build.scala

@@ -85,9 +85,6 @@ object Build {
   // Run tests with filter through vulpix test suite
   val testCompilation = inputKey[Unit]("runs integration test with the supplied filter")
 
-  // Run TASTY tests with filter through vulpix test suite
-  val testFromTasty = inputKey[Unit]("runs tasty integration test with the supplied filter")
-
   // Spawns a repl with the correct classpath
   val repl = inputKey[Unit]("run the REPL with correct classpath")
 
@@ -438,15 +435,6 @@ object Build {
     case Bootstrapped => `dotty-doc-bootstrapped`
   }
 
-  def testOnlyFiltered(test: String, options: String) = Def.inputTaskDyn {
-    val args = spaceDelimited("<arg>").parsed
-    val cmd = s" $test -- $options" + {
-      if (args.nonEmpty) " -Ddotty.tests.filter=" + args.mkString(" ")
-      else ""
-    }
-    (testOnly in Test).toTask(cmd)
-  }
-
   def findLib(attList: Seq[Attributed[File]], name: String) = attList
     .map(_.data.getAbsolutePath)
     .find(_.contains(name))
@@ -547,8 +535,17 @@ object Build {
         jarOpts ::: tuning ::: agentOptions ::: ci_build
       },
 
-      testCompilation := testOnlyFiltered("dotty.tools.dotc.*CompilationTests", "--exclude-categories=dotty.SlowTests").evaluated,
-      testFromTasty := testOnlyFiltered("dotty.tools.dotc.FromTastyTests", "").evaluated,
+      testCompilation := Def.inputTaskDyn {
+        val args = spaceDelimited("<arg>").parsed
+        val updateCheckfile = args.contains("--update-checkfiles")
+        val fromTasty = args.contains("--from-tasty")
+        val args1 = if (updateCheckfile | fromTasty) args.filter(x => x != "--update-checkfiles" && x != "--from-tasty") else args
+        val test = if (fromTasty) "dotty.tools.dotc.FromTastyTests" else "dotty.tools.dotc.*CompilationTests"
+        val cmd = s" $test -- --exclude-categories=dotty.SlowTests" +
+          (if (updateCheckfile) " -Ddotty.tests.updateCheckfiles=true" else "") +
+          (if (args1.nonEmpty) " -Ddotty.tests.filter=" + args1.mkString(" ") else "")
+        (testOnly in Test).toTask(cmd)
+      }.evaluated,
 
       dotr := {
         val args: List[String] = spaceDelimited("<arg>").parsed.toList

tests/pos/classWithCompObj.decompiled

@@ -1,3 +1,2 @@
-/** Decompiled from out/posTestFromTasty/pos/classWithCompObj/Foo.class */
 class Foo()
-object Foo
+object Foo

tests/pos/conforms.decompiled

@@ -1,4 +1,3 @@
-/** Decompiled from out/posTestFromTasty/pos/conforms/Test.class */
 object Test {
   def f[A, B](x: A)(implicit e: scala.Predef.<:<[A, B]): B = e.apply(x)
-}
+}

tests/pos/i0306.decompiled

@@ -1,4 +1,3 @@
-/** Decompiled from out/posTestFromTasty/pos/i0306/bar.class */
 object bar {
   class C[T <: scala.Seq[_ >: scala.Nothing <: scala.Any]]()
   val x: scala.AnyRef = new bar.C[scala.collection.Seq[_ >: scala.Nothing <: scala.Any]]()

tests/pos/i1181.decompiled

@@ -1,4 +1,3 @@
-/** Decompiled from out/posTestFromTasty/pos/i1181/Test.class */
 object Test {
   def foo[M[_$1]](x: M[scala.Int]): M[scala.Int] = x
   type Alias[A] = scala.Tuple2[A, A]

tests/pos/i1444.decompiled

@@ -1,4 +1,3 @@
-/** Decompiled from out/posTestFromTasty/pos/i1444/Test.class */
 object Test {
   class Cls(implicit x: Test.X)
   class ClsImpl() extends Test.Cls()(Test.AnX)
@@ -8,4 +7,4 @@ object Test {
   class Tr2Impl() extends Test.Tr2()(Test.AnX)
   trait X() extends java.lang.Object
   object AnX extends Test.X
-}
+}

tests/pos/i1570.decompiled

@@ -1,4 +1,3 @@
-/** Decompiled from out/posTestFromTasty/pos/i1570/Test.class */
 object Test {
   inline def foo(n: scala.Int): scala.Int = Test.bar(n)
   inline def bar(n: scala.Int): scala.Int = n

tests/pos/i2104b.decompiled

@@ -1,10 +1,8 @@
-/** Decompiled from out/posTestFromTasty/pos/i2104b/Cons.tasty */
 trait Cons[+H, +T]() extends java.lang.Object
 object Cons {
   def apply[H, T](h: H, t: T): Cons[H, T] = scala.Predef.???
   def unapply[H, T](t: Cons[H, T]): scala.Option[Pair[H, T]] = scala.Predef.???
 }
-/** Decompiled from out/posTestFromTasty/pos/i2104b/Pair.tasty */
 case class Pair[A, B](_1: A, _2: B) {
   override def hashCode(): scala.Int = {
     var acc: scala.Int = 2479866
@@ -32,10 +30,9 @@ case class Pair[A, B](_1: A, _2: B) {
   }
 }
 object Pair extends scala.AnyRef
-/** Decompiled from out/posTestFromTasty/pos/i2104b/Test.tasty */
 object Test {
   def main(args: scala.Array[scala.Predef.String]): scala.Unit = Cons.apply[scala.Option[scala.Int], scala.None.type](scala.Option.apply[scala.Int](1), scala.None) match {
     case Cons(scala.Some(i), scala.None) =>
       ()
   }
-}
+}

tests/pos/i4526-2.decompiled

@@ -1,2 +1 @@
-/** Decompiled from out/posTestFromTasty/pos/i4526-2/Foo.class */
 class Foo(x: scala.Int, y: scala.Int)

tests/pos/i4526a.decompiled

@@ -1,4 +1,3 @@
-/** Decompiled from out/posTestFromTasty/pos/i4526a/bar/Foo.class */
 package bar {
   class Foo() {
     protected[bar] def foo(): scala.Int = 0

tests/pos/i4526b.decompiled

@@ -1,4 +1,3 @@
-/** Decompiled from out/posTestFromTasty/pos/i4526b/Foo.class */
 class Foo() {
   def justdoit(f: scala.Either[scala.Int, scala.Predef.String]): scala.Predef.String = f match {
     case scala.Left(i) =>

tests/pos/i4678.decompiled

@@ -1,14 +1,8 @@
-/** Decompiled from out/posTestFromTasty/pos/i4678/Foo.tasty */
 class Foo() {
   val x: scala.Int = (1: @annot1 @annot2 @annot3 @annot4 @annot5)
 }
-/** Decompiled from out/posTestFromTasty/pos/i4678/annot1.tasty */
 class annot1() extends scala.annotation.Annotation
-/** Decompiled from out/posTestFromTasty/pos/i4678/annot2.tasty */
 class annot2() extends scala.annotation.Annotation
-/** Decompiled from out/posTestFromTasty/pos/i4678/annot3.tasty */
 class annot3() extends scala.annotation.Annotation
-/** Decompiled from out/posTestFromTasty/pos/i4678/annot4.tasty */
 class annot4() extends scala.annotation.Annotation
-/** Decompiled from out/posTestFromTasty/pos/i4678/annot5.tasty */
-class annot5() extends scala.annotation.Annotation
+class annot5() extends scala.annotation.Annotation

tests/pos/i566.decompiled

@@ -1,4 +1,3 @@
-/** Decompiled from out/posTestFromTasty/pos/i566/Test.class */
 class Test() {
   type T = scala.Predef.String
   type U

tests/pos/lambda.decompiled

@@ -1,4 +1,3 @@
-/** Decompiled from out/posTestFromTasty/pos/lambda/foo/Foo.class */
 package foo {
   class Foo() {
     ((x: scala.Int) => 2)

tests/pos/methodTypes.decompiled

@@ -1,4 +1,3 @@
-/** Decompiled from out/posTestFromTasty/pos/methodTypes/Foo.class */
 class Foo() {
   val x: scala.Int = 1
   def y: scala.Int = 2

tests/pos/selftypes.decompiled

@@ -1,4 +1,3 @@
-/** Decompiled from out/posTestFromTasty/pos/selftypes/selftypes.class */
 object selftypes {
   trait A() extends java.lang.Object { self: selftypes.AB =>
     type AA = scala.List[this.BX]