Skip to content

Commit 87bd50c

Browse files
bishaboshabishabosha
committed
wip: 2 pass parallel compile rebootstrap post -Yjava-tasty
1 parent 1c3cc11 commit 87bd50c

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

47 files changed

+745
-183
lines changed

‎build.sbt

+2
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
ThisBuild / usePipelining := true
2+
13
val scala3 = Build.scala3
24
val `scala3-bootstrapped` = Build.`scala3-bootstrapped`
35
val `scala3-interfaces` = Build.`scala3-interfaces`

‎compiler/src/dotty/tools/dotc/CompilationUnit.scala

+5
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@ import StdNames.nme
1818
import scala.annotation.internal.sharable
1919
import scala.util.control.NoStackTrace
2020
import transform.MacroAnnotations
21+
import dotty.tools.dotc.interfaces.AbstractFile
22+
import dotty.tools.io.NoAbstractFile
2123

2224
class CompilationUnit protected (val source: SourceFile, val info: CompilationUnitInfo | Null) {
2325

@@ -45,6 +47,7 @@ class CompilationUnit protected (val source: SourceFile, val info: CompilationUn
4547

4648
/** Pickled TASTY binaries, indexed by class. */
4749
var pickled: Map[ClassSymbol, () => Array[Byte]] = Map()
50+
var outlinePickled: Map[ClassSymbol, () => Array[Byte]] = Map()
4851

4952
/** The fresh name creator for the current unit.
5053
* FIXME(#7661): This is not fine-grained enough to enable reproducible builds,
@@ -106,6 +109,8 @@ class CompilationUnit protected (val source: SourceFile, val info: CompilationUn
106109
ctx.run.nn.suspendedUnits += this
107110
if ctx.phase == Phases.inliningPhase then
108111
suspendedAtInliningPhase = true
112+
else if ctx.settings.YearlyTastyOutput.value != NoAbstractFile then
113+
report.error(i"Compilation units may not be suspended before inlining with -Ypickle-write")
109114
throw CompilationUnit.SuspendException()
110115

111116
private var myAssignmentSpans: Map[Int, List[Span]] | Null = null

‎compiler/src/dotty/tools/dotc/Compiler.scala

+9-8
Original file line numberDiff line numberDiff line change
@@ -40,20 +40,21 @@ class Compiler {
4040
List(new sbt.ExtractDependencies) :: // Sends information on classes' dependencies to sbt via callbacks
4141
List(new semanticdb.ExtractSemanticDB.ExtractSemanticInfo) :: // Extract info into .semanticdb files
4242
List(new PostTyper) :: // Additional checks and cleanups after type checking
43+
// List(new sbt.ExtractAPI.Outline) :: // [runs in outline] Sends a representation of the API of classes to sbt via callbacks
4344
List(new sjs.PrepJSInterop) :: // Additional checks and transformations for Scala.js (Scala.js only)
4445
List(new SetRootTree) :: // Set the `rootTreeOrProvider` on class symbols
4546
Nil
4647

4748
/** Phases dealing with TASTY tree pickling and unpickling */
4849
protected def picklerPhases: List[List[Phase]] =
49-
List(new Pickler) :: // Generate TASTY info
50-
List(new sbt.ExtractAPI) :: // Sends a representation of the API of classes to sbt via callbacks
51-
List(new Inlining) :: // Inline and execute macros
52-
List(new PostInlining) :: // Add mirror support for inlined code
53-
List(new CheckUnused.PostInlining) :: // Check for unused elements
54-
List(new Staging) :: // Check staging levels and heal staged types
55-
List(new Splicing) :: // Replace level 1 splices with holes
56-
List(new PickleQuotes) :: // Turn quoted trees into explicit run-time data structures
50+
List(new Pickler) :: // Generate TASTY info
51+
List(new sbt.ExtractAPI) :: // [runs when not outline] Sends a representation of the API of classes to sbt via callbacks
52+
List(new Inlining) :: // Inline and execute macros
53+
List(new PostInlining) :: // Add mirror support for inlined code
54+
List(new CheckUnused.PostInlining) :: // Check for unused elements
55+
List(new Staging) :: // Check staging levels and heal staged types
56+
List(new Splicing) :: // Replace level 1 splices with holes
57+
List(new PickleQuotes) :: // Turn quoted trees into explicit run-time data structures
5758
Nil
5859

5960
/** Phases dealing with the transformation from pickled trees to backend trees */

‎compiler/src/dotty/tools/dotc/Driver.scala

+181-1
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,25 @@ import config.Feature
1313

1414
import scala.util.control.NonFatal
1515
import fromtasty.{TASTYCompiler, TastyFileUtil}
16+
import dotty.tools.io.NoAbstractFile
17+
import dotty.tools.io.{VirtualFile, VirtualDirectory}
18+
19+
import java.nio.file.Path as JPath
20+
import scala.concurrent.*
21+
import scala.annotation.internal.sharable
22+
import scala.concurrent.duration.*
23+
import scala.util.{Success, Failure}
24+
import scala.annotation.targetName
25+
import dotty.tools.dotc.classpath.FileUtils.hasScalaExtension
26+
import dotty.tools.dotc.core.Symbols
27+
28+
object Driver {
29+
@sharable lazy val executor =
30+
// fixed pool of 8 threads, because we have 8 cores
31+
val pool = java.util.concurrent.Executors.newFixedThreadPool(8).nn
32+
sys.addShutdownHook(pool.shutdown())
33+
ExecutionContext.fromExecutor(pool)
34+
}
1635

1736
/** Run the Dotty compiler.
1837
*
@@ -28,6 +47,167 @@ class Driver {
2847

2948
protected def emptyReporter: Reporter = new StoreReporter(null)
3049

50+
protected def doCompile(files: List[AbstractFile])(using ictx: Context): Reporter =
51+
val isOutline = ictx.settings.Youtline.value(using ictx)
52+
53+
if !isOutline then inContext(ictx):
54+
report.echo(s"basic compilation enabled on files ${files.headOption.map(f => s"$f...").getOrElse("[]")}")(using ictx)
55+
doCompile(newCompiler, files) // standard compilation
56+
else
57+
report.echo(s"Outline compilation enabled on files ${files.headOption.map(f => s"$f...").getOrElse("[]")}")(using ictx)
58+
val maxParallelism = ictx.settings.YmaxParallelism.valueIn(ictx.settingsState)
59+
val absParallelism = math.abs(maxParallelism)
60+
val isParallel = maxParallelism >= 0
61+
val parallelism =
62+
val ceiling = math.max(1, Runtime.getRuntime().nn.availableProcessors() - 1)
63+
if absParallelism > 0 then math.min(absParallelism, ceiling)
64+
else ceiling
65+
66+
// NOTE: sbt will delete this potentially as soon as you call `apiPhaseCompleted`
67+
val pickleWriteOutput = ictx.settings.YearlyTastyOutput.valueIn(ictx.settingsState)
68+
69+
val pickleWriteSource =
70+
pickleWriteOutput.underlyingSource match
71+
case Some(source) =>
72+
source.file.asInstanceOf[java.io.File | Null] match
73+
case f: java.io.File => Some(source)
74+
case null =>
75+
report.warning(s"Could not resolve file of ${source} (of class ${source.getClass.getName})")
76+
None
77+
case None =>
78+
if pickleWriteOutput.isInstanceOf[dotty.tools.io.JarArchive] then
79+
report.warning(s"Could not resolve underlying source of jar ${pickleWriteOutput} (of class ${pickleWriteOutput.getClass.getName})")
80+
None
81+
else
82+
report.warning(s"Could not resolve underlying source of ${pickleWriteOutput} (of class ${pickleWriteOutput.getClass.getName})")
83+
Some(pickleWriteOutput)
84+
85+
val outlineOutput = new VirtualDirectory("<outline-classpath>") {
86+
override def underlyingSource: Option[AbstractFile] = pickleWriteSource
87+
}
88+
89+
if pickleWriteOutput == NoAbstractFile then
90+
report.error("Requested outline compilation with `-Youtline` " +
91+
"but no output directory for TASTY files was specified with `-Ypickle-write`.")(using ictx)
92+
return ictx.reporter
93+
94+
val firstPassCtx = ictx.fresh
95+
.setSetting(ictx.settings.YoutlineClasspath, outlineOutput)
96+
inContext(firstPassCtx):
97+
doCompile(newCompiler, files)
98+
99+
def secondPassCtx(group: List[AbstractFile], promise: scala.concurrent.Promise[Unit]): Context =
100+
val baseCtx = initCtx.fresh
101+
.setSettings(ictx.settingsState) // copy over the classpath arguments also
102+
.setSetting(ictx.settings.YsecondPass, true)
103+
.setSetting(ictx.settings.YoutlineClasspath, outlineOutput)
104+
.setCallbacks(ictx.store)
105+
.setDepsFinishPromise(promise)
106+
.setReporter(if isParallel then new StoreReporter(ictx.reporter) else ictx.reporter)
107+
108+
// if ictx.settings.YoutlineClasspath.valueIn(ictx.settingsState).isEmpty then
109+
// baseCtx.setSetting(baseCtx.settings.YoutlineClasspath, pickleWriteAsClasspath)
110+
val fileNames: Array[String] =
111+
if sourcesRequired then group.map(_.toString).toArray else Array.empty
112+
setup(fileNames, baseCtx) match
113+
case Some((_, ctx)) =>
114+
assert(ctx.incCallback != null, s"cannot run outline without incremental callback")
115+
assert(ctx.depsFinishPromiseOpt.isDefined, s"cannot run outline without dependencies promise")
116+
ctx
117+
case None => baseCtx
118+
end secondPassCtx
119+
120+
val scalaFiles = files.filter(_.hasScalaExtension)
121+
122+
// 516 units, 8 cores => maxGroupSize = 65, unitGroups = 8, compilers = 8
123+
if !firstPassCtx.reporter.hasErrors && scalaFiles.nonEmpty then
124+
val maxGroupSize = Math.ceil(scalaFiles.length.toDouble / parallelism).toInt
125+
val fileGroups = scalaFiles.grouped(maxGroupSize).toList
126+
val compilers = fileGroups.length
127+
128+
129+
130+
def userRequestedSingleGroup = maxParallelism == 1
131+
132+
// TODO: probably not good to warn here because maybe compile is incremental
133+
// if compilers == 1 && !userRequestedSingleGroup then
134+
// val knownParallelism = maxParallelism > 0
135+
// val requestedParallelism = s"Requested parallelism with `-Ymax-parallelism` was ${maxParallelism}"
136+
// val computedAddedum =
137+
// if knownParallelism then "."
138+
// else s""",
139+
// | therefore operating with computed parallelism of ${parallelism}.""".stripMargin
140+
// val message =
141+
// s"""Outline compilation second pass will run with a single compile group.
142+
// | ${requestedParallelism}$computedAddedum
143+
// | With ${scalaUnits.length} units to compile I can only batch them into a single group.
144+
// | This will increase build times.
145+
// | Perhaps consider turning off -Youtline for this project.""".stripMargin
146+
// report.warning(message)(using firstPassCtx)
147+
148+
val promises = fileGroups.map(_ => scala.concurrent.Promise[Unit]())
149+
150+
locally:
151+
import scala.concurrent.ExecutionContext.Implicits.global
152+
Future.sequence(promises.map(_.future)).andThen {
153+
case Success(_) =>
154+
ictx.withIncCallback(_.dependencyPhaseCompleted())
155+
case Failure(ex) =>
156+
ex.printStackTrace()
157+
report.error(s"Exception during parallel compilation: ${ex.getMessage}")(using firstPassCtx)
158+
}
159+
160+
report.echo(s"Compiling $compilers groups of files ${if isParallel then "in parallel" else "sequentially"}")(using firstPassCtx)
161+
162+
def compileEager(
163+
promise: Promise[Unit],
164+
fileGroup: List[AbstractFile]
165+
): Reporter = {
166+
if ctx.settings.verbose.value then
167+
report.echo("#Compiling: " + fileGroup.take(3).mkString("", ", ", "..."))
168+
val secondCtx = secondPassCtx(fileGroup, promise)
169+
val reporter = inContext(secondCtx):
170+
doCompile(newCompiler, fileGroup) // second pass
171+
if !secondCtx.reporter.hasErrors then
172+
assert(promise.isCompleted, s"promise was not completed")
173+
if ctx.settings.verbose.value then
174+
report.echo("#Done: " + fileGroup.mkString(" "))
175+
reporter
176+
}
177+
178+
def compileFuture(
179+
promise: Promise[Unit],
180+
fileGroup: List[AbstractFile]
181+
)(using ExecutionContext): Future[Reporter] =
182+
Future {
183+
// println("#Compiling: " + fileGroup.mkString(" "))
184+
val secondCtx = secondPassCtx(fileGroup, promise)
185+
val reporter = inContext(secondCtx):
186+
doCompile(newCompiler, fileGroup) // second pass
187+
// println("#Done: " + fileGroup.mkString(" "))
188+
reporter
189+
}
190+
191+
if isParallel then
192+
// val executor = java.util.concurrent.Executors.newFixedThreadPool(compilers).nn
193+
given ec: ExecutionContext = Driver.executor // ExecutionContext.fromExecutor(executor)
194+
val futureReporters = Future.sequence(promises.lazyZip(fileGroups).map(compileFuture)).andThen {
195+
case Success(reporters) =>
196+
reporters.foreach(_.flush()(using firstPassCtx))
197+
case Failure(ex) =>
198+
ex.printStackTrace
199+
report.error(s"Exception during parallel compilation: ${ex.getMessage}")(using firstPassCtx)
200+
}
201+
Await.ready(futureReporters, Duration.Inf)
202+
// executor.shutdown()
203+
else
204+
promises.lazyZip(fileGroups).map(compileEager)
205+
firstPassCtx.reporter
206+
else
207+
ictx.withIncCallback(_.dependencyPhaseCompleted()) // may be just java files compiled
208+
firstPassCtx.reporter
209+
end doCompile
210+
31211
protected def doCompile(compiler: Compiler, files: List[AbstractFile])(using Context): Reporter =
32212
if files.nonEmpty then
33213
var runOrNull = ctx.run
@@ -194,7 +374,7 @@ class Driver {
194374
def process(args: Array[String], rootCtx: Context): Reporter = {
195375
setup(args, rootCtx) match
196376
case Some((files, compileCtx)) =>
197-
doCompile(newCompiler(using compileCtx), files)(using compileCtx)
377+
doCompile(files)(using compileCtx)
198378
case None =>
199379
rootCtx.reporter
200380
}

‎compiler/src/dotty/tools/dotc/Run.scala

+2-1
Original file line numberDiff line numberDiff line change
@@ -287,9 +287,10 @@ class Run(comp: Compiler, ictx: Context) extends ImplicitRunInfo with Constraint
287287
then ActiveProfile(ctx.settings.VprofileDetails.value.max(0).min(1000))
288288
else NoProfile
289289

290-
// If testing pickler, make sure to stop after pickling phase:
290+
// If testing pickler, of outline first pass, make sure to stop after pickling phase:
291291
val stopAfter =
292292
if (ctx.settings.YtestPickler.value) List("pickler")
293+
else if (ctx.isOutlineFirstPass) List("sbt-api")
293294
else ctx.settings.YstopAfter.value
294295

295296
val pluginPlan = ctx.base.addPluginPhases(ctx.base.phasePlan)

‎compiler/src/dotty/tools/dotc/ast/tpd.scala

+11
Original file line numberDiff line numberDiff line change
@@ -1376,6 +1376,17 @@ object tpd extends Trees.Instance[Type] with TypedTreeInfo {
13761376
def runtimeCall(name: TermName, args: List[Tree])(using Context): Tree =
13771377
Ident(defn.ScalaRuntimeModule.requiredMethod(name).termRef).appliedToTermArgs(args)
13781378

1379+
object ElidedTree:
1380+
def from(tree: Tree)(using Context): Tree =
1381+
Typed(tree, TypeTree(AnnotatedType(tree.tpe, Annotation(defn.ElidedTreeAnnot, tree.span))))
1382+
1383+
def isElided(tree: Tree)(using Context): Boolean = unapply(tree)
1384+
1385+
def unapply(tree: Tree)(using Context): Boolean = tree match
1386+
case Typed(_, tpt) => tpt.tpe.hasAnnotation(defn.ElidedTreeAnnot)
1387+
case _ => false
1388+
1389+
13791390
/** An extractor that pulls out type arguments */
13801391
object MaybePoly:
13811392
def unapply(tree: Tree): Option[(Tree, List[Tree])] = tree match

‎compiler/src/dotty/tools/dotc/classpath/FileUtils.scala

+2
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@ object FileUtils {
2323

2424
def hasTastyExtension: Boolean = file.ext.isTasty
2525

26+
def hasScalaExtension: Boolean = file.ext.isScala
27+
2628
def isTasty: Boolean = !file.isDirectory && hasTastyExtension
2729

2830
def isScalaBinary: Boolean = file.isClass || file.isTasty

‎compiler/src/dotty/tools/dotc/config/PathResolver.scala

+19-9
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@ import PartialFunction.condOpt
1212
import core.Contexts.*
1313
import Settings.*
1414
import dotty.tools.io.File
15+
import dotty.tools.io.NoAbstractFile
16+
import dotty.tools.dotc.classpath.FileUtils.isJarOrZip
1517

1618
object PathResolver {
1719

@@ -208,21 +210,28 @@ class PathResolver(using c: Context) {
208210
if (!settings.classpath.isDefault) settings.classpath.value
209211
else sys.env.getOrElse("CLASSPATH", ".")
210212

213+
def outlineClasspath: List[dotty.tools.io.AbstractFile] =
214+
if c.isOutlineFirstPass then Nil
215+
else c.settings.YoutlineClasspath.value match
216+
case NoAbstractFile => Nil
217+
case file => file :: Nil
218+
211219
import classPathFactory.*
212220

213221
// Assemble the elements!
214222
def basis: List[Iterable[ClassPath]] =
215223
val release = Option(ctx.settings.javaOutputVersion.value).filter(_.nonEmpty)
216224

217225
List(
218-
JrtClassPath(release), // 1. The Java 9+ classpath (backed by the jrt:/ virtual system, if available)
219-
classesInPath(javaBootClassPath), // 2. The Java bootstrap class path.
220-
contentsOfDirsInPath(javaExtDirs), // 3. The Java extension class path.
221-
classesInExpandedPath(javaUserClassPath), // 4. The Java application class path.
222-
classesInPath(scalaBootClassPath), // 5. The Scala boot class path.
223-
contentsOfDirsInPath(scalaExtDirs), // 6. The Scala extension class path.
224-
classesInExpandedPath(userClassPath), // 7. The Scala application class path.
225-
sourcesInPath(sourcePath) // 8. The Scala source path.
226+
outlineClasspath.map(newClassPath), // 1. The outline classpath (backed by the -Youtline-classpath option)
227+
JrtClassPath(release), // 2. The Java 9+ classpath (backed by the jrt:/ virtual system, if available)
228+
classesInPath(javaBootClassPath), // 3. The Java bootstrap class path.
229+
contentsOfDirsInPath(javaExtDirs), // 4. The Java extension class path.
230+
classesInExpandedPath(javaUserClassPath), // 5. The Java application class path.
231+
classesInPath(scalaBootClassPath), // 6. The Scala boot class path.
232+
contentsOfDirsInPath(scalaExtDirs), // 7. The Scala extension class path.
233+
classesInExpandedPath(userClassPath), // 8. The Scala application class path.
234+
sourcesInPath(sourcePath) // 9. The Scala source path.
226235
)
227236

228237
lazy val containers: List[ClassPath] = basis.flatten.distinct
@@ -238,12 +247,13 @@ class PathResolver(using c: Context) {
238247
| scalaExtDirs = %s
239248
| userClassPath = %s
240249
| sourcePath = %s
250+
| outlineClasspath = %s
241251
|}""".trim.stripMargin.format(
242252
scalaHome,
243253
ppcp(javaBootClassPath), ppcp(javaExtDirs), ppcp(javaUserClassPath),
244254
useJavaClassPath,
245255
ppcp(scalaBootClassPath), ppcp(scalaExtDirs), ppcp(userClassPath),
246-
ppcp(sourcePath)
256+
ppcp(sourcePath), outlineClasspath.map("\n" + _).mkString
247257
)
248258
}
249259

0 commit comments

Comments
 (0)