Skip to content

Commit e4ee13b

Browse files
committed
gather profiling data on second-pass workers
1 parent 87bd50c commit e4ee13b

File tree

2 files changed

+28
-8
lines changed

2 files changed

+28
-8
lines changed

compiler/src/dotty/tools/dotc/Driver.scala

+27-8
Original file line numberDiff line numberDiff line change
@@ -27,10 +27,13 @@ import dotty.tools.dotc.core.Symbols
2727

2828
object Driver {
2929
@sharable lazy val executor =
30-
// fixed pool of 8 threads, because we have 8 cores
31-
val pool = java.util.concurrent.Executors.newFixedThreadPool(8).nn
30+
// TODO: systemParallelism may change over time - is it possible to update the pool size?
31+
val pool = java.util.concurrent.Executors.newFixedThreadPool(systemParallelism()).nn
3232
sys.addShutdownHook(pool.shutdown())
3333
ExecutionContext.fromExecutor(pool)
34+
35+
/** 1 less than the system's own processor count (minimum 1) */
36+
def systemParallelism() = math.max(1, Runtime.getRuntime().nn.availableProcessors() - 1)
3437
}
3538

3639
/** Run the Dotty compiler.
@@ -59,12 +62,13 @@ class Driver {
5962
val absParallelism = math.abs(maxParallelism)
6063
val isParallel = maxParallelism >= 0
6164
val parallelism =
62-
val ceiling = math.max(1, Runtime.getRuntime().nn.availableProcessors() - 1)
65+
val ceiling = Driver.systemParallelism()
6366
if absParallelism > 0 then math.min(absParallelism, ceiling)
6467
else ceiling
6568

6669
// NOTE: sbt will delete this potentially as soon as you call `apiPhaseCompleted`
6770
val pickleWriteOutput = ictx.settings.YearlyTastyOutput.valueIn(ictx.settingsState)
71+
val profileDestination = ictx.settings.YprofileDestination.valueIn(ictx.settingsState)
6872

6973
val pickleWriteSource =
7074
pickleWriteOutput.underlyingSource match
@@ -96,7 +100,14 @@ class Driver {
96100
inContext(firstPassCtx):
97101
doCompile(newCompiler, files)
98102

99-
def secondPassCtx(group: List[AbstractFile], promise: scala.concurrent.Promise[Unit]): Context =
103+
def secondPassCtx(id: Int, group: List[AbstractFile], promise: scala.concurrent.Promise[Unit]): Context =
104+
val profileDestination0 =
105+
if profileDestination.nonEmpty then
106+
val ext = dotty.tools.io.Path.fileExtension(profileDestination)
107+
val filename = dotty.tools.io.Path.fileName(profileDestination)
108+
s"$filename-worker-$id${if ext.isEmpty then "" else s".$ext"}"
109+
else profileDestination
110+
100111
val baseCtx = initCtx.fresh
101112
.setSettings(ictx.settingsState) // copy over the classpath arguments also
102113
.setSetting(ictx.settings.YsecondPass, true)
@@ -105,6 +116,9 @@ class Driver {
105116
.setDepsFinishPromise(promise)
106117
.setReporter(if isParallel then new StoreReporter(ictx.reporter) else ictx.reporter)
107118

119+
if profileDestination0.nonEmpty then
120+
baseCtx.setSetting(ictx.settings.YprofileDestination, profileDestination0)
121+
108122
// if ictx.settings.YoutlineClasspath.valueIn(ictx.settingsState).isEmpty then
109123
// baseCtx.setSetting(baseCtx.settings.YoutlineClasspath, pickleWriteAsClasspath)
110124
val fileNames: Array[String] =
@@ -160,12 +174,13 @@ class Driver {
160174
report.echo(s"Compiling $compilers groups of files ${if isParallel then "in parallel" else "sequentially"}")(using firstPassCtx)
161175

162176
def compileEager(
177+
id: Int,
163178
promise: Promise[Unit],
164179
fileGroup: List[AbstractFile]
165180
): Reporter = {
166181
if ctx.settings.verbose.value then
167182
report.echo("#Compiling: " + fileGroup.take(3).mkString("", ", ", "..."))
168-
val secondCtx = secondPassCtx(fileGroup, promise)
183+
val secondCtx = secondPassCtx(id, fileGroup, promise)
169184
val reporter = inContext(secondCtx):
170185
doCompile(newCompiler, fileGroup) // second pass
171186
if !secondCtx.reporter.hasErrors then
@@ -176,22 +191,26 @@ class Driver {
176191
}
177192

178193
def compileFuture(
194+
id: Int,
179195
promise: Promise[Unit],
180196
fileGroup: List[AbstractFile]
181197
)(using ExecutionContext): Future[Reporter] =
182198
Future {
183199
// println("#Compiling: " + fileGroup.mkString(" "))
184-
val secondCtx = secondPassCtx(fileGroup, promise)
200+
val secondCtx = secondPassCtx(id, fileGroup, promise)
185201
val reporter = inContext(secondCtx):
186202
doCompile(newCompiler, fileGroup) // second pass
187203
// println("#Done: " + fileGroup.mkString(" "))
188204
reporter
189205
}
190206

207+
def fileGroupIds = LazyList.iterate(0)(_ + 1).take(compilers)
208+
def taggedGroups = fileGroupIds.lazyZip(promises).lazyZip(fileGroups)
209+
191210
if isParallel then
192211
// val executor = java.util.concurrent.Executors.newFixedThreadPool(compilers).nn
193212
given ec: ExecutionContext = Driver.executor // ExecutionContext.fromExecutor(executor)
194-
val futureReporters = Future.sequence(promises.lazyZip(fileGroups).map(compileFuture)).andThen {
213+
val futureReporters = Future.sequence(taggedGroups.map(compileFuture)).andThen {
195214
case Success(reporters) =>
196215
reporters.foreach(_.flush()(using firstPassCtx))
197216
case Failure(ex) =>
@@ -201,7 +220,7 @@ class Driver {
201220
Await.ready(futureReporters, Duration.Inf)
202221
// executor.shutdown()
203222
else
204-
promises.lazyZip(fileGroups).map(compileEager)
223+
taggedGroups.map(compileEager)
205224
firstPassCtx.reporter
206225
else
207226
ictx.withIncCallback(_.dependencyPhaseCompleted()) // may be just java files compiled

project/Build.scala

+1
Original file line numberDiff line numberDiff line change
@@ -405,6 +405,7 @@ object Build {
405405

406406
scalacOptions += "-Yexperimental-outline",
407407
scalacOptions ++= Seq("-Ymax-parallelism", "7"),
408+
scalacOptions ++= Seq("-Yprofile-enabled", "-Yprofile-destination", s"out/compile-${name.value}-profile.csv"),
408409
// scalacOptions += "-Ylog-classpath",
409410

410411
version := dottyVersion,

0 commit comments

Comments
 (0)