@@ -13,6 +13,28 @@ import config.Feature
1313
1414import scala .util .control .NonFatal
1515import fromtasty .{TASTYCompiler , TastyFileUtil }
16+ import dotty .tools .io .NoAbstractFile
17+ import dotty .tools .io .{VirtualFile , VirtualDirectory }
18+
19+ import java .nio .file .Path as JPath
20+ import scala .concurrent .*
21+ import scala .annotation .internal .sharable
22+ import scala .concurrent .duration .*
23+ import scala .util .{Success , Failure }
24+ import scala .annotation .targetName
25+ import dotty .tools .dotc .classpath .FileUtils .hasScalaExtension
26+ import dotty .tools .dotc .core .Symbols
27+
28+ object Driver {
29+ @ sharable lazy val executor =
30+ // TODO: systemParallelism may change over time - is it possible to update the pool size?
31+ val pool = java.util.concurrent.Executors .newFixedThreadPool(systemParallelism()).nn
32+ sys.addShutdownHook(pool.shutdown())
33+ ExecutionContext .fromExecutor(pool)
34+
35+ /** 1 less than the system's own processor count (minimum 1) */
36+ def systemParallelism () = math.max(1 , Runtime .getRuntime().nn.availableProcessors() - 1 )
37+ }
1638
1739/** Run the Dotty compiler.
1840 *
@@ -28,6 +50,183 @@ class Driver {
2850
2951 protected def emptyReporter : Reporter = new StoreReporter (null )
3052
53+ protected def doCompile (files : List [AbstractFile ])(using ictx : Context ): Reporter =
54+ val isOutline = ictx.settings.Youtline .value(using ictx)
55+
56+ if ! isOutline then inContext(ictx):
57+ report.echo(s " basic compilation enabled on files ${files.headOption.map(f => s " $f... " ).getOrElse(" []" " )(using ictx)
58+ doCompile(newCompiler, files) // standard compilation
59+ else
60+ report.echo(s " Outline compilation enabled on files ${files.headOption.map(f => s " $f... " ).getOrElse(" []" " )(using ictx)
61+ val maxParallelism = ictx.settings.YmaxParallelism .valueIn(ictx.settingsState)
62+ val absParallelism = math.abs(maxParallelism)
63+ val isParallel = maxParallelism >= 0
64+ val parallelism =
65+ val ceiling = Driver .systemParallelism()
66+ if absParallelism > 0 then math.min(absParallelism, ceiling)
67+ else ceiling
68+
69+ // NOTE: sbt will delete this potentially as soon as you call `apiPhaseCompleted`
70+ val pickleWriteOutput = ictx.settings.YearlyTastyOutput .valueIn(ictx.settingsState)
71+ val profileDestination = ictx.settings.YprofileDestination .valueIn(ictx.settingsState)
72+
73+ if pickleWriteOutput == NoAbstractFile then
74+ report.error(" Requested outline compilation with `-Yexperimental-outline` " +
75+ " but did not provide output directory for TASTY files (missing `-Yearly-tasty-output` flag)." using ictx)
76+ return ictx.reporter
77+
78+ val pickleWriteSource =
79+ pickleWriteOutput.underlyingSource match
80+ case Some (source) =>
81+ source.file.asInstanceOf [java.io.File | Null ] match
82+ case f : java.io.File => Some (source)
83+ case null =>
84+ report.warning(s " Could not resolve file of ${source} (of class ${source.getClass.getName}) " )
85+ None
86+ case None =>
87+ if pickleWriteOutput.isInstanceOf [dotty.tools.io.JarArchive ] then
88+ report.warning(s " Could not resolve underlying source of jar ${pickleWriteOutput} (of class ${pickleWriteOutput.getClass.getName}) " )
89+ None
90+ else
91+ report.warning(s " Could not resolve underlying source of ${pickleWriteOutput} (of class ${pickleWriteOutput.getClass.getName}) " )
92+ Some (pickleWriteOutput)
93+
94+ val outlineOutput = new VirtualDirectory (" <outline-classpath>"
95+ override def underlyingSource : Option [AbstractFile ] = pickleWriteSource
96+ }
97+
98+ val firstPassCtx = ictx.fresh
99+ .setSetting(ictx.settings.YoutlineClasspath , outlineOutput)
100+ inContext(firstPassCtx):
101+ doCompile(newCompiler, files)
102+
103+ def secondPassCtx (id : Int , group : List [AbstractFile ], promise : scala.concurrent.Promise [Unit ]): Context =
104+ val profileDestination0 =
105+ if profileDestination.nonEmpty then
106+ val ext = dotty.tools.io.Path .fileExtension(profileDestination)
107+ val filename = dotty.tools.io.Path .fileName(profileDestination)
108+ s " $filename-worker- $id${if ext.isEmpty then " " else s " . $ext" }"
109+ else profileDestination
110+
111+ val baseCtx = initCtx.fresh
112+ .setSettings(ictx.settingsState) // copy over the classpath arguments also
113+ .setSetting(ictx.settings.YsecondPass , true )
114+ .setSetting(ictx.settings.YoutlineClasspath , outlineOutput)
115+ .setCallbacks(ictx.store)
116+ .setDepsFinishPromise(promise)
117+ .setReporter(if isParallel then new StoreReporter (ictx.reporter) else ictx.reporter)
118+
119+ if profileDestination0.nonEmpty then
120+ baseCtx.setSetting(ictx.settings.YprofileDestination , profileDestination0)
121+
122+ // if ictx.settings.YoutlineClasspath.valueIn(ictx.settingsState).isEmpty then
123+ // baseCtx.setSetting(baseCtx.settings.YoutlineClasspath, pickleWriteAsClasspath)
124+ val fileNames : Array [String ] =
125+ if sourcesRequired then group.map(_.toString).toArray else Array .empty
126+ setup(fileNames, baseCtx) match
127+ case Some ((_, ctx)) =>
128+ assert(ctx.incCallback != null , s " cannot run outline without incremental callback " )
129+ assert(ctx.depsFinishPromiseOpt.isDefined, s " cannot run outline without dependencies promise " )
130+ ctx
131+ case None => baseCtx
132+ end secondPassCtx
133+
134+ val scalaFiles = files.filter(_.hasScalaExtension)
135+
136+ // 516 units, 8 cores => maxGroupSize = 65, unitGroups = 8, compilers = 8
137+ if ! firstPassCtx.reporter.hasErrors && scalaFiles.nonEmpty then
138+ val maxGroupSize = Math .ceil(scalaFiles.length.toDouble / parallelism).toInt
139+ val fileGroups = scalaFiles.grouped(maxGroupSize).toList
140+ val compilers = fileGroups.length
141+
142+
143+
144+ def userRequestedSingleGroup = maxParallelism == 1
145+
146+ // TODO: probably not good to warn here because maybe compile is incremental
147+ // if compilers == 1 && !userRequestedSingleGroup then
148+ // val knownParallelism = maxParallelism > 0
149+ // val requestedParallelism = s"Requested parallelism with `-Ymax-parallelism` was ${maxParallelism}"
150+ // val computedAddedum =
151+ // if knownParallelism then "."
152+ // else s""",
153+ // | therefore operating with computed parallelism of ${parallelism}.""".stripMargin
154+ // val message =
155+ // s"""Outline compilation second pass will run with a single compile group.
156+ // | ${requestedParallelism}$computedAddedum
157+ // | With ${scalaUnits.length} units to compile I can only batch them into a single group.
158+ // | This will increase build times.
159+ // | Perhaps consider turning off -Youtline for this project.""".stripMargin
160+ // report.warning(message)(using firstPassCtx)
161+
162+ val promises = fileGroups.map(_ => scala.concurrent.Promise [Unit ]())
163+
164+ locally :
165+ import scala .concurrent .ExecutionContext .Implicits .global
166+ Future .sequence(promises.map(_.future)).andThen {
167+ case Success (_) =>
168+ ictx.withIncCallback(_.dependencyPhaseCompleted())
169+ case Failure (ex) =>
170+ ex.printStackTrace()
171+ report.error(s " Exception during parallel compilation: ${ex.getMessage}" )(using firstPassCtx)
172+ }
173+
174+ report.echo(s " Compiling $compilers groups of files ${if isParallel then " in parallel" else " sequentially" " )(using firstPassCtx)
175+
176+ def compileEager (
177+ id : Int ,
178+ promise : Promise [Unit ],
179+ fileGroup : List [AbstractFile ]
180+ ): Reporter = {
181+ if ctx.settings.verbose.value then
182+ report.echo(" #Compiling: " + fileGroup.take(3 ).mkString(" " " , " " ..."
183+ val secondCtx = secondPassCtx(id, fileGroup, promise)
184+ val reporter = inContext(secondCtx):
185+ doCompile(newCompiler, fileGroup) // second pass
186+ if ! secondCtx.reporter.hasErrors then
187+ assert(promise.isCompleted, s " promise was not completed " )
188+ if ctx.settings.verbose.value then
189+ report.echo(" #Done: " + fileGroup.mkString(" "
190+ reporter
191+ }
192+
193+ def compileFuture (
194+ id : Int ,
195+ promise : Promise [Unit ],
196+ fileGroup : List [AbstractFile ]
197+ )(using ExecutionContext ): Future [Reporter ] =
198+ Future {
199+ // println("#Compiling: " + fileGroup.mkString(" "))
200+ val secondCtx = secondPassCtx(id, fileGroup, promise)
201+ val reporter = inContext(secondCtx):
202+ doCompile(newCompiler, fileGroup) // second pass
203+ // println("#Done: " + fileGroup.mkString(" "))
204+ reporter
205+ }
206+
207+ def fileGroupIds = LazyList .iterate(0 )(_ + 1 ).take(compilers)
208+ def taggedGroups = fileGroupIds.lazyZip(promises).lazyZip(fileGroups)
209+
210+ if isParallel then
211+ // val executor = java.util.concurrent.Executors.newFixedThreadPool(compilers).nn
212+ given ec : ExecutionContext = Driver .executor // ExecutionContext.fromExecutor(executor)
213+ val futureReporters = Future .sequence(taggedGroups.map(compileFuture)).andThen {
214+ case Success (reporters) =>
215+ reporters.foreach(_.flush()(using firstPassCtx))
216+ case Failure (ex) =>
217+ ex.printStackTrace
218+ report.error(s " Exception during parallel compilation: ${ex.getMessage}" )(using firstPassCtx)
219+ }
220+ Await .ready(futureReporters, Duration .Inf )
221+ // executor.shutdown()
222+ else
223+ taggedGroups.map(compileEager)
224+ firstPassCtx.reporter
225+ else
226+ ictx.withIncCallback(_.dependencyPhaseCompleted()) // may be just java files compiled
227+ firstPassCtx.reporter
228+ end doCompile
229+
31230 protected def doCompile (compiler : Compiler , files : List [AbstractFile ])(using Context ): Reporter =
32231 if files.nonEmpty then
33232 var runOrNull = ctx.run
@@ -193,7 +392,7 @@ class Driver {
193392 def process (args : Array [String ], rootCtx : Context ): Reporter = {
194393 setup(args, rootCtx) match
195394 case Some ((files, compileCtx)) =>
196- doCompile(newCompiler( using compileCtx), files)(using compileCtx)
395+ doCompile(files)(using compileCtx)
197396 case None =>
198397 rootCtx.reporter
199398 }
0 commit comments