Skip to content

Commit d96e9e4

Browse files
bishaboshabishabosha
authored
Store source file in TASTY attributes (#18948)
Add source file to TASTy attributes. This is a first step towards removing the `@SourceFile` annotation. #### Release notes Tools that read TASTy need to know that the source file must be unpickled from TASTy attributes.
2 parents 6bb23dd + e942cd0 commit d96e9e4

16 files changed

+158
-80
lines changed

‎compiler/src/dotty/tools/dotc/core/Annotations.scala

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -244,9 +244,6 @@ object Annotations {
244244
}
245245
else None
246246
}
247-
248-
def makeSourceFile(path: String, span: Span)(using Context): Annotation =
249-
apply(defn.SourceFileAnnot, Literal(Constant(path)), span)
250247
}
251248

252249
@sharable val EmptyAnnotation = Annotation(EmptyTree)

‎compiler/src/dotty/tools/dotc/core/SymbolLoaders.scala

Lines changed: 3 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -420,20 +420,9 @@ class TastyLoader(val tastyFile: AbstractFile) extends SymbolLoader {
420420
private val unpickler: tasty.DottyUnpickler =
421421
handleUnpicklingExceptions:
422422
val tastyBytes = tastyFile.toByteArray
423-
new tasty.DottyUnpickler(tastyBytes) // reads header and name table
424-
425-
val compilationUnitInfo: CompilationUnitInfo | Null =
426-
val tastyHeader = unpickler.unpickler.header
427-
val tastyVersion = TastyVersion(
428-
tastyHeader.majorVersion,
429-
tastyHeader.minorVersion,
430-
tastyHeader.experimentalVersion,
431-
)
432-
val attributes = unpickler.tastyAttributes
433-
new CompilationUnitInfo(
434-
tastyFile,
435-
tastyInfo = Some(TastyInfo(tastyVersion, attributes)),
436-
)
423+
new tasty.DottyUnpickler(tastyFile, tastyBytes) // reads header and name table
424+
425+
val compilationUnitInfo: CompilationUnitInfo | Null = unpickler.compilationUnitInfo
437426

438427
def description(using Context): String = "TASTy file " + tastyFile.toString
439428

‎compiler/src/dotty/tools/dotc/core/Symbols.scala

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -486,6 +486,12 @@ object Symbols extends SymUtils {
486486
mySource = ctx.getSource(file)
487487
else
488488
mySource = defn.patchSource(this)
489+
if !mySource.exists then
490+
val compUnitInfo = compilationUnitInfo
491+
if compUnitInfo != null then
492+
compUnitInfo.tastyInfo.flatMap(_.attributes.sourceFile) match
493+
case Some(path) => mySource = ctx.getSource(path)
494+
case _ =>
489495
if !mySource.exists then
490496
mySource = atPhaseNoLater(flattenPhase) {
491497
denot.topLevelClass.unforcedAnnotation(defn.SourceFileAnnot) match

‎compiler/src/dotty/tools/dotc/core/tasty/AttributePickler.scala

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ package dotty.tools.dotc.core.tasty
33
import dotty.tools.dotc.ast.{tpd, untpd}
44

55
import dotty.tools.tasty.TastyBuffer
6-
import dotty.tools.tasty.TastyFormat, TastyFormat.AttributesSection
6+
import dotty.tools.tasty.TastyFormat.*
77

88
object AttributePickler:
99

@@ -12,11 +12,26 @@ object AttributePickler:
1212
pickler: TastyPickler,
1313
buf: TastyBuffer
1414
): Unit =
15-
if attributes.booleanTags.nonEmpty then
16-
pickler.newSection(AttributesSection, buf)
15+
pickler.newSection(AttributesSection, buf)
16+
17+
var lastTag = -1
18+
def assertTagOrder(tag: Int): Unit =
19+
assert(tag != lastTag, s"duplicate attribute tag: $tag")
20+
assert(tag > lastTag, s"attribute tags are not ordered: $tag after $lastTag")
21+
lastTag = tag
1722

1823
for tag <- attributes.booleanTags do
24+
assert(isBooleanAttrTag(tag), "Not a boolean attribute tag: " + tag)
25+
assertTagOrder(tag)
26+
buf.writeByte(tag)
27+
28+
assert(attributes.stringTagValues.exists(_._1 == SOURCEFILEattr))
29+
for (tag, value) <- attributes.stringTagValues do
30+
assert(isStringAttrTag(tag), "Not a string attribute tag: " + tag)
31+
assertTagOrder(tag)
32+
val utf8Ref = pickler.nameBuffer.utf8Index(value)
1933
buf.writeByte(tag)
34+
buf.writeNat(utf8Ref.index)
2035

2136
end pickleAttributes
2237

‎compiler/src/dotty/tools/dotc/core/tasty/AttributeUnpickler.scala

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,19 +3,36 @@ package core.tasty
33

44
import scala.language.unsafeNulls
55
import scala.collection.immutable.BitSet
6+
import scala.collection.immutable.TreeMap
67

7-
import dotty.tools.tasty.{TastyFormat, TastyReader, TastyBuffer}
8+
import dotty.tools.tasty.{TastyFormat, TastyReader, TastyBuffer}, TastyFormat.{isBooleanAttrTag, isStringAttrTag}
9+
import dotty.tools.dotc.core.tasty.TastyUnpickler.NameTable
810

9-
class AttributeUnpickler(reader: TastyReader):
11+
class AttributeUnpickler(reader: TastyReader, nameAtRef: NameTable):
1012
import reader._
1113

1214
lazy val attributes: Attributes = {
1315
val booleanTags = BitSet.newBuilder
16+
val stringTagValue = List.newBuilder[(Int, String)]
1417

18+
var lastTag = -1
1519
while !isAtEnd do
16-
booleanTags += readByte()
20+
val tag = readByte()
21+
if isBooleanAttrTag(tag) then
22+
booleanTags += tag
23+
else if isStringAttrTag(tag) then
24+
val utf8Ref = readNameRef()
25+
val value = nameAtRef(utf8Ref).toString
26+
stringTagValue += tag -> value
27+
else
28+
assert(false, "unknown attribute tag: " + tag)
1729

18-
new Attributes(booleanTags.result())
30+
assert(tag != lastTag, s"duplicate attribute tag: $tag")
31+
assert(tag > lastTag, s"attribute tags are not ordered: $tag after $lastTag")
32+
lastTag = tag
33+
end while
34+
35+
new Attributes(booleanTags.result(), stringTagValue.result())
1936
}
2037

2138
end AttributeUnpickler

‎compiler/src/dotty/tools/dotc/core/tasty/Attributes.scala

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,20 +3,24 @@ package dotty.tools.dotc.core.tasty
33
import dotty.tools.tasty.TastyFormat.*
44

55
import scala.collection.immutable.BitSet
6+
import scala.collection.immutable.TreeMap
67

78
class Attributes private[tasty](
89
private[tasty] val booleanTags: BitSet,
10+
private[tasty] val stringTagValues: List[(Int, String)],
911
) {
1012
def scala2StandardLibrary: Boolean = booleanTags(SCALA2STANDARDLIBRARYattr)
1113
def explicitNulls: Boolean = booleanTags(EXPLICITNULLSattr)
1214
def captureChecked: Boolean = booleanTags(CAPTURECHECKEDattr)
1315
def withPureFuns: Boolean = booleanTags(WITHPUREFUNSattr)
1416
def isJava: Boolean = booleanTags(JAVAattr)
1517
def isOutline: Boolean = booleanTags(OUTLINEattr)
18+
def sourceFile: Option[String] = stringTagValues.find(_._1 == SOURCEFILEattr).map(_._2)
1619
}
1720

1821
object Attributes:
1922
def apply(
23+
sourceFile: String,
2024
scala2StandardLibrary: Boolean,
2125
explicitNulls: Boolean,
2226
captureChecked: Boolean,
@@ -31,8 +35,12 @@ object Attributes:
3135
if withPureFuns then booleanTags += WITHPUREFUNSattr
3236
if isJava then booleanTags += JAVAattr
3337
if isOutline then booleanTags += OUTLINEattr
34-
new Attributes(booleanTags.result())
38+
39+
val stringTagValues = List.newBuilder[(Int, String)]
40+
stringTagValues += SOURCEFILEattr -> sourceFile
41+
42+
new Attributes(booleanTags.result(), stringTagValues.result())
3543
end apply
3644

3745
val empty: Attributes =
38-
new Attributes(BitSet.empty)
46+
new Attributes(BitSet.empty, Nil)

‎compiler/src/dotty/tools/dotc/core/tasty/DottyUnpickler.scala

Lines changed: 19 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -16,15 +16,17 @@ import dotty.tools.tasty.TastyReader
1616
import dotty.tools.tasty.TastyFormat.{ASTsSection, PositionsSection, CommentsSection, AttributesSection}
1717
import dotty.tools.tasty.TastyVersion
1818

19+
import dotty.tools.io.AbstractFile
20+
1921
object DottyUnpickler {
2022

2123
/** Exception thrown if classfile is corrupted */
2224
class BadSignature(msg: String) extends RuntimeException(msg)
2325

24-
class TreeSectionUnpickler(posUnpickler: Option[PositionUnpickler], commentUnpickler: Option[CommentUnpickler], attributeUnpickler: Option[AttributeUnpickler])
26+
class TreeSectionUnpickler(compilationUnitInfo: CompilationUnitInfo, posUnpickler: Option[PositionUnpickler], commentUnpickler: Option[CommentUnpickler])
2527
extends SectionUnpickler[TreeUnpickler](ASTsSection) {
2628
def unpickle(reader: TastyReader, nameAtRef: NameTable): TreeUnpickler =
27-
new TreeUnpickler(reader, nameAtRef, posUnpickler, commentUnpickler, attributeUnpickler)
29+
new TreeUnpickler(reader, nameAtRef, compilationUnitInfo, posUnpickler, commentUnpickler)
2830
}
2931

3032
class PositionsSectionUnpickler extends SectionUnpickler[PositionUnpickler](PositionsSection) {
@@ -39,26 +41,33 @@ object DottyUnpickler {
3941

4042
class AttributesSectionUnpickler extends SectionUnpickler[AttributeUnpickler](AttributesSection) {
4143
def unpickle(reader: TastyReader, nameAtRef: NameTable): AttributeUnpickler =
42-
new AttributeUnpickler(reader)
44+
new AttributeUnpickler(reader, nameAtRef)
4345
}
4446
}
4547

4648
/** A class for unpickling Tasty trees and symbols.
49+
* @param tastyFile tasty file from which we unpickle (used for CompilationUnitInfo)
4750
* @param bytes the bytearray containing the Tasty file from which we unpickle
4851
* @param mode the tasty file contains package (TopLevel), an expression (Term) or a type (TypeTree)
4952
*/
50-
class DottyUnpickler(bytes: Array[Byte], mode: UnpickleMode = UnpickleMode.TopLevel) extends ClassfileParser.Embedded with tpd.TreeProvider {
53+
class DottyUnpickler(tastyFile: AbstractFile, bytes: Array[Byte], mode: UnpickleMode = UnpickleMode.TopLevel) extends ClassfileParser.Embedded with tpd.TreeProvider {
5154
import tpd.*
5255
import DottyUnpickler.*
5356

5457
val unpickler: TastyUnpickler = new TastyUnpickler(bytes)
58+
59+
val tastyAttributes: Attributes =
60+
unpickler.unpickle(new AttributesSectionUnpickler)
61+
.map(_.attributes).getOrElse(Attributes.empty)
62+
val compilationUnitInfo: CompilationUnitInfo =
63+
import unpickler.header.{majorVersion, minorVersion, experimentalVersion}
64+
val tastyVersion = TastyVersion(majorVersion, minorVersion, experimentalVersion)
65+
val tastyInfo = TastyInfo(tastyVersion, tastyAttributes)
66+
new CompilationUnitInfo(tastyFile, Some(tastyInfo))
67+
5568
private val posUnpicklerOpt = unpickler.unpickle(new PositionsSectionUnpickler)
5669
private val commentUnpicklerOpt = unpickler.unpickle(new CommentsSectionUnpickler)
57-
private val attributeUnpicklerOpt = unpickler.unpickle(new AttributesSectionUnpickler)
58-
private val treeUnpickler = unpickler.unpickle(treeSectionUnpickler(posUnpicklerOpt, commentUnpicklerOpt, attributeUnpicklerOpt)).get
59-
60-
def tastyAttributes: Attributes =
61-
attributeUnpicklerOpt.map(_.attributes).getOrElse(Attributes.empty)
70+
private val treeUnpickler = unpickler.unpickle(treeSectionUnpickler(posUnpicklerOpt, commentUnpicklerOpt)).get
6271

6372
/** Enter all toplevel classes and objects into their scopes
6473
* @param roots a set of SymDenotations that should be overwritten by unpickling
@@ -69,9 +78,8 @@ class DottyUnpickler(bytes: Array[Byte], mode: UnpickleMode = UnpickleMode.TopLe
6978
protected def treeSectionUnpickler(
7079
posUnpicklerOpt: Option[PositionUnpickler],
7180
commentUnpicklerOpt: Option[CommentUnpickler],
72-
attributeUnpicklerOpt: Option[AttributeUnpickler]
7381
): TreeSectionUnpickler =
74-
new TreeSectionUnpickler(posUnpicklerOpt, commentUnpicklerOpt, attributeUnpicklerOpt)
82+
new TreeSectionUnpickler(compilationUnitInfo, posUnpicklerOpt, commentUnpicklerOpt)
7583

7684
protected def computeRootTrees(using Context): List[Tree] = treeUnpickler.unpickle(mode)
7785

‎compiler/src/dotty/tools/dotc/core/tasty/NameBuffer.scala

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,10 @@ class NameBuffer extends TastyBuffer(10000) {
4949
}
5050
}
5151

52+
def utf8Index(value: String): NameRef =
53+
import Decorators.toTermName
54+
nameIndex(value.toTermName)
55+
5256
private inline def withLength(inline op: Unit, lengthWidth: Int = 1): Unit = {
5357
val lengthAddr = currentAddr
5458
var i = 0

‎compiler/src/dotty/tools/dotc/core/tasty/TastyPrinter.scala

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -228,11 +228,17 @@ class TastyPrinter(bytes: Array[Byte]) {
228228
import dotty.tools.tasty.TastyFormat.*
229229
def unpickle(reader: TastyReader, tastyName: NameTable): Unit = {
230230
import reader.*
231-
val attributes = new AttributeUnpickler(reader).attributes
232231
sb.append(s"\n\nAttributes (${reader.endAddr.index - reader.startAddr.index} bytes, starting from $base):\n")
233-
234-
for tag <- attributes.booleanTags do
235-
sb.append(" ").append(attributeTagToString(tag)).append("\n")
232+
while !isAtEnd do
233+
val tag = readByte()
234+
sb.append(" ").append(attributeTagToString(tag))
235+
if isBooleanAttrTag(tag) then ()
236+
else if isStringAttrTag(tag) then
237+
val utf8Ref = readNameRef()
238+
val value = nameAtRef(utf8Ref).toString
239+
sb.append(nameStr(s" ${utf8Ref.index} [$value]"))
240+
sb.append("\n")
241+
sb.result
236242
}
237243
}
238244

‎compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -49,15 +49,16 @@ import scala.compiletime.uninitialized
4949

5050
/** Unpickler for typed trees
5151
* @param reader the reader from which to unpickle
52+
* @param compilationUnitInfo the compilation unit info of the TASTy
5253
* @param posUnpicklerOpt the unpickler for positions, if it exists
5354
* @param commentUnpicklerOpt the unpickler for comments, if it exists
5455
* @param attributeUnpicklerOpt the unpickler for attributes, if it exists
5556
*/
5657
class TreeUnpickler(reader: TastyReader,
5758
nameAtRef: NameTable,
59+
compilationUnitInfo: CompilationUnitInfo,
5860
posUnpicklerOpt: Option[PositionUnpickler],
59-
commentUnpicklerOpt: Option[CommentUnpickler],
60-
attributeUnpicklerOpt: Option[AttributeUnpickler]) {
61+
commentUnpicklerOpt: Option[CommentUnpickler]) {
6162
import TreeUnpickler.*
6263
import tpd.*
6364

@@ -92,22 +93,21 @@ class TreeUnpickler(reader: TastyReader,
9293
/** The root owner tree. See `OwnerTree` class definition. Set by `enterTopLevel`. */
9394
private var ownerTree: OwnerTree = uninitialized
9495

96+
/** TASTy attributes */
97+
private val attributes: Attributes = compilationUnitInfo.tastyInfo.get.attributes
98+
9599
/** Was unpickled class compiled with capture checks? */
96-
private val withCaptureChecks: Boolean =
97-
attributeUnpicklerOpt.exists(_.attributes.captureChecked)
100+
private val withCaptureChecks: Boolean = attributes.captureChecked
98101

99-
private val unpicklingScala2Library =
100-
attributeUnpicklerOpt.exists(_.attributes.scala2StandardLibrary)
102+
private val unpicklingScala2Library = attributes.scala2StandardLibrary
101103

102104
/** This dependency was compiled with explicit nulls enabled */
103105
// TODO Use this to tag the symbols of this dependency as compiled with explicit nulls (see use of unpicklingScala2Library).
104-
private val explicitNulls =
105-
attributeUnpicklerOpt.exists(_.attributes.explicitNulls)
106+
private val explicitNulls = attributes.explicitNulls
106107

107-
private val unpicklingJava =
108-
attributeUnpicklerOpt.exists(_.attributes.isJava)
108+
private val unpicklingJava = attributes.isJava
109109

110-
private val isOutline = attributeUnpicklerOpt.exists(_.attributes.isOutline)
110+
private val isOutline = attributes.isOutline
111111

112112
private def registerSym(addr: Addr, sym: Symbol) =
113113
symAtAddr(addr) = sym
@@ -636,8 +636,8 @@ class TreeUnpickler(reader: TastyReader,
636636
rootd.symbol
637637
case _ =>
638638
val completer = adjustIfModule(new Completer(subReader(start, end)))
639-
if (isClass)
640-
newClassSymbol(ctx.owner, name.asTypeName, flags, completer, privateWithin, coord)
639+
if isClass then
640+
newClassSymbol(ctx.owner, name.asTypeName, flags, completer, privateWithin, coord, compilationUnitInfo)
641641
else
642642
newSymbol(ctx.owner, name, flags, completer, privateWithin, coord)
643643
}

‎compiler/src/dotty/tools/dotc/fromtasty/ReadTasty.scala

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import Denotations.staticRef
1111
import NameOps.*
1212
import ast.Trees.Tree
1313
import Phases.Phase
14+
import core.tasty.Attributes
1415

1516
/** Load trees from TASTY files */
1617
class ReadTasty extends Phase {

‎compiler/src/dotty/tools/dotc/quoted/PickledQuotes.scala

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import scala.quoted.runtime.impl.*
2121
import scala.collection.mutable
2222

2323
import QuoteUtils.*
24+
import dotty.tools.io.NoAbstractFile
2425

2526
object PickledQuotes {
2627
import tpd.*
@@ -268,7 +269,7 @@ object PickledQuotes {
268269
quotePickling.println(s"**** unpickling quote from TASTY\n${TastyPrinter.showContents(bytes, ctx.settings.color.value == "never")}")
269270

270271
val mode = if (isType) UnpickleMode.TypeTree else UnpickleMode.Term
271-
val unpickler = new DottyUnpickler(bytes, mode)
272+
val unpickler = new DottyUnpickler(NoAbstractFile, bytes, mode)
272273
unpickler.enter(Set.empty)
273274

274275
val tree = unpickler.tree

‎compiler/src/dotty/tools/dotc/transform/Pickler.scala

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -96,12 +96,16 @@ class Pickler extends Phase {
9696
do
9797
if ctx.settings.YtestPickler.value then beforePickling(cls) = tree.show
9898

99+
val sourceRelativePath =
100+
val reference = ctx.settings.sourceroot.value
101+
util.SourceFile.relativePath(unit.source, reference)
99102
val isJavaAttr = unit.isJava // we must always set JAVAattr when pickling Java sources
100103
if isJavaAttr then
101104
// assert that Java sources didn't reach Pickler without `-Yjava-tasty`.
102105
assert(ctx.settings.YjavaTasty.value, "unexpected Java source file without -Yjava-tasty")
103106
val isOutline = isJavaAttr // TODO: later we may want outline for Scala sources too
104107
val attributes = Attributes(
108+
sourceFile = sourceRelativePath,
105109
scala2StandardLibrary = ctx.settings.YcompileScala2Library.value,
106110
explicitNulls = ctx.settings.YexplicitNulls.value,
107111
captureChecked = Feature.ccEnabled,
@@ -231,7 +235,7 @@ class Pickler extends Phase {
231235
ctx.initialize()
232236
val unpicklers =
233237
for ((cls, (unit, bytes)) <- pickledBytes) yield {
234-
val unpickler = new DottyUnpickler(bytes)
238+
val unpickler = new DottyUnpickler(unit.source.file, bytes)
235239
unpickler.enter(roots = Set.empty)
236240
cls -> (unit, unpickler)
237241
}

‎compiler/src/dotty/tools/dotc/transform/PostTyper.scala

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -417,12 +417,13 @@ class PostTyper extends MacroTransform with InfoTransformer { thisPhase =>
417417
if illegalRefs.nonEmpty then
418418
report.error(
419419
em"The type of a class parent cannot refer to constructor parameters, but ${parent.tpe} refers to ${illegalRefs.map(_.name.show).mkString(",")}", parent.srcPos)
420-
// Add SourceFile annotation to top-level classes
421420
if sym.owner.is(Package) then
421+
// Add SourceFile annotation to top-level classes
422+
// TODO remove this annotation once the reference compiler uses the TASTy source file attribute.
422423
if ctx.compilationUnit.source.exists && sym != defn.SourceFileAnnot then
423424
val reference = ctx.settings.sourceroot.value
424425
val relativePath = util.SourceFile.relativePath(ctx.compilationUnit.source, reference)
425-
sym.addAnnotation(Annotation.makeSourceFile(relativePath, tree.span))
426+
sym.addAnnotation(Annotation(defn.SourceFileAnnot, Literal(Constants.Constant(relativePath)), tree.span))
426427
else
427428
if !sym.is(Param) && !sym.owner.isOneOf(AbstractOrTrait) then
428429
Checking.checkGoodBounds(tree.symbol)

0 commit comments

Comments
 (0)