Add @binaryAPI

The `@binaryAPI`  annotation will make any package private or protected term definition public in the generated bytecode. Definitions that override an `@binaryAPI` will also become public.

We cannot annotate `private[this]` definitions because we might have different definitions with the same name and signature in a subclass. These would clash if made public. Instead of using private/private[this], the user can write `private[C]` where `C` is the enclosing class.

This is useful in combination with inline definitions. If an inline definition refers to a `private`/`protected` definition marked as `@binaryAPI` it does not need to use an accessor. We still generate the accessors for binary compatibility but do not use them.

Author: nicolasstucki

compiler/src/dotty/tools/dotc/Compiler.scala

@@ -72,7 +72,8 @@ class Compiler {
          new ElimRepeated,           // Rewrite vararg parameters and arguments
          new RefChecks) ::           // Various checks mostly related to abstract members and overriding
     List(new init.Checker) ::        // Check initialization of objects
-    List(new ProtectedAccessors,     // Add accessors for protected members
+    List(new BinaryAPIAnnotations,   // Makes @binaryAPI definitions public
+         new ProtectedAccessors,     // Add accessors for protected members
          new ExtensionMethods,       // Expand methods of value classes with extension methods
          new UncacheGivenAliases,    // Avoid caching RHS of simple parameterless given aliases
          new ElimByName,             // Map by-name parameters to functions

compiler/src/dotty/tools/dotc/core/Definitions.scala

@@ -1039,6 +1039,7 @@ class Definitions {
   @tu lazy val RequiresCapabilityAnnot: ClassSymbol = requiredClass("scala.annotation.internal.requiresCapability")
   @tu lazy val RetainsAnnot: ClassSymbol = requiredClass("scala.annotation.retains")
   @tu lazy val RetainsByNameAnnot: ClassSymbol = requiredClass("scala.annotation.retainsByName")
+  @tu lazy val BinaryAPIAnnot: ClassSymbol = requiredClass("scala.annotation.binaryAPI")
 
   @tu lazy val JavaRepeatableAnnot: ClassSymbol = requiredClass("java.lang.annotation.Repeatable")
 

compiler/src/dotty/tools/dotc/core/SymDenotations.scala

@@ -1038,6 +1038,13 @@ object SymDenotations {
       isOneOf(EffectivelyErased)
       || is(Inline) && !isRetainedInline && !hasAnnotation(defn.ScalaStaticAnnot)
 
+    /** Is this a member that will become public in the generated binary */
+    def isBinaryAPI(using Context): Boolean =
+      isTerm && (
+        hasAnnotation(defn.BinaryAPIAnnot) ||
+        allOverriddenSymbols.exists(_.hasAnnotation(defn.BinaryAPIAnnot))
+      )
+
     /** ()T and => T types should be treated as equivalent for this symbol.
      *  Note: For the moment, we treat Scala-2 compiled symbols as loose matching,
      *  because the Scala library does not always follow the right conventions.

compiler/src/dotty/tools/dotc/inlines/PrepareInlineable.scala

@@ -54,6 +54,9 @@ object PrepareInlineable {
     /** A tree map which inserts accessors for non-public term members accessed from inlined code.
      */
     abstract class MakeInlineableMap(val inlineSym: Symbol) extends TreeMap with Insert {
+
+      def useBinaryAPI: Boolean
+
       def accessorNameOf(name: TermName, site: Symbol)(using Context): TermName =
         val accName = InlineAccessorName(name)
         if site.isExtensibleClass then accName.expandedName(site) else accName
@@ -71,6 +74,7 @@ object PrepareInlineable {
       def needsAccessor(sym: Symbol)(using Context): Boolean =
         sym.isTerm &&
         (sym.isOneOf(AccessFlags) || sym.privateWithin.exists) &&
+        (!useBinaryAPI || !sym.isBinaryAPI) &&
         !sym.isContainedIn(inlineSym) &&
         !(sym.isStableMember && sym.info.widenTermRefExpr.isInstanceOf[ConstantType]) &&
         !sym.isInlineMethod &&
@@ -94,7 +98,7 @@ object PrepareInlineable {
      *  possible if the receiver is essentially this or an outer this, which is indicated
      *  by the test that we can find a host for the accessor.
      */
-    class MakeInlineableDirect(inlineSym: Symbol) extends MakeInlineableMap(inlineSym) {
+    class MakeInlineableDirect(inlineSym: Symbol, val useBinaryAPI: Boolean) extends MakeInlineableMap(inlineSym) {
       def preTransform(tree: Tree)(using Context): Tree = tree match {
         case tree: RefTree if needsAccessor(tree.symbol) =>
           if (tree.symbol.isConstructor) {
@@ -118,13 +122,14 @@ object PrepareInlineable {
      *      private[inlines] def next[U](y: U): (T, U) = (x, y)
      *    }
      *    class TestPassing {
-     *      inline def foo[A](x: A): (A, Int) = {
-     *      val c = new C[A](x)
-     *      c.next(1)
-     *    }
-     *    inline def bar[A](x: A): (A, String) = {
-     *      val c = new C[A](x)
-     *      c.next("")
+     *        inline def foo[A](x: A): (A, Int) = {
+     *        val c = new C[A](x)
+     *        c.next(1)
+     *      }
+     *      inline def bar[A](x: A): (A, String) = {
+     *        val c = new C[A](x)
+     *        c.next("")
+     *      }
      *    }
      *
      *  `C` could be compiled separately, so we cannot place the inline accessor in it.
@@ -137,7 +142,7 @@ object PrepareInlineable {
      *  Since different calls might have different receiver types, we need to generate one
      *  such accessor per call, so they need to have unique names.
      */
-    class MakeInlineablePassing(inlineSym: Symbol) extends MakeInlineableMap(inlineSym) {
+    class MakeInlineablePassing(inlineSym: Symbol, val useBinaryAPI: Boolean) extends MakeInlineableMap(inlineSym) {
 
       def preTransform(tree: Tree)(using Context): Tree = tree match {
         case _: Apply | _: TypeApply | _: RefTree
@@ -220,8 +225,13 @@ object PrepareInlineable {
         // so no accessors are needed for them.
         tree
       else
-        new MakeInlineablePassing(inlineSym).transform(
-          new MakeInlineableDirect(inlineSym).transform(tree))
+        // Make sure the old accessors are generated for binary compatibility
+        new MakeInlineablePassing(inlineSym, useBinaryAPI = false).transform(
+          new MakeInlineableDirect(inlineSym, useBinaryAPI = false).transform(tree))
+
+        // TODO: warn if MakeInlineablePassing or MakeInlineableDirect generate accessors when useBinaryAPI in enabled
+        new MakeInlineablePassing(inlineSym, useBinaryAPI = true).transform(
+          new MakeInlineableDirect(inlineSym, useBinaryAPI = true).transform(tree))
     }
   }
 

compiler/src/dotty/tools/dotc/transform/BinaryAPIAnnotations.scala

@@ -0,0 +1,32 @@
+package dotty.tools.dotc.transform
+
+import dotty.tools.dotc.core.Contexts.*
+import dotty.tools.dotc.core.DenotTransformers.SymTransformer
+import dotty.tools.dotc.core.Flags.*
+import dotty.tools.dotc.core.Symbols.NoSymbol
+import dotty.tools.dotc.core.SymDenotations.SymDenotation
+import dotty.tools.dotc.transform.MegaPhase.MiniPhase
+import dotty.tools.dotc.typer.RefChecks
+
+/** Makes @binaryAPI definitions public */
+class BinaryAPIAnnotations extends MiniPhase with SymTransformer:
+
+  override def runsAfterGroupsOf: Set[String] = Set(RefChecks.name)
+
+  override def phaseName: String = BinaryAPIAnnotations.name
+  override def description: String = BinaryAPIAnnotations.description
+
+  def transformSym(d: SymDenotation)(using Context): SymDenotation = {
+    if d.isBinaryAPI then
+      d.resetFlag(Protected)
+      d.setPrivateWithin(NoSymbol)
+      if d.is(Module) then
+        val moduleClass = d.moduleClass
+        moduleClass.resetFlag(Protected)
+        moduleClass.setPrivateWithin(NoSymbol)
+    d
+  }
+
+object BinaryAPIAnnotations:
+  val name: String = "binaryAPIAnnotations"
+  val description: String = "makes @binaryAPI definitions public"

compiler/src/dotty/tools/dotc/transform/PostTyper.scala

@@ -165,10 +165,13 @@ class PostTyper extends MacroTransform with IdentityDenotTransformer { thisPhase
                   atPhase(thisPhase)(cls.annotationsCarrying(Set(defn.CompanionMethodMetaAnnot)))
                     ++ sym.annotations)
           else
+            val binaryAPIAnnotOpt = sym.getAnnotation(defn.BinaryAPIAnnot)
             if sym.is(Param) then
               sym.keepAnnotationsCarrying(thisPhase, Set(defn.ParamMetaAnnot), orNoneOf = defn.NonBeanMetaAnnots)
             else if sym.is(ParamAccessor) then
-              sym.keepAnnotationsCarrying(thisPhase, Set(defn.GetterMetaAnnot, defn.FieldMetaAnnot))
+              // FIXME: copyAndKeepAnnotationsCarrying is dropping defn.BinaryAPIAnnot
+              sym.keepAnnotationsCarrying(thisPhase, Set(defn.GetterMetaAnnot, defn.FieldMetaAnnot, defn.BinaryAPIAnnot))
+              for binaryAPIAnnot <- binaryAPIAnnotOpt do sym.addAnnotation(binaryAPIAnnot)
             else
               sym.keepAnnotationsCarrying(thisPhase, Set(defn.GetterMetaAnnot, defn.FieldMetaAnnot), orNoneOf = defn.NonBeanMetaAnnots)
           if sym.isScala2Macro && !ctx.settings.XignoreScala2Macros.value then

compiler/src/dotty/tools/dotc/typer/Checking.scala

@@ -526,6 +526,11 @@ object Checking {
         fail(em"Inline methods cannot be @tailrec")
     if sym.hasAnnotation(defn.TargetNameAnnot) && sym.isClass && sym.isTopLevelClass then
       fail(TargetNameOnTopLevelClass(sym))
+    if sym.hasAnnotation(defn.BinaryAPIAnnot) then
+      if sym.is(Enum) then fail(em"@binaryAPI cannot be used on enum definitions.")
+      else if sym.isType && !sym.is(Module) && !(sym.is(Given) || sym.companionModule.is(Given)) then fail(em"@binaryAPI cannot be used on ${sym.showKind} definitions")
+      else if !sym.owner.isClass && !(sym.is(Param) && sym.owner.isConstructor) then fail(em"@binaryAPI cannot be used on local definitions.")
+      else if sym.is(Private) then fail(em"@binaryAPI cannot be used on private definitions.\n\nCould use private[${sym.owner.name}] or protected instead.")
     if (sym.hasAnnotation(defn.NativeAnnot)) {
       if (!sym.is(Deferred))
         fail(NativeMembersMayNotHaveImplementation(sym))

compiler/src/dotty/tools/dotc/typer/TypeAssigner.scala

@@ -99,7 +99,9 @@ trait TypeAssigner {
     val tpe1 = accessibleType(tpe, superAccess)
     if tpe1.exists then tpe1
     else tpe match
-      case tpe: NamedType => inaccessibleErrorType(tpe, superAccess, pos)
+      case tpe: NamedType =>
+        if tpe.termSymbol.isBinaryAPI then tpe
+        else inaccessibleErrorType(tpe, superAccess, pos)
       case NoType => tpe
 
   /** Return a potentially skolemized version of `qualTpe` to be used