An implementation of flexible types for explicit nulls

Author: noti0na1

compiler/src/dotty/tools/dotc/config/ScalaSettings.scala

@@ -31,7 +31,7 @@ object ScalaSettings extends ScalaSettings
 
 // Kept as seperate type to avoid breaking backward compatibility
 abstract class ScalaSettings extends SettingGroup, AllScalaSettings:
-  val settingsByCategory: Map[SettingCategory, List[Setting[_]]] = 
+  val settingsByCategory: Map[SettingCategory, List[Setting[_]]] =
     allSettings.groupBy(_.category)
       .view.mapValues(_.toList).toMap
       .withDefaultValue(Nil)
@@ -43,7 +43,7 @@ abstract class ScalaSettings extends SettingGroup, AllScalaSettings:
   val verboseSettings: List[Setting[_]] = settingsByCategory(VerboseSetting).sortBy(_.name)
   val settingsByAliases: Map[String, Setting[_]] = allSettings.flatMap(s => s.aliases.map(_ -> s)).toMap
 
-  
+
 trait AllScalaSettings extends CommonScalaSettings, PluginSettings, VerboseSettings, WarningSettings, XSettings, YSettings:
   self: SettingGroup =>
 
@@ -416,6 +416,7 @@ private sealed trait YSettings:
   // Experimental language features
   val YnoKindPolymorphism: Setting[Boolean] = BooleanSetting(ForkSetting, "Yno-kind-polymorphism", "Disable kind polymorphism.")
   val YexplicitNulls: Setting[Boolean] = BooleanSetting(ForkSetting, "Yexplicit-nulls", "Make reference types non-nullable. Nullable types can be expressed with unions: e.g. String|Null.")
+  val YnoFlexibleTypes: Setting[Boolean] = BooleanSetting("-Yno-flexible-types", "Disable turning nullable Java return types and parameter types into flexible types, which behaves like abstract types with a nullable lower bound and non-nullable upper bound.")
   val YcheckInit: Setting[Boolean] = BooleanSetting(ForkSetting, "Ysafe-init", "Ensure safe initialization of objects.")
   val YcheckInitGlobal: Setting[Boolean] = BooleanSetting(ForkSetting, "Ysafe-init-global", "Check safe initialization of global objects.")
   val YrequireTargetName: Setting[Boolean] = BooleanSetting(ForkSetting, "Yrequire-targetName", "Warn if an operator is defined without a @targetName annotation.")

compiler/src/dotty/tools/dotc/core/Contexts.scala

@@ -472,6 +472,9 @@ object Contexts {
     /** Is the explicit nulls option set? */
     def explicitNulls: Boolean = base.settings.YexplicitNulls.value
 
+    /** Is the flexible types option set? */
+    def flexibleTypes: Boolean = base.settings.YexplicitNulls.value && !base.settings.YnoFlexibleTypes.value
+
     /** A fresh clone of this context embedded in this context. */
     def fresh: FreshContext = freshOver(this)
 

compiler/src/dotty/tools/dotc/core/JavaNullInterop.scala

@@ -78,11 +78,11 @@ object JavaNullInterop {
    *  but the result type is not nullable.
    */
   private def nullifyExceptReturnType(tp: Type)(using Context): Type =
-    new JavaNullMap(true)(tp)
+    new JavaNullMap(outermostLevelAlreadyNullable = true)(tp)
 
   /** Nullifies a Java type by adding `| Null` in the relevant places. */
   private def nullifyType(tp: Type)(using Context): Type =
-    new JavaNullMap(false)(tp)
+    new JavaNullMap(outermostLevelAlreadyNullable = false)(tp)
 
   /** A type map that implements the nullification function on types. Given a Java-sourced type, this adds `| Null`
    *  in the right places to make the nulls explicit in Scala.
@@ -96,25 +96,29 @@ object JavaNullInterop {
    *                                       to `(A & B) | Null`, instead of `(A | Null & B | Null) | Null`.
    */
   private class JavaNullMap(var outermostLevelAlreadyNullable: Boolean)(using Context) extends TypeMap {
+    def nullify(tp: Type): Type = if ctx.flexibleTypes then FlexibleType(tp) else OrNull(tp)
+
     /** Should we nullify `tp` at the outermost level? */
     def needsNull(tp: Type): Boolean =
-      !outermostLevelAlreadyNullable && (tp match {
+      !(outermostLevelAlreadyNullable || (tp match {
         case tp: TypeRef =>
           // We don't modify value types because they're non-nullable even in Java.
-          !tp.symbol.isValueClass &&
+          tp.symbol.isValueClass
+          // We don't modify unit types.
+          || tp.isRef(defn.UnitClass)
           // We don't modify `Any` because it's already nullable.
-          !tp.isRef(defn.AnyClass) &&
+          || tp.isRef(defn.AnyClass)
           // We don't nullify Java varargs at the top level.
           // Example: if `setNames` is a Java method with signature `void setNames(String... names)`,
           // then its Scala signature will be `def setNames(names: (String|Null)*): Unit`.
           // This is because `setNames(null)` passes as argument a single-element array containing the value `null`,
           // and not a `null` array.
-          !tp.isRef(defn.RepeatedParamClass)
-        case _ => true
-      })
+          || !ctx.flexibleTypes && tp.isRef(defn.RepeatedParamClass)
+        case _ => false
+      }))
 
     override def apply(tp: Type): Type = tp match {
-      case tp: TypeRef if needsNull(tp) => OrNull(tp)
+      case tp: TypeRef if needsNull(tp) => nullify(tp)
       case appTp @ AppliedType(tycon, targs) =>
         val oldOutermostNullable = outermostLevelAlreadyNullable
         // We don't make the outmost levels of type arguments nullable if tycon is Java-defined.
@@ -124,7 +128,7 @@ object JavaNullInterop {
         val targs2 = targs map this
         outermostLevelAlreadyNullable = oldOutermostNullable
         val appTp2 = derivedAppliedType(appTp, tycon, targs2)
-        if needsNull(tycon) then OrNull(appTp2) else appTp2
+        if needsNull(tycon) then nullify(appTp2) else appTp2
       case ptp: PolyType =>
         derivedLambdaType(ptp)(ptp.paramInfos, this(ptp.resType))
       case mtp: MethodType =>
@@ -138,12 +142,12 @@ object JavaNullInterop {
         // nullify(A & B) = (nullify(A) & nullify(B)) | Null, but take care not to add
         // duplicate `Null`s at the outermost level inside `A` and `B`.
         outermostLevelAlreadyNullable = true
-        OrNull(derivedAndType(tp, this(tp.tp1), this(tp.tp2)))
-      case tp: TypeParamRef if needsNull(tp) => OrNull(tp)
+        nullify(derivedAndType(tp, this(tp.tp1), this(tp.tp2)))
+      case tp: TypeParamRef if needsNull(tp) => nullify(tp)
       // In all other cases, return the type unchanged.
       // In particular, if the type is a ConstantType, then we don't nullify it because it is the
       // type of a final non-nullable field.
       case _ => tp
     }
   }
-}
+}

compiler/src/dotty/tools/dotc/core/NullOpsDecorator.scala

@@ -8,6 +8,10 @@ import Types.*
 object NullOpsDecorator:
 
   extension (self: Type)
+    def stripFlexible(using Context): Type = self match
+      case FlexibleType(tp) => tp
+      case _ => self
+
     /** Syntactically strips the nullability from this type.
      *  If the type is `T1 | ... | Tn`, and `Ti` references to `Null`,
      *  then return `T1 | ... | Ti-1 | Ti+1 | ... | Tn`.
@@ -33,6 +37,7 @@ object NullOpsDecorator:
             if (tp1s ne tp1) && (tp2s ne tp2) then
               tp.derivedAndType(tp1s, tp2s)
             else tp
+          case tp: FlexibleType => tp.hi
           case tp @ TypeBounds(lo, hi) =>
             tp.derivedTypeBounds(strip(lo), strip(hi))
           case tp => tp

compiler/src/dotty/tools/dotc/core/OrderingConstraint.scala

@@ -564,6 +564,9 @@ class OrderingConstraint(private val boundsMap: ParamBounds,
       case CapturingType(parent, refs) =>
         val parent1 = recur(parent)
         if parent1 ne parent then tp.derivedCapturingType(parent1, refs) else tp
+      case tp: FlexibleType =>
+        val underlying = recur(tp.underlying)
+        if underlying ne tp.underlying then tp.derivedFlexibleType(underlying) else tp
       case tp: AnnotatedType =>
         val parent1 = recur(tp.parent)
         if parent1 ne tp.parent then tp.derivedAnnotatedType(parent1, tp.annot) else tp

compiler/src/dotty/tools/dotc/core/PatternTypeConstrainer.scala

@@ -163,7 +163,7 @@ trait PatternTypeConstrainer { self: TypeComparer =>
       }
     }
 
-    def dealiasDropNonmoduleRefs(tp: Type) = tp.dealias match {
+    def dealiasDropNonmoduleRefs(tp: Type): Type = tp.dealias match {
       case tp: TermRef =>
         // we drop TermRefs that don't have a class symbol, as they can't
         // meaningfully participate in GADT reasoning and just get in the way.
@@ -172,6 +172,7 @@ trait PatternTypeConstrainer { self: TypeComparer =>
         // additional trait - argument-less enum cases desugar to vals.
         // See run/enum-Tree.scala.
         if tp.classSymbol.exists then tp else tp.info
+      case FlexibleType(tp) => dealiasDropNonmoduleRefs(tp)
       case tp => tp
     }
 

compiler/src/dotty/tools/dotc/core/TypeComparer.scala

@@ -24,6 +24,7 @@ import reporting.trace
 import annotation.constructorOnly
 import cc.*
 import NameKinds.WildcardParamName
+import NullOpsDecorator.stripFlexible
 
 /** Provides methods to compare types.
  */
@@ -864,6 +865,8 @@ class TypeComparer(@constructorOnly initctx: Context) extends ConstraintHandling
             false
         }
         compareClassInfo
+      case tp2: FlexibleType =>
+        recur(tp1, tp2.lo)
       case _ =>
         fourthTry
     }
@@ -1059,6 +1062,8 @@ class TypeComparer(@constructorOnly initctx: Context) extends ConstraintHandling
       case tp1: ExprType if ctx.phaseId > gettersPhase.id =>
         // getters might have converted T to => T, need to compensate.
         recur(tp1.widenExpr, tp2)
+      case tp1: FlexibleType =>
+        recur(tp1.hi, tp2)
       case _ =>
         false
     }
@@ -2516,15 +2521,18 @@ class TypeComparer(@constructorOnly initctx: Context) extends ConstraintHandling
     case _ =>
       tp
 
-  private def andTypeGen(tp1: Type, tp2: Type, op: (Type, Type) => Type,
-      original: (Type, Type) => Type = _ & _, isErased: Boolean = ctx.erasedTypes): Type = trace(s"andTypeGen(${tp1.show}, ${tp2.show})", subtyping, show = true) {
+  private def andTypeGen(tp1orig: Type, tp2orig: Type, op: (Type, Type) => Type,
+      original: (Type, Type) => Type = _ & _, isErased: Boolean = ctx.erasedTypes): Type = trace(s"andTypeGen(${tp1orig.show}, ${tp2orig.show})", subtyping, show = true) {
+    val tp1 = tp1orig.stripFlexible
+    val tp2 = tp2orig.stripFlexible
     val t1 = distributeAnd(tp1, tp2)
-    if (t1.exists) t1
-    else {
-      val t2 = distributeAnd(tp2, tp1)
-      if (t2.exists) t2
-      else if (isErased) erasedGlb(tp1, tp2)
-      else liftIfHK(tp1, tp2, op, original, _ | _)
+    val ret =
+      if t1.exists then t1
+      else
+        val t2 = distributeAnd(tp2, tp1)
+        if t2.exists then t2
+        else if isErased then erasedGlb(tp1, tp2)
+        else liftIfHK(tp1, tp2, op, original, _ | _)
         // The ` | ` on variances is needed since variances are associated with bounds
         // not lambdas. Example:
         //
@@ -2534,7 +2542,9 @@ class TypeComparer(@constructorOnly initctx: Context) extends ConstraintHandling
         //
         // Here, `F` is treated as bivariant in `O`. That is, only bivariant implementation
         // of `F` are allowed. See neg/hk-variance2s.scala test.
-    }
+
+    if tp1orig.isInstanceOf[FlexibleType] && tp2orig.isInstanceOf[FlexibleType]
+    then FlexibleType(ret) else ret
   }
 
   /** Form a normalized conjunction of two types.

compiler/src/dotty/tools/dotc/core/Types.scala

@@ -307,6 +307,7 @@ object Types extends TypeUtils {
       isRef(defn.ObjectClass) && (typeSymbol eq defn.FromJavaObjectSymbol)
 
     def containsFromJavaObject(using Context): Boolean = this match
+      case tp: FlexibleType => tp.original.containsFromJavaObject
       case tp: OrType => tp.tp1.containsFromJavaObject || tp.tp2.containsFromJavaObject
       case tp: AndType => tp.tp1.containsFromJavaObject && tp.tp2.containsFromJavaObject
       case _ => isFromJavaObject
@@ -345,6 +346,7 @@ object Types extends TypeUtils {
     /** Is this type guaranteed not to have `null` as a value? */
     final def isNotNull(using Context): Boolean = this match {
       case tp: ConstantType => tp.value.value != null
+      case tp: FlexibleType => false
       case tp: ClassInfo => !tp.cls.isNullableClass && tp.cls != defn.NothingClass
       case tp: AppliedType => tp.superType.isNotNull
       case tp: TypeBounds => tp.lo.isNotNull
@@ -374,6 +376,7 @@ object Types extends TypeUtils {
       case AppliedType(tycon, args) => tycon.unusableForInference || args.exists(_.unusableForInference)
       case RefinedType(parent, _, rinfo) => parent.unusableForInference || rinfo.unusableForInference
       case TypeBounds(lo, hi) => lo.unusableForInference || hi.unusableForInference
+      case FlexibleType(underlying) => underlying.unusableForInference
       case tp: AndOrType => tp.tp1.unusableForInference || tp.tp2.unusableForInference
       case tp: LambdaType => tp.resultType.unusableForInference || tp.paramInfos.exists(_.unusableForInference)
       case WildcardType(optBounds) => optBounds.unusableForInference
@@ -1616,6 +1619,9 @@ object Types extends TypeUtils {
 
     /** If this is a repeated type, its element type, otherwise the type itself */
     def repeatedToSingle(using Context): Type = this match {
+      case tp: FlexibleType =>
+        val underlyingSingle = tp.underlying.repeatedToSingle
+        if underlyingSingle ne tp.underlying then underlyingSingle else tp
       case tp @ ExprType(tp1) => tp.derivedExprType(tp1.repeatedToSingle)
       case _                  => if (isRepeatedParam) this.argTypesHi.head else this
     }
@@ -3440,6 +3446,56 @@ object Types extends TypeUtils {
     }
   }
 
+  // --- FlexibleType -----------------------------------------------------------------
+
+  /* A flexible type is a type with a custom subtyping relationship.
+   * It is used by explicit nulls to represent a type coming from Java which can be
+   * consider as nullable or non-nullable depending on the context, in a similar way to Platform
+   * Types in Kotlin. A `FlexibleType(T)` generally behaves like a type variable with special bounds
+   * `T | Null .. T`, so that `T | Null <: FlexibleType(T) <: T`.
+   * A flexible type will be erased to its original type `T`.
+   */
+  case class FlexibleType(original: Type, lo: Type, hi: Type) extends CachedProxyType with ValueType {
+    def underlying(using Context): Type = original
+
+    override def superType(using Context): Type = hi
+
+    def derivedFlexibleType(original: Type)(using Context): Type =
+      if this.original eq original then this else FlexibleType(original)
+
+    override def computeHash(bs: Binders): Int = doHash(bs, original)
+
+    override final def baseClasses(using Context): List[ClassSymbol] = original.baseClasses
+  }
+
+  object FlexibleType {
+    def apply(original: Type)(using Context): Type = original match {
+      case ft: FlexibleType => ft
+      case _ =>
+        // val original1 = original.stripNull
+        // if original1.isNullType then
+        //   // (Null)? =:= ? >: Null <: (Object & Null)
+        //   FlexibleType(defn.NullType, original, AndType(defn.ObjectType, defn.NullType))
+        // else
+        //   // (T | Null)? =:= ? >: T | Null <: T
+        //   // (T)? =:= ? >: T | Null <: T
+        //   val hi = original1
+        //   val lo = if hi eq original then OrNull(hi) else original
+        //   FlexibleType(original, lo, hi)
+        //
+        // The commented out code does more work to analyze the original type to ensure the
+        // flexible type is always a subtype of the original type and the Object type.
+        // It is not necessary according to the use cases, so we choose to use a simpler
+        // rule.
+        FlexibleType(original, OrNull(original), original)
+    }
+
+    def unapply(tp: Type)(using Context): Option[Type] = tp match {
+      case ft: FlexibleType => Some(ft.original)
+      case _ => None
+    }
+  }
+
   // --- AndType/OrType ---------------------------------------------------------------
 
   abstract class AndOrType extends CachedGroundType with ValueType {
@@ -5941,6 +5997,8 @@ object Types extends TypeUtils {
         samClass(tp.underlying)
       case tp: AnnotatedType =>
         samClass(tp.underlying)
+      case tp: FlexibleType =>
+        samClass(tp.superType)
       case _ =>
         NoSymbol
 
@@ -6071,6 +6129,8 @@ object Types extends TypeUtils {
       tp.derivedJavaArrayType(elemtp)
     protected def derivedExprType(tp: ExprType, restpe: Type): Type =
       tp.derivedExprType(restpe)
+    protected def derivedFlexibleType(tp: FlexibleType, under: Type): Type =
+      tp.derivedFlexibleType(under)
     // note: currying needed  because Scala2 does not support param-dependencies
     protected def derivedLambdaType(tp: LambdaType)(formals: List[tp.PInfo], restpe: Type): Type =
       tp.derivedLambdaType(tp.paramNames, formals, restpe)
@@ -6194,6 +6254,9 @@ object Types extends TypeUtils {
         case tp: OrType =>
           derivedOrType(tp, this(tp.tp1), this(tp.tp2))
 
+        case tp: FlexibleType =>
+          derivedFlexibleType(tp, this(tp.underlying))
+
         case tp: MatchType =>
           val bound1 = this(tp.bound)
           val scrut1 = atVariance(0)(this(tp.scrutinee))
@@ -6481,6 +6544,14 @@ object Types extends TypeUtils {
           if (underlying.isExactlyNothing) underlying
           else tp.derivedAnnotatedType(underlying, annot)
       }
+    override protected def derivedFlexibleType(tp: FlexibleType, underlying: Type): Type =
+      underlying match {
+        case Range(lo, hi) =>
+          range(tp.derivedFlexibleType(lo), tp.derivedFlexibleType(hi))
+        case _ =>
+          if (underlying.isExactlyNothing) underlying
+          else tp.derivedFlexibleType(underlying)
+      }
     override protected def derivedCapturingType(tp: Type, parent: Type, refs: CaptureSet): Type =
       parent match // TODO ^^^ handle ranges in capture sets as well
         case Range(lo, hi) =>
@@ -6610,6 +6681,9 @@ object Types extends TypeUtils {
       case tp: TypeVar =>
         this(x, tp.underlying)
 
+      case tp: FlexibleType =>
+        this(x, tp.underlying)
+
       case ExprType(restpe) =>
         this(x, restpe)
 

compiler/src/dotty/tools/dotc/core/tasty/TreePickler.scala

@@ -272,6 +272,9 @@ class TreePickler(pickler: TastyPickler, attributes: Attributes) {
     case tpe: OrType =>
       writeByte(ORtype)
       withLength { pickleType(tpe.tp1, richTypes); pickleType(tpe.tp2, richTypes) }
+    case tpe: FlexibleType =>
+      writeByte(FLEXIBLEtype)
+      withLength { pickleType(tpe.underlying, richTypes)  }
     case tpe: ExprType =>
       writeByte(BYNAMEtype)
       pickleType(tpe.underlying)

compiler/src/dotty/tools/dotc/core/tasty/TreeUnpickler.scala

@@ -444,6 +444,8 @@ class TreeUnpickler(reader: TastyReader,
               readTypeRef() match {
                 case binder: LambdaType => binder.paramRefs(readNat())
               }
+            case FLEXIBLEtype =>
+              FlexibleType(readType())
           }
         assert(currentAddr == end, s"$start $currentAddr $end ${astTagToString(tag)}")
         result

compiler/src/dotty/tools/dotc/printing/PlainPrinter.scala

@@ -294,6 +294,8 @@ class PlainPrinter(_ctx: Context) extends Printer {
             && !printDebug
         then atPrec(GlobalPrec)( Str("into ") ~ toText(tpe) )
         else toTextLocal(tpe) ~ " " ~ toText(annot)
+      case FlexibleType(tpe) =>
+        "(" ~ toText(tpe) ~ ")?"
       case tp: TypeVar =>
         def toTextCaret(tp: Type) = if printDebug then toTextLocal(tp) ~ Str("^") else toText(tp)
         if (tp.isInstantiated)