Fix cycle detection for type aliases with wildcard arguments

compiler/src/dotty/tools/dotc/core/SymDenotations.scala

@@ -658,6 +658,9 @@ object SymDenotations {
       containsOpaques ||
       is(Module, butNot = Package) && owner.seesOpaques
 
+    def isProvisional(using Context): Boolean =
+      flagsUNSAFE.is(Provisional) // do not force the info to check the flag
+
     /** Is this the denotation of a self symbol of some class?
      *  This is the case if one of two conditions holds:
      *  1. It is the symbol referred to in the selfInfo part of the ClassInfo

compiler/src/dotty/tools/dotc/core/TypeApplications.scala

@@ -9,7 +9,7 @@ import SymDenotations.LazyType
 import Decorators._
 import util.Stats._
 import Names._
-import Flags.Module
+import Flags.{Module, Provisional}
 import dotty.tools.dotc.config.Config
 
 object TypeApplications {
@@ -284,7 +284,8 @@ class TypeApplications(val self: Type) extends AnyVal {
 
   /** Dealias type if it can be done without forcing the TypeRef's info */
   def safeDealias(using Context): Type = self match {
-    case self: TypeRef if self.denot.exists && self.symbol.isAliasType =>
+    case self: TypeRef
+    if self.denot.exists && self.symbol.isAliasType && !self.symbol.isProvisional =>
       self.superType.stripTypeVar.safeDealias
     case _ =>
       self

compiler/src/dotty/tools/dotc/core/Types.scala

@@ -118,7 +118,7 @@ object Types {
             case t: TypeRef =>
               !t.currentSymbol.isStatic && {
                 (t: Type).mightBeProvisional = false // break cycles
-                t.symbol.flagsUNSAFE.is(Provisional)
+                t.symbol.isProvisional
                 || test(t.prefix, theAcc)
                 || t.denot.infoOrCompleter.match
                     case info: LazyType => true

compiler/src/dotty/tools/dotc/typer/Typer.scala

@@ -2027,12 +2027,19 @@ class Typer(@constructorOnly nestingLevel: Int = 0) extends Namer
       var checkedArgs = preCheckKinds(args1, paramBounds)
         // check that arguments conform to bounds is done in phase PostTyper
       val tycon = tpt1.symbol
-      if (tycon == defn.andType)
-        checkedArgs = checkedArgs.mapconserve(arg =>
-          checkSimpleKinded(checkNoWildcard(arg)))
-      else if (tycon == defn.orType)
+      if tycon == defn.andType || tycon == defn.orType then
         checkedArgs = checkedArgs.mapconserve(arg =>
           checkSimpleKinded(checkNoWildcard(arg)))
+      else if tycon.isProvisional then
+        // A type with Provisional flag is either an alias or abstract type.
+        // If it is an alias type, it would mean the type is cyclic
+        // If it is an abstract type, it would mean the type is an irreducible
+        // application of a higher-kinded type to a wildcard argument.
+        // Either way, the wildcard argument is illegal. The early test of
+        // `checkNoWildcard` here is needed, so that we do not accidentally reduce
+        // an application of a Provisional type away, which would mean that the type constructor
+        // is no longer present on the right hand side. See neg/i15507.scala.
+        checkedArgs = checkedArgs.mapconserve(checkNoWildcard)
       else if tycon == defn.throwsAlias
           && checkedArgs.length == 2
           && checkedArgs(1).tpe.derivesFrom(defn.RuntimeExceptionClass)

tests/neg/i15507.scala

@@ -0,0 +1,15 @@
+object TestNested:
+  type _NestedSet1[X] = Set[_NestedSet1[?]] // error // error
+  type _NestedSet2[X] <: Set[_NestedSet2[?]] // error
+  type _NestedSet3[X] <: Set[_NestedSet3[X]] // ok
+  type _NestedSet4[X] >: Set[_NestedSet4[X]] // error
+  type _NestedSet5[X] = Set[_NestedSet5[X]] // error
+  type _NestedSet6[X] = Set[_NestedSet6[Int]] // error
+
+  type _NestedList1[X] = List[_NestedList1[?]] // error // error
+  type _NestedList2[X] <: List[_NestedList2[?]] // error
+  type _NestedList3[X] <: List[_NestedList3[X]] // ok
+  type _NestedList4[X] >: List[_NestedList4[X]] // error
+  type _NestedList5[X] = List[_NestedList5[X]] // error
+  type _NestedList6[X] = List[_NestedList6[Int]] // error
+