cmd/compile/internal/types2: ensure named types are expanded after type-checking

This is a clean port of CL 356490 from go/types to types2.

Fixes #48703.
Fixes #48974.

Change-Id: I08c0db0b92250cbb043325541b21a577726b40ca
Reviewed-on: https://go-review.googlesource.com/c/go/+/356515
Trust: Robert Griesemer <[email protected]>
Reviewed-by: Robert Findley <[email protected]>

Author: griesemer

src/cmd/compile/internal/types2/check.go

@@ -132,6 +132,7 @@ type Checker struct {
 	untyped  map[syntax.Expr]exprInfo // map of expressions without final type
 	delayed  []action                 // stack of delayed action segments; segments are processed in FIFO order
 	objPath  []Object                 // path of object dependencies during type inference (for cycle reporting)
+	defTypes []*Named                 // defined types created during type checking, for final validation.
 
 	// context within which the current object is type-checked
 	// (valid only for the duration of type-checking a specific object)
@@ -302,6 +303,9 @@ func (check *Checker) checkFiles(files []*syntax.File) (err error) {
 	print("== processDelayed ==")
 	check.processDelayed(0) // incl. all functions
 
+	print("== expandDefTypes ==")
+	check.expandDefTypes()
+
 	print("== initOrder ==")
 	check.initOrder()
 
@@ -321,6 +325,7 @@ func (check *Checker) checkFiles(files []*syntax.File) (err error) {
 	check.pkgPathMap = nil
 	check.seenPkgMap = nil
 	check.recvTParamMap = nil
+	check.defTypes = nil
 
 	// TODO(gri) There's more memory we should release at this point.
 
@@ -347,6 +352,29 @@ func (check *Checker) processDelayed(top int) {
 	check.delayed = check.delayed[:top]
 }
 
+func (check *Checker) expandDefTypes() {
+	// Ensure that every defined type created in the course of type-checking has
+	// either non-*Named underlying, or is unresolved.
+	//
+	// This guarantees that we don't leak any types whose underlying is *Named,
+	// because any unresolved instances will lazily compute their underlying by
+	// substituting in the underlying of their origin. The origin must have
+	// either been imported or type-checked and expanded here, and in either case
+	// its underlying will be fully expanded.
+	for i := 0; i < len(check.defTypes); i++ {
+		n := check.defTypes[i]
+		switch n.underlying.(type) {
+		case nil:
+			if n.resolver == nil {
+				panic("nil underlying")
+			}
+		case *Named:
+			n.under() // n.under may add entries to check.defTypes
+		}
+		n.check = nil
+	}
+}
+
 func (check *Checker) record(x *operand) {
 	// convert x into a user-friendly set of values
 	// TODO(gri) this code can be simplified

src/cmd/compile/internal/types2/named.go

@@ -65,22 +65,9 @@ func (check *Checker) newNamed(obj *TypeName, orig *Named, underlying Type, tpar
 	if obj.typ == nil {
 		obj.typ = typ
 	}
-	// Ensure that typ is always expanded, at which point the check field can be
-	// nilled out.
-	//
-	// Note that currently we cannot nil out check inside typ.under(), because
-	// it's possible that typ is expanded multiple times.
-	//
-	// TODO(gri): clean this up so that under is the only function mutating
-	//            named types.
+	// Ensure that typ is always expanded and sanity-checked.
 	if check != nil {
-		check.later(func() {
-			switch typ.under().(type) {
-			case *Named:
-				panic("unexpanded underlying type")
-			}
-			typ.check = nil
-		})
+		check.defTypes = append(check.defTypes, typ)
 	}
 	return typ
 }
@@ -239,6 +226,12 @@ func expandNamed(ctxt *Context, n *Named, instPos syntax.Pos) (tparams *TypePara
 
 	check := n.check
 
+	if _, unexpanded := n.orig.underlying.(*Named); unexpanded {
+		// We should only get an unexpanded underlying here during type checking
+		// (for example, in recursive type declarations).
+		assert(check != nil)
+	}
+
 	// Mismatching arg and tparam length may be checked elsewhere.
 	if n.orig.tparams.Len() == n.targs.Len() {
 		// We must always have a context, to avoid infinite recursion.

src/cmd/compile/internal/types2/testdata/fixedbugs/issue48703.go2

@@ -0,0 +1,27 @@
+// Copyright 2021 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package p
+
+import "unsafe"
+
+// The actual example from the issue.
+type List[P any] struct{}
+
+func (_ List[P]) m() (_ List[List[P]]) { return }
+
+// Other types of recursion through methods.
+type R[P any] int
+
+func (*R[R /* ERROR must be an identifier */ [int]]) m0() {}
+func (R[P]) m1(R[R[P]])                                   {}
+func (R[P]) m2(R[*P])                                     {}
+func (R[P]) m3([unsafe.Sizeof(new(R[P]))]int)             {}
+func (R[P]) m4([unsafe.Sizeof(new(R[R[P]]))]int)          {}
+
+// Mutual recursion
+type M[P any] int
+
+func (R[P]) m5(M[M[P]]) {}
+func (M[P]) m(R[R[P]])  {}

src/cmd/compile/internal/types2/testdata/fixedbugs/issue48974.go2

@@ -0,0 +1,22 @@
+// Copyright 2021 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package p
+
+type Fooer interface {
+	Foo()
+}
+
+type Fooable[F Fooer] struct {
+	ptr F
+}
+
+func (f *Fooable[F]) Adapter() *Fooable[*FooerImpl[F]] {
+	return &Fooable[*FooerImpl[F]]{&FooerImpl[F]{}}
+}
+
+type FooerImpl[F Fooer] struct {
+}
+
+func (fi *FooerImpl[F]) Foo() {}