go/internal/gcimporter: add support for the Go 1.18 export data version

Add support for accepting the Go 1.18 export data version (2) to the
x/tools gcimporter, while preserving support for generic code at version
1.

For now, the exporter still outputs version 1.

For golang/go#49040

Change-Id: Ic4547e385ced72b88212d150bf16acaf200a010e
Reviewed-on: https://go-review.googlesource.com/c/tools/+/358034
Trust: Robert Findley <[email protected]>
Trust: Dan Scales <[email protected]>
Run-TryBot: Robert Findley <[email protected]>
gopls-CI: kokoro <[email protected]>
TryBot-Result: Go Bot <[email protected]>
Reviewed-by: Dan Scales <[email protected]>
Reviewed-by: Robert Griesemer <[email protected]>

Author: findleyr

go/internal/gcimporter/iexport.go

@@ -35,15 +35,15 @@ const bundleVersion = 0
 // The package path of the top-level package will not be recorded,
 // so that calls to IImportData can override with a provided package path.
 func IExportData(out io.Writer, fset *token.FileSet, pkg *types.Package) error {
-	return iexportCommon(out, fset, false, []*types.Package{pkg})
+	return iexportCommon(out, fset, false, iexportVersion, []*types.Package{pkg})
 }
 
 // IExportBundle writes an indexed export bundle for pkgs to out.
 func IExportBundle(out io.Writer, fset *token.FileSet, pkgs []*types.Package) error {
-	return iexportCommon(out, fset, true, pkgs)
+	return iexportCommon(out, fset, true, iexportVersion, pkgs)
 }
 
-func iexportCommon(out io.Writer, fset *token.FileSet, bundle bool, pkgs []*types.Package) (err error) {
+func iexportCommon(out io.Writer, fset *token.FileSet, bundle bool, version int, pkgs []*types.Package) (err error) {
 	if !debug {
 		defer func() {
 			if e := recover(); e != nil {
@@ -59,6 +59,7 @@ func iexportCommon(out io.Writer, fset *token.FileSet, bundle bool, pkgs []*type
 
 	p := iexporter{
 		fset:        fset,
+		version:     version,
 		allPkgs:     map[*types.Package]bool{},
 		stringIndex: map[string]uint64{},
 		declIndex:   map[types.Object]uint64{},
@@ -122,7 +123,7 @@ func iexportCommon(out io.Writer, fset *token.FileSet, bundle bool, pkgs []*type
 	if bundle {
 		hdr.uint64(bundleVersion)
 	}
-	hdr.uint64(iexportVersion)
+	hdr.uint64(uint64(p.version))
 	hdr.uint64(uint64(p.strings.Len()))
 	hdr.uint64(dataLen)
 
@@ -200,8 +201,9 @@ func (p *iexporter) indexName(obj types.Object) (res string) {
 }
 
 type iexporter struct {
-	fset *token.FileSet
-	out  *bytes.Buffer
+	fset    *token.FileSet
+	out     *bytes.Buffer
+	version int
 
 	localpkg *types.Package
 
@@ -224,7 +226,7 @@ type iexporter struct {
 }
 
 func (p *iexporter) trace(format string, args ...interface{}) {
-	if !debug {
+	if !trace {
 		// Call sites should also be guarded, but having this check here allows
 		// easily enabling/disabling debug trace statements.
 		return
@@ -330,7 +332,15 @@ func (p *iexporter) doDecl(obj types.Object) {
 		if tparam, ok := t.(*typeparams.TypeParam); ok {
 			w.tag('P')
 			w.pos(obj.Pos())
-			w.typ(tparam.Constraint(), obj.Pkg())
+			constraint := tparam.Constraint()
+			if p.version >= iexportVersionGo1_18 {
+				implicit := false
+				if iface, _ := constraint.(*types.Interface); iface != nil {
+					implicit = typeparams.IsImplicit(iface)
+				}
+				w.bool(implicit)
+			}
+			w.typ(constraint, obj.Pkg())
 			break
 		}
 
@@ -397,7 +407,7 @@ func (w *exportWriter) tag(tag byte) {
 }
 
 func (w *exportWriter) pos(pos token.Pos) {
-	if iexportVersion >= iexportVersionPosCol {
+	if w.p.version >= iexportVersionPosCol {
 		w.posV1(pos)
 	} else {
 		w.posV0(pos)
@@ -484,7 +494,6 @@ func (w *exportWriter) qualifiedIdent(obj types.Object) {
 
 	// Ensure any referenced declarations are written out too.
 	w.p.pushDecl(obj)
-	w.p.trace("writing ident %s for %s", name, obj)
 	w.string(name)
 	w.pkg(obj.Pkg())
 }

go/internal/gcimporter/iexport_common_test.go

@@ -0,0 +1,16 @@
+// Copyright 2021 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package gcimporter
+
+// Temporarily expose version-related functionality so that we can test at
+// specific export data versions.
+
+var IExportCommon = iexportCommon
+
+const (
+	IExportVersion         = iexportVersion
+	IExportVersionGenerics = iexportVersionGenerics
+	IExportVersionGo1_18   = iexportVersionGo1_18
+)

go/internal/gcimporter/iexport_go118_test.go

@@ -50,26 +50,39 @@ func testExportSrc(t *testing.T, src []byte) {
 		t.Skipf("gc-built packages not available (compiler = %s)", runtime.Compiler)
 	}
 
-	fset := token.NewFileSet()
-	f, err := parser.ParseFile(fset, "g.go", src, 0)
-	if err != nil {
-		t.Fatal(err)
-	}
-	conf := types.Config{
-		Importer: importer.Default(),
-	}
-	pkg, err := conf.Check("", fset, []*ast.File{f}, nil)
-	if err != nil {
-		t.Fatal(err)
+	// Test at both stages of the 1.18 export data format change.
+	tests := []struct {
+		name    string
+		version int
+	}{
+		{"legacy generics", gcimporter.IExportVersionGenerics},
+		{"go1.18", gcimporter.IExportVersionGo1_18},
 	}
 
-	// export
-	var b bytes.Buffer
-	if err := gcimporter.IExportData(&b, fset, pkg); err != nil {
-		t.Fatal(err)
-	}
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			fset := token.NewFileSet()
+			f, err := parser.ParseFile(fset, "g.go", src, 0)
+			if err != nil {
+				t.Fatal(err)
+			}
+			conf := types.Config{
+				Importer: importer.Default(),
+			}
+			pkg, err := conf.Check("", fset, []*ast.File{f}, nil)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			// export
+			data, err := iexport(fset, test.version, pkg)
+			if err != nil {
+				t.Fatal(err)
+			}
 
-	testPkgData(t, fset, pkg, b.Bytes())
+			testPkgData(t, fset, test.version, pkg, data)
+		})
+	}
 }
 
 func TestImportTypeparamTests(t *testing.T) {

go/internal/gcimporter/iexport_test.go

@@ -55,9 +55,9 @@ func readExportFile(filename string) ([]byte, error) {
 	return ioutil.ReadAll(buf)
 }
 
-func iexport(fset *token.FileSet, pkg *types.Package) ([]byte, error) {
+func iexport(fset *token.FileSet, version int, pkg *types.Package) ([]byte, error) {
 	var buf bytes.Buffer
-	if err := gcimporter.IExportData(&buf, fset, pkg); err != nil {
+	if err := gcimporter.IExportCommon(&buf, fset, false, version, []*types.Package{pkg}); err != nil {
 		return nil, err
 	}
 	return buf.Bytes(), nil
@@ -132,11 +132,12 @@ type UnknownType undefined
 		return sorted[i].Path() < sorted[j].Path()
 	})
 
+	version := gcimporter.IExportVersion
 	for _, pkg := range sorted {
-		if exportdata, err := iexport(conf.Fset, pkg); err != nil {
+		if exportdata, err := iexport(conf.Fset, version, pkg); err != nil {
 			t.Error(err)
 		} else {
-			testPkgData(t, conf.Fset, pkg, exportdata)
+			testPkgData(t, conf.Fset, version, pkg, exportdata)
 		}
 
 		if pkg.Name() == "main" || pkg.Name() == "haserrors" {
@@ -146,7 +147,7 @@ type UnknownType undefined
 		} else if exportdata, err := readExportFile(bp.PkgObj); err != nil {
 			t.Log("warning:", err)
 		} else {
-			testPkgData(t, conf.Fset, pkg, exportdata)
+			testPkgData(t, conf.Fset, version, pkg, exportdata)
 		}
 	}
 
@@ -162,23 +163,23 @@ type UnknownType undefined
 	}
 
 	for i, pkg := range sorted {
-		testPkg(t, conf.Fset, pkg, fset2, pkgs2[i])
+		testPkg(t, conf.Fset, version, pkg, fset2, pkgs2[i])
 	}
 }
 
-func testPkgData(t *testing.T, fset *token.FileSet, pkg *types.Package, exportdata []byte) {
+func testPkgData(t *testing.T, fset *token.FileSet, version int, pkg *types.Package, exportdata []byte) {
 	imports := make(map[string]*types.Package)
 	fset2 := token.NewFileSet()
 	_, pkg2, err := gcimporter.IImportData(fset2, imports, exportdata, pkg.Path())
 	if err != nil {
 		t.Errorf("IImportData(%s): %v", pkg.Path(), err)
 	}
 
-	testPkg(t, fset, pkg, fset2, pkg2)
+	testPkg(t, fset, version, pkg, fset2, pkg2)
 }
 
-func testPkg(t *testing.T, fset *token.FileSet, pkg *types.Package, fset2 *token.FileSet, pkg2 *types.Package) {
-	if _, err := iexport(fset2, pkg2); err != nil {
+func testPkg(t *testing.T, fset *token.FileSet, version int, pkg *types.Package, fset2 *token.FileSet, pkg2 *types.Package) {
+	if _, err := iexport(fset2, version, pkg2); err != nil {
 		t.Errorf("reexport %q: %v", pkg.Path(), err)
 	}
 
@@ -226,7 +227,7 @@ func TestIExportData_long(t *testing.T) {
 	}
 
 	// export
-	exportdata, err := iexport(fset1, pkg)
+	exportdata, err := iexport(fset1, gcimporter.IExportVersion, pkg)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -269,7 +270,7 @@ func TestIExportData_typealiases(t *testing.T) {
 
 	// export
 	// use a nil fileset here to confirm that it doesn't panic
-	exportdata, err := iexport(nil, pkg1)
+	exportdata, err := iexport(nil, gcimporter.IExportVersion, pkg1)
 	if err != nil {
 		t.Fatal(err)
 	}

go/internal/gcimporter/iimport.go

@@ -45,13 +45,20 @@ func (r *intReader) uint64() uint64 {
 }
 
 // Keep this in sync with constants in iexport.go.
+//
+// Temporarily, the x/tools importer accepts generic code at both version 1 and
+// 2. However, version 2 contains some breaking changes on top of version 1:
+//   - the 'implicit' bit is added to exported constraints
+//   - a 'kind' byte is added to constant values (not yet done)
+//
+// Once we've completed the bump to version 2 in the standard library, we'll
+// remove support for generics here at version 1.
 const (
 	iexportVersionGo1_11 = 0
 	iexportVersionPosCol = 1
+	iexportVersionGo1_18 = 2
 	// TODO: before release, change this back to 2.
 	iexportVersionGenerics = iexportVersionPosCol
-
-	iexportVersionCurrent = iexportVersionGenerics
 )
 
 type ident struct {
@@ -124,9 +131,9 @@ func iimportCommon(fset *token.FileSet, imports map[string]*types.Package, data
 
 	version = int64(r.uint64())
 	switch version {
-	case /* iexportVersionGenerics, */ iexportVersionPosCol, iexportVersionGo1_11:
+	case iexportVersionGo1_18, iexportVersionPosCol, iexportVersionGo1_11:
 	default:
-		if version > iexportVersionGenerics {
+		if version > iexportVersionGo1_18 {
 			errorf("unstable iexport format version %d, just rebuild compiler and std library", version)
 		} else {
 			errorf("unknown iexport format version %d", version)
@@ -437,8 +444,19 @@ func (r *importReader) obj(name string) {
 		// bound, save the partial type in tparamIndex before reading the bounds.
 		id := ident{r.currPkg.Name(), name}
 		r.p.tparamIndex[id] = t
-
-		typeparams.SetTypeParamConstraint(t, r.typ())
+		var implicit bool
+		if r.p.exportVersion >= iexportVersionGo1_18 {
+			implicit = r.bool()
+		}
+		constraint := r.typ()
+		if implicit {
+			iface, _ := constraint.(*types.Interface)
+			if iface == nil {
+				errorf("non-interface constraint marked implicit")
+			}
+			typeparams.MarkImplicit(iface)
+		}
+		typeparams.SetTypeParamConstraint(t, constraint)
 
 	case 'V':
 		typ := r.typ()

internal/typeparams/typeparams_go117.go

@@ -137,6 +137,10 @@ func IsImplicit(*types.Interface) bool {
 	return false
 }
 
+// MarkImplicit does nothing, because this Go version does not have implicit
+// interfaces.
+func MarkImplicit(*types.Interface) {}
+
 // ForNamed returns an empty type parameter list, as type parameters are not
 // supported at this Go version.
 func ForNamed(*types.Named) *TypeParamList {

internal/typeparams/typeparams_go118.go

@@ -130,6 +130,11 @@ func IsImplicit(iface *types.Interface) bool {
 	return iface.IsImplicit()
 }
 
+// MarkImplicit calls iface.MarkImplicit().
+func MarkImplicit(iface *types.Interface) {
+	iface.MarkImplicit()
+}
+
 // ForNamed extracts the (possibly empty) type parameter object list from
 // named.
 func ForNamed(named *types.Named) *TypeParamList {