internal/gcimporter: API for shallow export data

This change adds an internal API for marshalling and
unmarshalling a types.Package to "shallow" export data,
which does not index packages other than the main one.
The import function accepts a function that loads symbols
on demand (e.g. by recursively reading export data for
indirect dependencies).

The CL includes a test that the entire standard
library can be type-checked using shallow data.

Also:
- break dependency on go/ast.
- narrow the name and type of qualifiedObject.
- add (test) dependency on errgroup, and tidy go.mod.

Change-Id: I92d31efd343cf5dd6fca6d7b918a23749e2d1e83
Reviewed-on: https://go-review.googlesource.com/c/tools/+/447737
Run-TryBot: Alan Donovan <[email protected]>
Reviewed-by: Matthew Dempsky <[email protected]>
TryBot-Result: Gopher Robot <[email protected]>
gopls-CI: kokoro <[email protected]>
Reviewed-by: Robert Findley <[email protected]>

Author: adonovan

go.mod

@@ -8,3 +8,5 @@ require (
 	golang.org/x/net v0.1.0
 	golang.org/x/sys v0.1.0
 )
+
+require golang.org/x/sync v0.1.0

go.sum

@@ -13,6 +13,8 @@ golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0=
 golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

gopls/go.mod

@@ -8,7 +8,7 @@ require (
 	github.com/jba/templatecheck v0.6.0
 	github.com/sergi/go-diff v1.1.0
 	golang.org/x/mod v0.6.0
-	golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4
+	golang.org/x/sync v0.1.0
 	golang.org/x/sys v0.1.0
 	golang.org/x/text v0.4.0
 	golang.org/x/tools v0.1.13-0.20220928184430-f80e98464e27

gopls/go.sum

@@ -55,8 +55,9 @@ golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

internal/gcimporter/bexport.go

@@ -12,7 +12,6 @@ import (
 	"bytes"
 	"encoding/binary"
 	"fmt"
-	"go/ast"
 	"go/constant"
 	"go/token"
 	"go/types"
@@ -145,7 +144,7 @@ func BExportData(fset *token.FileSet, pkg *types.Package) (b []byte, err error)
 	objcount := 0
 	scope := pkg.Scope()
 	for _, name := range scope.Names() {
-		if !ast.IsExported(name) {
+		if !token.IsExported(name) {
 			continue
 		}
 		if trace {
@@ -482,7 +481,7 @@ func (p *exporter) method(m *types.Func) {
 
 	p.pos(m)
 	p.string(m.Name())
-	if m.Name() != "_" && !ast.IsExported(m.Name()) {
+	if m.Name() != "_" && !token.IsExported(m.Name()) {
 		p.pkg(m.Pkg(), false)
 	}
 
@@ -501,7 +500,7 @@ func (p *exporter) fieldName(f *types.Var) {
 		// 3) field name doesn't match base type name (alias name)
 		bname := basetypeName(f.Type())
 		if name == bname {
-			if ast.IsExported(name) {
+			if token.IsExported(name) {
 				name = "" // 1) we don't need to know the field name or package
 			} else {
 				name = "?" // 2) use unexported name "?" to force package export
@@ -514,7 +513,7 @@ func (p *exporter) fieldName(f *types.Var) {
 	}
 
 	p.string(name)
-	if name != "" && !ast.IsExported(name) {
+	if name != "" && !token.IsExported(name) {
 		p.pkg(f.Pkg(), false)
 	}
 }

internal/gcimporter/bexport_test.go

@@ -109,7 +109,7 @@ type UnknownType undefined
 
 		// Compare the packages' corresponding members.
 		for _, name := range pkg.Scope().Names() {
-			if !ast.IsExported(name) {
+			if !token.IsExported(name) {
 				continue
 			}
 			obj1 := pkg.Scope().Lookup(name)

internal/gcimporter/iexport.go

@@ -12,7 +12,6 @@ import (
 	"bytes"
 	"encoding/binary"
 	"fmt"
-	"go/ast"
 	"go/constant"
 	"go/token"
 	"go/types"
@@ -26,6 +25,41 @@ import (
 	"golang.org/x/tools/internal/typeparams"
 )
 
+// IExportShallow encodes "shallow" export data for the specified package.
+//
+// No promises are made about the encoding other than that it can be
+// decoded by the same version of IIExportShallow. If you plan to save
+// export data in the file system, be sure to include a cryptographic
+// digest of the executable in the key to avoid version skew.
+func IExportShallow(fset *token.FileSet, pkg *types.Package) ([]byte, error) {
+	// In principle this operation can only fail if out.Write fails,
+	// but that's impossible for bytes.Buffer---and as a matter of
+	// fact iexportCommon doesn't even check for I/O errors.
+	// TODO(adonovan): handle I/O errors properly.
+	// TODO(adonovan): use byte slices throughout, avoiding copying.
+	const bundle, shallow = false, true
+	var out bytes.Buffer
+	err := iexportCommon(&out, fset, bundle, shallow, iexportVersion, []*types.Package{pkg})
+	return out.Bytes(), err
+}
+
+// IImportShallow decodes "shallow" types.Package data encoded by IExportShallow
+// in the same executable. This function cannot import data from
+// cmd/compile or gcexportdata.Write.
+func IImportShallow(fset *token.FileSet, imports map[string]*types.Package, data []byte, path string, insert InsertType) (*types.Package, error) {
+	const bundle = false
+	pkgs, err := iimportCommon(fset, imports, data, bundle, path, insert)
+	if err != nil {
+		return nil, err
+	}
+	return pkgs[0], nil
+}
+
+// InsertType is the type of a function that creates a types.TypeName
+// object for a named type and inserts it into the scope of the
+// specified Package.
+type InsertType = func(pkg *types.Package, name string)
+
 // Current bundled export format version. Increase with each format change.
 // 0: initial implementation
 const bundleVersion = 0
@@ -36,15 +70,17 @@ const bundleVersion = 0
 // The package path of the top-level package will not be recorded,
 // so that calls to IImportData can override with a provided package path.
 func IExportData(out io.Writer, fset *token.FileSet, pkg *types.Package) error {
-	return iexportCommon(out, fset, false, iexportVersion, []*types.Package{pkg})
+	const bundle, shallow = false, false
+	return iexportCommon(out, fset, bundle, shallow, iexportVersion, []*types.Package{pkg})
 }
 
 // IExportBundle writes an indexed export bundle for pkgs to out.
 func IExportBundle(out io.Writer, fset *token.FileSet, pkgs []*types.Package) error {
-	return iexportCommon(out, fset, true, iexportVersion, pkgs)
+	const bundle, shallow = true, false
+	return iexportCommon(out, fset, bundle, shallow, iexportVersion, pkgs)
 }
 
-func iexportCommon(out io.Writer, fset *token.FileSet, bundle bool, version int, pkgs []*types.Package) (err error) {
+func iexportCommon(out io.Writer, fset *token.FileSet, bundle, shallow bool, version int, pkgs []*types.Package) (err error) {
 	if !debug {
 		defer func() {
 			if e := recover(); e != nil {
@@ -61,6 +97,7 @@ func iexportCommon(out io.Writer, fset *token.FileSet, bundle bool, version int,
 	p := iexporter{
 		fset:        fset,
 		version:     version,
+		shallow:     shallow,
 		allPkgs:     map[*types.Package]bool{},
 		stringIndex: map[string]uint64{},
 		declIndex:   map[types.Object]uint64{},
@@ -82,7 +119,7 @@ func iexportCommon(out io.Writer, fset *token.FileSet, bundle bool, version int,
 	for _, pkg := range pkgs {
 		scope := pkg.Scope()
 		for _, name := range scope.Names() {
-			if ast.IsExported(name) {
+			if token.IsExported(name) {
 				p.pushDecl(scope.Lookup(name))
 			}
 		}
@@ -205,7 +242,8 @@ type iexporter struct {
 	out     *bytes.Buffer
 	version int
 
-	localpkg *types.Package
+	shallow  bool           // don't put types from other packages in the index
+	localpkg *types.Package // (nil in bundle mode)
 
 	// allPkgs tracks all packages that have been referenced by
 	// the export data, so we can ensure to include them in the
@@ -256,6 +294,11 @@ func (p *iexporter) pushDecl(obj types.Object) {
 		panic("cannot export package unsafe")
 	}
 
+	// Shallow export data: don't index decls from other packages.
+	if p.shallow && obj.Pkg() != p.localpkg {
+		return
+	}
+
 	if _, ok := p.declIndex[obj]; ok {
 		return
 	}
@@ -497,7 +540,7 @@ func (w *exportWriter) pkg(pkg *types.Package) {
 	w.string(w.exportPath(pkg))
 }
 
-func (w *exportWriter) qualifiedIdent(obj types.Object) {
+func (w *exportWriter) qualifiedType(obj *types.TypeName) {
 	name := w.p.exportName(obj)
 
 	// Ensure any referenced declarations are written out too.
@@ -556,11 +599,11 @@ func (w *exportWriter) doTyp(t types.Type, pkg *types.Package) {
 			return
 		}
 		w.startType(definedType)
-		w.qualifiedIdent(t.Obj())
+		w.qualifiedType(t.Obj())
 
 	case *typeparams.TypeParam:
 		w.startType(typeParamType)
-		w.qualifiedIdent(t.Obj())
+		w.qualifiedType(t.Obj())
 
 	case *types.Pointer:
 		w.startType(pointerType)

internal/gcimporter/iexport_test.go

@@ -59,7 +59,8 @@ func readExportFile(filename string) ([]byte, error) {
 
 func iexport(fset *token.FileSet, version int, pkg *types.Package) ([]byte, error) {
 	var buf bytes.Buffer
-	if err := gcimporter.IExportCommon(&buf, fset, false, version, []*types.Package{pkg}); err != nil {
+	const bundle, shallow = false, false
+	if err := gcimporter.IExportCommon(&buf, fset, bundle, shallow, version, []*types.Package{pkg}); err != nil {
 		return nil, err
 	}
 	return buf.Bytes(), nil
@@ -197,7 +198,7 @@ func testPkg(t *testing.T, fset *token.FileSet, version int, pkg *types.Package,
 
 	// Compare the packages' corresponding members.
 	for _, name := range pkg.Scope().Names() {
-		if !ast.IsExported(name) {
+		if !token.IsExported(name) {
 			continue
 		}
 		obj1 := pkg.Scope().Lookup(name)

internal/gcimporter/iimport.go

@@ -85,7 +85,7 @@ const (
 // If the export data version is not recognized or the format is otherwise
 // compromised, an error is returned.
 func IImportData(fset *token.FileSet, imports map[string]*types.Package, data []byte, path string) (int, *types.Package, error) {
-	pkgs, err := iimportCommon(fset, imports, data, false, path)
+	pkgs, err := iimportCommon(fset, imports, data, false, path, nil)
 	if err != nil {
 		return 0, nil, err
 	}
@@ -94,10 +94,10 @@ func IImportData(fset *token.FileSet, imports map[string]*types.Package, data []
 
 // IImportBundle imports a set of packages from the serialized package bundle.
 func IImportBundle(fset *token.FileSet, imports map[string]*types.Package, data []byte) ([]*types.Package, error) {
-	return iimportCommon(fset, imports, data, true, "")
+	return iimportCommon(fset, imports, data, true, "", nil)
 }
 
-func iimportCommon(fset *token.FileSet, imports map[string]*types.Package, data []byte, bundle bool, path string) (pkgs []*types.Package, err error) {
+func iimportCommon(fset *token.FileSet, imports map[string]*types.Package, data []byte, bundle bool, path string, insert InsertType) (pkgs []*types.Package, err error) {
 	const currentVersion = iexportVersionCurrent
 	version := int64(-1)
 	if !debug {
@@ -147,6 +147,7 @@ func iimportCommon(fset *token.FileSet, imports map[string]*types.Package, data
 	p := iimporter{
 		version: int(version),
 		ipath:   path,
+		insert:  insert,
 
 		stringData:  stringData,
 		stringCache: make(map[uint64]string),
@@ -187,11 +188,18 @@ func iimportCommon(fset *token.FileSet, imports map[string]*types.Package, data
 		} else if pkg.Name() != pkgName {
 			errorf("conflicting names %s and %s for package %q", pkg.Name(), pkgName, path)
 		}
+		if i == 0 && !bundle {
+			p.localpkg = pkg
+		}
 
 		p.pkgCache[pkgPathOff] = pkg
 
+		// Read index for package.
 		nameIndex := make(map[string]uint64)
-		for nSyms := r.uint64(); nSyms > 0; nSyms-- {
+		nSyms := r.uint64()
+		// In shallow mode we don't expect an index for other packages.
+		assert(nSyms == 0 || p.localpkg == pkg || p.insert == nil)
+		for ; nSyms > 0; nSyms-- {
 			name := p.stringAt(r.uint64())
 			nameIndex[name] = r.uint64()
 		}
@@ -267,6 +275,9 @@ type iimporter struct {
 	version int
 	ipath   string
 
+	localpkg *types.Package
+	insert   func(pkg *types.Package, name string) // "shallow" mode only
+
 	stringData  []byte
 	stringCache map[uint64]string
 	pkgCache    map[uint64]*types.Package
@@ -310,6 +321,13 @@ func (p *iimporter) doDecl(pkg *types.Package, name string) {
 
 	off, ok := p.pkgIndex[pkg][name]
 	if !ok {
+		// In "shallow" mode, call back to the application to
+		// find the object and insert it into the package scope.
+		if p.insert != nil {
+			assert(pkg != p.localpkg)
+			p.insert(pkg, name) // "can't fail"
+			return
+		}
 		errorf("%v.%v not in index", pkg, name)
 	}