gopls/internal/lsp/source/typerefs: collapse SCCs of ref graph

This CL uses Tarjan's algorithm to collapse the strong components
of the intra-package reference graph. It is the first of several
graph optimizations I plan to evaluate for the reference graph
to make it compactly serializable and efficiently searchable.

By eliminating cycles, the number of edges that must be traversed
by the reachability query is reduced, but more importantly,
the resulting graph becomes acyclic, which means that there
is no need to revisit any node, whereas previously a complete
traversal was initiated from every node.

Also:
- use two-pass algorithm in Refs to gather decls and then
  build cross-references;
- document the algorithm;
- unexport various fields of Package
- stop indexing PackageIDs in the local preprocessing
  (it's not necessary).

Details:

k8s benchmark on M1 MBP shows a 60% reduction in real time:
xtools$ go test ./gopls/internal/lsp/source/typerefs/ -run=NONE -bench=BuildP -dir=$(pwd)/../kubernetes/ -verify=false -query=./...
Before:
BenchmarkBuildPackageGraph-8   	       1	2997634916 ns/op
BenchmarkBuildPackageGraph-8   	       1	2623176209 ns/op
After:
BenchmarkBuildPackageGraph-8   	       1	1142697000 ns/op
BenchmarkBuildPackageGraph-8   	       1	1144183250 ns/op

Next steps: simplify the reference graph; serialize it.

Change-Id: Ic8a6154c0d8d7174529b90774a8683ea9450f484
Reviewed-on: https://go-review.googlesource.com/c/tools/+/481784
TryBot-Result: Gopher Robot <[email protected]>
Run-TryBot: Alan Donovan <[email protected]>
Reviewed-by: Robert Findley <[email protected]>

Author: adonovan

gopls/internal/lsp/source/typerefs/doc.go

@@ -59,6 +59,51 @@
 // analysis. Notably, we also skip identifiers that refer to type parameters in
 // generic declarations.
 //
+// # Graph optimizations
+//
+// The references extracted from the syntax are used to construct
+// edges between declNodes. Edges are of two kinds: internal
+// references, from one package-level declaration to another; and
+// external references, from a symbol in this package to a symbol
+// imported from a direct dependency.
+//
+// Once the symbol reference graph is constructed, we find its
+// strongly connected components (SCCs) using Tarjan's algorithm. A
+// node from each SCC is chosen arbitrarily to be its representative,
+// and all the edges (internal and external) of the SCC are
+// accumulated into the representative, thus forming the strong
+// component graph, which is acyclic. This property simplifies the
+// logic and improves the efficiency of the reachability query.
+//
+// TODO(adonovan): opt: subsequent planned optimizations include:
+//
+//   - The Hash-Value Numbering optimization described in
+//     Hardekopf and Lin; see golang.org/x/go/pointer/hvn.go for an
+//     implementation. (Like pointer analysis, our problem is
+//     fundamentally one of graph reachability.)
+//
+//     The "pointer equivalence" (PE) portion of this algorithm uses a
+//     hash table to create a mapping from unique sets of external
+//     references to small integers. Each of the n external symbols
+//     referenced by the package is assigned a integer from 1 to n;
+//     this number stands for a singleton set. Higher numbers refer to
+//     unions of strictly smaller sets. The PE algorithm allows us to
+//     coalesce redundant graph nodes. For example, all functions that
+//     ultimately reference only {fmt.Println,fmt.Sprintf} would be
+//     marked as equivalent to each other, and to the union of
+//     the sets of {fmt.Sprint} and {fmt.Println}.
+//
+//     This reduces the worst-case size of the Refs() result. Consider
+//     M decls that each reference type t, which references N imported
+//     types. The source code has O(M + N) lines but the Refs result
+//     is current O(M*N). Preserving the essential structure of the
+//     reference graph (as a DAG of union operations) will reduce the
+//     asymptote.
+//
+//   - Serializing the SC graph obtained each package and saving it in
+//     the file cache. Once we have a DAG of unions, we can serialize
+//     it easily and amortize the cost of the local preprocessing.
+//
 // # API
 //
 // The main entry point for this analysis is the [Refs] function, which

gopls/internal/lsp/source/typerefs/pkgrefs.go

@@ -6,7 +6,6 @@ package typerefs
 
 import (
 	"context"
-	"go/token"
 	"runtime"
 	"sync"
 
@@ -24,42 +23,20 @@ const (
 
 // A Package holds reference information for a single package.
 type Package struct {
-	idx packageIdx // memoized index of this package's ID, to save map lookups
+	// metadata holds metadata about this package and its dependencies.
+	metadata *source.Metadata
 
-	// Metadata holds metadata about this package and its dependencies.
-	Metadata *source.Metadata
-
-	// Refs records syntactic edges between declarations in this package and
-	// declarations in this package or another package. See the package
-	// documentation for a detailed description of what these edges do (and do
-	// not) represent.
-	Refs map[string][]Ref
-
-	// TransitiveRefs records, for each declaration in the package, the
+	// transitiveRefs records, for each exported declaration in the package, the
 	// transitive set of packages within the containing graph that are
 	// transitively reachable through references, starting with the given decl.
-	TransitiveRefs map[string]*PackageSet
+	transitiveRefs map[string]*PackageSet
 
 	// ReachesViaDeps records the set of packages in the containing graph whose
 	// syntax may affect the current package's types. See the package
 	// documentation for more details of what this means.
 	ReachesByDeps *PackageSet
 }
 
-// A Ref is a referenced declaration.
-//
-// Unpack it using the Unpack method, with the PackageIndex instance that was
-// used to construct the references.
-type Ref struct {
-	pkg  packageIdx
-	name string
-}
-
-// UnpackRef unpacks the actual PackageID an name encoded in ref.
-func (r Ref) Unpack(index *PackageIndex) (PackageID source.PackageID, name string) {
-	return index.id(r.pkg), r.name
-}
-
 // A PackageGraph represents a fully analyzed graph of packages and their
 // dependencies.
 type PackageGraph struct {
@@ -139,111 +116,78 @@ func (g *PackageGraph) Package(ctx context.Context, id source.PackageID) (*Packa
 // only be called from Package.
 func (g *PackageGraph) buildPackage(ctx context.Context, id source.PackageID) (*Package, error) {
 	p := &Package{
-		idx:            g.pkgIndex.idx(id),
-		Metadata:       g.meta.Metadata(id),
-		Refs:           make(map[string][]Ref),
-		TransitiveRefs: make(map[string]*PackageSet),
+		metadata:       g.meta.Metadata(id),
+		transitiveRefs: make(map[string]*PackageSet),
 	}
 	var files []*source.ParsedGoFile
-	for _, filename := range p.Metadata.CompiledGoFiles {
+	for _, filename := range p.metadata.CompiledGoFiles {
 		f, err := g.parse(ctx, filename)
 		if err != nil {
 			return nil, err
 		}
 		files = append(files, f)
 	}
 	imports := make(map[source.ImportPath]*source.Metadata)
-	for impPath, depID := range p.Metadata.DepsByImpPath {
+	for impPath, depID := range p.metadata.DepsByImpPath {
 		if depID != "" {
 			imports[impPath] = g.meta.Metadata(depID)
 		}
 	}
-	p.Refs = Refs(files, id, imports, g.pkgIndex)
-
-	// Compute packages reachable from each exported symbol of this package.
-	for name := range p.Refs {
-		if token.IsExported(name) {
-			set := g.pkgIndex.New()
-			g.reachableByName(ctx, p, name, set, make(map[string]bool))
-			p.TransitiveRefs[name] = set
-		}
-	}
 
-	var err error
-	p.ReachesByDeps, err = g.reachesByDeps(ctx, p.Metadata)
-	if err != nil {
-		return nil, err
-	}
-	return p, nil
-}
-
-// ExternalRefs returns a new map whose keys are the exported symbols
-// of the package (of the specified id, pkgIndex, and refs). The
-// corresponding value of each key is the set of exported symbols
-// indirectly referenced by it.
-//
-// TODO(adonovan): simplify the API once the SCC-based optimization lands.
-func ExternalRefs(pkgIndex *PackageIndex, id source.PackageID, refs map[string][]Ref) map[string]map[Ref]bool {
-	// (This intrapackage recursion will go away in a follow-up CL.)
-	var visit func(name string, res map[Ref]bool, seen map[string]bool)
-	visit = func(name string, res map[Ref]bool, seen map[string]bool) {
-		if !seen[name] {
-			seen[name] = true
-			for _, ref := range refs[name] {
-				if pkgIndex.id(ref.pkg) == id {
-					visit(ref.name, res, seen) // intrapackage recursion
-				} else {
-					res[ref] = true // cross-package ref
-				}
-			}
-		}
-	}
+	// Compute the symbol-level dependencies through this package.
+	//
+	// refs records syntactic edges between declarations in this
+	// package and declarations in this package or another
+	// package. See the package documentation for a detailed
+	// description of what these edges do (and do not) represent.
+	//
+	// TODO(adonovan): opt: serialize and deserialize the refs
+	// result computed above and persist it in the filecache.
+	refs := Refs(files, id, imports)
 
-	results := make(map[string]map[Ref]bool)
-	for name := range refs {
-		if token.IsExported(name) {
-			res := make(map[Ref]bool)
-			seen := make(map[string]bool)
-			visit(name, res, seen)
-			results[name] = res
-		}
-	}
-	return results
-}
+	//      This point separates the local preprocessing
+	//  --  of a single package (above) from the global   --
+	//      transitive reachability query (below).
 
-// reachableByName computes the set of packages that are reachable through
-// references, starting with the declaration for name in package p.
-func (g *PackageGraph) reachableByName(ctx context.Context, p *Package, name string, set *PackageSet, seen map[string]bool) error {
-	if seen[name] {
-		return nil
-	}
-	seen[name] = true
-
-	// Opt: when we compact reachable edges inside the Refs algorithm, we handle
-	// all edges to a given package in a batch, so they should be adjacent to
-	// each other in the resulting slice. Therefore remembering the last P here
-	// can save on lookups.
-	depP := p
-	for _, node := range p.Refs[name] {
-		if node.pkg == p.idx {
-			// same package
-			g.reachableByName(ctx, p, node.name, set, seen)
-		} else {
-			// cross-package ref
-			if depP.idx != node.pkg {
-				id := g.pkgIndex.id(node.pkg)
+	// Now compute the transitive closure of packages reachable
+	// from any exported symbol of this package.
+	//
+	// TODO(adonovan): opt: many elements of refs[name] are
+	// identical, so this does redundant work. Choose a data type
+	// for the result of Refs() that expresses the M:N structure
+	// explicitly.
+	for name, nodes := range refs {
+		set := g.pkgIndex.New()
+
+		// The nodes slice is sorted by (package, name),
+		// so we can economize by calling g.Package only
+		// when the package id changes.
+		depP := p
+		for _, node := range nodes {
+			assert(node.PkgID != id, "intra-package edge")
+			if depP.metadata.ID != node.PkgID {
+				// package changed
 				var err error
-				depP, err = g.Package(ctx, id)
+				depP, err = g.Package(ctx, node.PkgID)
 				if err != nil {
-					return err
+					return nil, err
 				}
 			}
-			set.add(node.pkg)
-			set.Union(depP.TransitiveRefs[node.name])
+			set.add(g.pkgIndex.idx(node.PkgID))
+			set.Union(depP.transitiveRefs[node.Name])
 		}
+		p.transitiveRefs[name] = set
+	}
+
+	// Finally compute the union of transitiveRefs
+	// across the direct deps of this package.
+	byDeps, err := g.reachesByDeps(ctx, p.metadata)
+	if err != nil {
+		return nil, err
 	}
+	p.ReachesByDeps = byDeps
 
-	return nil
+	return p, nil
 }
 
 // reachesByDeps computes the set of packages that are reachable through
@@ -255,11 +199,9 @@ func (g *PackageGraph) reachesByDeps(ctx context.Context, m *source.Metadata) (*
 		if err != nil {
 			return nil, err
 		}
-		transitive.add(dep.idx)
-		for name, set := range dep.TransitiveRefs {
-			if token.IsExported(name) {
-				transitive.Union(set)
-			}
+		transitive.add(g.pkgIndex.idx(dep.metadata.ID))
+		for _, set := range dep.transitiveRefs {
+			transitive.Union(set)
 		}
 	}
 	return transitive, nil

gopls/internal/lsp/source/typerefs/pkgrefs_test.go

@@ -195,7 +195,6 @@ func TestBuildPackageGraph(t *testing.T) {
 		}
 		fmt.Printf("%52s%8.1f%8.1f%8.1f%8.1f%8.1f%8.1f\n", "averages:", avg(imports), avg(importedBy), avg(reaches), avg(reachedBy), avg(loads), avg(loadedBy))
 	}
-
 }
 
 func importMap(id PackageID, meta MetadataSource) map[PackagePath]PackageID {