internal/lsp: track and parse non-compiled go files

When packages.Load'ing cgo packages, the authored files show up in
GoFiles, and the generated files show up in CompiledGoFiles. We need the
AST and type information for the latter, since they're the only thing we
can type check. But we also need the contents (and column mapper) for
the authored file so that we can navigate into it.

Store GoFiles in package metadata and checked Packages. Parse the extra
files, just for their mappers. Refactor the View functions a little bit,
since there's only one place that actually needs to find the mapper for
a file.

Updates golang/go#35720.

Change-Id: I9f96872a9a592bf0e11da27ebd8976c6db8752c9
Reviewed-on: https://go-review.googlesource.com/c/tools/+/208502
Run-TryBot: Heschi Kreinick <[email protected]>
Reviewed-by: Ian Cottrell <[email protected]>
Reviewed-by: Rebecca Stambler <[email protected]>

Author: heschi

internal/lsp/cache/check.go

@@ -18,6 +18,7 @@ import (
 	"golang.org/x/tools/internal/lsp/source"
 	"golang.org/x/tools/internal/lsp/telemetry"
 	"golang.org/x/tools/internal/memoize"
+	"golang.org/x/tools/internal/span"
 	"golang.org/x/tools/internal/telemetry/log"
 	"golang.org/x/tools/internal/telemetry/trace"
 	errors "golang.org/x/xerrors"
@@ -27,6 +28,8 @@ import (
 type checkPackageHandle struct {
 	handle *memoize.Handle
 
+	goFiles []source.ParseGoHandle
+
 	// compiledGoFiles are the ParseGoHandles that compose the package.
 	compiledGoFiles []source.ParseGoHandle
 
@@ -77,7 +80,8 @@ func (s *snapshot) checkPackageHandle(ctx context.Context, id packageID, mode so
 	//
 
 	m := cph.m
-	files := cph.compiledGoFiles
+	goFiles := cph.goFiles
+	compiledGoFiles := cph.compiledGoFiles
 	key := cph.key
 	fset := s.view.session.cache.fset
 
@@ -89,7 +93,7 @@ func (s *snapshot) checkPackageHandle(ctx context.Context, id packageID, mode so
 			}(dep)
 		}
 		data := &checkPackageData{}
-		data.pkg, data.err = typeCheck(ctx, fset, m, mode, files, deps)
+		data.pkg, data.err = typeCheck(ctx, fset, m, mode, goFiles, compiledGoFiles, deps)
 		return data
 	})
 	cph.handle = h
@@ -106,13 +110,18 @@ func (s *snapshot) buildKey(ctx context.Context, id packageID, mode source.Parse
 	if m == nil {
 		return nil, nil, errors.Errorf("no metadata for %s", id)
 	}
-	phs, err := s.compiledParseGoHandles(ctx, m, mode)
+	goFiles, err := s.parseGoHandles(ctx, m.goFiles, mode)
+	if err != nil {
+		return nil, nil, err
+	}
+	compiledGoFiles, err := s.parseGoHandles(ctx, m.compiledGoFiles, mode)
 	if err != nil {
 		return nil, nil, err
 	}
 	cph := &checkPackageHandle{
 		m:               m,
-		compiledGoFiles: phs,
+		goFiles:         goFiles,
+		compiledGoFiles: compiledGoFiles,
 		mode:            mode,
 	}
 
@@ -206,9 +215,9 @@ func (cph *checkPackageHandle) cached() (*pkg, error) {
 	return data.pkg, data.err
 }
 
-func (s *snapshot) compiledParseGoHandles(ctx context.Context, m *metadata, mode source.ParseMode) ([]source.ParseGoHandle, error) {
-	phs := make([]source.ParseGoHandle, 0, len(m.compiledGoFiles))
-	for _, uri := range m.compiledGoFiles {
+func (s *snapshot) parseGoHandles(ctx context.Context, files []span.URI, mode source.ParseMode) ([]source.ParseGoHandle, error) {
+	phs := make([]source.ParseGoHandle, 0, len(files))
+	for _, uri := range files {
 		f, err := s.view.GetFile(ctx, uri)
 		if err != nil {
 			return nil, err
@@ -219,7 +228,7 @@ func (s *snapshot) compiledParseGoHandles(ctx context.Context, m *metadata, mode
 	return phs, nil
 }
 
-func typeCheck(ctx context.Context, fset *token.FileSet, m *metadata, mode source.ParseMode, phs []source.ParseGoHandle, deps map[packagePath]*checkPackageHandle) (*pkg, error) {
+func typeCheck(ctx context.Context, fset *token.FileSet, m *metadata, mode source.ParseMode, goFiles []source.ParseGoHandle, compiledGoFiles []source.ParseGoHandle, deps map[packagePath]*checkPackageHandle) (*pkg, error) {
 	ctx, done := trace.StartSpan(ctx, "cache.importer.typeCheck", telemetry.Package.Of(m.id))
 	defer done()
 
@@ -232,7 +241,8 @@ func typeCheck(ctx context.Context, fset *token.FileSet, m *metadata, mode sourc
 		id:              m.id,
 		pkgPath:         m.pkgPath,
 		mode:            mode,
-		compiledGoFiles: phs,
+		goFiles:         goFiles,
+		compiledGoFiles: compiledGoFiles,
 		imports:         make(map[packagePath]*pkg),
 		typesSizes:      m.typesSizes,
 		typesInfo: &types.Info{
@@ -252,11 +262,18 @@ func typeCheck(ctx context.Context, fset *token.FileSet, m *metadata, mode sourc
 	for i, ph := range pkg.compiledGoFiles {
 		wg.Add(1)
 		go func(i int, ph source.ParseGoHandle) {
-			defer wg.Done()
-
 			files[i], _, parseErrors[i], _ = ph.Parse(ctx)
+			wg.Done()
 		}(i, ph)
 	}
+	for _, ph := range pkg.goFiles {
+		wg.Add(1)
+		// We need to parse the non-compiled go files, but we don't care about their errors.
+		go func(ph source.ParseGoHandle) {
+			ph.Parse(ctx)
+			wg.Done()
+		}(ph)
+	}
 	wg.Wait()
 
 	for _, e := range parseErrors {

internal/lsp/cache/load.go

@@ -23,6 +23,7 @@ type metadata struct {
 	id              packageID
 	pkgPath         packagePath
 	name            string
+	goFiles         []span.URI
 	compiledGoFiles []span.URI
 	typesSizes      types.Sizes
 	errors          []packages.Error
@@ -210,14 +211,19 @@ func (s *snapshot) updateImports(ctx context.Context, pkgPath packagePath, pkg *
 		errors:     pkg.Errors,
 		config:     cfg,
 	}
-	seen[id] = struct{}{}
+
 	for _, filename := range pkg.CompiledGoFiles {
 		uri := span.FileURI(filename)
 		m.compiledGoFiles = append(m.compiledGoFiles, uri)
-
+		s.addID(uri, m.id)
+	}
+	for _, filename := range pkg.GoFiles {
+		uri := span.FileURI(filename)
+		m.goFiles = append(m.goFiles, uri)
 		s.addID(uri, m.id)
 	}
 
+	seen[id] = struct{}{}
 	copied := make(map[packageID]struct{})
 	for k, v := range seen {
 		copied[k] = v

internal/lsp/cache/pkg.go

@@ -22,6 +22,7 @@ type pkg struct {
 	pkgPath packagePath
 	mode    source.ParseMode
 
+	goFiles         []source.ParseGoHandle
 	compiledGoFiles []source.ParseGoHandle
 	errors          []*source.Error
 	imports         map[packagePath]*pkg
@@ -49,7 +50,12 @@ func (p *pkg) CompiledGoFiles() []source.ParseGoHandle {
 }
 
 func (p *pkg) File(uri span.URI) (source.ParseGoHandle, error) {
-	for _, ph := range p.CompiledGoFiles() {
+	for _, ph := range p.compiledGoFiles {
+		if ph.File().Identity().URI == uri {
+			return ph, nil
+		}
+	}
+	for _, ph := range p.goFiles {
 		if ph.File().Identity().URI == uri {
 			return ph, nil
 		}

internal/lsp/cache/view.go

@@ -432,12 +432,12 @@ func (v *view) mapFile(uri span.URI, f viewFile) {
 	}
 }
 
-func (v *view) FindPosInPackage(searchpkg source.Package, pos token.Pos) (*ast.File, *protocol.ColumnMapper, source.Package, error) {
+func (v *view) FindPosInPackage(searchpkg source.Package, pos token.Pos) (*ast.File, source.Package, error) {
 	tok := v.session.cache.fset.File(pos)
 	if tok == nil {
-		return nil, nil, nil, errors.Errorf("no file for pos in package %s", searchpkg.ID())
+		return nil, nil, errors.Errorf("no file for pos in package %s", searchpkg.ID())
 	}
-	uri := span.FileURI(tok.Position(pos).Filename)
+	uri := span.FileURI(tok.Name())
 
 	// Special case for ignored files.
 	var (
@@ -451,16 +451,37 @@ func (v *view) FindPosInPackage(searchpkg source.Package, pos token.Pos) (*ast.F
 		ph, pkg, err = findFileInPackage(searchpkg, uri)
 	}
 	if err != nil {
-		return nil, nil, nil, err
+		return nil, nil, err
 	}
-	file, m, _, err := ph.Cached()
+	file, _, _, err := ph.Cached()
 	if err != nil {
-		return nil, nil, nil, err
+		return nil, nil, err
 	}
 	if !(file.Pos() <= pos && pos <= file.End()) {
-		return nil, nil, nil, err
+		return nil, nil, fmt.Errorf("pos %v, apparently in file %q, is not between %v and %v", pos, ph.File().Identity().URI, file.Pos(), file.End())
 	}
-	return file, m, pkg, nil
+	return file, pkg, nil
+}
+
+func (v *view) FindMapperInPackage(searchpkg source.Package, uri span.URI) (*protocol.ColumnMapper, error) {
+	// Special case for ignored files.
+	var (
+		ph  source.ParseGoHandle
+		err error
+	)
+	if v.Ignore(uri) {
+		ph, _, err = v.findIgnoredFile(uri)
+	} else {
+		ph, _, err = findFileInPackage(searchpkg, uri)
+	}
+	if err != nil {
+		return nil, err
+	}
+	_, m, _, err := ph.Cached()
+	if err != nil {
+		return nil, err
+	}
+	return m, nil
 }
 
 func (v *view) findIgnoredFile(uri span.URI) (source.ParseGoHandle, source.Package, error) {
@@ -482,10 +503,8 @@ func findFileInPackage(pkg source.Package, uri span.URI) (source.ParseGoHandle,
 		queue = queue[1:]
 		seen[pkg.ID()] = true
 
-		for _, ph := range pkg.CompiledGoFiles() {
-			if ph.File().Identity().URI == uri {
-				return ph, pkg, nil
-			}
+		if f, err := pkg.File(uri); err == nil {
+			return f, pkg, nil
 		}
 		for _, dep := range pkg.Imports() {
 			if !seen[dep.ID()] {

internal/lsp/source/completion_format.go

@@ -160,7 +160,7 @@ func (c *completer) item(cand candidate) (CompletionItem, error) {
 	if cand.imp != nil && cand.imp.pkg != nil {
 		searchPkg = cand.imp.pkg
 	}
-	file, _, pkg, err := c.snapshot.View().FindPosInPackage(searchPkg, obj.Pos())
+	file, pkg, err := c.snapshot.View().FindPosInPackage(searchPkg, obj.Pos())
 	if err != nil {
 		return item, nil
 	}

internal/lsp/source/identifier.go

@@ -242,7 +242,7 @@ func hasErrorType(obj types.Object) bool {
 }
 
 func objToNode(v View, pkg Package, obj types.Object) (ast.Decl, error) {
-	declAST, _, _, err := v.FindPosInPackage(pkg, obj.Pos())
+	declAST, _, err := v.FindPosInPackage(pkg, obj.Pos())
 	if err != nil {
 		return nil, err
 	}

internal/lsp/source/implementation.go

@@ -65,7 +65,7 @@ func (i *IdentifierInfo) Implementation(ctx context.Context) ([]protocol.Locatio
 		if pkgs[obj] == nil || len(pkg.CompiledGoFiles()) == 0 {
 			continue
 		}
-		file, _, _, err := i.Snapshot.View().FindPosInPackage(pkgs[obj], obj.Pos())
+		file, _, err := i.Snapshot.View().FindPosInPackage(pkgs[obj], obj.Pos())
 		if err != nil {
 			return nil, err
 		}

internal/lsp/source/util.go

@@ -169,7 +169,8 @@ func nodeToMappedRange(view View, m *protocol.ColumnMapper, n ast.Node) (mappedR
 }
 
 func posToMappedRange(v View, pkg Package, pos, end token.Pos) (mappedRange, error) {
-	_, m, _, err := v.FindPosInPackage(pkg, pos)
+	logicalFilename := v.Session().Cache().FileSet().File(pos).Position(pos).Filename
+	m, err := v.FindMapperInPackage(pkg, span.FileURI(logicalFilename))
 	if err != nil {
 		return mappedRange{}, err
 	}

internal/lsp/source/view.go

@@ -126,7 +126,9 @@ type View interface {
 
 	// FindFileInPackage returns the AST and type information for a file that may
 	// belong to or be part of a dependency of the given package.
-	FindPosInPackage(pkg Package, pos token.Pos) (*ast.File, *protocol.ColumnMapper, Package, error)
+	FindPosInPackage(pkg Package, pos token.Pos) (*ast.File, Package, error)
+
+	FindMapperInPackage(pkg Package, uri span.URI) (*protocol.ColumnMapper, error)
 
 	// Snapshot returns the current snapshot for the view.
 	Snapshot() Snapshot