internal/lsp: build the packages config on demand from proper configuration

This moves the fileset down to the base cache, the overlays down to the session
and stores the environment on the view.
packages.Config is no longer part of any public API, and the config is build on
demand by combining all the layers of cache.
Also added some documentation to the main source pacakge interfaces.

Change-Id: I058092ad2275d433864d1f58576fc55e194607a6
Reviewed-on: https://go-review.googlesource.com/c/tools/+/178017
Run-TryBot: Ian Cottrell <[email protected]>
TryBot-Result: Gobot Gobot <[email protected]>
Reviewed-by: Rebecca Stambler <[email protected]>

Author: ianthehat

cmd/gopls/main.go

@@ -17,5 +17,5 @@ import (
 )
 
 func main() {
-	tool.Main(context.Background(), cmd.New(nil), os.Args[1:])
+	tool.Main(context.Background(), cmd.New("", nil), os.Args[1:])
 }

internal/lsp/cache/cache.go

@@ -5,20 +5,31 @@
 package cache
 
 import (
+	"go/token"
+
 	"golang.org/x/tools/internal/lsp/source"
 	"golang.org/x/tools/internal/lsp/xlog"
+	"golang.org/x/tools/internal/span"
 )
 
 func New() source.Cache {
-	return &cache{}
+	return &cache{
+		fset: token.NewFileSet(),
+	}
 }
 
 type cache struct {
+	fset *token.FileSet
 }
 
 func (c *cache) NewSession(log xlog.Logger) source.Session {
 	return &session{
-		cache: c,
-		log:   log,
+		cache:    c,
+		log:      log,
+		overlays: make(map[span.URI][]byte),
 	}
 }
+
+func (c *cache) FileSet() *token.FileSet {
+	return c.fset
+}

internal/lsp/cache/check.go

@@ -10,6 +10,7 @@ import (
 	"go/ast"
 	"go/parser"
 	"go/scanner"
+	"go/token"
 	"go/types"
 
 	"golang.org/x/tools/go/analysis"
@@ -49,6 +50,7 @@ func (v *view) parse(ctx context.Context, file source.File) ([]packages.Error, e
 		view: v,
 		seen: make(map[string]struct{}),
 		ctx:  ctx,
+		fset: f.FileSet(),
 	}
 	// Start prefetching direct imports.
 	for importPath := range f.meta.children {
@@ -69,12 +71,11 @@ func (v *view) parse(ctx context.Context, file source.File) ([]packages.Error, e
 
 func (v *view) checkMetadata(ctx context.Context, f *goFile) ([]packages.Error, error) {
 	if v.reparseImports(ctx, f, f.filename()) {
-		cfg := v.config
-		cfg.Mode = packages.LoadImports | packages.NeedTypesSizes
-		pkgs, err := packages.Load(&cfg, fmt.Sprintf("file=%s", f.filename()))
+		cfg := v.buildConfig()
+		pkgs, err := packages.Load(cfg, fmt.Sprintf("file=%s", f.filename()))
 		if len(pkgs) == 0 {
 			if err == nil {
-				err = fmt.Errorf("no packages found for %s", f.filename())
+				err = fmt.Errorf("%s: no packages found", f.filename())
 			}
 			// Return this error as a diagnostic to the user.
 			return []packages.Error{
@@ -104,7 +105,7 @@ func (v *view) reparseImports(ctx context.Context, f *goFile, filename string) b
 	}
 	// Get file content in case we don't already have it?
 	f.read(ctx)
-	parsed, _ := parser.ParseFile(v.config.Fset, filename, f.content, parser.ImportsOnly)
+	parsed, _ := parser.ParseFile(f.FileSet(), filename, f.content, parser.ImportsOnly)
 	if parsed == nil {
 		return true
 	}
@@ -173,7 +174,8 @@ type importer struct {
 	// If we have seen a package that is already in this map, we have a circular import.
 	seen map[string]struct{}
 
-	ctx context.Context
+	ctx  context.Context
+	fset *token.FileSet
 }
 
 func (imp *importer) Import(pkgPath string) (*types.Package, error) {
@@ -255,9 +257,10 @@ func (imp *importer) typeCheck(pkgPath string) (*pkg, error) {
 			view: imp.view,
 			seen: seen,
 			ctx:  imp.ctx,
+			fset: imp.fset,
 		},
 	}
-	check := types.NewChecker(cfg, imp.view.config.Fset, pkg.types, pkg.typesInfo)
+	check := types.NewChecker(cfg, imp.fset, pkg.types, pkg.typesInfo)
 	check.Files(pkg.syntax)
 
 	// Add every file in this package to our cache.
@@ -273,7 +276,7 @@ func (v *view) cachePackage(ctx context.Context, pkg *pkg, meta *metadata) {
 			v.Session().Logger().Errorf(ctx, "invalid position for file %v", file.Name)
 			continue
 		}
-		tok := v.config.Fset.File(file.Pos())
+		tok := v.Session().Cache().FileSet().File(file.Pos())
 		if tok == nil {
 			v.Session().Logger().Errorf(ctx, "no token.File for %v", file.Name)
 			continue
@@ -341,7 +344,7 @@ func (v *view) appendPkgError(pkg *pkg, err error) {
 		}
 	case types.Error:
 		errs = append(errs, packages.Error{
-			Pos:  v.config.Fset.Position(err.Pos).String(),
+			Pos:  v.Session().Cache().FileSet().Position(err.Pos).String(),
 			Msg:  err.Msg,
 			Kind: packages.TypeError,
 		})

internal/lsp/cache/file.go

@@ -80,8 +80,8 @@ func (f *fileBase) GetContent(ctx context.Context) []byte {
 	return f.content
 }
 
-func (f *fileBase) GetFileSet(ctx context.Context) *token.FileSet {
-	return f.view.config.Fset
+func (f *fileBase) FileSet() *token.FileSet {
+	return f.view.Session().Cache().FileSet()
 }
 
 func (f *goFile) GetToken(ctx context.Context) *token.File {
@@ -144,7 +144,9 @@ func (f *fileBase) read(ctx context.Context) {
 		}
 	}
 	// We might have the content saved in an overlay.
-	if content, ok := f.view.config.Overlay[f.filename()]; ok {
+	f.view.session.overlayMu.Lock()
+	defer f.view.session.overlayMu.Unlock()
+	if content, ok := f.view.session.overlays[f.URI()]; ok {
 		f.content = content
 		return
 	}

internal/lsp/cache/parse.go

@@ -11,7 +11,6 @@ import (
 	"go/parser"
 	"go/scanner"
 	"go/token"
-	"io/ioutil"
 	"os"
 	"path/filepath"
 	"strings"
@@ -20,6 +19,10 @@ import (
 	"golang.org/x/tools/internal/span"
 )
 
+func parseFile(fset *token.FileSet, filename string, src []byte) (*ast.File, error) {
+	return parser.ParseFile(fset, filename, src, parser.AllErrors|parser.ParseComments)
+}
+
 // We use a counting semaphore to limit
 // the number of parallel I/O calls per process.
 var ioLimit = make(chan bool, 20)
@@ -37,53 +40,43 @@ func (imp *importer) parseFiles(filenames []string) ([]*ast.File, []error) {
 	parsed := make([]*ast.File, n)
 	errors := make([]error, n)
 	for i, filename := range filenames {
-		if imp.view.config.Context.Err() != nil {
+		if imp.ctx.Err() != nil {
 			parsed[i] = nil
-			errors[i] = imp.view.config.Context.Err()
+			errors[i] = imp.ctx.Err()
 			continue
 		}
 
 		// First, check if we have already cached an AST for this file.
 		f, err := imp.view.findFile(span.FileURI(filename))
-		if err != nil {
+		if err != nil || f == nil {
 			parsed[i], errors[i] = nil, err
+			continue
 		}
-		var fAST *ast.File
-		if f != nil {
-			if gof, ok := f.(*goFile); ok {
-				fAST = gof.ast
-			}
+		gof, ok := f.(*goFile)
+		if !ok {
+			parsed[i], errors[i] = nil, fmt.Errorf("Non go file in parse call: %v", filename)
+			continue
 		}
 
 		wg.Add(1)
 		go func(i int, filename string) {
 			ioLimit <- true // wait
 
-			if fAST != nil {
-				parsed[i], errors[i] = fAST, nil
+			if gof.ast != nil {
+				parsed[i], errors[i] = gof.ast, nil
 			} else {
 				// We don't have a cached AST for this file.
-				var src []byte
-				// Check for an available overlay.
-				for f, contents := range imp.view.config.Overlay {
-					if sameFile(f, filename) {
-						src = contents
-					}
-				}
-				var err error
-				// We don't have an overlay, so we must read the file's contents.
+				gof.read(imp.ctx)
+				src := gof.content
 				if src == nil {
-					src, err = ioutil.ReadFile(filename)
-				}
-				if err != nil {
-					parsed[i], errors[i] = nil, err
+					parsed[i], errors[i] = nil, fmt.Errorf("No source for %v", filename)
 				} else {
 					// ParseFile may return both an AST and an error.
-					parsed[i], errors[i] = imp.view.config.ParseFile(imp.view.config.Fset, filename, src)
+					parsed[i], errors[i] = parseFile(imp.fset, filename, src)
 
 					// Fix any badly parsed parts of the AST.
 					if file := parsed[i]; file != nil {
-						tok := imp.view.config.Fset.File(file.Pos())
+						tok := imp.fset.File(file.Pos())
 						imp.view.fix(imp.ctx, parsed[i], tok, src)
 					}
 				}

internal/lsp/cache/session.go

@@ -10,7 +10,6 @@ import (
 	"strings"
 	"sync"
 
-	"golang.org/x/tools/go/packages"
 	"golang.org/x/tools/internal/lsp/source"
 	"golang.org/x/tools/internal/lsp/xlog"
 	"golang.org/x/tools/internal/span"
@@ -24,6 +23,9 @@ type session struct {
 	viewMu  sync.Mutex
 	views   []*view
 	viewMap map[span.URI]source.View
+
+	overlayMu sync.Mutex
+	overlays  map[span.URI][]byte
 }
 
 func (s *session) Shutdown(ctx context.Context) {
@@ -40,7 +42,7 @@ func (s *session) Cache() source.Cache {
 	return s.cache
 }
 
-func (s *session) NewView(name string, folder span.URI, config *packages.Config) source.View {
+func (s *session) NewView(name string, folder span.URI) source.View {
 	s.viewMu.Lock()
 	defer s.viewMu.Unlock()
 	ctx := context.Background()
@@ -49,9 +51,7 @@ func (s *session) NewView(name string, folder span.URI, config *packages.Config)
 		session:        s,
 		baseCtx:        ctx,
 		backgroundCtx:  backgroundCtx,
-		builtinPkg:     builtinPkg(*config),
 		cancel:         cancel,
-		config:         *config,
 		name:           name,
 		folder:         folder,
 		filesByURI:     make(map[span.URI]viewFile),
@@ -65,9 +65,6 @@ func (s *session) NewView(name string, folder span.URI, config *packages.Config)
 		},
 		ignoredURIs: make(map[span.URI]struct{}),
 	}
-	for filename := range v.builtinPkg.Files {
-		v.ignoredURIs[span.NewURI(filename)] = struct{}{}
-	}
 	s.views = append(s.views, v)
 	// we always need to drop the view map
 	s.viewMap = make(map[span.URI]source.View)
@@ -155,3 +152,26 @@ func (s *session) removeView(ctx context.Context, view *view) error {
 func (s *session) Logger() xlog.Logger {
 	return s.log
 }
+
+func (s *session) setOverlay(uri span.URI, content []byte) {
+	s.overlayMu.Lock()
+	defer s.overlayMu.Unlock()
+	//TODO: we also need to invalidate anything that depended on this "file"
+	if content == nil {
+		delete(s.overlays, uri)
+		return
+	}
+	s.overlays[uri] = content
+}
+
+func (s *session) buildOverlay() map[string][]byte {
+	s.overlayMu.Lock()
+	defer s.overlayMu.Unlock()
+	overlay := make(map[string][]byte)
+	for uri, content := range s.overlays {
+		if filename, err := uri.Filename(); err == nil {
+			overlay[filename] = content
+		}
+	}
+	return overlay
+}