Speed up packages loading

go.mod

@@ -50,3 +50,6 @@ require (
 	mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b // indirect
 	mvdan.cc/unparam v0.0.0-20190209190245-fbb59629db34
 )
+
+// https://github.com/golang/tools/pull/160
+replace golang.org/x/tools => github.com/golangci/tools v0.0.0-20190914130248-e9260b99c8f1

go.sum

@@ -106,6 +106,8 @@ github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21 h1:leSNB7iYzLYSS
 github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21/go.mod h1:tf5+bzsHdTM0bsB7+8mt0GUMvjCgwLpTapNZHU8AajI=
 github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0 h1:HVfrLniijszjS1aiNg8JbBMO2+E1WIQ+j/gL4SQqGPg=
 github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4=
+github.com/golangci/tools v0.0.0-20190914130248-e9260b99c8f1 h1:8eGJVbBRoAvCh/YZq6n11s9WJkIgWKa/iTNq+R0UrNw=
+github.com/golangci/tools v0.0.0-20190914130248-e9260b99c8f1/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4 h1:zwtduBRr5SSWhqsYNgcuWO2kFlpdOZbP0+yRjmvPGys=
 github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg8RkN3rCIMLGE9CyYmU9pY2Jer6DgANEnZ/L/cQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
@@ -283,21 +285,6 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20181117154741-2ddaf7f79a09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190110163146-51295c7ec13a/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190121143147-24cd39ecf745/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190311215038-5c2858a9cfe5/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190521203540-521d6ed310dd/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190910044552-dd2b5c81c578/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20190911022129-16c5e0f7d110/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20190912215617-3720d1ec3678 h1:rM1Udd0CgtYI3KUIhu9ROz0QCqjW+n/ODp/hH7c60Xc=
-golang.org/x/tools v0.0.0-20190912215617-3720d1ec3678/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=

pkg/commands/executor.go

@@ -31,6 +31,7 @@ type Executor struct {
 	goenv             *goutil.Env
 	fileCache         *fsutils.FileCache
 	lineCache         *fsutils.LineCache
+	debugf            logutils.DebugFunc
 }
 
 func NewExecutor(version, commit, date string) *Executor {
@@ -40,8 +41,10 @@ func NewExecutor(version, commit, date string) *Executor {
 		commit:    commit,
 		date:      date,
 		DBManager: lintersdb.NewManager(nil),
+		debugf:    logutils.Debug("exec"),
 	}
 
+	e.debugf("Starting execution...")
 	e.log = report.NewLogWrapper(logutils.NewStderrLog(""), &e.reportData)
 
 	// to setup log level early we need to parse config from command line extra time to
@@ -100,10 +103,12 @@ func NewExecutor(version, commit, date string) *Executor {
 	e.fileCache = fsutils.NewFileCache()
 	e.lineCache = fsutils.NewLineCache(e.fileCache)
 	e.contextLoader = lint.NewContextLoader(e.cfg, e.log.Child("loader"), e.goenv, e.lineCache, e.fileCache)
-
+	e.debugf("Initialized executor")
 	return e
 }
 
 func (e *Executor) Execute() error {
-	return e.rootCmd.Execute()
+	err := e.rootCmd.Execute()
+	e.debugf("Finished execution")
+	return err
 }

pkg/commands/help.go

@@ -46,7 +46,7 @@ func printLinterConfigs(lcs []*linter.Config) {
 			altNamesStr = fmt.Sprintf(" (%s)", strings.Join(lc.AlternativeNames, ", "))
 		}
 		fmt.Fprintf(logutils.StdOut, "%s%s: %s [fast: %t, auto-fix: %t]\n", color.YellowString(lc.Name()),
-			altNamesStr, lc.Linter.Desc(), !lc.NeedsSSARepr, lc.CanAutoFix)
+			altNamesStr, lc.Linter.Desc(), !lc.NeedsDepsTypeInfo, lc.CanAutoFix)
 	}
 }
 

pkg/commands/run.go

@@ -406,7 +406,7 @@ func (e *Executor) executeRun(_ *cobra.Command, args []string) {
 	defer cancel()
 
 	if needTrackResources {
-		go watchResources(ctx, trackResourcesEndCh, e.log)
+		go watchResources(ctx, trackResourcesEndCh, e.log, e.debugf)
 	}
 
 	if err := e.runAndPrint(ctx, args); err != nil {
@@ -447,11 +447,12 @@ func (e *Executor) setupExitCode(ctx context.Context) {
 	}
 }
 
-func watchResources(ctx context.Context, done chan struct{}, logger logutils.Log) {
+func watchResources(ctx context.Context, done chan struct{}, logger logutils.Log, debugf logutils.DebugFunc) {
 	startedAt := time.Now()
+	debugf("Started tracking time")
 
 	var rssValues []uint64
-	ticker := time.NewTicker(100 * time.Millisecond)
+	ticker := time.NewTicker(10 * time.Millisecond)
 	defer ticker.Stop()
 
 	logEveryRecord := os.Getenv("GL_MEM_LOG_EVERY") == "1"
@@ -474,6 +475,7 @@ func watchResources(ctx context.Context, done chan struct{}, logger logutils.Log
 		select {
 		case <-ctx.Done():
 			stop = true
+			debugf("Stopped resources tracking")
 		case <-ticker.C: // track every second
 		}
 

pkg/golinters/govet.go

@@ -11,12 +11,12 @@ import (
 	"golang.org/x/tools/go/analysis/passes/asmdecl"
 	"golang.org/x/tools/go/analysis/passes/assign"
 	"golang.org/x/tools/go/analysis/passes/atomic"
-	"golang.org/x/tools/go/analysis/passes/atomicalign"
 	"golang.org/x/tools/go/analysis/passes/bools"
 	"golang.org/x/tools/go/analysis/passes/buildtag"
 	"golang.org/x/tools/go/analysis/passes/cgocall"
 	"golang.org/x/tools/go/analysis/passes/composite"
 	"golang.org/x/tools/go/analysis/passes/copylock"
+	"golang.org/x/tools/go/analysis/passes/errorsas"
 	"golang.org/x/tools/go/analysis/passes/httpresponse"
 	"golang.org/x/tools/go/analysis/passes/loopclosure"
 	"golang.org/x/tools/go/analysis/passes/lostcancel"
@@ -31,8 +31,6 @@ import (
 	"golang.org/x/tools/go/analysis/passes/unreachable"
 	"golang.org/x/tools/go/analysis/passes/unsafeptr"
 	"golang.org/x/tools/go/analysis/passes/unusedresult"
-
-	"github.com/golangci/golangci-lint/pkg/golinters/goanalysis/passes/nilness"
 )
 
 func NewGovet(cfg *config.GovetSettings) *goanalysis.Linter {
@@ -41,12 +39,12 @@ func NewGovet(cfg *config.GovetSettings) *goanalysis.Linter {
 		asmdecl.Analyzer,
 		assign.Analyzer,
 		atomic.Analyzer,
-		atomicalign.Analyzer,
 		bools.Analyzer,
 		buildtag.Analyzer,
 		cgocall.Analyzer,
 		composite.Analyzer,
 		copylock.Analyzer,
+		errorsas.Analyzer,
 		httpresponse.Analyzer,
 		loopclosure.Analyzer,
 		lostcancel.Analyzer,
@@ -60,13 +58,6 @@ func NewGovet(cfg *config.GovetSettings) *goanalysis.Linter {
 		unreachable.Analyzer,
 		unsafeptr.Analyzer,
 		unusedresult.Analyzer,
-
-		// for debugging:
-		// findcall.Analyzer,
-		// pkgfact.Analyzer,
-
-		// uses SSA:
-		nilness.Analyzer,
 	}
 
 	var settings map[string]map[string]interface{}

pkg/golinters/megacheck.go

@@ -144,15 +144,16 @@ func (MegacheckMetalinter) BuildLinterConfig(enabledChildren []string) (*linter.
 
 	// TODO: merge linter.Config and linter.Linter or refactor it in another way
 	return &linter.Config{
-		Linter:           m,
-		EnabledByDefault: false,
-		NeedsTypeInfo:    true,
-		NeedsSSARepr:     true,
-		InPresets:        []string{linter.PresetStyle, linter.PresetBugs, linter.PresetUnused},
-		Speed:            1,
-		AlternativeNames: nil,
-		OriginalURL:      "",
-		ParentLinterName: "",
+		Linter:            m,
+		EnabledByDefault:  false,
+		NeedsTypeInfo:     true,
+		NeedsDepsTypeInfo: true,
+		NeedsSSARepr:      true,
+		InPresets:         []string{linter.PresetStyle, linter.PresetBugs, linter.PresetUnused},
+		Speed:             1,
+		AlternativeNames:  nil,
+		OriginalURL:       "",
+		ParentLinterName:  "",
 	}, nil
 }
 

pkg/goutil/env.go

@@ -5,12 +5,21 @@ import (
 	"encoding/json"
 	"os"
 	"os/exec"
+	"strings"
+	"time"
 
 	"github.com/pkg/errors"
 
 	"github.com/golangci/golangci-lint/pkg/logutils"
 )
 
+type EnvKey string
+
+const (
+	EnvGoCache EnvKey = "GOCACHE"
+	EnvGoRoot  EnvKey = "GOROOT"
+)
+
 type Env struct {
 	vars   map[string]string
 	log    logutils.Log
@@ -26,24 +35,27 @@ func NewEnv(log logutils.Log) *Env {
 }
 
 func (e *Env) Discover(ctx context.Context) error {
-	out, err := exec.CommandContext(ctx, "go", "env", "-json").Output()
+	startedAt := time.Now()
+	args := []string{"env", "-json"}
+	args = append(args, string(EnvGoCache), string(EnvGoRoot))
+	out, err := exec.CommandContext(ctx, "go", args...).Output()
 	if err != nil {
 		return errors.Wrap(err, "failed to run 'go env'")
 	}
 
 	if err = json.Unmarshal(out, &e.vars); err != nil {
-		return errors.Wrap(err, "failed to parse go env json")
+		return errors.Wrapf(err, "failed to parse 'go %s' json", strings.Join(args, " "))
 	}
 
-	e.debugf("Read go env: %#v", e.vars)
+	e.debugf("Read go env for %s: %#v", time.Since(startedAt), e.vars)
 	return nil
 }
 
-func (e Env) Get(k string) string {
-	envValue := os.Getenv(k)
+func (e Env) Get(k EnvKey) string {
+	envValue := os.Getenv(string(k))
 	if envValue != "" {
 		return envValue
 	}
 
-	return e.vars[k]
+	return e.vars[string(k)]
 }

pkg/lint/linter/config.go

@@ -13,8 +13,9 @@ type Config struct {
 	Linter           Linter
 	EnabledByDefault bool
 
-	NeedsTypeInfo bool
-	NeedsSSARepr  bool
+	NeedsTypeInfo     bool
+	NeedsDepsTypeInfo bool
+	NeedsSSARepr      bool
 
 	InPresets        []string
 	Speed            int // more value means faster execution of linter
@@ -30,6 +31,12 @@ func (lc *Config) WithTypeInfo() *Config {
 	return lc
 }
 
+func (lc *Config) WithDepsTypeInfo() *Config {
+	lc.NeedsTypeInfo = true
+	lc.NeedsDepsTypeInfo = true
+	return lc
+}
+
 func (lc *Config) WithSSA() *Config {
 	lc.NeedsTypeInfo = true
 	lc.NeedsSSARepr = true

pkg/lint/lintersdb/enabled_set.go

@@ -51,7 +51,7 @@ func (es EnabledSet) build(lcfg *config.Linters, enabledByDefaultLinters []*lint
 	// It should be before --enable and --disable to be able to enable or disable specific linter.
 	if lcfg.Fast {
 		for name := range resultLintersSet {
-			if es.m.GetLinterConfig(name).NeedsSSARepr {
+			if es.m.GetLinterConfig(name).NeedsDepsTypeInfo {
 				delete(resultLintersSet, name)
 			}
 		}

pkg/lint/lintersdb/manager.go

@@ -85,12 +85,13 @@ func (m Manager) GetAllSupportedLinterConfigs() []*linter.Config {
 	}
 	lcs := []*linter.Config{
 		linter.NewConfig(golinters.NewGovet(govetCfg)).
-			WithSSA(). // TODO: extract from the linter config and don't build SSA, just use LoadAllSyntax mode
+			WithDepsTypeInfo().
 			WithPresets(linter.PresetBugs).
 			WithSpeed(4).
 			WithAlternativeNames("vet", "vetshadow").
 			WithURL("https://golang.org/cmd/vet/"),
 		linter.NewConfig(golinters.NewBodyclose()).
+			WithDepsTypeInfo().
 			WithSSA().
 			WithPresets(linter.PresetPerformance, linter.PresetBugs).
 			WithSpeed(4).
@@ -106,21 +107,25 @@ func (m Manager) GetAllSupportedLinterConfigs() []*linter.Config {
 			WithURL("https://github.com/golang/lint"),
 
 		linter.NewConfig(golinters.NewStaticcheck()).
+			WithDepsTypeInfo().
 			WithSSA().
 			WithPresets(linter.PresetBugs).
 			WithSpeed(2).
 			WithURL("https://staticcheck.io/"),
 		linter.NewConfig(golinters.NewUnused()).
+			WithDepsTypeInfo().
 			WithSSA().
 			WithPresets(linter.PresetUnused).
 			WithSpeed(5).
 			WithURL("https://github.com/dominikh/go-tools/tree/master/cmd/unused"),
 		linter.NewConfig(golinters.NewGosimple()).
+			WithDepsTypeInfo().
 			WithSSA().
 			WithPresets(linter.PresetStyle).
 			WithSpeed(5).
 			WithURL("https://github.com/dominikh/go-tools/tree/master/cmd/gosimple"),
 		linter.NewConfig(golinters.NewStylecheck()).
+			WithDepsTypeInfo().
 			WithSSA().
 			WithPresets(linter.PresetStyle).
 			WithSpeed(5).
@@ -143,6 +148,7 @@ func (m Manager) GetAllSupportedLinterConfigs() []*linter.Config {
 			WithSpeed(10).
 			WithURL("https://github.com/opennota/check"),
 		linter.NewConfig(golinters.Interfacer{}).
+			WithDepsTypeInfo().
 			WithSSA().
 			WithPresets(linter.PresetStyle).
 			WithSpeed(6).
@@ -211,6 +217,7 @@ func (m Manager) GetAllSupportedLinterConfigs() []*linter.Config {
 		linter.NewConfig(golinters.Unparam{}).
 			WithPresets(linter.PresetUnused).
 			WithSpeed(3).
+			WithDepsTypeInfo().
 			WithSSA().
 			WithURL("https://github.com/mvdan/unparam"),
 		linter.NewConfig(golinters.Nakedret{}).

pkg/lint/load.go

@@ -53,7 +53,7 @@ func NewContextLoader(cfg *config.Config, log logutils.Log, goenv *goutil.Env,
 func (cl ContextLoader) prepareBuildContext() {
 	// Set GOROOT to have working cross-compilation: cross-compiled binaries
 	// have invalid GOROOT. XXX: can't use runtime.GOROOT().
-	goroot := cl.goenv.Get("GOROOT")
+	goroot := cl.goenv.Get(goutil.EnvGoRoot)
 	if goroot == "" {
 		return
 	}
@@ -149,7 +149,7 @@ func (cl ContextLoader) findLoadMode(linters []*linter.Config) packages.LoadMode
 		if lc.NeedsTypeInfo {
 			loadMode |= packages.NeedImports | packages.NeedTypes | packages.NeedTypesSizes | packages.NeedTypesInfo | packages.NeedSyntax
 		}
-		if lc.NeedsSSARepr {
+		if lc.NeedsDepsTypeInfo {
 			loadMode |= packages.NeedDeps
 		}
 	}
@@ -349,6 +349,15 @@ func (cl ContextLoader) filterDuplicatePackages(pkgs []*packages.Package) []*pac
 	return retPkgs
 }
 
+func needSSA(linters []*linter.Config) bool {
+	for _, lc := range linters {
+		if lc.NeedsSSARepr {
+			return true
+		}
+	}
+	return false
+}
+
 //nolint:gocyclo
 func (cl ContextLoader) Load(ctx context.Context, linters []*linter.Config) (*linter.Context, error) {
 	loadMode := cl.findLoadMode(linters)
@@ -369,7 +378,7 @@ func (cl ContextLoader) Load(ctx context.Context, linters []*linter.Config) (*li
 	}
 
 	var ssaProg *ssa.Program
-	if loadMode&packages.NeedDeps != 0 {
+	if needSSA(linters) {
 		ssaProg = cl.buildSSAProgram(deduplicatedPkgs)
 	}
 

pkg/logutils/logutils.go

@@ -30,9 +30,10 @@ func Debug(tag string) DebugFunc {
 		return nopDebugf
 	}
 
+	logger := NewStderrLog(tag)
+	logger.SetLevel(LogLevelDebug)
+
 	return func(format string, args ...interface{}) {
-		logger := NewStderrLog(tag)
-		logger.SetLevel(LogLevelDebug)
 		logger.Debugf(format, args...)
 	}
 }