cmd/go: smooth out upgrade paths for lazy loading

This change adds two possible upgrade paths for lazy loading:

1. Run 'go mod tidy -go=1.17'.

2. Starting in a module with no existing 'go' directive,
   run any 'go' command that updates the go.mod file.

In the latter case, commands other than 'go mod tidy'
may leave the go.mod file *very* untidy if it had non-trivial
dependencies. (The 'go' invocation will promote all
implicit eager dependencies to explicit lazy ones,
which preserves the original module graph — most of which is
not actually relevant.)

'go mod tidy -go=1.17' can be used to enable lazy loading without
accidentally downgrading existing transitive dependencies.

'go mod tidy -go=1.16' can be used to disable lazy loading and clear
away redundant roots in a single step (if reducing the go version), or
to prune away dependencies of tests-of-external-tests (if increasing
the go version).

'go mod tidy -go=1.15' can be used to add dependencies of
tests-of-external-tests, although there isn't much point to that.

DO NOT MERGE

This change still needs an explicit test and a release note.

Fixes #45094
For #36460

Change-Id: I68f057e39489dfd6a667cd11dc1e320c1ee1aec1
Reviewed-on: https://go-review.googlesource.com/c/go/+/315210
Trust: Bryan C. Mills <[email protected]>
Run-TryBot: Bryan C. Mills <[email protected]>
TryBot-Result: Go Bot <[email protected]>
Reviewed-by: Michael Matloob <[email protected]>

Author: Bryan C. Mills

doc/go1.17.html

@@ -58,6 +58,17 @@ <h4 id="lazy-loading">Lazy module loading</h4>
   <!-- TODO(bcmills): replace the design-doc link with proper documentation. -->
 </p>
 
+<p><!-- golang.org/issue/45094 --> To facilitate the upgrade to lazy loading,
+  the <code>go</code> <code>mod</code> <code>tidy</code> subcommand now supports
+  a <code>-go</code> flag to set or change the <code>go</code> version in
+  the <code>go.mod</code> file. To enable lazy loading for an existing module
+  without changing the selected versions of its dependencies, run:
+</p>
+
+<pre>
+  go mod tidy -go=1.17
+</pre>
+
 <h4 id="module-deprecation-comments">Module deprecation comments</h4>
 
 <p><!-- golang.org/issue/40357 -->

src/cmd/go/alldocs.go

@@ -12,10 +12,12 @@ import (
 	"cmd/go/internal/imports"
 	"cmd/go/internal/modload"
 	"context"
+
+	"golang.org/x/mod/modfile"
 )
 
 var cmdTidy = &base.Command{
-	UsageLine: "go mod tidy [-e] [-v]",
+	UsageLine: "go mod tidy [-e] [-v] [-go=version]",
 	Short:     "add missing and remove unused modules",
 	Long: `
 Tidy makes sure go.mod matches the source code in the module.
@@ -30,16 +32,26 @@ to standard error.
 The -e flag causes tidy to attempt to proceed despite errors
 encountered while loading packages.
 
+The -go flag causes tidy to update the 'go' directive in the go.mod
+file to the given version, which may change which module dependencies
+are retained as explicit requirements in the go.mod file.
+(Go versions 1.17 and higher retain more requirements in order to
+support lazy module loading.)
+
 See https://golang.org/ref/mod#go-mod-tidy for more about 'go mod tidy'.
 	`,
 	Run: runTidy,
 }
 
-var tidyE bool // if true, report errors but proceed anyway.
+var (
+	tidyE  bool   // if true, report errors but proceed anyway.
+	tidyGo string // go version to write to the tidied go.mod file (toggles lazy loading)
+)
 
 func init() {
 	cmdTidy.Flag.BoolVar(&cfg.BuildV, "v", false, "")
 	cmdTidy.Flag.BoolVar(&tidyE, "e", false, "")
+	cmdTidy.Flag.StringVar(&tidyGo, "go", "", "")
 	base.AddModCommonFlags(&cmdTidy.Flag)
 }
 
@@ -48,6 +60,12 @@ func runTidy(ctx context.Context, cmd *base.Command, args []string) {
 		base.Fatalf("go mod tidy: no arguments allowed")
 	}
 
+	if tidyGo != "" {
+		if !modfile.GoVersionRE.MatchString(tidyGo) {
+			base.Fatalf(`go mod: invalid -go option %q; expecting something like "-go 1.17"`, tidyGo)
+		}
+	}
+
 	// Tidy aims to make 'go test' reproducible for any package in 'all', so we
 	// need to include test dependencies. For modules that specify go 1.15 or
 	// earlier this is a no-op (because 'all' saturates transitive test
@@ -62,6 +80,7 @@ func runTidy(ctx context.Context, cmd *base.Command, args []string) {
 	modload.RootMode = modload.NeedRoot
 
 	modload.LoadPackages(ctx, modload.PackageOpts{
+		GoVersion:                tidyGo,
 		Tags:                     imports.AnyTags(),
 		Tidy:                     true,
 		VendorModulesInGOROOTSrc: true,

src/cmd/go/internal/modcmd/tidy.go

@@ -259,8 +259,8 @@ func moduleInfo(ctx context.Context, rs *Requirements, m module.Version, mode Li
 		if m.GoVersion == "" && checksumOk("/go.mod") {
 			// Load the go.mod file to determine the Go version, since it hasn't
 			// already been populated from rawGoVersion.
-			if summary, err := rawGoModSummary(mod); err == nil && summary.goVersionV != "" {
-				m.GoVersion = summary.goVersionV[1:]
+			if summary, err := rawGoModSummary(mod); err == nil && summary.goVersion != "" {
+				m.GoVersion = summary.goVersion
 			}
 		}
 

src/cmd/go/internal/modload/build.go

@@ -299,7 +299,7 @@ func readModGraph(ctx context.Context, depth modDepth, roots []module.Version) (
 			// sufficient to build the packages it contains. We must load its full
 			// transitive dependency graph to be sure that we see all relevant
 			// dependencies.
-			if depth == eager || summary.depth() == eager {
+			if depth == eager || summary.depth == eager {
 				for _, r := range summary.require {
 					enqueue(r, eager)
 				}
@@ -393,7 +393,7 @@ func LoadModGraph(ctx context.Context) *ModuleGraph {
 		base.Fatalf("go: %v", err)
 	}
 
-	commitRequirements(ctx, rs)
+	commitRequirements(ctx, modFileGoVersion(), rs)
 	return mg
 }
 
@@ -459,7 +459,7 @@ func EditBuildList(ctx context.Context, add, mustSelect []module.Version) (chang
 	if err != nil {
 		return false, err
 	}
-	commitRequirements(ctx, rs)
+	commitRequirements(ctx, modFileGoVersion(), rs)
 	return changed, err
 }
 
@@ -943,10 +943,40 @@ func updateEagerRoots(ctx context.Context, direct map[string]bool, rs *Requireme
 	if err != nil {
 		return rs, err
 	}
-	if reflect.DeepEqual(min, rs.rootModules) && reflect.DeepEqual(direct, rs.direct) {
-		// The root set is unchanged, so keep rs to preserve its cached ModuleGraph
-		// (if any).
+	if rs.depth == eager && reflect.DeepEqual(min, rs.rootModules) && reflect.DeepEqual(direct, rs.direct) {
+		// The root set is unchanged and rs was already eager, so keep rs to
+		// preserve its cached ModuleGraph (if any).
 		return rs, nil
 	}
-	return newRequirements(rs.depth, min, direct), nil
+	return newRequirements(eager, min, direct), nil
+}
+
+// convertDepth returns a version of rs with the given depth.
+// If rs already has the given depth, convertDepth returns rs unmodified.
+func convertDepth(ctx context.Context, rs *Requirements, depth modDepth) (*Requirements, error) {
+	if rs.depth == depth {
+		return rs, nil
+	}
+
+	if depth == eager {
+		// We are converting a lazy module to an eager one. The roots of an eager
+		// module graph are a superset of the roots of a lazy graph, so we don't
+		// need to add any new roots — we just need to prune away the ones that are
+		// redundant given eager loading, which is exactly what updateEagerRoots
+		// does.
+		return updateEagerRoots(ctx, rs.direct, rs, nil)
+	}
+
+	// We are converting an eager module to a lazy one. The module graph of an
+	// eager module includes the transitive dependencies of every module in the
+	// build list.
+	//
+	// Hey, we can express that as a lazy root set! “Include the transitive
+	// dependencies of every module in the build list” is exactly what happens in
+	// a lazy module if we promote every module in the build list to a root!
+	mg, err := rs.Graph(ctx)
+	if err != nil {
+		return rs, err
+	}
+	return newRequirements(lazy, mg.BuildList()[1:], rs.direct), nil
 }

src/cmd/go/internal/modload/buildlist.go

@@ -229,7 +229,7 @@ func raiseLimitsForUpgrades(ctx context.Context, maxVersion map[string]string, d
 			if err != nil {
 				return err
 			}
-			if summary.depth() == eager {
+			if summary.depth == eager {
 				// For efficiency, we'll load all of the eager upgrades as one big
 				// graph, rather than loading the (potentially-overlapping) subgraph for
 				// each upgrade individually.
@@ -522,7 +522,7 @@ func (l *versionLimiter) check(m module.Version, depth modDepth) dqState {
 		return l.disqualify(m, dqState{err: err})
 	}
 
-	if summary.depth() == eager {
+	if summary.depth == eager {
 		depth = eager
 	}
 	for _, r := range summary.require {

src/cmd/go/internal/modload/edit.go

@@ -381,7 +381,7 @@ var errGoModDirty error = goModDirtyError{}
 func LoadModFile(ctx context.Context) *Requirements {
 	rs, needCommit := loadModFile(ctx)
 	if needCommit {
-		commitRequirements(ctx, rs)
+		commitRequirements(ctx, modFileGoVersion(), rs)
 	}
 	return rs
 }
@@ -401,8 +401,9 @@ func loadModFile(ctx context.Context) (rs *Requirements, needCommit bool) {
 	if modRoot == "" {
 		Target = module.Version{Path: "command-line-arguments"}
 		targetPrefix = "command-line-arguments"
-		rawGoVersion.Store(Target, latestGoVersion())
-		requirements = newRequirements(index.depth(), nil, nil)
+		goVersion := latestGoVersion()
+		rawGoVersion.Store(Target, goVersion)
+		requirements = newRequirements(modDepthFromGoVersion(goVersion), nil, nil)
 		return requirements, false
 	}
 
@@ -432,35 +433,31 @@ func loadModFile(ctx context.Context) (rs *Requirements, needCommit bool) {
 	}
 
 	setDefaultBuildMod() // possibly enable automatic vendoring
-	rs = requirementsFromModFile(ctx, f)
+	rs = requirementsFromModFile(ctx)
 
 	if cfg.BuildMod == "vendor" {
 		readVendorList()
 		checkVendorConsistency()
 		rs.initVendor(vendorList)
 	}
 	if index.goVersionV == "" {
-		// The main module necessarily has a go.mod file, and that file lacks a
-		// 'go' directive. The 'go' command has been adding that directive
-		// automatically since Go 1.12, so this module either dates to Go 1.11 or
-		// has been erroneously hand-edited.
-		//
-		// The semantics of the go.mod file are more-or-less the same from Go 1.11
-		// through Go 1.16, changing at 1.17 for lazy loading. So even though a
-		// go.mod file without a 'go' directive is theoretically a Go 1.11 file,
-		// scripts may assume that it ends up as a Go 1.16 module. We can't go
-		// higher than that, because we don't know which semantics the user intends.
-		//
-		// (Note that 'go mod init' always adds the latest version, so scripts that
-		// use 'go mod init' will result in current-version modules instead of Go
-		// 1.16 modules.)
-		//
-		// If we are able to modify the go.mod file, we will add a 'go' directive
-		// to at least make the situation explicit going forward.
-		if cfg.BuildMod == "mod" {
-			addGoStmt("1.16")
+		// TODO(#45551): Do something more principled instead of checking
+		// cfg.CmdName directly here.
+		if cfg.BuildMod == "mod" && cfg.CmdName != "mod graph" && cfg.CmdName != "mod why" {
+			addGoStmt(latestGoVersion())
+			if go117EnableLazyLoading {
+				// We need to add a 'go' version to the go.mod file, but we must assume
+				// that its existing contents match something between Go 1.11 and 1.16.
+				// Go 1.11 through 1.16 have eager requirements, but the latest Go
+				// version uses lazy requirements instead — so we need to cnvert the
+				// requirements to be lazy.
+				rs, err = convertDepth(ctx, rs, lazy)
+				if err != nil {
+					base.Fatalf("go: %v", err)
+				}
+			}
 		} else {
-			rawGoVersion.Store(Target, "1.16")
+			rawGoVersion.Store(Target, modFileGoVersion())
 		}
 	}
 
@@ -509,7 +506,7 @@ func CreateModFile(ctx context.Context, modPath string) {
 		base.Fatalf("go: %v", err)
 	}
 
-	commitRequirements(ctx, requirementsFromModFile(ctx, modFile))
+	commitRequirements(ctx, modFileGoVersion(), requirementsFromModFile(ctx))
 
 	// Suggest running 'go mod tidy' unless the project is empty. Even if we
 	// imported all the correct requirements above, we're probably missing
@@ -661,12 +658,13 @@ func initTarget(m module.Version) {
 	}
 }
 
-// requirementsFromModFile returns the set of non-excluded requirements from f.
-func requirementsFromModFile(ctx context.Context, f *modfile.File) *Requirements {
-	roots := make([]module.Version, 0, len(f.Require))
+// requirementsFromModFile returns the set of non-excluded requirements from
+// the global modFile.
+func requirementsFromModFile(ctx context.Context) *Requirements {
+	roots := make([]module.Version, 0, len(modFile.Require))
 	mPathCount := map[string]int{Target.Path: 1}
 	direct := map[string]bool{}
-	for _, r := range f.Require {
+	for _, r := range modFile.Require {
 		if index != nil && index.exclude[r.Mod] {
 			if cfg.BuildMod == "mod" {
 				fmt.Fprintf(os.Stderr, "go: dropping requirement on excluded version %s %s\n", r.Mod.Path, r.Mod.Version)
@@ -683,7 +681,7 @@ func requirementsFromModFile(ctx context.Context, f *modfile.File) *Requirements
 		}
 	}
 	module.Sort(roots)
-	rs := newRequirements(index.depth(), roots, direct)
+	rs := newRequirements(modDepthFromGoVersion(modFileGoVersion()), roots, direct)
 
 	// If any module path appears more than once in the roots, we know that the
 	// go.mod file needs to be updated even though we have not yet loaded any
@@ -988,12 +986,12 @@ func WriteGoMod(ctx context.Context) {
 	if !allowWriteGoMod {
 		panic("WriteGoMod called while disallowed")
 	}
-	commitRequirements(ctx, LoadModFile(ctx))
+	commitRequirements(ctx, modFileGoVersion(), LoadModFile(ctx))
 }
 
 // commitRequirements writes sets the global requirements variable to rs and
 // writes its contents back to the go.mod file on disk.
-func commitRequirements(ctx context.Context, rs *Requirements) {
+func commitRequirements(ctx context.Context, goVersion string, rs *Requirements) {
 	requirements = rs
 
 	if !allowWriteGoMod {
@@ -1014,6 +1012,9 @@ func commitRequirements(ctx context.Context, rs *Requirements) {
 		})
 	}
 	modFile.SetRequire(list)
+	if goVersion != "" {
+		modFile.AddGoStmt(goVersion)
+	}
 	modFile.Cleanup()
 
 	dirty := index.modFileIsDirty(modFile)

src/cmd/go/internal/modload/init.go

@@ -72,7 +72,7 @@ func ListModules(ctx context.Context, args []string, mode ListMode) ([]*modinfo.
 	}
 
 	if err == nil {
-		commitRequirements(ctx, rs)
+		commitRequirements(ctx, modFileGoVersion(), rs)
 	}
 	return mods, err
 }