cmd/go/internal/modfetch: cache info and gomod on disk

This on-disk caching was previously done inside package vgo,
but that means that other code calling modfetch directly
does not benefit from the cache entries and does unnecessary
network operations. Fix that.

This CL also renames the local cache directory root from
$GOPATH/src/v to $GOPATH/src/mod.

The "v" stood for versioned, but "mod" seems clearer
(it's the downloaded modules).

This CL also fixes a deadlock in the caching code: one Repo's
GoMod may need to call another repo's Stat and vice versa,
so it can't be that a Repo must only be used from one goroutine
at a time, or else we'll end up with a deadlock. Redefine that
any Repo must be allowed to be used from multiple goroutines
simultaneously and update the code accordingly. This eliminates
the potential deadlock.

This CL also makes the gitrepo implementation work a bit
harder to use its local information before doing any network
operations. In particular, locally-cached tags or commits are
now resolved without consulting the remote repo.

Fixes golang/go#25919.

Change-Id: I3443ffc97a1e3e465c8780fb81f263be3b6d77ae
Reviewed-on: https://go-review.googlesource.com/119475
Run-TryBot: Russ Cox <[email protected]>
Reviewed-by: Bryan C. Mills <[email protected]>

Author: rsc

vendor/cmd/go/internal/modcmd/verify.go

@@ -29,9 +29,9 @@ func runVerify() {
 
 func verifyMod(mod module.Version) bool {
 	ok := true
-	zip := filepath.Join(vgo.SrcV, "cache", mod.Path, "/@v/", mod.Version+".zip")
+	zip := filepath.Join(vgo.SrcMod, "cache", mod.Path, "/@v/", mod.Version+".zip")
 	_, zipErr := os.Stat(zip)
-	dir := filepath.Join(vgo.SrcV, mod.Path+"@"+mod.Version)
+	dir := filepath.Join(vgo.SrcMod, mod.Path+"@"+mod.Version)
 	_, dirErr := os.Stat(dir)
 	data, err := ioutil.ReadFile(zip + "hash")
 	if err != nil {

vendor/cmd/go/internal/modfetch/cache.go

@@ -5,11 +5,24 @@
 package modfetch
 
 import (
-	"sync"
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
 
+	"cmd/go/internal/modconv"
+	"cmd/go/internal/modfetch/codehost"
 	"cmd/go/internal/par"
+	"cmd/go/internal/semver"
 )
 
+var QuietLookup bool // do not print about lookups
+
+var CacheRoot string // $GOPATH/src/mod/cache
+
 // A cachingRepo is a cache around an underlying Repo,
 // avoiding redundant calls to ModulePath, Versions, Stat, Latest, and GoMod (but not Zip).
 // It is also safe for simultaneous use by multiple goroutines
@@ -18,9 +31,7 @@ import (
 type cachingRepo struct {
 	path  string
 	cache par.Cache // cache for all operations
-
-	mu sync.Mutex // protects r's methods
-	r  Repo
+	r     Repo
 }
 
 func newCachingRepo(r Repo) *cachingRepo {
@@ -40,8 +51,6 @@ func (r *cachingRepo) Versions(prefix string) ([]string, error) {
 		err  error
 	}
 	c := r.cache.Do("versions:"+prefix, func() interface{} {
-		r.mu.Lock()
-		defer r.mu.Unlock()
 		list, err := r.r.Versions(prefix)
 		return cached{list, err}
 	}).(cached)
@@ -52,17 +61,36 @@ func (r *cachingRepo) Versions(prefix string) ([]string, error) {
 	return append([]string(nil), c.list...), nil
 }
 
+type cachedInfo struct {
+	info *RevInfo
+	err  error
+}
+
 func (r *cachingRepo) Stat(rev string) (*RevInfo, error) {
-	type cached struct {
-		info *RevInfo
-		err  error
-	}
 	c := r.cache.Do("stat:"+rev, func() interface{} {
-		r.mu.Lock()
-		defer r.mu.Unlock()
-		info, err := r.r.Stat(rev)
-		return cached{info, err}
-	}).(cached)
+		file, info, err := readDiskStat(r.path, rev)
+		if err == nil {
+			return cachedInfo{info, nil}
+		}
+
+		if !QuietLookup {
+			fmt.Fprintf(os.Stderr, "vgo: finding %s %s\n", r.path, rev)
+		}
+		info, err = r.r.Stat(rev)
+		if err == nil {
+			if err := writeDiskStat(file, info); err != nil {
+				fmt.Fprintf(os.Stderr, "go: writing stat cache: %v\n", err)
+			}
+			// If we resolved, say, 1234abcde to v0.0.0-20180604122334-1234abcdef78,
+			// then save the information under the proper version, for future use.
+			if info.Version != rev {
+				r.cache.Do("stat:"+info.Version, func() interface{} {
+					return cachedInfo{info, err}
+				})
+			}
+		}
+		return cachedInfo{info, err}
+	}).(cachedInfo)
 
 	if c.err != nil {
 		return nil, c.err
@@ -72,16 +100,28 @@ func (r *cachingRepo) Stat(rev string) (*RevInfo, error) {
 }
 
 func (r *cachingRepo) Latest() (*RevInfo, error) {
-	type cached struct {
+	type cachedInfo struct {
 		info *RevInfo
 		err  error
 	}
 	c := r.cache.Do("latest:", func() interface{} {
-		r.mu.Lock()
-		defer r.mu.Unlock()
+		if !QuietLookup {
+			fmt.Fprintf(os.Stderr, "vgo: finding %s latest\n", r.path)
+		}
 		info, err := r.r.Latest()
-		return cached{info, err}
-	}).(cached)
+
+		// Save info for likely future Stat call.
+		if err == nil {
+			r.cache.Do("stat:"+info.Version, func() interface{} {
+				return cachedInfo{info, err}
+			})
+			if file, _, err := readDiskStat(r.path, info.Version); err != nil {
+				writeDiskStat(file, info)
+			}
+		}
+
+		return cachedInfo{info, err}
+	}).(cachedInfo)
 
 	if c.err != nil {
 		return nil, c.err
@@ -96,9 +136,17 @@ func (r *cachingRepo) GoMod(rev string) ([]byte, error) {
 		err  error
 	}
 	c := r.cache.Do("gomod:"+rev, func() interface{} {
-		r.mu.Lock()
-		defer r.mu.Unlock()
-		text, err := r.r.GoMod(rev)
+		file, text, err := readDiskGoMod(r.path, rev)
+		if err == nil {
+			return cached{text, nil}
+		}
+
+		text, err = r.r.GoMod(rev)
+		if err == nil {
+			if err := writeDiskGoMod(file, text); err != nil {
+				fmt.Fprintf(os.Stderr, "go: writing go.mod cache: %v\n", err)
+			}
+		}
 		return cached{text, err}
 	}).(cached)
 
@@ -109,7 +157,186 @@ func (r *cachingRepo) GoMod(rev string) ([]byte, error) {
 }
 
 func (r *cachingRepo) Zip(version, tmpdir string) (string, error) {
-	r.mu.Lock()
-	defer r.mu.Unlock()
 	return r.r.Zip(version, tmpdir)
 }
+
+// Stat is like Lookup(path).Stat(rev) but avoids the
+// repository path resolution in Lookup if the result is
+// already cached on local disk.
+func Stat(path, rev string) (*RevInfo, error) {
+	_, info, err := readDiskStat(path, rev)
+	if err == nil {
+		return info, nil
+	}
+	repo, err := Lookup(path)
+	if err != nil {
+		return nil, err
+	}
+	return repo.Stat(rev)
+}
+
+// GoMod is like Lookup(path).GoMod(rev) but avoids the
+// repository path resolution in Lookup if the result is
+// already cached on local disk.
+func GoMod(path, rev string) ([]byte, error) {
+	// Convert commit hash to pseudo-version
+	// to increase cache hit rate.
+	if !semver.IsValid(rev) {
+		info, err := Stat(path, rev)
+		if err != nil {
+			return nil, err
+		}
+		rev = info.Version
+	}
+	_, data, err := readDiskGoMod(path, rev)
+	if err == nil {
+		return data, nil
+	}
+	repo, err := Lookup(path)
+	if err != nil {
+		return nil, err
+	}
+	return repo.GoMod(rev)
+}
+
+var errNotCached = fmt.Errorf("not in cache")
+
+// readDiskStat reads a cached stat result from disk,
+// returning the name of the cache file and the result.
+// If the read fails, the caller can use
+// writeDiskStat(file, info) to write a new cache entry.
+func readDiskStat(path, rev string) (file string, info *RevInfo, err error) {
+	file, data, err := readDiskCache(path, rev, "info")
+	if err != nil {
+		if file, info, err := readDiskStatByHash(path, rev); err == nil {
+			return file, info, nil
+		}
+		return file, nil, err
+	}
+	info = new(RevInfo)
+	if err := json.Unmarshal(data, info); err != nil {
+		return file, nil, errNotCached
+	}
+	return file, info, nil
+}
+
+// readDiskStatByHash is a fallback for readDiskStat for the case
+// where rev is a commit hash instead of a proper semantic version.
+// In that case, we look for a cached pseudo-version that matches
+// the commit hash. If we find one, we use it.
+// This matters most for converting legacy package management
+// configs, when we are often looking up commits by full hash.
+// Without this check we'd be doing network I/O to the remote repo
+// just to find out about a commit we already know about
+// (and have cached under its pseudo-version).
+func readDiskStatByHash(path, rev string) (file string, info *RevInfo, err error) {
+	if !codehost.AllHex(rev) || len(rev) < 12 {
+		return "", nil, errNotCached
+	}
+	rev = rev[:12]
+	dir, err := os.Open(filepath.Join(CacheRoot, path, "@v"))
+	if err != nil {
+		return "", nil, errNotCached
+	}
+	names, err := dir.Readdirnames(-1)
+	dir.Close()
+	if err != nil {
+		return "", nil, errNotCached
+	}
+	suffix := "-" + rev + ".info"
+	for _, name := range names {
+		if strings.HasSuffix(name, suffix) && isPseudoVersion(strings.TrimSuffix(name, ".info")) {
+			return readDiskStat(path, strings.TrimSuffix(name, ".info"))
+		}
+	}
+	return "", nil, errNotCached
+}
+
+var vgoVersion = []byte(modconv.Prefix)
+
+// readDiskGoMod reads a cached stat result from disk,
+// returning the name of the cache file and the result.
+// If the read fails, the caller can use
+// writeDiskGoMod(file, data) to write a new cache entry.
+func readDiskGoMod(path, rev string) (file string, data []byte, err error) {
+	file, data, err = readDiskCache(path, rev, "mod")
+
+	// If go.mod has a //vgo comment at the start,
+	// it was auto-converted from a legacy lock file.
+	// The auto-conversion details may have bugs and
+	// may be fixed in newer versions of vgo.
+	// We ignore cached go.mod files if they do not match
+	// our own vgoVersion.
+	// This use of "vgo" appears in disk files and must be preserved
+	// even once we excise most of the mentions of vgo from the code.
+	if err == nil && bytes.HasPrefix(data, vgoVersion[:len("//vgo")]) && !bytes.HasPrefix(data, vgoVersion) {
+		err = errNotCached
+		data = nil
+	}
+
+	return file, data, err
+}
+
+// readDiskCache is the generic "read from a cache file" implementation.
+// It takes the revision and an identifying suffix for the kind of data being cached.
+// It returns the name of the cache file and the content of the file.
+// If the read fails, the caller can use
+// writeDiskCache(file, data) to write a new cache entry.
+func readDiskCache(path, rev, suffix string) (file string, data []byte, err error) {
+	if !semver.IsValid(rev) || CacheRoot == "" {
+		return "", nil, errNotCached
+	}
+	file = filepath.Join(CacheRoot, path, "@v", rev+"."+suffix)
+	data, err = ioutil.ReadFile(file)
+	if err != nil {
+		return file, nil, errNotCached
+	}
+	return file, data, nil
+}
+
+// writeDiskStat writes a stat result cache entry.
+// The file name must have been returned by a previous call to readDiskStat.
+func writeDiskStat(file string, info *RevInfo) error {
+	if file == "" {
+		return nil
+	}
+	js, err := json.Marshal(info)
+	if err != nil {
+		return err
+	}
+	return writeDiskCache(file, js)
+}
+
+// writeDiskGoMod writes a go.mod cache entry.
+// The file name must have been returned by a previous call to readDiskGoMod.
+func writeDiskGoMod(file string, text []byte) error {
+	return writeDiskCache(file, text)
+}
+
+// writeDiskCache is the generic "write to a cache file" implementation.
+// The file must have been returned by a previous call to readDiskCache.
+func writeDiskCache(file string, data []byte) error {
+	if file == "" {
+		return nil
+	}
+	// Make sure directory for file exists.
+	if err := os.MkdirAll(filepath.Dir(file), 0777); err != nil {
+		return err
+	}
+	// Write data to temp file next to target file.
+	f, err := ioutil.TempFile(filepath.Dir(file), filepath.Base(file)+".tmp*")
+	if err != nil {
+		return err
+	}
+	defer os.Remove(f.Name())
+	defer f.Close()
+	if _, err := f.Write(data); err != nil {
+		return err
+	}
+	if err := f.Close(); err != nil {
+		return err
+	}
+	// Rename temp file onto cache file,
+	// so that the cache file is always a complete file.
+	return os.Rename(f.Name(), file)
+}