From c76d2adfaee7523900c8b659fbc07166062abb4f Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sun, 15 Nov 2020 19:20:06 +0000 Subject: [PATCH 01/19] Move last commit cache back into modules/git Signed-off-by: Andrew Thornton --- modules/cache/cache.go | 23 +++++++++++ modules/git/cache.go | 13 ------ modules/git/commit_info.go | 23 +++++++---- modules/git/commit_info_test.go | 4 +- .../last_commit_cache.go} | 41 +++++++++++-------- modules/repository/cache.go | 8 ++-- routers/repo/view.go | 4 +- templates/repo/view_list.tmpl | 11 ++--- 8 files changed, 76 insertions(+), 51 deletions(-) delete mode 100644 modules/git/cache.go rename modules/{cache/last_commit.go => git/last_commit_cache.go} (55%) diff --git a/modules/cache/cache.go b/modules/cache/cache.go index 60865d8335db6..42227f928953b 100644 --- a/modules/cache/cache.go +++ b/modules/cache/cache.go @@ -27,6 +27,24 @@ func newCache(cacheConfig setting.Cache) (mc.Cache, error) { }) } +// Cache is the interface that operates the cache data. +type Cache interface { + // Put puts value into cache with key and expire time. + Put(key string, val interface{}, timeout int64) error + // Get gets cached value by given key. + Get(key string) interface{} + // Delete deletes cached value by given key. + Delete(key string) error + // Incr increases cached int-type value by given key as a counter. + Incr(key string) error + // Decr decreases cached int-type value by given key as a counter. + Decr(key string) error + // IsExist returns true if cached value exists. + IsExist(key string) bool + // Flush deletes all cached data. + Flush() error +} + // NewContext start cache service func NewContext() error { var err error @@ -40,6 +58,11 @@ func NewContext() error { return err } +// GetCache returns the currently configured cache +func GetCache() Cache { + return conn +} + // GetString returns the key value from cache with callback when no key exists in cache func GetString(key string, getFunc func() (string, error)) (string, error) { if conn == nil || setting.CacheService.TTL == 0 { diff --git a/modules/git/cache.go b/modules/git/cache.go deleted file mode 100644 index a1f0f8a57b1fc..0000000000000 --- a/modules/git/cache.go +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2019 The Gitea Authors. All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. - -package git - -import "github.com/go-git/go-git/v5/plumbing/object" - -// LastCommitCache cache -type LastCommitCache interface { - Get(ref, entryPath string) (*object.Commit, error) - Put(ref, entryPath, commitID string) error -} diff --git a/modules/git/commit_info.go b/modules/git/commit_info.go index e03ea00fc69b9..9f5badf6ca3a8 100644 --- a/modules/git/commit_info.go +++ b/modules/git/commit_info.go @@ -13,8 +13,15 @@ import ( cgobject "github.com/go-git/go-git/v5/plumbing/object/commitgraph" ) +// CommitInfo describes the first commit with the provided entry +type CommitInfo struct { + Entry *TreeEntry + Commit *Commit + SubModuleFile *SubModuleFile +} + // GetCommitsInfo gets information of all commits that are corresponding to these entries -func (tes Entries) GetCommitsInfo(commit *Commit, treePath string, cache LastCommitCache) ([][]interface{}, *Commit, error) { +func (tes Entries) GetCommitsInfo(commit *Commit, treePath string, cache *LastCommitCache) ([]CommitInfo, *Commit, error) { entryPaths := make([]string, len(tes)+1) // Get the commit for the treePath itself entryPaths[0] = "" @@ -61,10 +68,14 @@ func (tes Entries) GetCommitsInfo(commit *Commit, treePath string, cache LastCom commit.repo.gogitStorage.Close() - commitsInfo := make([][]interface{}, len(tes)) + commitsInfo := make([]CommitInfo, len(tes)) for i, entry := range tes { + commitsInfo[i] = CommitInfo{ + Entry: entry, + } if rev, ok := revs[entry.Name()]; ok { entryCommit := convertCommit(rev) + commitsInfo[i].Commit = entryCommit if entry.IsSubModule() { subModuleURL := "" var fullPath string @@ -79,12 +90,8 @@ func (tes Entries) GetCommitsInfo(commit *Commit, treePath string, cache LastCom subModuleURL = subModule.URL } subModuleFile := NewSubModuleFile(entryCommit, subModuleURL, entry.ID.String()) - commitsInfo[i] = []interface{}{entry, subModuleFile} - } else { - commitsInfo[i] = []interface{}{entry, entryCommit} + commitsInfo[i].SubModuleFile = subModuleFile } - } else { - commitsInfo[i] = []interface{}{entry, nil} } } @@ -151,7 +158,7 @@ func getFileHashes(c cgobject.CommitNode, treePath string, paths []string) (map[ return hashes, nil } -func getLastCommitForPathsByCache(commitID, treePath string, paths []string, cache LastCommitCache) (map[string]*object.Commit, []string, error) { +func getLastCommitForPathsByCache(commitID, treePath string, paths []string, cache *LastCommitCache) (map[string]*object.Commit, []string, error) { var unHitEntryPaths []string var results = make(map[string]*object.Commit) for _, p := range paths { diff --git a/modules/git/commit_info_test.go b/modules/git/commit_info_test.go index 8bdf1a769bed6..6bd104ec3bbe7 100644 --- a/modules/git/commit_info_test.go +++ b/modules/git/commit_info_test.go @@ -67,8 +67,8 @@ func testGetCommitsInfo(t *testing.T, repo1 *Repository) { assert.NoError(t, err) assert.Len(t, commitsInfo, len(testCase.ExpectedIDs)) for _, commitInfo := range commitsInfo { - entry := commitInfo[0].(*TreeEntry) - commit := commitInfo[1].(*Commit) + entry := commitInfo.Entry + commit := commitInfo.Commit expectedID, ok := testCase.ExpectedIDs[entry.Name()] if !assert.True(t, ok) { continue diff --git a/modules/cache/last_commit.go b/modules/git/last_commit_cache.go similarity index 55% rename from modules/cache/last_commit.go rename to modules/git/last_commit_cache.go index 660a9250d63a8..bc8279b71ed44 100644 --- a/modules/cache/last_commit.go +++ b/modules/git/last_commit_cache.go @@ -2,51 +2,58 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -package cache +package git import ( "crypto/sha256" "fmt" - "code.gitea.io/gitea/modules/git" - "code.gitea.io/gitea/modules/log" - - mc "gitea.com/macaron/cache" "github.com/go-git/go-git/v5/plumbing/object" ) +// Cache represents a caching interface +type Cache interface { + // Put puts value into cache with key and expire time. + Put(key string, val interface{}, timeout int64) error + // Get gets cached value by given key. + Get(key string) interface{} +} + // LastCommitCache represents a cache to store last commit type LastCommitCache struct { repoPath string ttl int64 - repo *git.Repository + repo *Repository commitCache map[string]*object.Commit - mc.Cache + cache Cache } // NewLastCommitCache creates a new last commit cache for repo -func NewLastCommitCache(repoPath string, gitRepo *git.Repository, ttl int64) *LastCommitCache { +func NewLastCommitCache(repoPath string, gitRepo *Repository, ttl int64, cache Cache) *LastCommitCache { + if cache == nil { + return nil + } return &LastCommitCache{ repoPath: repoPath, repo: gitRepo, commitCache: make(map[string]*object.Commit), ttl: ttl, - Cache: conn, + cache: cache, } } -func (c LastCommitCache) getCacheKey(repoPath, ref, entryPath string) string { +func (c *LastCommitCache) getCacheKey(repoPath, ref, entryPath string) string { hashBytes := sha256.Sum256([]byte(fmt.Sprintf("%s:%s:%s", repoPath, ref, entryPath))) return fmt.Sprintf("last_commit:%x", hashBytes) } // Get get the last commit information by commit id and entry path -func (c LastCommitCache) Get(ref, entryPath string) (*object.Commit, error) { - v := c.Cache.Get(c.getCacheKey(c.repoPath, ref, entryPath)) +func (c *LastCommitCache) Get(ref, entryPath string) (*object.Commit, error) { + v := c.cache.Get(c.getCacheKey(c.repoPath, ref, entryPath)) if vs, ok := v.(string); ok { - log.Trace("LastCommitCache hit level 1: [%s:%s:%s]", ref, entryPath, vs) + log("LastCommitCache hit level 1: [%s:%s:%s]", ref, entryPath, vs) if commit, ok := c.commitCache[vs]; ok { - log.Trace("LastCommitCache hit level 2: [%s:%s:%s]", ref, entryPath, vs) + log("LastCommitCache hit level 2: [%s:%s:%s]", ref, entryPath, vs) return commit, nil } id, err := c.repo.ConvertToSHA1(vs) @@ -64,7 +71,7 @@ func (c LastCommitCache) Get(ref, entryPath string) (*object.Commit, error) { } // Put put the last commit id with commit and entry path -func (c LastCommitCache) Put(ref, entryPath, commitID string) error { - log.Trace("LastCommitCache save: [%s:%s:%s]", ref, entryPath, commitID) - return c.Cache.Put(c.getCacheKey(c.repoPath, ref, entryPath), commitID, c.ttl) +func (c *LastCommitCache) Put(ref, entryPath, commitID string) error { + log("LastCommitCache save: [%s:%s:%s]", ref, entryPath, commitID) + return c.cache.Put(c.getCacheKey(c.repoPath, ref, entryPath), commitID, c.ttl) } diff --git a/modules/repository/cache.go b/modules/repository/cache.go index 508e5bec0b761..4d8d7866e7dff 100644 --- a/modules/repository/cache.go +++ b/modules/repository/cache.go @@ -16,7 +16,7 @@ import ( cgobject "github.com/go-git/go-git/v5/plumbing/object/commitgraph" ) -func recusiveCache(gitRepo *git.Repository, c cgobject.CommitNode, tree *git.Tree, treePath string, ca *cache.LastCommitCache, level int) error { +func recursiveCache(gitRepo *git.Repository, c cgobject.CommitNode, tree *git.Tree, treePath string, ca *git.LastCommitCache, level int) error { if level == 0 { return nil } @@ -47,7 +47,7 @@ func recusiveCache(gitRepo *git.Repository, c cgobject.CommitNode, tree *git.Tre if err != nil { return err } - if err := recusiveCache(gitRepo, c, subTree, entry, ca, level-1); err != nil { + if err := recursiveCache(gitRepo, c, subTree, entry, ca, level-1); err != nil { return err } } @@ -91,7 +91,7 @@ func CacheRef(repo *models.Repository, gitRepo *git.Repository, fullRefName stri return err } - ca := cache.NewLastCommitCache(repo.FullName(), gitRepo, int64(setting.CacheService.LastCommit.TTL.Seconds())) + ca := git.NewLastCommitCache(repo.FullName(), gitRepo, int64(setting.CacheService.LastCommit.TTL.Seconds()), cache.GetCache()) - return recusiveCache(gitRepo, c, &commit.Tree, "", ca, 1) + return recursiveCache(gitRepo, c, &commit.Tree, "", ca, 1) } diff --git a/routers/repo/view.go b/routers/repo/view.go index 2df5b30ce8f95..7d69ee4cf8851 100644 --- a/routers/repo/view.go +++ b/routers/repo/view.go @@ -137,9 +137,9 @@ func renderDirectory(ctx *context.Context, treeLink string) { } entries.CustomSort(base.NaturalSortLess) - var c git.LastCommitCache + var c *git.LastCommitCache if setting.CacheService.LastCommit.Enabled && ctx.Repo.CommitsCount >= setting.CacheService.LastCommit.CommitsCount { - c = cache.NewLastCommitCache(ctx.Repo.Repository.FullName(), ctx.Repo.GitRepo, int64(setting.CacheService.LastCommit.TTL.Seconds())) + c = git.NewLastCommitCache(ctx.Repo.Repository.FullName(), ctx.Repo.GitRepo, int64(setting.CacheService.LastCommit.TTL.Seconds()), cache.GetCache()) } var latestCommit *git.Commit diff --git a/templates/repo/view_list.tmpl b/templates/repo/view_list.tmpl index b4f2260fe2bdd..7549122dc5aab 100644 --- a/templates/repo/view_list.tmpl +++ b/templates/repo/view_list.tmpl @@ -40,18 +40,19 @@ {{end}} {{range $item := .Files}} - {{$entry := index $item 0}} - {{$commit := index $item 1}} + {{$entry := $item.Entry}} + {{$commit := $item.Commit}} + {{$subModuleFile := $item.SubModuleFile}} {{if $entry.IsSubModule}} {{svg "octicon-file-submodule"}} - {{$refURL := $commit.RefURL AppUrl $.Repository.FullName $.SSHDomain}} + {{$refURL := $subModuleFile.RefURL AppUrl $.Repository.FullName $.SSHDomain}} {{if $refURL}} - {{$entry.Name}}@{{ShortSha $commit.RefID}} + {{$entry.Name}}@{{ShortSha $subModuleFile.RefID}} {{else}} - {{$entry.Name}}@{{ShortSha $commit.RefID}} + {{$entry.Name}}@{{ShortSha $subModuleFile.RefID}} {{end}} {{else}} {{if $entry.IsDir}} From fd69b5bfdc66622309342f1dbbde2e9e9d42badc Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sun, 15 Nov 2020 15:26:51 +0000 Subject: [PATCH 02/19] Remove go-git from the interface for last commit cache Signed-off-by: Andrew Thornton --- modules/git/commit_info.go | 2 +- modules/git/last_commit_cache.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/git/commit_info.go b/modules/git/commit_info.go index 9f5badf6ca3a8..5df10cf46d145 100644 --- a/modules/git/commit_info.go +++ b/modules/git/commit_info.go @@ -167,7 +167,7 @@ func getLastCommitForPathsByCache(commitID, treePath string, paths []string, cac return nil, nil, err } if lastCommit != nil { - results[p] = lastCommit + results[p] = lastCommit.(*object.Commit) continue } diff --git a/modules/git/last_commit_cache.go b/modules/git/last_commit_cache.go index bc8279b71ed44..1f5f19e9e0207 100644 --- a/modules/git/last_commit_cache.go +++ b/modules/git/last_commit_cache.go @@ -48,7 +48,7 @@ func (c *LastCommitCache) getCacheKey(repoPath, ref, entryPath string) string { } // Get get the last commit information by commit id and entry path -func (c *LastCommitCache) Get(ref, entryPath string) (*object.Commit, error) { +func (c *LastCommitCache) Get(ref, entryPath string) (interface{}, error) { v := c.cache.Get(c.getCacheKey(c.repoPath, ref, entryPath)) if vs, ok := v.(string); ok { log("LastCommitCache hit level 1: [%s:%s:%s]", ref, entryPath, vs) From 20139553b7194322a96c57fee1713822b039ee07 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sun, 15 Nov 2020 20:01:52 +0000 Subject: [PATCH 03/19] move cacheref to last_commit_cache Signed-off-by: Andrew Thornton --- modules/git/last_commit_cache.go | 55 ++++++++++++++++++++++++++++++++ modules/repository/cache.go | 54 ++----------------------------- 2 files changed, 57 insertions(+), 52 deletions(-) diff --git a/modules/git/last_commit_cache.go b/modules/git/last_commit_cache.go index 1f5f19e9e0207..b69b1d868ac28 100644 --- a/modules/git/last_commit_cache.go +++ b/modules/git/last_commit_cache.go @@ -7,8 +7,10 @@ package git import ( "crypto/sha256" "fmt" + "path" "github.com/go-git/go-git/v5/plumbing/object" + cgobject "github.com/go-git/go-git/v5/plumbing/object/commitgraph" ) // Cache represents a caching interface @@ -75,3 +77,56 @@ func (c *LastCommitCache) Put(ref, entryPath, commitID string) error { log("LastCommitCache save: [%s:%s:%s]", ref, entryPath, commitID) return c.cache.Put(c.getCacheKey(c.repoPath, ref, entryPath), commitID, c.ttl) } + +// CacheCommit will cache the commit from the gitRepository +func (c *LastCommitCache) CacheCommit(gitRepo *Repository, commit *Commit) error { + + commitNodeIndex, _ := gitRepo.CommitNodeIndex() + + index, err := commitNodeIndex.Get(commit.ID) + if err != nil { + return err + } + + return c.recursiveCache(gitRepo, index, &commit.Tree, "", 1) +} + +func (c *LastCommitCache) recursiveCache(gitRepo *Repository, index cgobject.CommitNode, tree *Tree, treePath string, level int) error { + if level == 0 { + return nil + } + + entries, err := tree.ListEntries() + if err != nil { + return err + } + + entryPaths := make([]string, len(entries)) + entryMap := make(map[string]*TreeEntry) + for i, entry := range entries { + entryPaths[i] = entry.Name() + entryMap[entry.Name()] = entry + } + + commits, err := GetLastCommitForPaths(index, treePath, entryPaths) + if err != nil { + return err + } + + for entry, cm := range commits { + if err := c.Put(index.ID().String(), path.Join(treePath, entry), cm.ID().String()); err != nil { + return err + } + if entryMap[entry].IsDir() { + subTree, err := tree.SubTree(entry) + if err != nil { + return err + } + if err := c.recursiveCache(gitRepo, index, subTree, entry, level-1); err != nil { + return err + } + } + } + + return nil +} diff --git a/modules/repository/cache.go b/modules/repository/cache.go index 4d8d7866e7dff..1ac1bdd277eed 100644 --- a/modules/repository/cache.go +++ b/modules/repository/cache.go @@ -5,57 +5,14 @@ package repository import ( - "path" "strings" "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/cache" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/setting" - - cgobject "github.com/go-git/go-git/v5/plumbing/object/commitgraph" ) -func recursiveCache(gitRepo *git.Repository, c cgobject.CommitNode, tree *git.Tree, treePath string, ca *git.LastCommitCache, level int) error { - if level == 0 { - return nil - } - - entries, err := tree.ListEntries() - if err != nil { - return err - } - - entryPaths := make([]string, len(entries)) - entryMap := make(map[string]*git.TreeEntry) - for i, entry := range entries { - entryPaths[i] = entry.Name() - entryMap[entry.Name()] = entry - } - - commits, err := git.GetLastCommitForPaths(c, treePath, entryPaths) - if err != nil { - return err - } - - for entry, cm := range commits { - if err := ca.Put(c.ID().String(), path.Join(treePath, entry), cm.ID().String()); err != nil { - return err - } - if entryMap[entry].IsDir() { - subTree, err := tree.SubTree(entry) - if err != nil { - return err - } - if err := recursiveCache(gitRepo, c, subTree, entry, ca, level-1); err != nil { - return err - } - } - } - - return nil -} - func getRefName(fullRefName string) string { if strings.HasPrefix(fullRefName, git.TagPrefix) { return fullRefName[len(git.TagPrefix):] @@ -84,14 +41,7 @@ func CacheRef(repo *models.Repository, gitRepo *git.Repository, fullRefName stri return nil } - commitNodeIndex, _ := gitRepo.CommitNodeIndex() - - c, err := commitNodeIndex.Get(commit.ID) - if err != nil { - return err - } - - ca := git.NewLastCommitCache(repo.FullName(), gitRepo, int64(setting.CacheService.LastCommit.TTL.Seconds()), cache.GetCache()) + commitCache := git.NewLastCommitCache(repo.FullName(), gitRepo, int64(setting.CacheService.LastCommit.TTL.Seconds()), cache.GetCache()) - return recursiveCache(gitRepo, c, &commit.Tree, "", ca, 1) + return commitCache.CacheCommit(gitRepo, commit) } From 58f0579d050c198923ab411f2db24687480a770b Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sun, 15 Nov 2020 20:07:02 +0000 Subject: [PATCH 04/19] Remove go-git from routers/private/hook Signed-off-by: Andrew Thornton --- modules/git/commit_reader.go | 8 +++----- routers/private/hook.go | 3 +-- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/modules/git/commit_reader.go b/modules/git/commit_reader.go index fdcb6dca84a9b..666c2c23c82cf 100644 --- a/modules/git/commit_reader.go +++ b/modules/git/commit_reader.go @@ -9,13 +9,11 @@ import ( "bytes" "io" "strings" - - "github.com/go-git/go-git/v5/plumbing" ) // CommitFromReader will generate a Commit from a provided reader // We will need this to interpret commits from cat-file -func CommitFromReader(gitRepo *Repository, sha plumbing.Hash, reader io.Reader) (*Commit, error) { +func CommitFromReader(gitRepo *Repository, sha SHA1, reader io.Reader) (*Commit, error) { commit := &Commit{ ID: sha, } @@ -72,10 +70,10 @@ func CommitFromReader(gitRepo *Repository, sha plumbing.Hash, reader io.Reader) switch string(split[0]) { case "tree": - commit.Tree = *NewTree(gitRepo, plumbing.NewHash(string(data))) + commit.Tree = *NewTree(gitRepo, MustIDFromString(string(data))) _, _ = payloadSB.Write(line) case "parent": - commit.Parents = append(commit.Parents, plumbing.NewHash(string(data))) + commit.Parents = append(commit.Parents, MustIDFromString(string(data))) _, _ = payloadSB.Write(line) case "author": commit.Author = &Signature{} diff --git a/routers/private/hook.go b/routers/private/hook.go index dac39407562fc..1b9ab000e4be8 100644 --- a/routers/private/hook.go +++ b/routers/private/hook.go @@ -25,7 +25,6 @@ import ( repo_service "code.gitea.io/gitea/services/repository" "gitea.com/macaron/macaron" - "github.com/go-git/go-git/v5/plumbing" ) func verifyCommits(oldCommitID, newCommitID string, repo *git.Repository, env []string) error { @@ -82,7 +81,7 @@ func readAndVerifyCommit(sha string, repo *git.Repository, env []string) error { _ = stdoutReader.Close() _ = stdoutWriter.Close() }() - hash := plumbing.NewHash(sha) + hash := git.MustIDFromString(sha) return git.NewCommand("cat-file", "commit", sha). RunInDirTimeoutEnvFullPipelineFunc(env, -1, repo.Path, From d45a18faf263ef752b7571c3f8bfdc47577e46e3 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sun, 15 Nov 2020 21:01:01 +0000 Subject: [PATCH 05/19] Move FindLFSFiles to pipeline Signed-off-by: Andrew Thornton --- modules/git/pipeline/lfs.go | 157 ++++++++++++++++++++++++++++++++++++ modules/git/sha1.go | 5 ++ routers/repo/lfs.go | 146 ++------------------------------- 3 files changed, 168 insertions(+), 140 deletions(-) create mode 100644 modules/git/pipeline/lfs.go diff --git a/modules/git/pipeline/lfs.go b/modules/git/pipeline/lfs.go new file mode 100644 index 0000000000000..a106cc8c86e24 --- /dev/null +++ b/modules/git/pipeline/lfs.go @@ -0,0 +1,157 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package pipeline + +import ( + "bufio" + "fmt" + "io" + "sort" + "strings" + "sync" + "time" + + "code.gitea.io/gitea/modules/git" + gogit "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing/object" +) + +// LFSResult represents commits found using a provided pointer file hash +type LFSResult struct { + Name string + SHA string + Summary string + When time.Time + ParentHashes []git.SHA1 + BranchName string + FullCommitName string +} + +type lfsResultSlice []*LFSResult + +func (a lfsResultSlice) Len() int { return len(a) } +func (a lfsResultSlice) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a lfsResultSlice) Less(i, j int) bool { return a[j].When.After(a[i].When) } + +// FindLFSFile finds commits that contain a provided pointer file hash +func FindLFSFile(repo *git.Repository, hash git.SHA1) ([]*LFSResult, error) { + resultsMap := map[string]*LFSResult{} + results := make([]*LFSResult, 0) + + basePath := repo.Path + gogitRepo := repo.GoGitRepo() + + commitsIter, err := gogitRepo.Log(&gogit.LogOptions{ + Order: gogit.LogOrderCommitterTime, + All: true, + }) + if err != nil { + return nil, fmt.Errorf("Failed to get GoGit CommitsIter. Error: %w", err) + } + + err = commitsIter.ForEach(func(gitCommit *object.Commit) error { + tree, err := gitCommit.Tree() + if err != nil { + return err + } + treeWalker := object.NewTreeWalker(tree, true, nil) + defer treeWalker.Close() + for { + name, entry, err := treeWalker.Next() + if err == io.EOF { + break + } + if entry.Hash == hash { + result := LFSResult{ + Name: name, + SHA: gitCommit.Hash.String(), + Summary: strings.Split(strings.TrimSpace(gitCommit.Message), "\n")[0], + When: gitCommit.Author.When, + ParentHashes: gitCommit.ParentHashes, + } + resultsMap[gitCommit.Hash.String()+":"+name] = &result + } + } + return nil + }) + if err != nil && err != io.EOF { + return nil, fmt.Errorf("Failure in CommitIter.ForEach: %w", err) + } + + for _, result := range resultsMap { + hasParent := false + for _, parentHash := range result.ParentHashes { + if _, hasParent = resultsMap[parentHash.String()+":"+result.Name]; hasParent { + break + } + } + if !hasParent { + results = append(results, result) + } + } + + sort.Sort(lfsResultSlice(results)) + + // Should really use a go-git function here but name-rev is not completed and recapitulating it is not simple + shasToNameReader, shasToNameWriter := io.Pipe() + nameRevStdinReader, nameRevStdinWriter := io.Pipe() + errChan := make(chan error, 1) + wg := sync.WaitGroup{} + wg.Add(3) + + go func() { + defer wg.Done() + scanner := bufio.NewScanner(nameRevStdinReader) + i := 0 + for scanner.Scan() { + line := scanner.Text() + if len(line) == 0 { + continue + } + result := results[i] + result.FullCommitName = line + result.BranchName = strings.Split(line, "~")[0] + i++ + } + }() + go NameRevStdin(shasToNameReader, nameRevStdinWriter, &wg, basePath) + go func() { + defer wg.Done() + defer shasToNameWriter.Close() + for _, result := range results { + i := 0 + if i < len(result.SHA) { + n, err := shasToNameWriter.Write([]byte(result.SHA)[i:]) + if err != nil { + errChan <- err + break + } + i += n + } + n := 0 + for n < 1 { + n, err = shasToNameWriter.Write([]byte{'\n'}) + if err != nil { + errChan <- err + break + } + + } + + } + }() + + wg.Wait() + + select { + case err, has := <-errChan: + if has { + return nil, fmt.Errorf("Unable to obtain name for LFS files. Error: %w", err) + } + default: + } + + return results, nil +} diff --git a/modules/git/sha1.go b/modules/git/sha1.go index 06c8ad14b55c0..d3563060f78cb 100644 --- a/modules/git/sha1.go +++ b/modules/git/sha1.go @@ -60,3 +60,8 @@ func NewIDFromString(s string) (SHA1, error) { } return NewID(b) } + +// ComputeBlobHash compute the hash for a given blob content +func ComputeBlobHash(content []byte) SHA1 { + return plumbing.ComputeHash(plumbing.BlobObject, content) +} diff --git a/routers/repo/lfs.go b/routers/repo/lfs.go index be95e56d3b150..c74b088e2e4fd 100644 --- a/routers/repo/lfs.go +++ b/routers/repo/lfs.go @@ -12,11 +12,9 @@ import ( "io" "io/ioutil" "path" - "sort" "strconv" "strings" "sync" - "time" "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/base" @@ -29,9 +27,6 @@ import ( "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/storage" - gogit "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/plumbing/object" "github.com/unknwon/com" ) @@ -363,22 +358,6 @@ func LFSDelete(ctx *context.Context) { ctx.Redirect(ctx.Repo.RepoLink + "/settings/lfs") } -type lfsResult struct { - Name string - SHA string - Summary string - When time.Time - ParentHashes []plumbing.Hash - BranchName string - FullCommitName string -} - -type lfsResultSlice []*lfsResult - -func (a lfsResultSlice) Len() int { return len(a) } -func (a lfsResultSlice) Swap(i, j int) { a[i], a[j] = a[j], a[i] } -func (a lfsResultSlice) Less(i, j int) bool { return a[j].When.After(a[i].When) } - // LFSFileFind guesses a sha for the provided oid (or uses the provided sha) and then finds the commits that contain this sha func LFSFileFind(ctx *context.Context) { if !setting.LFS.StartServer { @@ -394,140 +373,27 @@ func LFSFileFind(ctx *context.Context) { sha := ctx.Query("sha") ctx.Data["Title"] = oid ctx.Data["PageIsSettingsLFS"] = true - var hash plumbing.Hash + var hash git.SHA1 if len(sha) == 0 { meta := models.LFSMetaObject{Oid: oid, Size: size} pointer := meta.Pointer() - hash = plumbing.ComputeHash(plumbing.BlobObject, []byte(pointer)) + hash = git.ComputeBlobHash([]byte(pointer)) sha = hash.String() } else { - hash = plumbing.NewHash(sha) + hash = git.MustIDFromString(sha) } ctx.Data["LFSFilesLink"] = ctx.Repo.RepoLink + "/settings/lfs" ctx.Data["Oid"] = oid ctx.Data["Size"] = size ctx.Data["SHA"] = sha - resultsMap := map[string]*lfsResult{} - results := make([]*lfsResult, 0) - - basePath := ctx.Repo.Repository.RepoPath() - gogitRepo := ctx.Repo.GitRepo.GoGitRepo() - - commitsIter, err := gogitRepo.Log(&gogit.LogOptions{ - Order: gogit.LogOrderCommitterTime, - All: true, - }) - if err != nil { - log.Error("Failed to get GoGit CommitsIter: %v", err) - ctx.ServerError("LFSFind: Iterate Commits", err) - return - } - - err = commitsIter.ForEach(func(gitCommit *object.Commit) error { - tree, err := gitCommit.Tree() - if err != nil { - return err - } - treeWalker := object.NewTreeWalker(tree, true, nil) - defer treeWalker.Close() - for { - name, entry, err := treeWalker.Next() - if err == io.EOF { - break - } - if entry.Hash == hash { - result := lfsResult{ - Name: name, - SHA: gitCommit.Hash.String(), - Summary: strings.Split(strings.TrimSpace(gitCommit.Message), "\n")[0], - When: gitCommit.Author.When, - ParentHashes: gitCommit.ParentHashes, - } - resultsMap[gitCommit.Hash.String()+":"+name] = &result - } - } - return nil - }) + results, err := pipeline.FindLFSFile(ctx.Repo.GitRepo, hash) if err != nil && err != io.EOF { - log.Error("Failure in CommitIter.ForEach: %v", err) - ctx.ServerError("LFSFind: IterateCommits ForEach", err) + log.Error("Failure in FindLFSFile: %v", err) + ctx.ServerError("LFSFind: FindLFSFile.", err) return } - for _, result := range resultsMap { - hasParent := false - for _, parentHash := range result.ParentHashes { - if _, hasParent = resultsMap[parentHash.String()+":"+result.Name]; hasParent { - break - } - } - if !hasParent { - results = append(results, result) - } - } - - sort.Sort(lfsResultSlice(results)) - - // Should really use a go-git function here but name-rev is not completed and recapitulating it is not simple - shasToNameReader, shasToNameWriter := io.Pipe() - nameRevStdinReader, nameRevStdinWriter := io.Pipe() - errChan := make(chan error, 1) - wg := sync.WaitGroup{} - wg.Add(3) - - go func() { - defer wg.Done() - scanner := bufio.NewScanner(nameRevStdinReader) - i := 0 - for scanner.Scan() { - line := scanner.Text() - if len(line) == 0 { - continue - } - result := results[i] - result.FullCommitName = line - result.BranchName = strings.Split(line, "~")[0] - i++ - } - }() - go pipeline.NameRevStdin(shasToNameReader, nameRevStdinWriter, &wg, basePath) - go func() { - defer wg.Done() - defer shasToNameWriter.Close() - for _, result := range results { - i := 0 - if i < len(result.SHA) { - n, err := shasToNameWriter.Write([]byte(result.SHA)[i:]) - if err != nil { - errChan <- err - break - } - i += n - } - n := 0 - for n < 1 { - n, err = shasToNameWriter.Write([]byte{'\n'}) - if err != nil { - errChan <- err - break - } - - } - - } - }() - - wg.Wait() - - select { - case err, has := <-errChan: - if has { - ctx.ServerError("LFSPointerFiles", err) - } - default: - } - ctx.Data["Results"] = results ctx.HTML(200, tplSettingsLFSFileFind) } From 164a02ed8b69174b718ed547efdff58eafff33a2 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Thu, 12 Nov 2020 20:43:56 +0000 Subject: [PATCH 06/19] Make no-go-git variants Signed-off-by: Andrew Thornton --- modules/convert/git_commit_test.go | 3 +- modules/git/blob.go | 2 + modules/git/blob_nogogit.go | 149 ++++++ modules/git/command.go | 2 +- modules/git/commit.go | 59 +-- modules/git/commit_gogit.go | 70 +++ modules/git/commit_info.go | 2 + modules/git/commit_info_nogogit.go | 548 +++++++++++++++++++++ modules/git/commit_info_test.go | 12 +- modules/git/commit_reader.go | 34 +- modules/git/last_commit_cache.go | 2 + modules/git/last_commit_cache_nogogit.go | 124 +++++ modules/git/notes.go | 2 + modules/git/notes_nogogit.go | 69 +++ modules/git/parse.go | 2 + modules/git/parse_nogogit.go | 78 +++ modules/git/parse_test.go | 2 + modules/git/pipeline/lfs.go | 2 + modules/git/pipeline/lfs_nogogit.go | 257 ++++++++++ modules/git/repo.go | 2 + modules/git/repo_blob.go | 2 + modules/git/repo_blob_nogogit.go | 26 + modules/git/repo_branch.go | 2 + modules/git/repo_branch_nogogit.go | 229 +++++++++ modules/git/repo_commit.go | 2 + modules/git/repo_commit_nogogit.go | 519 +++++++++++++++++++ modules/git/repo_commitgraph.go | 2 + modules/git/repo_language_stats.go | 2 + modules/git/repo_language_stats_nogogit.go | 112 +++++ modules/git/repo_nogogit.go | 403 +++++++++++++++ modules/git/repo_object.go | 5 + modules/git/repo_ref.go | 2 + modules/git/repo_ref_nogogit.go | 89 ++++ modules/git/repo_tag.go | 2 + modules/git/repo_tag_nogogit.go | 264 ++++++++++ modules/git/repo_tree.go | 4 +- modules/git/repo_tree_nogogit.go | 157 ++++++ modules/git/sha1.go | 2 + modules/git/sha1_nogogit.go | 108 ++++ modules/git/signature.go | 2 + modules/git/signature_nogogit.go | 99 ++++ modules/git/tag.go | 31 +- modules/git/tree.go | 2 + modules/git/tree_blob.go | 2 + modules/git/tree_blob_nogogit.go | 64 +++ modules/git/tree_entry.go | 20 +- modules/git/tree_entry_mode.go | 36 ++ modules/git/tree_entry_nogogit.go | 246 +++++++++ modules/git/tree_entry_test.go | 2 + modules/git/tree_nogogit.go | 106 ++++ modules/git/utils.go | 32 +- modules/indexer/stats/db.go | 3 + 52 files changed, 3887 insertions(+), 111 deletions(-) create mode 100644 modules/git/blob_nogogit.go create mode 100644 modules/git/commit_gogit.go create mode 100644 modules/git/commit_info_nogogit.go create mode 100644 modules/git/last_commit_cache_nogogit.go create mode 100644 modules/git/notes_nogogit.go create mode 100644 modules/git/parse_nogogit.go create mode 100644 modules/git/pipeline/lfs_nogogit.go create mode 100644 modules/git/repo_blob_nogogit.go create mode 100644 modules/git/repo_branch_nogogit.go create mode 100644 modules/git/repo_commit_nogogit.go create mode 100644 modules/git/repo_language_stats_nogogit.go create mode 100644 modules/git/repo_nogogit.go create mode 100644 modules/git/repo_ref_nogogit.go create mode 100644 modules/git/repo_tag_nogogit.go create mode 100644 modules/git/repo_tree_nogogit.go create mode 100644 modules/git/sha1_nogogit.go create mode 100644 modules/git/signature_nogogit.go create mode 100644 modules/git/tree_blob_nogogit.go create mode 100644 modules/git/tree_entry_mode.go create mode 100644 modules/git/tree_entry_nogogit.go create mode 100644 modules/git/tree_nogogit.go diff --git a/modules/convert/git_commit_test.go b/modules/convert/git_commit_test.go index 2158d0d77796d..aa35571706e29 100644 --- a/modules/convert/git_commit_test.go +++ b/modules/convert/git_commit_test.go @@ -13,7 +13,6 @@ import ( api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/util" - "github.com/go-git/go-git/v5/plumbing/object" "github.com/stretchr/testify/assert" ) @@ -21,7 +20,7 @@ func TestToCommitMeta(t *testing.T) { assert.NoError(t, models.PrepareTestDatabase()) headRepo := models.AssertExistsAndLoadBean(t, &models.Repository{ID: 1}).(*models.Repository) sha1, _ := git.NewIDFromString("0000000000000000000000000000000000000000") - signature := &object.Signature{Name: "Test Signature", Email: "test@email.com", When: time.Unix(0, 0)} + signature := &git.Signature{Name: "Test Signature", Email: "test@email.com", When: time.Unix(0, 0)} tag := &git.Tag{ Name: "Test Tag", ID: sha1, diff --git a/modules/git/blob.go b/modules/git/blob.go index 98545f2f909b0..9a23518522172 100644 --- a/modules/git/blob.go +++ b/modules/git/blob.go @@ -3,6 +3,8 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. +// +build !nogogit + package git import ( diff --git a/modules/git/blob_nogogit.go b/modules/git/blob_nogogit.go new file mode 100644 index 0000000000000..2eba974067a04 --- /dev/null +++ b/modules/git/blob_nogogit.go @@ -0,0 +1,149 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build nogogit + +package git + +import ( + "bufio" + "bytes" + "encoding/base64" + "io" + "io/ioutil" + "strconv" + "strings" +) + +// Blob represents a Git object. +type Blob struct { + ID SHA1 + + gotSize bool + size int64 + repoPath string + name string +} + +// DataAsync gets a ReadCloser for the contents of a blob without reading it all. +// Calling the Close function on the result will discard all unread output. +func (b *Blob) DataAsync() (io.ReadCloser, error) { + stdoutReader, stdoutWriter := io.Pipe() + var err error + + go func() { + stderr := &strings.Builder{} + err = NewCommand("cat-file", "--batch").RunInDirFullPipeline(b.repoPath, stdoutWriter, stderr, strings.NewReader(b.ID.String()+"\n")) + if err != nil { + err = ConcatenateError(err, stderr.String()) + stdoutWriter.CloseWithError(err) + } else { + stdoutWriter.Close() + } + }() + + bufReader := bufio.NewReader(stdoutReader) + _, _, size, err := ReadBatchLine(bufReader) + if err != nil { + stdoutReader.Close() + return nil, err + } + + return &LimitedReaderCloser{ + R: bufReader, + C: stdoutReader, + N: int64(size), + }, err +} + +// Size returns the uncompressed size of the blob +func (b *Blob) Size() int64 { + if b.gotSize { + return b.size + } + + size, err := NewCommand("cat-file", "-s", b.ID.String()).RunInDir(b.repoPath) + if err != nil { + log("error whilst reading size for %s in %s. Error: %v", b.ID.String(), b.repoPath, err) + return 0 + } + + b.size, err = strconv.ParseInt(size[:len(size)-1], 10, 64) + if err != nil { + log("error whilst parsing size %s for %s in %s. Error: %v", size, b.ID.String(), b.repoPath, err) + return 0 + } + b.gotSize = true + + return b.size +} + +// Name returns name of the tree entry this blob object was created from (or empty string) +func (b *Blob) Name() string { + return b.name +} + +// GetBlobContent Gets the content of the blob as raw text +func (b *Blob) GetBlobContent() (string, error) { + dataRc, err := b.DataAsync() + if err != nil { + return "", err + } + defer dataRc.Close() + buf := make([]byte, 1024) + n, _ := dataRc.Read(buf) + buf = buf[:n] + return string(buf), nil +} + +// GetBlobLineCount gets line count of lob as raw text +func (b *Blob) GetBlobLineCount() (int, error) { + reader, err := b.DataAsync() + if err != nil { + return 0, err + } + defer reader.Close() + buf := make([]byte, 32*1024) + count := 0 + lineSep := []byte{'\n'} + for { + c, err := reader.Read(buf) + count += bytes.Count(buf[:c], lineSep) + switch { + case err == io.EOF: + return count, nil + case err != nil: + return count, err + } + } +} + +// GetBlobContentBase64 Reads the content of the blob with a base64 encode and returns the encoded string +func (b *Blob) GetBlobContentBase64() (string, error) { + dataRc, err := b.DataAsync() + if err != nil { + return "", err + } + defer dataRc.Close() + + pr, pw := io.Pipe() + encoder := base64.NewEncoder(base64.StdEncoding, pw) + + go func() { + _, err := io.Copy(encoder, dataRc) + _ = encoder.Close() + + if err != nil { + _ = pw.CloseWithError(err) + } else { + _ = pw.Close() + } + }() + + out, err := ioutil.ReadAll(pr) + if err != nil { + return "", err + } + return string(out), nil +} diff --git a/modules/git/command.go b/modules/git/command.go index d40c0bfa2322b..2328ccd9a391f 100644 --- a/modules/git/command.go +++ b/modules/git/command.go @@ -188,7 +188,7 @@ func (c *Command) RunInDirTimeoutEnv(env []string, timeout time.Duration, dir st stdout := new(bytes.Buffer) stderr := new(bytes.Buffer) if err := c.RunInDirTimeoutEnvPipeline(env, timeout, dir, stdout, stderr); err != nil { - return nil, concatenateError(err, stderr.String()) + return nil, ConcatenateError(err, stderr.String()) } if stdout.Len() > 0 { diff --git a/modules/git/commit.go b/modules/git/commit.go index 6425345ea8ee0..ce82c2f58209c 100644 --- a/modules/git/commit.go +++ b/modules/git/commit.go @@ -19,8 +19,6 @@ import ( "net/http" "strconv" "strings" - - "github.com/go-git/go-git/v5/plumbing/object" ) // Commit represents a git commit. @@ -43,61 +41,6 @@ type CommitGPGSignature struct { Payload string //TODO check if can be reconstruct from the rest of commit information to not have duplicate data } -func convertPGPSignature(c *object.Commit) *CommitGPGSignature { - if c.PGPSignature == "" { - return nil - } - - var w strings.Builder - var err error - - if _, err = fmt.Fprintf(&w, "tree %s\n", c.TreeHash.String()); err != nil { - return nil - } - - for _, parent := range c.ParentHashes { - if _, err = fmt.Fprintf(&w, "parent %s\n", parent.String()); err != nil { - return nil - } - } - - if _, err = fmt.Fprint(&w, "author "); err != nil { - return nil - } - - if err = c.Author.Encode(&w); err != nil { - return nil - } - - if _, err = fmt.Fprint(&w, "\ncommitter "); err != nil { - return nil - } - - if err = c.Committer.Encode(&w); err != nil { - return nil - } - - if _, err = fmt.Fprintf(&w, "\n\n%s", c.Message); err != nil { - return nil - } - - return &CommitGPGSignature{ - Signature: c.PGPSignature, - Payload: w.String(), - } -} - -func convertCommit(c *object.Commit) *Commit { - return &Commit{ - ID: c.Hash, - CommitMessage: c.Message, - Committer: &c.Committer, - Author: &c.Author, - Signature: convertPGPSignature(c), - Parents: c.ParentHashes, - } -} - // Message returns the commit message. Same as retrieving CommitMessage directly. func (c *Commit) Message() string { return c.CommitMessage @@ -576,7 +519,7 @@ func GetCommitFileStatus(repoPath, commitID string) (*CommitFileStatus, error) { err := NewCommand("show", "--name-status", "--pretty=format:''", commitID).RunInDirPipeline(repoPath, w, stderr) w.Close() // Close writer to exit parsing goroutine if err != nil { - return nil, concatenateError(err, stderr.String()) + return nil, ConcatenateError(err, stderr.String()) } <-done diff --git a/modules/git/commit_gogit.go b/modules/git/commit_gogit.go new file mode 100644 index 0000000000000..51152b58c2049 --- /dev/null +++ b/modules/git/commit_gogit.go @@ -0,0 +1,70 @@ +// Copyright 2015 The Gogs Authors. All rights reserved. +// Copyright 2018 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build !nogogit + +package git + +import ( + "fmt" + "strings" + + "github.com/go-git/go-git/v5/plumbing/object" +) + +func convertPGPSignature(c *object.Commit) *CommitGPGSignature { + if c.PGPSignature == "" { + return nil + } + + var w strings.Builder + var err error + + if _, err = fmt.Fprintf(&w, "tree %s\n", c.TreeHash.String()); err != nil { + return nil + } + + for _, parent := range c.ParentHashes { + if _, err = fmt.Fprintf(&w, "parent %s\n", parent.String()); err != nil { + return nil + } + } + + if _, err = fmt.Fprint(&w, "author "); err != nil { + return nil + } + + if err = c.Author.Encode(&w); err != nil { + return nil + } + + if _, err = fmt.Fprint(&w, "\ncommitter "); err != nil { + return nil + } + + if err = c.Committer.Encode(&w); err != nil { + return nil + } + + if _, err = fmt.Fprintf(&w, "\n\n%s", c.Message); err != nil { + return nil + } + + return &CommitGPGSignature{ + Signature: c.PGPSignature, + Payload: w.String(), + } +} + +func convertCommit(c *object.Commit) *Commit { + return &Commit{ + ID: c.Hash, + CommitMessage: c.Message, + Committer: &c.Committer, + Author: &c.Author, + Signature: convertPGPSignature(c), + Parents: c.ParentHashes, + } +} diff --git a/modules/git/commit_info.go b/modules/git/commit_info.go index 5df10cf46d145..e3224b24af2af 100644 --- a/modules/git/commit_info.go +++ b/modules/git/commit_info.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. +// +build !nogogit + package git import ( diff --git a/modules/git/commit_info_nogogit.go b/modules/git/commit_info_nogogit.go new file mode 100644 index 0000000000000..a82ca581e9025 --- /dev/null +++ b/modules/git/commit_info_nogogit.go @@ -0,0 +1,548 @@ +// Copyright 2017 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build nogogit + +package git + +import ( + "bufio" + "bytes" + "io" + "math" + "path" + "sort" + "strconv" + "strings" +) + +// CommitInfo describes the first commit with the provided entry +type CommitInfo struct { + Entry *TreeEntry + Commit *Commit + SubModuleFile *SubModuleFile +} + +// ReadBatchLine reads the header line from cat-file --batch +// We expect: +// SP SP LF +func ReadBatchLine(rd *bufio.Reader) (sha []byte, typ string, size int64, err error) { + sha, err = rd.ReadBytes(' ') + if err != nil { + return + } + sha = sha[:len(sha)-1] + + typ, err = rd.ReadString(' ') + if err != nil { + return + } + typ = typ[:len(typ)-1] + + var sizeStr string + sizeStr, err = rd.ReadString('\n') + if err != nil { + return + } + + size, err = strconv.ParseInt(sizeStr[:len(sizeStr)-1], 10, 64) + return +} + +func GetTagObjectID(rd *bufio.Reader, size int64) (string, error) { + id := "" + var n int64 +headerLoop: + for { + line, err := rd.ReadBytes('\n') + if err != nil { + return "", err + } + n += int64(len(line)) + idx := bytes.Index(line, []byte{' '}) + if idx < 0 { + continue + } + + if string(line[:idx]) == "object" { + id = string(line[idx+1 : len(line)-1]) + break headerLoop + } + } + + // Discard the rest of the tag + discard := size - n + for discard > math.MaxInt32 { + _, err := rd.Discard(math.MaxInt32) + if err != nil { + return id, err + } + discard -= math.MaxInt32 + } + _, err := rd.Discard(int(discard)) + return id, err +} + +func ParseTreeLine(rd *bufio.Reader) (mode, fname, sha string, n int, err error) { + mode, err = rd.ReadString(' ') + if err != nil { + return + } + n += len(mode) + mode = mode[:len(mode)-1] + + fname, err = rd.ReadString('\x00') + if err != nil { + return + } + n += len(fname) + fname = fname[:len(fname)-1] + + shaBytes := make([]byte, 20) + read, err := rd.Read(shaBytes) + if err != nil { + return + } + n += read + sha = MustID(shaBytes).String() + return +} + +// GetCommitsInfo gets information of all commits that are corresponding to these entries +func (tes Entries) GetCommitsInfo(commit *Commit, treePath string, cache *LastCommitCache) ([]CommitInfo, *Commit, error) { + entryPaths := make([]string, len(tes)+1) + // Get the commit for the treePath itself + entryPaths[0] = "" + for i, entry := range tes { + entryPaths[i+1] = entry.Name() + } + + var err error + + var revs map[string]*Commit + if cache != nil { + var unHitPaths []string + revs, unHitPaths, err = getLastCommitForPathsByCache(commit.ID.String(), treePath, entryPaths, cache) + if err != nil { + return nil, nil, err + } + if len(unHitPaths) > 0 { + sort.Strings(unHitPaths) + commits, err := GetLastCommitForPaths(commit, treePath, unHitPaths) + if err != nil { + return nil, nil, err + } + + for i, found := range commits { + if err := cache.Put(commit.ID.String(), path.Join(treePath, unHitPaths[i]), found.ID.String()); err != nil { + return nil, nil, err + } + revs[unHitPaths[i]] = found + } + } + } else { + sort.Strings(entryPaths) + revs = map[string]*Commit{} + var foundCommits []*Commit + foundCommits, err = GetLastCommitForPaths(commit, treePath, entryPaths) + for i, found := range foundCommits { + revs[entryPaths[i]] = found + } + } + if err != nil { + return nil, nil, err + } + + commitsInfo := make([]CommitInfo, len(tes)) + for i, entry := range tes { + commitsInfo[i] = CommitInfo{ + Entry: entry, + } + if entryCommit, ok := revs[entry.Name()]; ok { + commitsInfo[i].Commit = entryCommit + if entry.IsSubModule() { + subModuleURL := "" + var fullPath string + if len(treePath) > 0 { + fullPath = treePath + "/" + entry.Name() + } else { + fullPath = entry.Name() + } + if subModule, err := commit.GetSubModule(fullPath); err != nil { + return nil, nil, err + } else if subModule != nil { + subModuleURL = subModule.URL + } + subModuleFile := NewSubModuleFile(entryCommit, subModuleURL, entry.ID.String()) + commitsInfo[i].SubModuleFile = subModuleFile + } + } + } + + // Retrieve the commit for the treePath itself (see above). We basically + // get it for free during the tree traversal and it's used for listing + // pages to display information about newest commit for a given path. + var treeCommit *Commit + var ok bool + if treePath == "" { + treeCommit = commit + } else if treeCommit, ok = revs[""]; ok { + treeCommit.repo = commit.repo + } + return commitsInfo, treeCommit, nil +} + +func getLastCommitForPathsByCache(commitID, treePath string, paths []string, cache *LastCommitCache) (map[string]*Commit, []string, error) { + var unHitEntryPaths []string + var results = make(map[string]*Commit) + for _, p := range paths { + lastCommit, err := cache.Get(commitID, path.Join(treePath, p)) + if err != nil { + return nil, nil, err + } + if lastCommit != nil { + results[p] = lastCommit.(*Commit) + continue + } + + unHitEntryPaths = append(unHitEntryPaths, p) + } + + return results, unHitEntryPaths, nil +} + +// GetLastCommitForPaths returns last commit information +func GetLastCommitForPaths(commit *Commit, treePath string, paths []string) ([]*Commit, error) { + // We read backwards from the commit to obtain all of the commits + + // We'll do this by using rev-list to provide us with parent commits in order + revListReader, revListWriter := io.Pipe() + defer func() { + _ = revListWriter.Close() + _ = revListReader.Close() + }() + + go func() { + stderr := strings.Builder{} + err := NewCommand("rev-list", commit.ID.String()).RunInDirPipeline(commit.repo.Path, revListWriter, &stderr) + if err != nil { + _ = revListWriter.CloseWithError(ConcatenateError(err, (&stderr).String())) + } else { + _ = revListWriter.Close() + } + }() + + // We feed the commits in order into cat-file --batch, followed by their trees and sub trees as necessary. + // so let's create a batch stdin and stdout + batchStdinReader, batchStdinWriter := io.Pipe() + batchStdoutReader, batchStdoutWriter := io.Pipe() + defer func() { + _ = batchStdinReader.Close() + _ = batchStdinWriter.Close() + _ = batchStdoutReader.Close() + _ = batchStdoutWriter.Close() + }() + + go func() { + stderr := strings.Builder{} + err := NewCommand("cat-file", "--batch").RunInDirFullPipeline(commit.repo.Path, batchStdoutWriter, &stderr, batchStdinReader) + if err != nil { + _ = revListWriter.CloseWithError(ConcatenateError(err, (&stderr).String())) + } else { + _ = revListWriter.Close() + } + }() + + // For simplicities sake we'll us a buffered reader + batchReader := bufio.NewReader(batchStdoutReader) + + // commits is the returnable commits matching the paths provided + commits := make([]*Commit, len(paths)) + // ids are the blob/tree ids for the paths + ids := make([]string, len(paths)) + // found is a shortcut to help break out of parsing early + found := make([]bool, len(paths)) + + // We'll use a scanner for the revList because it's simpler than a bufio.Reader + scan := bufio.NewScanner(revListReader) +revListLoop: + for scan.Scan() { + // Get the next parent commit ID + commitID := scan.Bytes() + + // push the commit to the cat-file --batch process + _, err := batchStdinWriter.Write(commitID) + if err != nil { + return nil, err + } + _, err = batchStdinWriter.Write([]byte{'\n'}) + if err != nil { + return nil, err + } + + var curCommit *Commit + + currentPath := "" + + i := 0 + commitDone := false + commitReadingLoop: + for !commitDone { + _, typ, size, err := ReadBatchLine(batchReader) + if err != nil { + return nil, err + } + + switch typ { + case "tag": + // This shouldn't happen but if it does well just get the commit and try again + id, err := GetTagObjectID(batchReader, size) + if err != nil { + return nil, err + } + _, err = batchStdinWriter.Write([]byte(id + "\n")) + if err != nil { + return nil, err + } + continue + case "commit": + // Read in the commit to get its tree and in case this is one of the last used commits + curCommit, err = CommitFromReader(commit.repo, MustIDFromString(string(commitID)), io.LimitReader(batchReader, int64(size))) + if err != nil { + return nil, err + } + + // Get the tree for the commit + id := curCommit.Tree.ID.String() + // OK if the target tree path is "" and the "" is in the paths just set this now + if treePath == "" && paths[0] == "" { + if i == 0 { + i++ + } + // If this is the first time we see this set the id appropriate for this paths to this tree and set the last commit to curCommit + if ids[0] == "" { + log("setting initial id to: %s on commit %s", string(id), curCommit.ID.String()) + ids[0] = id + commits[0] = curCommit + } else if ids[0] != id { + // Else if the last id doesn't match this we've found the last commit that added this + found[0] = true + + // check if that completes our list + done := true + for _, find := range found { + if !find { + done = false + break + } + } + if done { + break revListLoop + } + } else if !found[0] { + // Finally if we haven't found the commit set the curCommit to this + commits[0] = curCommit + } + } + + // in the unlikely event that we've now done all the paths + if i >= len(paths) { + commitDone = true + continue + } + + // Finally add the tree back in to batch reader + _, err = batchStdinWriter.Write([]byte(id + "\n")) + if err != nil { + return nil, err + } + continue + case "tree": + // Handle trees + + // n is counter for file position in the tree file + var n int64 + + // Two options: currentPath is the targetTreepath + if treePath == currentPath { + // We are in the right directory + // Parse each tree line in turn. (don't care about mode here.) + for n < size { + _, fname, sha, count, err := ParseTreeLine(batchReader) + if err != nil { + return nil, err + } + n += int64(count) + + // now fname and paths are both sorted and in order + // if fname == paths[i] => we're looking at the current path + if fname == paths[i] { + // Now if this is the first time round set the initial Blob(ish) SHA ID and the commit + if ids[i] == "" { + ids[i] = sha + commits[i] = curCommit + } else if !found[i] { + // if we've not already found this path's commit + if ids[i] != sha { + // if the SHA is different we've now found the commit for this path + found[i] = true + + // check if we've found all the paths + done := true + for _, find := range found { + if !find { + done = false + break + } + } + if done { + break revListLoop + } + } else { + commits[i] = curCommit + } + } + // Step the path index counter up + i++ + } + + done := false + + // if fname > paths[i] well paths[i] must not be present in this set it to found + for i < len(paths) && (fname > paths[i] || n >= size) { + found[i] = true + done = true + i++ + } + + // check if we're really done + if done { + for _, find := range found { + if !find { + done = false + break + } + } + if done { + break revListLoop + } + } + + // Now if i >= len(paths) we're beyond the paths we're looking at - discard the rest of the tree if there is any left + if i >= len(paths) { + // set the commit as done + commitDone = true + if n < size { + discard := size - n + for discard > math.MaxInt32 { + _, err := batchReader.Discard(math.MaxInt32) + if err != nil { + return nil, err + } + discard -= math.MaxInt32 + } + _, err := batchReader.Discard(int(discard)) + if err != nil { + return nil, err + } + } + // break out of the commit reading loop + break commitReadingLoop + } + } + // We should not be able to get here... + break revListLoop + } + + // We're in the wrong directory + // Find target directory in this directory + idx := len(currentPath) + if idx > 0 { + idx++ + } + target := strings.SplitN(treePath[idx:], "/", 2)[0] + + treeID := "" + for n < size { + // Read each tree entry in turn + mode, fname, sha, count, err := ParseTreeLine(batchReader) + if err != nil { + return nil, err + } + n += int64(count) + + // if we have found the target directory + if fname == target && ToEntryMode(mode) == EntryModeTree { + treeID = sha + break + } else if fname > target { + break + } + } + // if we haven't found a treeID for the target directory our search is over + if treeID == "" { + break revListLoop + } + + if n < size { + // Discard any remaining entries in the current tree + discard := size - n + for discard > math.MaxInt32 { + _, err := batchReader.Discard(math.MaxInt32) + if err != nil { + return nil, err + } + discard -= math.MaxInt32 + } + _, err := batchReader.Discard(int(discard)) + if err != nil { + return nil, err + } + } + + // add the target to the current path + if idx > 0 { + currentPath += "/" + } + currentPath += target + + // if we've now found the curent path check its sha id and commit status + if treePath == currentPath && paths[0] == "" { + if i == 0 { + i++ + } + if ids[0] == "" { + ids[0] = treeID + commits[0] = curCommit + } else if !found[0] { + if ids[0] != treeID { + found[0] = true + + // check if that completes our list + done := true + for _, find := range found { + if !find { + done = false + break + } + } + if done { + break revListLoop + } + } else if !found[0] { + commits[0] = curCommit + } + } + } + _, err = batchStdinWriter.Write([]byte(treeID + "\n")) + if err != nil { + return nil, err + } + continue + } + } + } + + return commits, scan.Err() +} diff --git a/modules/git/commit_info_test.go b/modules/git/commit_info_test.go index 6bd104ec3bbe7..3966419bc146f 100644 --- a/modules/git/commit_info_test.go +++ b/modules/git/commit_info_test.go @@ -58,13 +58,23 @@ func testGetCommitsInfo(t *testing.T, repo1 *Repository) { for _, testCase := range testCases { commit, err := repo1.GetCommit(testCase.CommitID) assert.NoError(t, err) + assert.NotNil(t, commit) + assert.NotNil(t, commit.Tree) + assert.NotNil(t, commit.Tree.repo) + tree, err := commit.Tree.SubTree(testCase.Path) + assert.NotNil(t, tree, "tree is nil for testCase CommitID %s in Path %s", testCase.CommitID, testCase.Path) + assert.NotNil(t, tree.repo, "repo is nil for testCase CommitID %s in Path %s", testCase.CommitID, testCase.Path) + assert.NoError(t, err) entries, err := tree.ListEntries() assert.NoError(t, err) commitsInfo, treeCommit, err := entries.GetCommitsInfo(commit, testCase.Path, nil) - assert.Equal(t, testCase.ExpectedTreeCommit, treeCommit.ID.String()) assert.NoError(t, err) + if err != nil { + t.FailNow() + } + assert.Equal(t, testCase.ExpectedTreeCommit, treeCommit.ID.String()) assert.Len(t, commitsInfo, len(testCase.ExpectedIDs)) for _, commitInfo := range commitsInfo { entry := commitInfo.Entry diff --git a/modules/git/commit_reader.go b/modules/git/commit_reader.go index 666c2c23c82cf..9f95ca2a50d1c 100644 --- a/modules/git/commit_reader.go +++ b/modules/git/commit_reader.go @@ -24,26 +24,20 @@ func CommitFromReader(gitRepo *Repository, sha SHA1, reader io.Reader) (*Commit, message := false pgpsig := false - scanner := bufio.NewScanner(reader) - // Split by '\n' but include the '\n' - scanner.Split(func(data []byte, atEOF bool) (advance int, token []byte, err error) { - if atEOF && len(data) == 0 { - return 0, nil, nil - } - if i := bytes.IndexByte(data, '\n'); i >= 0 { - // We have a full newline-terminated line. - return i + 1, data[0 : i+1], nil - } - // If we're at EOF, we have a final, non-terminated line. Return it. - if atEOF { - return len(data), data, nil - } - // Request more data. - return 0, nil, nil - }) + bufReader, ok := reader.(*bufio.Reader) + if !ok { + bufReader = bufio.NewReader(reader) + } - for scanner.Scan() { - line := scanner.Bytes() +readLoop: + for { + line, err := bufReader.ReadBytes('\n') + if err != nil { + if err == io.EOF { + break readLoop + } + return nil, err + } if pgpsig { if len(line) > 0 && line[0] == ' ' { _, _ = signatureSB.Write(line[1:]) @@ -102,5 +96,5 @@ func CommitFromReader(gitRepo *Repository, sha SHA1, reader io.Reader) (*Commit, commit.Signature = nil } - return commit, scanner.Err() + return commit, nil } diff --git a/modules/git/last_commit_cache.go b/modules/git/last_commit_cache.go index b69b1d868ac28..9394df5b05266 100644 --- a/modules/git/last_commit_cache.go +++ b/modules/git/last_commit_cache.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. +// +build !nogogit + package git import ( diff --git a/modules/git/last_commit_cache_nogogit.go b/modules/git/last_commit_cache_nogogit.go new file mode 100644 index 0000000000000..b9d024c808a7a --- /dev/null +++ b/modules/git/last_commit_cache_nogogit.go @@ -0,0 +1,124 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build nogogit + +package git + +import ( + "crypto/sha256" + "fmt" + "path" +) + +// Cache represents a caching interface +type Cache interface { + // Put puts value into cache with key and expire time. + Put(key string, val interface{}, timeout int64) error + // Get gets cached value by given key. + Get(key string) interface{} +} + +// LastCommitCache represents a cache to store last commit +type LastCommitCache struct { + repoPath string + ttl int64 + repo *Repository + commitCache map[string]*Commit + cache Cache +} + +// NewLastCommitCache creates a new last commit cache for repo +func NewLastCommitCache(repoPath string, gitRepo *Repository, ttl int64, cache Cache) *LastCommitCache { + if cache == nil { + return nil + } + return &LastCommitCache{ + repoPath: repoPath, + repo: gitRepo, + commitCache: make(map[string]*Commit), + ttl: ttl, + cache: cache, + } +} + +func (c *LastCommitCache) getCacheKey(repoPath, ref, entryPath string) string { + hashBytes := sha256.Sum256([]byte(fmt.Sprintf("%s:%s:%s", repoPath, ref, entryPath))) + return fmt.Sprintf("last_commit:%x", hashBytes) +} + +// Get get the last commit information by commit id and entry path +func (c *LastCommitCache) Get(ref, entryPath string) (interface{}, error) { + v := c.cache.Get(c.getCacheKey(c.repoPath, ref, entryPath)) + if vs, ok := v.(string); ok { + log("LastCommitCache hit level 1: [%s:%s:%s]", ref, entryPath, vs) + if commit, ok := c.commitCache[vs]; ok { + log("LastCommitCache hit level 2: [%s:%s:%s]", ref, entryPath, vs) + return commit, nil + } + id, err := c.repo.ConvertToSHA1(vs) + if err != nil { + return nil, err + } + commit, err := c.repo.getCommit(id) + if err != nil { + return nil, err + } + c.commitCache[vs] = commit + return commit, nil + } + return nil, nil +} + +// Put put the last commit id with commit and entry path +func (c *LastCommitCache) Put(ref, entryPath, commitID string) error { + log("LastCommitCache save: [%s:%s:%s]", ref, entryPath, commitID) + return c.cache.Put(c.getCacheKey(c.repoPath, ref, entryPath), commitID, c.ttl) +} + +// CacheCommit will cache the commit from the gitRepository +func (c *LastCommitCache) CacheCommit(gitRepo *Repository, commit *Commit) error { + return c.recursiveCache(gitRepo, commit, &commit.Tree, "", 1) +} + +func (c *LastCommitCache) recursiveCache(gitRepo *Repository, commit *Commit, tree *Tree, treePath string, level int) error { + if level == 0 { + return nil + } + + entries, err := tree.ListEntries() + if err != nil { + return err + } + + entryPaths := make([]string, len(entries)) + entryMap := make(map[string]*TreeEntry) + for i, entry := range entries { + entryPaths[i] = entry.Name() + entryMap[entry.Name()] = entry + } + + commits, err := GetLastCommitForPaths(commit, treePath, entryPaths) + if err != nil { + return err + } + + for i, entryCommit := range commits { + entry := entryPaths[i] + if err := c.Put(commit.ID.String(), path.Join(treePath, entryPaths[i]), entryCommit.ID.String()); err != nil { + return err + } + if entryMap[entry].IsDir() { + subTree, err := tree.SubTree(entry) + if err != nil { + return err + } + if err := c.recursiveCache(gitRepo, commit, subTree, entry, level-1); err != nil { + return err + } + } + } + + return nil +} diff --git a/modules/git/notes.go b/modules/git/notes.go index ba19fa4893414..4d8b9068f92b5 100644 --- a/modules/git/notes.go +++ b/modules/git/notes.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. +// +build !nogogit + package git import ( diff --git a/modules/git/notes_nogogit.go b/modules/git/notes_nogogit.go new file mode 100644 index 0000000000000..5dbfa12c23999 --- /dev/null +++ b/modules/git/notes_nogogit.go @@ -0,0 +1,69 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build nogogit + +package git + +import ( + "io/ioutil" +) + +// NotesRef is the git ref where Gitea will look for git-notes data. +// The value ("refs/notes/commits") is the default ref used by git-notes. +const NotesRef = "refs/notes/commits" + +// Note stores information about a note created using git-notes. +type Note struct { + Message []byte + Commit *Commit +} + +// GetNote retrieves the git-notes data for a given commit. +func GetNote(repo *Repository, commitID string, note *Note) error { + notes, err := repo.GetCommit(NotesRef) + if err != nil { + return err + } + + path := "" + + tree := ¬es.Tree + + var entry *TreeEntry + for len(commitID) > 2 { + entry, err = tree.GetTreeEntryByPath(commitID) + if err == nil { + path += commitID + break + } + if IsErrNotExist(err) { + tree, err = tree.SubTree(commitID[0:2]) + path += commitID[0:2] + "/" + commitID = commitID[2:] + } + if err != nil { + return err + } + } + + dataRc, err := entry.Blob().DataAsync() + if err != nil { + return err + } + defer dataRc.Close() + d, err := ioutil.ReadAll(dataRc) + if err != nil { + return err + } + note.Message = d + + lastCommits, err := GetLastCommitForPaths(notes, "", []string{path}) + if err != nil { + return err + } + note.Commit = lastCommits[0] + + return nil +} diff --git a/modules/git/parse.go b/modules/git/parse.go index 89b4488600ecb..99cb1306cccef 100644 --- a/modules/git/parse.go +++ b/modules/git/parse.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. +// +build !nogogit + package git import ( diff --git a/modules/git/parse_nogogit.go b/modules/git/parse_nogogit.go new file mode 100644 index 0000000000000..867de4bacdbbb --- /dev/null +++ b/modules/git/parse_nogogit.go @@ -0,0 +1,78 @@ +// Copyright 2018 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build nogogit + +package git + +import ( + "bytes" + "fmt" + "strconv" +) + +// ParseTreeEntries parses the output of a `git ls-tree` command. +func ParseTreeEntries(data []byte) ([]*TreeEntry, error) { + return parseTreeEntries(data, nil) +} + +func parseTreeEntries(data []byte, ptree *Tree) ([]*TreeEntry, error) { + entries := make([]*TreeEntry, 0, 10) + for pos := 0; pos < len(data); { + // expect line to be of the form " \t" + entry := new(TreeEntry) + entry.ptree = ptree + if pos+6 > len(data) { + return nil, fmt.Errorf("Invalid ls-tree output: %s", string(data)) + } + switch string(data[pos : pos+6]) { + case "100644": + entry.entryMode = EntryModeBlob + pos += 12 // skip over "100644 blob " + case "100755": + entry.entryMode = EntryModeExec + pos += 12 // skip over "100755 blob " + case "120000": + entry.entryMode = EntryModeSymlink + pos += 12 // skip over "120000 blob " + case "160000": + entry.entryMode = EntryModeCommit + pos += 14 // skip over "160000 object " + case "040000": + entry.entryMode = EntryModeTree + pos += 12 // skip over "040000 tree " + default: + return nil, fmt.Errorf("unknown type: %v", string(data[pos:pos+6])) + } + + if pos+40 > len(data) { + return nil, fmt.Errorf("Invalid ls-tree output: %s", string(data)) + } + id, err := NewIDFromString(string(data[pos : pos+40])) + if err != nil { + return nil, fmt.Errorf("Invalid ls-tree output: %v", err) + } + entry.ID = id + pos += 41 // skip over sha and trailing space + + end := pos + bytes.IndexByte(data[pos:], '\n') + if end < pos { + return nil, fmt.Errorf("Invalid ls-tree output: %s", string(data)) + } + + // In case entry name is surrounded by double quotes(it happens only in git-shell). + if data[pos] == '"' { + entry.name, err = strconv.Unquote(string(data[pos:end])) + if err != nil { + return nil, fmt.Errorf("Invalid ls-tree output: %v", err) + } + } else { + entry.name = string(data[pos:end]) + } + + pos = end + 1 + entries = append(entries, entry) + } + return entries, nil +} diff --git a/modules/git/parse_test.go b/modules/git/parse_test.go index 8e0be828b3c66..550754ba6a799 100644 --- a/modules/git/parse_test.go +++ b/modules/git/parse_test.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. +// +build !nogogit + package git import ( diff --git a/modules/git/pipeline/lfs.go b/modules/git/pipeline/lfs.go index a106cc8c86e24..36480174d953b 100644 --- a/modules/git/pipeline/lfs.go +++ b/modules/git/pipeline/lfs.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. +// +build !nogogit + package pipeline import ( diff --git a/modules/git/pipeline/lfs_nogogit.go b/modules/git/pipeline/lfs_nogogit.go new file mode 100644 index 0000000000000..d054adb8f8261 --- /dev/null +++ b/modules/git/pipeline/lfs_nogogit.go @@ -0,0 +1,257 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build nogogit + +package pipeline + +import ( + "bufio" + "fmt" + "io" + "sort" + "strings" + "sync" + "time" + + "code.gitea.io/gitea/modules/git" +) + +// LFSResult represents commits found using a provided pointer file hash +type LFSResult struct { + Name string + SHA string + Summary string + When time.Time + ParentHashes []git.SHA1 + BranchName string + FullCommitName string +} + +type lfsResultSlice []*LFSResult + +func (a lfsResultSlice) Len() int { return len(a) } +func (a lfsResultSlice) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a lfsResultSlice) Less(i, j int) bool { return a[j].When.After(a[i].When) } + +// FindLFSFile finds commits that contain a provided pointer file hash +func FindLFSFile(repo *git.Repository, hash git.SHA1) ([]*LFSResult, error) { + resultsMap := map[string]*LFSResult{} + results := make([]*LFSResult, 0) + + basePath := repo.Path + + hashStr := hash.String() + + // Use rev-list to provide us with all commits in order + revListReader, revListWriter := io.Pipe() + defer func() { + _ = revListWriter.Close() + _ = revListReader.Close() + }() + + go func() { + stderr := strings.Builder{} + err := git.NewCommand("rev-list", "--all").RunInDirPipeline(repo.Path, revListWriter, &stderr) + if err != nil { + _ = revListWriter.CloseWithError(git.ConcatenateError(err, (&stderr).String())) + } else { + _ = revListWriter.Close() + } + }() + + // Next feed the commits in order into cat-file --batch, followed by their trees and sub trees as necessary. + // so let's create a batch stdin and stdout + batchStdinReader, batchStdinWriter := io.Pipe() + batchStdoutReader, batchStdoutWriter := io.Pipe() + defer func() { + _ = batchStdinReader.Close() + _ = batchStdinWriter.Close() + _ = batchStdoutReader.Close() + _ = batchStdoutWriter.Close() + }() + + go func() { + stderr := strings.Builder{} + err := git.NewCommand("cat-file", "--batch").RunInDirFullPipeline(repo.Path, batchStdoutWriter, &stderr, batchStdinReader) + if err != nil { + _ = revListWriter.CloseWithError(git.ConcatenateError(err, (&stderr).String())) + } else { + _ = revListWriter.Close() + } + }() + + // For simplicities sake we'll us a buffered reader to read from the cat-file --batch + batchReader := bufio.NewReader(batchStdoutReader) + + // We'll use a scanner for the revList because it's simpler than a bufio.Reader + scan := bufio.NewScanner(revListReader) + trees := []string{} + paths := []string{} + + for scan.Scan() { + // Get the next commit ID + commitID := scan.Bytes() + + // push the commit to the cat-file --batch process + _, err := batchStdinWriter.Write(commitID) + if err != nil { + return nil, err + } + _, err = batchStdinWriter.Write([]byte{'\n'}) + if err != nil { + return nil, err + } + + var curCommit *git.Commit + curPath := "" + + commitReadingLoop: + for { + _, typ, size, err := git.ReadBatchLine(batchReader) + if err != nil { + return nil, err + } + + switch typ { + case "tag": + // This shouldn't happen but if it does well just get the commit and try again + id, err := git.GetTagObjectID(batchReader, size) + if err != nil { + return nil, err + } + _, err = batchStdinWriter.Write([]byte(id + "\n")) + if err != nil { + return nil, err + } + continue + case "commit": + // Read in the commit to get its tree and in case this is one of the last used commits + curCommit, err = git.CommitFromReader(repo, git.MustIDFromString(string(commitID)), io.LimitReader(batchReader, int64(size))) + if err != nil { + return nil, err + } + + _, err := batchStdinWriter.Write([]byte(curCommit.Tree.ID.String() + "\n")) + if err != nil { + return nil, err + } + curPath = "" + case "tree": + var n int64 + for n < size { + mode, fname, sha, count, err := git.ParseTreeLine(batchReader) + if err != nil { + return nil, err + } + n += int64(count) + if sha == hashStr { + result := LFSResult{ + Name: curPath + fname, + SHA: curCommit.ID.String(), + Summary: strings.Split(strings.TrimSpace(curCommit.CommitMessage), "\n")[0], + When: curCommit.Author.When, + ParentHashes: curCommit.Parents, + } + resultsMap[curCommit.ID.String()+":"+curPath+fname] = &result + } else if mode == git.EntryModeTree.String() { + trees = append(trees, sha) + paths = append(paths, curPath+fname+"/") + } + } + if len(trees) > 0 { + _, err := batchStdinWriter.Write([]byte(trees[len(trees)-1] + "\n")) + if err != nil { + return nil, err + } + curPath = paths[len(paths)-1] + trees = trees[:len(trees)-1] + paths = paths[:len(paths)-1] + } else { + break commitReadingLoop + } + } + } + } + + if err := scan.Err(); err != nil { + return nil, err + } + + for _, result := range resultsMap { + hasParent := false + for _, parentHash := range result.ParentHashes { + if _, hasParent = resultsMap[parentHash.String()+":"+result.Name]; hasParent { + break + } + } + if !hasParent { + results = append(results, result) + } + } + + sort.Sort(lfsResultSlice(results)) + + // Should really use a go-git function here but name-rev is not completed and recapitulating it is not simple + shasToNameReader, shasToNameWriter := io.Pipe() + nameRevStdinReader, nameRevStdinWriter := io.Pipe() + errChan := make(chan error, 1) + wg := sync.WaitGroup{} + wg.Add(3) + + go func() { + defer wg.Done() + scanner := bufio.NewScanner(nameRevStdinReader) + i := 0 + for scanner.Scan() { + line := scanner.Text() + if len(line) == 0 { + continue + } + result := results[i] + result.FullCommitName = line + result.BranchName = strings.Split(line, "~")[0] + i++ + } + }() + go NameRevStdin(shasToNameReader, nameRevStdinWriter, &wg, basePath) + go func() { + defer wg.Done() + defer shasToNameWriter.Close() + for _, result := range results { + i := 0 + if i < len(result.SHA) { + n, err := shasToNameWriter.Write([]byte(result.SHA)[i:]) + if err != nil { + errChan <- err + break + } + i += n + } + var err error + n := 0 + for n < 1 { + n, err = shasToNameWriter.Write([]byte{'\n'}) + if err != nil { + errChan <- err + break + } + + } + + } + }() + + wg.Wait() + + select { + case err, has := <-errChan: + if has { + return nil, fmt.Errorf("Unable to obtain name for LFS files. Error: %w", err) + } + default: + } + + return results, nil +} diff --git a/modules/git/repo.go b/modules/git/repo.go index ae370d3da973a..8d6e9eca9135f 100644 --- a/modules/git/repo.go +++ b/modules/git/repo.go @@ -3,6 +3,8 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. +// +build !nogogit + package git import ( diff --git a/modules/git/repo_blob.go b/modules/git/repo_blob.go index ce0ad6b50fbf7..204f4a339fe02 100644 --- a/modules/git/repo_blob.go +++ b/modules/git/repo_blob.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. +// +build !nogogit + package git import ( diff --git a/modules/git/repo_blob_nogogit.go b/modules/git/repo_blob_nogogit.go new file mode 100644 index 0000000000000..958efc48f44fa --- /dev/null +++ b/modules/git/repo_blob_nogogit.go @@ -0,0 +1,26 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build nogogit + +package git + +func (repo *Repository) getBlob(id SHA1) (*Blob, error) { + if id.IsZero() { + return nil, ErrNotExist{id.String(), ""} + } + return &Blob{ + ID: id, + repoPath: repo.Path, + }, nil +} + +// GetBlob finds the blob object in the repository. +func (repo *Repository) GetBlob(idStr string) (*Blob, error) { + id, err := NewIDFromString(idStr) + if err != nil { + return nil, err + } + return repo.getBlob(id) +} diff --git a/modules/git/repo_branch.go b/modules/git/repo_branch.go index cd30c191ea31e..d6a215a9b3702 100644 --- a/modules/git/repo_branch.go +++ b/modules/git/repo_branch.go @@ -3,6 +3,8 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. +// +build !nogogit + package git import ( diff --git a/modules/git/repo_branch_nogogit.go b/modules/git/repo_branch_nogogit.go new file mode 100644 index 0000000000000..a4c7f9c3efbcd --- /dev/null +++ b/modules/git/repo_branch_nogogit.go @@ -0,0 +1,229 @@ +// Copyright 2015 The Gogs Authors. All rights reserved. +// Copyright 2018 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build nogogit + +package git + +import ( + "bufio" + "fmt" + "io" + "strings" +) + +// BranchPrefix base dir of the branch information file store on git +const BranchPrefix = "refs/heads/" + +// IsReferenceExist returns true if given reference exists in the repository. +func IsReferenceExist(repoPath, name string) bool { + _, err := NewCommand("show-ref", "--verify", "--", name).RunInDir(repoPath) + return err == nil +} + +// IsBranchExist returns true if given branch exists in the repository. +func IsBranchExist(repoPath, name string) bool { + return IsReferenceExist(repoPath, BranchPrefix+name) +} + +// IsBranchExist returns true if given branch exists in current repository. +func (repo *Repository) IsBranchExist(name string) bool { + if name == "" { + return false + } + return IsReferenceExist(repo.Path, BranchPrefix+name) +} + +// Branch represents a Git branch. +type Branch struct { + Name string + Path string + + gitRepo *Repository +} + +// GetHEADBranch returns corresponding branch of HEAD. +func (repo *Repository) GetHEADBranch() (*Branch, error) { + if repo == nil { + return nil, fmt.Errorf("nil repo") + } + stdout, err := NewCommand("symbolic-ref", "HEAD").RunInDir(repo.Path) + if err != nil { + return nil, err + } + stdout = strings.TrimSpace(stdout) + + if !strings.HasPrefix(stdout, BranchPrefix) { + return nil, fmt.Errorf("invalid HEAD branch: %v", stdout) + } + + return &Branch{ + Name: stdout[len(BranchPrefix):], + Path: stdout, + gitRepo: repo, + }, nil +} + +// SetDefaultBranch sets default branch of repository. +func (repo *Repository) SetDefaultBranch(name string) error { + _, err := NewCommand("symbolic-ref", "HEAD", BranchPrefix+name).RunInDir(repo.Path) + return err +} + +// GetDefaultBranch gets default branch of repository. +func (repo *Repository) GetDefaultBranch() (string, error) { + return NewCommand("symbolic-ref", "HEAD").RunInDir(repo.Path) +} + +// GetBranches returns all branches of the repository. +func (repo *Repository) GetBranches() ([]string, error) { + return callShowRef(repo.Path, BranchPrefix, "--heads") +} + +func callShowRef(repoPath, prefix, arg string) ([]string, error) { + var branchNames []string + + stdoutReader, stdoutWriter := io.Pipe() + defer func() { + _ = stdoutReader.Close() + _ = stdoutWriter.Close() + }() + + go func() { + stderrBuilder := &strings.Builder{} + err := NewCommand("show-ref", arg).RunInDirPipeline(repoPath, stdoutWriter, stderrBuilder) + if err != nil { + if stderrBuilder.Len() == 0 { + _ = stdoutWriter.Close() + return + } + _ = stdoutWriter.CloseWithError(ConcatenateError(err, stderrBuilder.String())) + } else { + _ = stdoutWriter.Close() + } + }() + + bufReader := bufio.NewReader(stdoutReader) + for { + // The output of show-ref is simply a list: + // SP LF + _, err := bufReader.ReadSlice(' ') + for err == bufio.ErrBufferFull { + // This shouldn't happen but we'll tolerate it for the sake of peace + _, err = bufReader.ReadSlice(' ') + } + if err == io.EOF { + return branchNames, nil + } + if err != nil { + return nil, err + } + + branchName, err := bufReader.ReadString('\n') + if err == io.EOF { + // This shouldn't happen... but we'll tolerate it for the sake of peace + return branchNames, nil + } + if err != nil { + return nil, err + } + branchName = strings.TrimPrefix(branchName, prefix) + if len(branchName) > 0 { + branchName = branchName[:len(branchName)-1] + } + branchNames = append(branchNames, branchName) + } +} + +// GetBranch returns a branch by it's name +func (repo *Repository) GetBranch(branch string) (*Branch, error) { + if !repo.IsBranchExist(branch) { + return nil, ErrBranchNotExist{branch} + } + return &Branch{ + Path: repo.Path, + Name: branch, + gitRepo: repo, + }, nil +} + +// GetBranchesByPath returns a branch by it's path +func GetBranchesByPath(path string) ([]*Branch, error) { + gitRepo, err := OpenRepository(path) + if err != nil { + return nil, err + } + defer gitRepo.Close() + + brs, err := gitRepo.GetBranches() + if err != nil { + return nil, err + } + + branches := make([]*Branch, len(brs)) + for i := range brs { + branches[i] = &Branch{ + Path: path, + Name: brs[i], + gitRepo: gitRepo, + } + } + + return branches, nil +} + +// DeleteBranchOptions Option(s) for delete branch +type DeleteBranchOptions struct { + Force bool +} + +// DeleteBranch delete a branch by name on repository. +func (repo *Repository) DeleteBranch(name string, opts DeleteBranchOptions) error { + cmd := NewCommand("branch") + + if opts.Force { + cmd.AddArguments("-D") + } else { + cmd.AddArguments("-d") + } + + cmd.AddArguments("--", name) + _, err := cmd.RunInDir(repo.Path) + + return err +} + +// CreateBranch create a new branch +func (repo *Repository) CreateBranch(branch, oldbranchOrCommit string) error { + cmd := NewCommand("branch") + cmd.AddArguments("--", branch, oldbranchOrCommit) + + _, err := cmd.RunInDir(repo.Path) + + return err +} + +// AddRemote adds a new remote to repository. +func (repo *Repository) AddRemote(name, url string, fetch bool) error { + cmd := NewCommand("remote", "add") + if fetch { + cmd.AddArguments("-f") + } + cmd.AddArguments(name, url) + + _, err := cmd.RunInDir(repo.Path) + return err +} + +// RemoveRemote removes a remote from repository. +func (repo *Repository) RemoveRemote(name string) error { + _, err := NewCommand("remote", "rm", name).RunInDir(repo.Path) + return err +} + +// GetCommit returns the head commit of a branch +func (branch *Branch) GetCommit() (*Commit, error) { + return branch.gitRepo.GetBranchCommit(branch.Name) +} diff --git a/modules/git/repo_commit.go b/modules/git/repo_commit.go index ee3b05447b8fb..3aa96dac175f5 100644 --- a/modules/git/repo_commit.go +++ b/modules/git/repo_commit.go @@ -3,6 +3,8 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. +// +build !nogogit + package git import ( diff --git a/modules/git/repo_commit_nogogit.go b/modules/git/repo_commit_nogogit.go new file mode 100644 index 0000000000000..e3173918eff05 --- /dev/null +++ b/modules/git/repo_commit_nogogit.go @@ -0,0 +1,519 @@ +// Copyright 2015 The Gogs Authors. All rights reserved. +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build nogogit + +package git + +import ( + "bufio" + "bytes" + "container/list" + "fmt" + "io" + "io/ioutil" + "strconv" + "strings" +) + +// GetRefCommitID returns the last commit ID string of given reference (branch or tag). +func (repo *Repository) ResolveReference(name string) (string, error) { + stdout, err := NewCommand("show-ref", "--hash", name).RunInDir(repo.Path) + if err != nil { + if strings.Contains(err.Error(), "not a valid ref") { + return "", ErrNotExist{name, ""} + } + return "", err + } + stdout = strings.TrimSpace(stdout) + if stdout == "" { + return "", ErrNotExist{name, ""} + } + + return stdout, nil +} + +// GetRefCommitID returns the last commit ID string of given reference (branch or tag). +func (repo *Repository) GetRefCommitID(name string) (string, error) { + stdout, err := NewCommand("show-ref", "--verify", "--hash", name).RunInDir(repo.Path) + if err != nil { + if strings.Contains(err.Error(), "not a valid ref") { + return "", ErrNotExist{name, ""} + } + return "", err + } + + return strings.TrimSpace(stdout), nil +} + +// IsCommitExist returns true if given commit exists in current repository. +func (repo *Repository) IsCommitExist(name string) bool { + _, err := NewCommand("cat-file", "-e", name).RunInDir(repo.Path) + return err == nil +} + +// GetBranchCommitID returns last commit ID string of given branch. +func (repo *Repository) GetBranchCommitID(name string) (string, error) { + return repo.GetRefCommitID(BranchPrefix + name) +} + +// GetTagCommitID returns last commit ID string of given tag. +func (repo *Repository) GetTagCommitID(name string) (string, error) { + stdout, err := NewCommand("rev-list", "-n", "1", TagPrefix+name).RunInDir(repo.Path) + if err != nil { + if strings.Contains(err.Error(), "unknown revision or path") { + return "", ErrNotExist{name, ""} + } + return "", err + } + return strings.TrimSpace(stdout), nil +} + +func (repo *Repository) getCommit(id SHA1) (*Commit, error) { + stdoutReader, stdoutWriter := io.Pipe() + defer func() { + _ = stdoutReader.Close() + _ = stdoutWriter.Close() + }() + + go func() { + stderr := strings.Builder{} + err := NewCommand("cat-file", "--batch").RunInDirFullPipeline(repo.Path, stdoutWriter, &stderr, strings.NewReader(id.String()+"\n")) + if err != nil { + _ = stdoutWriter.CloseWithError(ConcatenateError(err, (&stderr).String())) + } else { + _ = stdoutWriter.Close() + } + }() + + bufReader := bufio.NewReader(stdoutReader) + _, typ, size, err := ReadBatchLine(bufReader) + if err != nil { + return nil, err + } + + switch typ { + case "tag": + // then we need to parse the tag + // and load the commit + data, err := ioutil.ReadAll(io.LimitReader(bufReader, size)) + if err != nil { + return nil, err + } + tag, err := parseTagData(data) + if err != nil { + return nil, err + } + tag.repo = repo + + commit, err := tag.Commit() + if err != nil { + return nil, err + } + + commit.CommitMessage = strings.TrimSpace(tag.Message) + commit.Author = tag.Tagger + commit.Signature = tag.Signature + + return commit, nil + case "commit": + return CommitFromReader(repo, id, io.LimitReader(bufReader, size)) + default: + stdoutReader.CloseWithError(fmt.Errorf("unknown typ: %s", typ)) + log("Unknown typ: %s", typ) + return nil, ErrNotExist{ + ID: id.String(), + } + } +} + +// ConvertToSHA1 returns a Hash object from a potential ID string +func (repo *Repository) ConvertToSHA1(commitID string) (SHA1, error) { + if len(commitID) != 40 { + var err error + actualCommitID, err := NewCommand("rev-parse", "--verify", commitID).RunInDir(repo.Path) + if err != nil { + if strings.Contains(err.Error(), "unknown revision or path") || + strings.Contains(err.Error(), "fatal: Needed a single revision") { + return SHA1{}, ErrNotExist{commitID, ""} + } + return SHA1{}, err + } + commitID = actualCommitID + } + return NewIDFromString(commitID) +} + +// GetCommit returns commit object of by ID string. +func (repo *Repository) GetCommit(commitID string) (*Commit, error) { + id, err := repo.ConvertToSHA1(commitID) + if err != nil { + return nil, err + } + + return repo.getCommit(id) +} + +// GetBranchCommit returns the last commit of given branch. +func (repo *Repository) GetBranchCommit(name string) (*Commit, error) { + commitID, err := repo.GetBranchCommitID(name) + if err != nil { + return nil, err + } + return repo.GetCommit(commitID) +} + +// GetTagCommit get the commit of the specific tag via name +func (repo *Repository) GetTagCommit(name string) (*Commit, error) { + commitID, err := repo.GetTagCommitID(name) + if err != nil { + return nil, err + } + return repo.GetCommit(commitID) +} + +func (repo *Repository) getCommitByPathWithID(id SHA1, relpath string) (*Commit, error) { + // File name starts with ':' must be escaped. + if relpath[0] == ':' { + relpath = `\` + relpath + } + + stdout, err := NewCommand("log", "-1", prettyLogFormat, id.String(), "--", relpath).RunInDir(repo.Path) + if err != nil { + return nil, err + } + + id, err = NewIDFromString(stdout) + if err != nil { + return nil, err + } + + return repo.getCommit(id) +} + +// GetCommitByPath returns the last commit of relative path. +func (repo *Repository) GetCommitByPath(relpath string) (*Commit, error) { + stdout, err := NewCommand("log", "-1", prettyLogFormat, "--", relpath).RunInDirBytes(repo.Path) + if err != nil { + return nil, err + } + + commits, err := repo.parsePrettyFormatLogToList(stdout) + if err != nil { + return nil, err + } + return commits.Front().Value.(*Commit), nil +} + +// CommitsRangeSize the default commits range size +var CommitsRangeSize = 50 + +func (repo *Repository) commitsByRange(id SHA1, page, pageSize int) (*list.List, error) { + stdout, err := NewCommand("log", id.String(), "--skip="+strconv.Itoa((page-1)*pageSize), + "--max-count="+strconv.Itoa(pageSize), prettyLogFormat).RunInDirBytes(repo.Path) + + if err != nil { + return nil, err + } + return repo.parsePrettyFormatLogToList(stdout) +} + +func (repo *Repository) searchCommits(id SHA1, opts SearchCommitsOptions) (*list.List, error) { + // create new git log command with limit of 100 commis + cmd := NewCommand("log", id.String(), "-100", prettyLogFormat) + // ignore case + args := []string{"-i"} + + // add authors if present in search query + if len(opts.Authors) > 0 { + for _, v := range opts.Authors { + args = append(args, "--author="+v) + } + } + + // add commiters if present in search query + if len(opts.Committers) > 0 { + for _, v := range opts.Committers { + args = append(args, "--committer="+v) + } + } + + // add time constraints if present in search query + if len(opts.After) > 0 { + args = append(args, "--after="+opts.After) + } + if len(opts.Before) > 0 { + args = append(args, "--before="+opts.Before) + } + + // pretend that all refs along with HEAD were listed on command line as + // https://git-scm.com/docs/git-log#Documentation/git-log.txt---all + // note this is done only for command created above + if opts.All { + cmd.AddArguments("--all") + } + + // add remaining keywords from search string + // note this is done only for command created above + if len(opts.Keywords) > 0 { + for _, v := range opts.Keywords { + cmd.AddArguments("--grep=" + v) + } + } + + // search for commits matching given constraints and keywords in commit msg + cmd.AddArguments(args...) + stdout, err := cmd.RunInDirBytes(repo.Path) + if err != nil { + return nil, err + } + if len(stdout) != 0 { + stdout = append(stdout, '\n') + } + + // if there are any keywords (ie not commiter:, author:, time:) + // then let's iterate over them + if len(opts.Keywords) > 0 { + for _, v := range opts.Keywords { + // ignore anything below 4 characters as too unspecific + if len(v) >= 4 { + // create new git log command with 1 commit limit + hashCmd := NewCommand("log", "-1", prettyLogFormat) + // add previous arguments except for --grep and --all + hashCmd.AddArguments(args...) + // add keyword as + hashCmd.AddArguments(v) + + // search with given constraints for commit matching sha hash of v + hashMatching, err := hashCmd.RunInDirBytes(repo.Path) + if err != nil || bytes.Contains(stdout, hashMatching) { + continue + } + stdout = append(stdout, hashMatching...) + stdout = append(stdout, '\n') + } + } + } + + return repo.parsePrettyFormatLogToList(bytes.TrimSuffix(stdout, []byte{'\n'})) +} + +func (repo *Repository) getFilesChanged(id1, id2 string) ([]string, error) { + stdout, err := NewCommand("diff", "--name-only", id1, id2).RunInDirBytes(repo.Path) + if err != nil { + return nil, err + } + return strings.Split(string(stdout), "\n"), nil +} + +// FileChangedBetweenCommits Returns true if the file changed between commit IDs id1 and id2 +// You must ensure that id1 and id2 are valid commit ids. +func (repo *Repository) FileChangedBetweenCommits(filename, id1, id2 string) (bool, error) { + stdout, err := NewCommand("diff", "--name-only", "-z", id1, id2, "--", filename).RunInDirBytes(repo.Path) + if err != nil { + return false, err + } + return len(strings.TrimSpace(string(stdout))) > 0, nil +} + +// FileCommitsCount return the number of files at a revison +func (repo *Repository) FileCommitsCount(revision, file string) (int64, error) { + return CommitsCountFiles(repo.Path, []string{revision}, []string{file}) +} + +// CommitsByFileAndRange return the commits according revison file and the page +func (repo *Repository) CommitsByFileAndRange(revision, file string, page int) (*list.List, error) { + stdout, err := NewCommand("log", revision, "--follow", "--skip="+strconv.Itoa((page-1)*50), + "--max-count="+strconv.Itoa(CommitsRangeSize), prettyLogFormat, "--", file).RunInDirBytes(repo.Path) + if err != nil { + return nil, err + } + return repo.parsePrettyFormatLogToList(stdout) +} + +// CommitsByFileAndRangeNoFollow return the commits according revison file and the page +func (repo *Repository) CommitsByFileAndRangeNoFollow(revision, file string, page int) (*list.List, error) { + stdout, err := NewCommand("log", revision, "--skip="+strconv.Itoa((page-1)*50), + "--max-count="+strconv.Itoa(CommitsRangeSize), prettyLogFormat, "--", file).RunInDirBytes(repo.Path) + if err != nil { + return nil, err + } + return repo.parsePrettyFormatLogToList(stdout) +} + +// FilesCountBetween return the number of files changed between two commits +func (repo *Repository) FilesCountBetween(startCommitID, endCommitID string) (int, error) { + stdout, err := NewCommand("diff", "--name-only", startCommitID+"..."+endCommitID).RunInDir(repo.Path) + if err != nil && strings.Contains(err.Error(), "no merge base") { + // git >= 2.28 now returns an error if startCommitID and endCommitID have become unrelated. + // previously it would return the results of git diff --name-only startCommitID endCommitID so let's try that... + stdout, err = NewCommand("diff", "--name-only", startCommitID, endCommitID).RunInDir(repo.Path) + } + if err != nil { + return 0, err + } + return len(strings.Split(stdout, "\n")) - 1, nil +} + +// CommitsBetween returns a list that contains commits between [last, before). +func (repo *Repository) CommitsBetween(last *Commit, before *Commit) (*list.List, error) { + var stdout []byte + var err error + if before == nil { + stdout, err = NewCommand("rev-list", last.ID.String()).RunInDirBytes(repo.Path) + } else { + stdout, err = NewCommand("rev-list", before.ID.String()+"..."+last.ID.String()).RunInDirBytes(repo.Path) + if err != nil && strings.Contains(err.Error(), "no merge base") { + // future versions of git >= 2.28 are likely to return an error if before and last have become unrelated. + // previously it would return the results of git rev-list before last so let's try that... + stdout, err = NewCommand("rev-list", before.ID.String(), last.ID.String()).RunInDirBytes(repo.Path) + } + } + if err != nil { + return nil, err + } + return repo.parsePrettyFormatLogToList(bytes.TrimSpace(stdout)) +} + +// CommitsBetweenLimit returns a list that contains at most limit commits skipping the first skip commits between [last, before) +func (repo *Repository) CommitsBetweenLimit(last *Commit, before *Commit, limit, skip int) (*list.List, error) { + var stdout []byte + var err error + if before == nil { + stdout, err = NewCommand("rev-list", "--max-count", strconv.Itoa(limit), "--skip", strconv.Itoa(skip), last.ID.String()).RunInDirBytes(repo.Path) + } else { + stdout, err = NewCommand("rev-list", "--max-count", strconv.Itoa(limit), "--skip", strconv.Itoa(skip), before.ID.String()+"..."+last.ID.String()).RunInDirBytes(repo.Path) + if err != nil && strings.Contains(err.Error(), "no merge base") { + // future versions of git >= 2.28 are likely to return an error if before and last have become unrelated. + // previously it would return the results of git rev-list --max-count n before last so let's try that... + stdout, err = NewCommand("rev-list", "--max-count", strconv.Itoa(limit), "--skip", strconv.Itoa(skip), before.ID.String(), last.ID.String()).RunInDirBytes(repo.Path) + } + } + if err != nil { + return nil, err + } + return repo.parsePrettyFormatLogToList(bytes.TrimSpace(stdout)) +} + +// CommitsBetweenIDs return commits between twoe commits +func (repo *Repository) CommitsBetweenIDs(last, before string) (*list.List, error) { + lastCommit, err := repo.GetCommit(last) + if err != nil { + return nil, err + } + if before == "" { + return repo.CommitsBetween(lastCommit, nil) + } + beforeCommit, err := repo.GetCommit(before) + if err != nil { + return nil, err + } + return repo.CommitsBetween(lastCommit, beforeCommit) +} + +// CommitsCountBetween return numbers of commits between two commits +func (repo *Repository) CommitsCountBetween(start, end string) (int64, error) { + count, err := CommitsCountFiles(repo.Path, []string{start + "..." + end}, []string{}) + if err != nil && strings.Contains(err.Error(), "no merge base") { + // future versions of git >= 2.28 are likely to return an error if before and last have become unrelated. + // previously it would return the results of git rev-list before last so let's try that... + return CommitsCountFiles(repo.Path, []string{start, end}, []string{}) + } + + return count, err +} + +// commitsBefore the limit is depth, not total number of returned commits. +func (repo *Repository) commitsBefore(id SHA1, limit int) (*list.List, error) { + cmd := NewCommand("log") + if limit > 0 { + cmd.AddArguments("-"+strconv.Itoa(limit), prettyLogFormat, id.String()) + } else { + cmd.AddArguments(prettyLogFormat, id.String()) + } + + stdout, err := cmd.RunInDirBytes(repo.Path) + if err != nil { + return nil, err + } + + formattedLog, err := repo.parsePrettyFormatLogToList(bytes.TrimSpace(stdout)) + if err != nil { + return nil, err + } + + commits := list.New() + for logEntry := formattedLog.Front(); logEntry != nil; logEntry = logEntry.Next() { + commit := logEntry.Value.(*Commit) + branches, err := repo.getBranches(commit, 2) + if err != nil { + return nil, err + } + + if len(branches) > 1 { + break + } + + commits.PushBack(commit) + } + + return commits, nil +} + +func (repo *Repository) getCommitsBefore(id SHA1) (*list.List, error) { + return repo.commitsBefore(id, 0) +} + +func (repo *Repository) getCommitsBeforeLimit(id SHA1, num int) (*list.List, error) { + return repo.commitsBefore(id, num) +} + +func (repo *Repository) getBranches(commit *Commit, limit int) ([]string, error) { + if CheckGitVersionAtLeast("2.7.0") == nil { + stdout, err := NewCommand("for-each-ref", "--count="+strconv.Itoa(limit), "--format=%(refname:strip=2)", "--contains", commit.ID.String(), BranchPrefix).RunInDir(repo.Path) + if err != nil { + return nil, err + } + + branches := strings.Fields(stdout) + return branches, nil + } + + stdout, err := NewCommand("branch", "--contains", commit.ID.String()).RunInDir(repo.Path) + if err != nil { + return nil, err + } + + refs := strings.Split(stdout, "\n") + + var max int + if len(refs) > limit { + max = limit + } else { + max = len(refs) - 1 + } + + branches := make([]string, max) + for i, ref := range refs[:max] { + parts := strings.Fields(ref) + + branches[i] = parts[len(parts)-1] + } + return branches, nil +} + +// GetCommitsFromIDs get commits from commit IDs +func (repo *Repository) GetCommitsFromIDs(commitIDs []string) (commits *list.List) { + commits = list.New() + + for _, commitID := range commitIDs { + commit, err := repo.GetCommit(commitID) + if err == nil && commit != nil { + commits.PushBack(commit) + } + } + + return commits +} diff --git a/modules/git/repo_commitgraph.go b/modules/git/repo_commitgraph.go index 00111f550370e..4e5f8a37786f5 100644 --- a/modules/git/repo_commitgraph.go +++ b/modules/git/repo_commitgraph.go @@ -3,6 +3,8 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. +// +build !nogogit + package git import ( diff --git a/modules/git/repo_language_stats.go b/modules/git/repo_language_stats.go index b721b996e498b..2ab12de4fc83f 100644 --- a/modules/git/repo_language_stats.go +++ b/modules/git/repo_language_stats.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. +// +build !nogogit + package git import ( diff --git a/modules/git/repo_language_stats_nogogit.go b/modules/git/repo_language_stats_nogogit.go new file mode 100644 index 0000000000000..1a2d45ca733ed --- /dev/null +++ b/modules/git/repo_language_stats_nogogit.go @@ -0,0 +1,112 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build nogogit + +package git + +import ( + "bytes" + "io" + "io/ioutil" + + "code.gitea.io/gitea/modules/analyze" + + "github.com/go-enry/go-enry/v2" +) + +const fileSizeLimit int64 = 16 * 1024 // 16 KiB +const bigFileSize int64 = 1024 * 1024 // 1 MiB + +// GetLanguageStats calculates language stats for git repository at specified commit +func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, error) { + // FIXME: We can be more efficient here... + // + // We're expecting that we will be reading a lot of blobs and the trees + // Thus we should use a shared `cat-file --batch` to get all of this data + // And keep the buffers around with resets as necessary. + // + // It's more complicated so... + commit, err := repo.GetCommit(commitID) + if err != nil { + log("Unable to get commit for: %s", commitID) + return nil, err + } + + tree := commit.Tree + + entries, err := tree.ListEntriesRecursive() + if err != nil { + return nil, err + } + + sizes := make(map[string]int64) + for _, f := range entries { + if f.Size() == 0 || enry.IsVendor(f.Name()) || enry.IsDotFile(f.Name()) || + enry.IsDocumentation(f.Name()) || enry.IsConfiguration(f.Name()) { + continue + } + + // If content can not be read or file is too big just do detection by filename + var content []byte + if f.Size() <= bigFileSize { + content, _ = readFile(f, fileSizeLimit) + } + if enry.IsGenerated(f.Name(), content) { + continue + } + + // TODO: Use .gitattributes file for linguist overrides + // FIXME: Why can't we split this and the IsGenerated tests to avoid reading the blob unless absolutely necessary? + // - eg. do the all the detection tests using filename first before reading content. + language := analyze.GetCodeLanguage(f.Name(), content) + if language == enry.OtherLanguage || language == "" { + continue + } + + // group languages, such as Pug -> HTML; SCSS -> CSS + group := enry.GetLanguageGroup(language) + if group != "" { + language = group + } + + sizes[language] += f.Size() + + continue + } + + // filter special languages unless they are the only language + if len(sizes) > 1 { + for language := range sizes { + langtype := enry.GetLanguageType(language) + if langtype != enry.Programming && langtype != enry.Markup { + delete(sizes, language) + } + } + } + + return sizes, nil +} + +func readFile(entry *TreeEntry, limit int64) ([]byte, error) { + // FIXME: We can probably be a little more efficient here... see above + r, err := entry.Blob().DataAsync() + if err != nil { + return nil, err + } + defer r.Close() + + if limit <= 0 { + return ioutil.ReadAll(r) + } + + size := entry.Size() + if limit > 0 && size > limit { + size = limit + } + buf := bytes.NewBuffer(nil) + buf.Grow(int(size)) + _, err = io.Copy(buf, io.LimitReader(r, limit)) + return buf.Bytes(), err +} diff --git a/modules/git/repo_nogogit.go b/modules/git/repo_nogogit.go new file mode 100644 index 0000000000000..efaa843ebf9bf --- /dev/null +++ b/modules/git/repo_nogogit.go @@ -0,0 +1,403 @@ +// Copyright 2015 The Gogs Authors. All rights reserved. +// Copyright 2017 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build nogogit + +package git + +import ( + "bytes" + "container/list" + "errors" + "fmt" + "os" + "path" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/unknwon/com" +) + +// Repository represents a Git repository. +type Repository struct { + Path string + + tagCache *ObjectCache + + gpgSettings *GPGSettings +} + +// GPGSettings represents the default GPG settings for this repository +type GPGSettings struct { + Sign bool + KeyID string + Email string + Name string + PublicKeyContent string +} + +const prettyLogFormat = `--pretty=format:%H` + +// GetAllCommitsCount returns count of all commits in repository +func (repo *Repository) GetAllCommitsCount() (int64, error) { + return AllCommitsCount(repo.Path, false) +} + +func (repo *Repository) parsePrettyFormatLogToList(logs []byte) (*list.List, error) { + l := list.New() + if len(logs) == 0 { + return l, nil + } + + parts := bytes.Split(logs, []byte{'\n'}) + + for _, commitID := range parts { + commit, err := repo.GetCommit(string(commitID)) + if err != nil { + return nil, err + } + l.PushBack(commit) + } + + return l, nil +} + +// IsRepoURLAccessible checks if given repository URL is accessible. +func IsRepoURLAccessible(url string) bool { + _, err := NewCommand("ls-remote", "-q", "-h", url, "HEAD").Run() + return err == nil +} + +// InitRepository initializes a new Git repository. +func InitRepository(repoPath string, bare bool) error { + err := os.MkdirAll(repoPath, os.ModePerm) + if err != nil { + return err + } + + cmd := NewCommand("init") + if bare { + cmd.AddArguments("--bare") + } + _, err = cmd.RunInDir(repoPath) + return err +} + +// OpenRepository opens the repository at the given path. +func OpenRepository(repoPath string) (*Repository, error) { + repoPath, err := filepath.Abs(repoPath) + if err != nil { + return nil, err + } else if !isDir(repoPath) { + return nil, errors.New("no such file or directory") + } + return &Repository{ + Path: repoPath, + tagCache: newObjectCache(), + }, nil +} + +// Close this repository, in particular close the underlying gogitStorage if this is not nil +func (repo *Repository) Close() { +} + +// IsEmpty Check if repository is empty. +func (repo *Repository) IsEmpty() (bool, error) { + var errbuf strings.Builder + if err := NewCommand("log", "-1").RunInDirPipeline(repo.Path, nil, &errbuf); err != nil { + if strings.Contains(errbuf.String(), "fatal: bad default revision 'HEAD'") || + strings.Contains(errbuf.String(), "fatal: your current branch 'master' does not have any commits yet") { + return true, nil + } + return true, fmt.Errorf("check empty: %v - %s", err, errbuf.String()) + } + + return false, nil +} + +// CloneRepoOptions options when clone a repository +type CloneRepoOptions struct { + Timeout time.Duration + Mirror bool + Bare bool + Quiet bool + Branch string + Shared bool + NoCheckout bool + Depth int +} + +// Clone clones original repository to target path. +func Clone(from, to string, opts CloneRepoOptions) (err error) { + cargs := make([]string, len(GlobalCommandArgs)) + copy(cargs, GlobalCommandArgs) + return CloneWithArgs(from, to, cargs, opts) +} + +// CloneWithArgs original repository to target path. +func CloneWithArgs(from, to string, args []string, opts CloneRepoOptions) (err error) { + toDir := path.Dir(to) + if err = os.MkdirAll(toDir, os.ModePerm); err != nil { + return err + } + + cmd := NewCommandNoGlobals(args...).AddArguments("clone") + if opts.Mirror { + cmd.AddArguments("--mirror") + } + if opts.Bare { + cmd.AddArguments("--bare") + } + if opts.Quiet { + cmd.AddArguments("--quiet") + } + if opts.Shared { + cmd.AddArguments("-s") + } + if opts.NoCheckout { + cmd.AddArguments("--no-checkout") + } + if opts.Depth > 0 { + cmd.AddArguments("--depth", strconv.Itoa(opts.Depth)) + } + + if len(opts.Branch) > 0 { + cmd.AddArguments("-b", opts.Branch) + } + cmd.AddArguments("--", from, to) + + if opts.Timeout <= 0 { + opts.Timeout = -1 + } + + _, err = cmd.RunTimeout(opts.Timeout) + return err +} + +// PullRemoteOptions options when pull from remote +type PullRemoteOptions struct { + Timeout time.Duration + All bool + Rebase bool + Remote string + Branch string +} + +// Pull pulls changes from remotes. +func Pull(repoPath string, opts PullRemoteOptions) error { + cmd := NewCommand("pull") + if opts.Rebase { + cmd.AddArguments("--rebase") + } + if opts.All { + cmd.AddArguments("--all") + } else { + cmd.AddArguments("--", opts.Remote, opts.Branch) + } + + if opts.Timeout <= 0 { + opts.Timeout = -1 + } + + _, err := cmd.RunInDirTimeout(opts.Timeout, repoPath) + return err +} + +// PushOptions options when push to remote +type PushOptions struct { + Remote string + Branch string + Force bool + Env []string +} + +// Push pushs local commits to given remote branch. +func Push(repoPath string, opts PushOptions) error { + cmd := NewCommand("push") + if opts.Force { + cmd.AddArguments("-f") + } + cmd.AddArguments("--", opts.Remote, opts.Branch) + var outbuf, errbuf strings.Builder + + err := cmd.RunInDirTimeoutEnvPipeline(opts.Env, -1, repoPath, &outbuf, &errbuf) + if err != nil { + if strings.Contains(errbuf.String(), "non-fast-forward") { + return &ErrPushOutOfDate{ + StdOut: outbuf.String(), + StdErr: errbuf.String(), + Err: err, + } + } else if strings.Contains(errbuf.String(), "! [remote rejected]") { + err := &ErrPushRejected{ + StdOut: outbuf.String(), + StdErr: errbuf.String(), + Err: err, + } + err.GenerateMessage() + return err + } + } + + if errbuf.Len() > 0 && err != nil { + return fmt.Errorf("%v - %s", err, errbuf.String()) + } + + return err +} + +// CheckoutOptions options when heck out some branch +type CheckoutOptions struct { + Timeout time.Duration + Branch string + OldBranch string +} + +// Checkout checkouts a branch +func Checkout(repoPath string, opts CheckoutOptions) error { + cmd := NewCommand("checkout") + if len(opts.OldBranch) > 0 { + cmd.AddArguments("-b") + } + + if opts.Timeout <= 0 { + opts.Timeout = -1 + } + + cmd.AddArguments(opts.Branch) + + if len(opts.OldBranch) > 0 { + cmd.AddArguments(opts.OldBranch) + } + + _, err := cmd.RunInDirTimeout(opts.Timeout, repoPath) + return err +} + +// ResetHEAD resets HEAD to given revision or head of branch. +func ResetHEAD(repoPath string, hard bool, revision string) error { + cmd := NewCommand("reset") + if hard { + cmd.AddArguments("--hard") + } + _, err := cmd.AddArguments(revision).RunInDir(repoPath) + return err +} + +// MoveFile moves a file to another file or directory. +func MoveFile(repoPath, oldTreeName, newTreeName string) error { + _, err := NewCommand("mv").AddArguments(oldTreeName, newTreeName).RunInDir(repoPath) + return err +} + +// CountObject represents repository count objects report +type CountObject struct { + Count int64 + Size int64 + InPack int64 + Packs int64 + SizePack int64 + PrunePack int64 + Garbage int64 + SizeGarbage int64 +} + +const ( + statCount = "count: " + statSize = "size: " + statInpack = "in-pack: " + statPacks = "packs: " + statSizePack = "size-pack: " + statPrunePackage = "prune-package: " + statGarbage = "garbage: " + statSizeGarbage = "size-garbage: " +) + +// CountObjects returns the results of git count-objects on the repoPath +func CountObjects(repoPath string) (*CountObject, error) { + cmd := NewCommand("count-objects", "-v") + stdout, err := cmd.RunInDir(repoPath) + if err != nil { + return nil, err + } + + return parseSize(stdout), nil +} + +// parseSize parses the output from count-objects and return a CountObject +func parseSize(objects string) *CountObject { + repoSize := new(CountObject) + for _, line := range strings.Split(objects, "\n") { + switch { + case strings.HasPrefix(line, statCount): + repoSize.Count = com.StrTo(line[7:]).MustInt64() + case strings.HasPrefix(line, statSize): + repoSize.Size = com.StrTo(line[6:]).MustInt64() * 1024 + case strings.HasPrefix(line, statInpack): + repoSize.InPack = com.StrTo(line[9:]).MustInt64() + case strings.HasPrefix(line, statPacks): + repoSize.Packs = com.StrTo(line[7:]).MustInt64() + case strings.HasPrefix(line, statSizePack): + repoSize.SizePack = com.StrTo(line[11:]).MustInt64() * 1024 + case strings.HasPrefix(line, statPrunePackage): + repoSize.PrunePack = com.StrTo(line[16:]).MustInt64() + case strings.HasPrefix(line, statGarbage): + repoSize.Garbage = com.StrTo(line[9:]).MustInt64() + case strings.HasPrefix(line, statSizeGarbage): + repoSize.SizeGarbage = com.StrTo(line[14:]).MustInt64() * 1024 + } + } + return repoSize +} + +// GetLatestCommitTime returns time for latest commit in repository (across all branches) +func GetLatestCommitTime(repoPath string) (time.Time, error) { + cmd := NewCommand("for-each-ref", "--sort=-committerdate", "refs/heads/", "--count", "1", "--format=%(committerdate)") + stdout, err := cmd.RunInDir(repoPath) + if err != nil { + return time.Time{}, err + } + commitTime := strings.TrimSpace(stdout) + return time.Parse(GitTimeLayout, commitTime) +} + +// DivergeObject represents commit count diverging commits +type DivergeObject struct { + Ahead int + Behind int +} + +func checkDivergence(repoPath string, baseBranch string, targetBranch string) (int, error) { + branches := fmt.Sprintf("%s..%s", baseBranch, targetBranch) + cmd := NewCommand("rev-list", "--count", branches) + stdout, err := cmd.RunInDir(repoPath) + if err != nil { + return -1, err + } + outInteger, errInteger := strconv.Atoi(strings.Trim(stdout, "\n")) + if errInteger != nil { + return -1, errInteger + } + return outInteger, nil +} + +// GetDivergingCommits returns the number of commits a targetBranch is ahead or behind a baseBranch +func GetDivergingCommits(repoPath string, baseBranch string, targetBranch string) (DivergeObject, error) { + // $(git rev-list --count master..feature) commits ahead of master + ahead, errorAhead := checkDivergence(repoPath, baseBranch, targetBranch) + if errorAhead != nil { + return DivergeObject{}, errorAhead + } + + // $(git rev-list --count feature..master) commits behind master + behind, errorBehind := checkDivergence(repoPath, targetBranch, baseBranch) + if errorBehind != nil { + return DivergeObject{}, errorBehind + } + + return DivergeObject{ahead, behind}, nil +} diff --git a/modules/git/repo_object.go b/modules/git/repo_object.go index d4d638a74344b..f054c349029e5 100644 --- a/modules/git/repo_object.go +++ b/modules/git/repo_object.go @@ -27,6 +27,11 @@ const ( ObjectBranch ObjectType = "branch" ) +// Bytes returns the byte array for the Object Type +func (o ObjectType) Bytes() []byte { + return []byte(o) +} + // HashObject takes a reader and returns SHA1 hash for that reader func (repo *Repository) HashObject(reader io.Reader) (SHA1, error) { idStr, err := repo.hashObject(reader) diff --git a/modules/git/repo_ref.go b/modules/git/repo_ref.go index be2a38c5f0af6..ef4f73a1ebbfe 100644 --- a/modules/git/repo_ref.go +++ b/modules/git/repo_ref.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. +// +build !nogogit + package git import ( diff --git a/modules/git/repo_ref_nogogit.go b/modules/git/repo_ref_nogogit.go new file mode 100644 index 0000000000000..74534046dd96e --- /dev/null +++ b/modules/git/repo_ref_nogogit.go @@ -0,0 +1,89 @@ +// Copyright 2018 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build nogogit + +package git + +import ( + "bufio" + "io" + "strings" +) + +// GetRefs returns all references of the repository. +func (repo *Repository) GetRefs() ([]*Reference, error) { + return repo.GetRefsFiltered("") +} + +// GetRefsFiltered returns all references of the repository that matches patterm exactly or starting with. +func (repo *Repository) GetRefsFiltered(pattern string) ([]*Reference, error) { + stdoutReader, stdoutWriter := io.Pipe() + defer func() { + _ = stdoutReader.Close() + _ = stdoutWriter.Close() + }() + + go func() { + stderrBuilder := &strings.Builder{} + err := NewCommand("for-each-ref").RunInDirPipeline(repo.Path, stdoutWriter, stderrBuilder) + if err != nil { + _ = stdoutWriter.CloseWithError(ConcatenateError(err, stderrBuilder.String())) + } else { + _ = stdoutWriter.Close() + } + }() + + refs := make([]*Reference, 0) + bufReader := bufio.NewReader(stdoutReader) + for { + // The output of for-each-ref is simply a list: + // SP TAB LF + sha, err := bufReader.ReadString(' ') + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + sha = sha[:len(sha)-1] + + typ, err := bufReader.ReadString('\t') + if err == io.EOF { + // This should not happen, but we'll tolerate it + break + } + if err != nil { + return nil, err + } + typ = typ[:len(typ)-1] + + refName, err := bufReader.ReadString('\n') + if err == io.EOF { + // This should not happen, but we'll tolerate it + break + } + if err != nil { + return nil, err + } + refName = refName[:len(refName)-1] + + // refName cannot be HEAD but can be remotes or stash + if strings.HasPrefix(refName, "/refs/remotes/") || refName == "/refs/stash" { + continue + } + + if pattern == "" || strings.HasPrefix(refName, pattern) { + r := &Reference{ + Name: refName, + Object: MustIDFromString(sha), + Type: typ, + repo: repo, + } + refs = append(refs, r) + } + } + + return refs, nil +} diff --git a/modules/git/repo_tag.go b/modules/git/repo_tag.go index 376a699502bac..5bd85b8462e09 100644 --- a/modules/git/repo_tag.go +++ b/modules/git/repo_tag.go @@ -3,6 +3,8 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. +// +build !nogogit + package git import ( diff --git a/modules/git/repo_tag_nogogit.go b/modules/git/repo_tag_nogogit.go new file mode 100644 index 0000000000000..52ed3da2eda9f --- /dev/null +++ b/modules/git/repo_tag_nogogit.go @@ -0,0 +1,264 @@ +// Copyright 2015 The Gogs Authors. All rights reserved. +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build nogogit + +package git + +import ( + "fmt" + "strings" +) + +// TagPrefix tags prefix path on the repository +const TagPrefix = "refs/tags/" + +// IsTagExist returns true if given tag exists in the repository. +func IsTagExist(repoPath, name string) bool { + return IsReferenceExist(repoPath, TagPrefix+name) +} + +// IsTagExist returns true if given tag exists in the repository. +func (repo *Repository) IsTagExist(name string) bool { + return IsReferenceExist(repo.Path, TagPrefix+name) +} + +// CreateTag create one tag in the repository +func (repo *Repository) CreateTag(name, revision string) error { + _, err := NewCommand("tag", "--", name, revision).RunInDir(repo.Path) + return err +} + +// CreateAnnotatedTag create one annotated tag in the repository +func (repo *Repository) CreateAnnotatedTag(name, message, revision string) error { + _, err := NewCommand("tag", "-a", "-m", message, "--", name, revision).RunInDir(repo.Path) + return err +} + +func (repo *Repository) getTag(id SHA1) (*Tag, error) { + t, ok := repo.tagCache.Get(id.String()) + if ok { + log("Hit cache: %s", id) + tagClone := *t.(*Tag) + return &tagClone, nil + } + + // Get tag name + name, err := repo.GetTagNameBySHA(id.String()) + if err != nil { + return nil, err + } + + tp, err := repo.GetTagType(id) + if err != nil { + return nil, err + } + + // Get the commit ID and tag ID (may be different for annotated tag) for the returned tag object + commitIDStr, err := repo.GetTagCommitID(name) + if err != nil { + // every tag should have a commit ID so return all errors + return nil, err + } + commitID, err := NewIDFromString(commitIDStr) + if err != nil { + return nil, err + } + + // tagID defaults to the commit ID as the tag ID and then tries to get a tag ID (only annotated tags) + tagID := commitID + if tagIDStr, err := repo.GetTagID(name); err != nil { + // if the err is NotExist then we can ignore and just keep tagID as ID (is lightweight tag) + // all other errors we return + if !IsErrNotExist(err) { + return nil, err + } + } else { + tagID, err = NewIDFromString(tagIDStr) + if err != nil { + return nil, err + } + } + + // If type is "commit, the tag is a lightweight tag + if ObjectType(tp) == ObjectCommit { + commit, err := repo.GetCommit(id.String()) + if err != nil { + return nil, err + } + tag := &Tag{ + Name: name, + ID: tagID, + Object: commitID, + Type: string(ObjectCommit), + Tagger: commit.Committer, + Message: commit.Message(), + repo: repo, + } + + repo.tagCache.Set(id.String(), tag) + return tag, nil + } + + // The tag is an annotated tag with a message. + data, err := NewCommand("cat-file", "-p", id.String()).RunInDirBytes(repo.Path) + if err != nil { + return nil, err + } + + tag, err := parseTagData(data) + if err != nil { + return nil, err + } + + tag.Name = name + tag.ID = id + tag.repo = repo + tag.Type = tp + + repo.tagCache.Set(id.String(), tag) + return tag, nil +} + +// GetTagNameBySHA returns the name of a tag from its tag object SHA or commit SHA +func (repo *Repository) GetTagNameBySHA(sha string) (string, error) { + if len(sha) < 5 { + return "", fmt.Errorf("SHA is too short: %s", sha) + } + + stdout, err := NewCommand("show-ref", "--tags", "-d").RunInDir(repo.Path) + if err != nil { + return "", err + } + + tagRefs := strings.Split(stdout, "\n") + for _, tagRef := range tagRefs { + if len(strings.TrimSpace(tagRef)) > 0 { + fields := strings.Fields(tagRef) + if strings.HasPrefix(fields[0], sha) && strings.HasPrefix(fields[1], TagPrefix) { + name := fields[1][len(TagPrefix):] + // annotated tags show up twice, we should only return if is not the ^{} ref + if !strings.HasSuffix(name, "^{}") { + return name, nil + } + } + } + } + return "", ErrNotExist{ID: sha} +} + +// GetTagID returns the object ID for a tag (annotated tags have both an object SHA AND a commit SHA) +func (repo *Repository) GetTagID(name string) (string, error) { + stdout, err := NewCommand("show-ref", "--tags", "--", name).RunInDir(repo.Path) + if err != nil { + return "", err + } + // Make sure exact match is used: "v1" != "release/v1" + for _, line := range strings.Split(stdout, "\n") { + fields := strings.Fields(line) + if len(fields) == 2 && fields[1] == "refs/tags/"+name { + return fields[0], nil + } + } + return "", ErrNotExist{ID: name} +} + +// GetTag returns a Git tag by given name. +func (repo *Repository) GetTag(name string) (*Tag, error) { + idStr, err := repo.GetTagID(name) + if err != nil { + return nil, err + } + + id, err := NewIDFromString(idStr) + if err != nil { + return nil, err + } + + tag, err := repo.getTag(id) + if err != nil { + return nil, err + } + return tag, nil +} + +// GetTagInfos returns all tag infos of the repository. +func (repo *Repository) GetTagInfos(page, pageSize int) ([]*Tag, error) { + // TODO this a slow implementation, makes one git command per tag + stdout, err := NewCommand("tag").RunInDir(repo.Path) + if err != nil { + return nil, err + } + + tagNames := strings.Split(strings.TrimRight(stdout, "\n"), "\n") + + if page != 0 { + skip := (page - 1) * pageSize + if skip >= len(tagNames) { + return nil, nil + } + if (len(tagNames) - skip) < pageSize { + pageSize = len(tagNames) - skip + } + tagNames = tagNames[skip : skip+pageSize] + } + + var tags = make([]*Tag, 0, len(tagNames)) + for _, tagName := range tagNames { + tagName = strings.TrimSpace(tagName) + if len(tagName) == 0 { + continue + } + + tag, err := repo.GetTag(tagName) + if err != nil { + return nil, err + } + tag.Name = tagName + tags = append(tags, tag) + } + sortTagsByTime(tags) + return tags, nil +} + +// GetTags returns all tags of the repository. +func (repo *Repository) GetTags() ([]string, error) { + return callShowRef(repo.Path, TagPrefix, "--tags") +} + +// GetTagType gets the type of the tag, either commit (simple) or tag (annotated) +func (repo *Repository) GetTagType(id SHA1) (string, error) { + // Get tag type + stdout, err := NewCommand("cat-file", "-t", id.String()).RunInDir(repo.Path) + if err != nil { + return "", err + } + if len(stdout) == 0 { + return "", ErrNotExist{ID: id.String()} + } + return strings.TrimSpace(stdout), nil +} + +// GetAnnotatedTag returns a Git tag by its SHA, must be an annotated tag +func (repo *Repository) GetAnnotatedTag(sha string) (*Tag, error) { + id, err := NewIDFromString(sha) + if err != nil { + return nil, err + } + + // Tag type must be "tag" (annotated) and not a "commit" (lightweight) tag + if tagType, err := repo.GetTagType(id); err != nil { + return nil, err + } else if ObjectType(tagType) != ObjectTag { + // not an annotated tag + return nil, ErrNotExist{ID: id.String()} + } + + tag, err := repo.getTag(id) + if err != nil { + return nil, err + } + return tag, nil +} diff --git a/modules/git/repo_tree.go b/modules/git/repo_tree.go index 0b08a10d554d9..562899bb12f9b 100644 --- a/modules/git/repo_tree.go +++ b/modules/git/repo_tree.go @@ -3,6 +3,8 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. +// +build !nogogit + package git import ( @@ -102,7 +104,7 @@ func (repo *Repository) CommitTree(author *Signature, committer *Signature, tree err = cmd.RunInDirTimeoutEnvFullPipeline(env, -1, repo.Path, stdout, stderr, messageBytes) if err != nil { - return SHA1{}, concatenateError(err, stderr.String()) + return SHA1{}, ConcatenateError(err, stderr.String()) } return NewIDFromString(strings.TrimSpace(stdout.String())) } diff --git a/modules/git/repo_tree_nogogit.go b/modules/git/repo_tree_nogogit.go new file mode 100644 index 0000000000000..f2bccac6c27db --- /dev/null +++ b/modules/git/repo_tree_nogogit.go @@ -0,0 +1,157 @@ +// Copyright 2015 The Gogs Authors. All rights reserved. +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build nogogit + +package git + +import ( + "bufio" + "bytes" + "fmt" + "io" + "io/ioutil" + "os" + "strings" + "time" +) + +func (repo *Repository) getTree(id SHA1) (*Tree, error) { + stdoutReader, stdoutWriter := io.Pipe() + defer func() { + _ = stdoutReader.Close() + _ = stdoutWriter.Close() + }() + + go func() { + stderr := &strings.Builder{} + err := NewCommand("cat-file", "--batch").RunInDirFullPipeline(repo.Path, stdoutWriter, stderr, strings.NewReader(id.String()+"\n")) + if err != nil { + stdoutWriter.CloseWithError(ConcatenateError(err, stderr.String())) + } else { + stdoutWriter.Close() + } + }() + + bufReader := bufio.NewReader(stdoutReader) + // ignore the SHA + _, typ, _, err := ReadBatchLine(bufReader) + if err != nil { + return nil, err + } + + switch typ { + case "tag": + resolvedID := id + data, err := ioutil.ReadAll(bufReader) + if err != nil { + return nil, err + } + tag, err := parseTagData(data) + if err != nil { + return nil, err + } + commit, err := tag.Commit() + if err != nil { + return nil, err + } + commit.Tree.ResolvedID = resolvedID + log("tag.commit.Tree: %s %v", commit.Tree.ID.String(), commit.Tree.repo) + return &commit.Tree, nil + case "commit": + commit, err := CommitFromReader(repo, id, bufReader) + if err != nil { + stdoutReader.CloseWithError(err) + return nil, err + } + commit.Tree.ResolvedID = commit.ID + log("commit.Tree: %s %v", commit.Tree.ID.String(), commit.Tree.repo) + return &commit.Tree, nil + case "tree": + stdoutReader.Close() + tree := NewTree(repo, id) + tree.ResolvedID = id + return tree, nil + default: + stdoutReader.CloseWithError(fmt.Errorf("unknown typ: %s", typ)) + return nil, ErrNotExist{ + ID: id.String(), + } + } +} + +// GetTree find the tree object in the repository. +func (repo *Repository) GetTree(idStr string) (*Tree, error) { + if len(idStr) != 40 { + res, err := NewCommand("rev-parse", "--verify", idStr).RunInDir(repo.Path) + if err != nil { + return nil, err + } + if len(res) > 0 { + idStr = res[:len(res)-1] + } + } + id, err := NewIDFromString(idStr) + if err != nil { + return nil, err + } + + return repo.getTree(id) +} + +// CommitTreeOpts represents the possible options to CommitTree +type CommitTreeOpts struct { + Parents []string + Message string + KeyID string + NoGPGSign bool + AlwaysSign bool +} + +// CommitTree creates a commit from a given tree id for the user with provided message +func (repo *Repository) CommitTree(author *Signature, committer *Signature, tree *Tree, opts CommitTreeOpts) (SHA1, error) { + err := LoadGitVersion() + if err != nil { + return SHA1{}, err + } + + commitTimeStr := time.Now().Format(time.RFC3339) + + // Because this may call hooks we should pass in the environment + env := append(os.Environ(), + "GIT_AUTHOR_NAME="+author.Name, + "GIT_AUTHOR_EMAIL="+author.Email, + "GIT_AUTHOR_DATE="+commitTimeStr, + "GIT_COMMITTER_NAME="+committer.Name, + "GIT_COMMITTER_EMAIL="+committer.Email, + "GIT_COMMITTER_DATE="+commitTimeStr, + ) + cmd := NewCommand("commit-tree", tree.ID.String()) + + for _, parent := range opts.Parents { + cmd.AddArguments("-p", parent) + } + + messageBytes := new(bytes.Buffer) + _, _ = messageBytes.WriteString(opts.Message) + _, _ = messageBytes.WriteString("\n") + + if CheckGitVersionAtLeast("1.7.9") == nil && (opts.KeyID != "" || opts.AlwaysSign) { + cmd.AddArguments(fmt.Sprintf("-S%s", opts.KeyID)) + } + + if CheckGitVersionAtLeast("2.0.0") == nil && opts.NoGPGSign { + cmd.AddArguments("--no-gpg-sign") + } + + stdout := new(bytes.Buffer) + stderr := new(bytes.Buffer) + err = cmd.RunInDirTimeoutEnvFullPipeline(env, -1, repo.Path, stdout, stderr, messageBytes) + + if err != nil { + return SHA1{}, ConcatenateError(err, stderr.String()) + } + return NewIDFromString(strings.TrimSpace(stdout.String())) +} diff --git a/modules/git/sha1.go b/modules/git/sha1.go index d3563060f78cb..f6dc006cb2f79 100644 --- a/modules/git/sha1.go +++ b/modules/git/sha1.go @@ -3,6 +3,8 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. +// +build !nogogit + package git import ( diff --git a/modules/git/sha1_nogogit.go b/modules/git/sha1_nogogit.go new file mode 100644 index 0000000000000..1de782bb152c9 --- /dev/null +++ b/modules/git/sha1_nogogit.go @@ -0,0 +1,108 @@ +// Copyright 2015 The Gogs Authors. All rights reserved. +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build nogogit + +package git + +import ( + "crypto/sha1" + "encoding/hex" + "fmt" + "hash" + "regexp" + "strconv" + "strings" +) + +// EmptySHA defines empty git SHA +const EmptySHA = "0000000000000000000000000000000000000000" + +// EmptyTreeSHA is the SHA of an empty tree +const EmptyTreeSHA = "4b825dc642cb6eb9a060e54bf8d69288fbee4904" + +// SHAPattern can be used to determine if a string is an valid sha +var SHAPattern = regexp.MustCompile(`^[0-9a-f]{4,40}$`) + +// SHA1 a git commit name +type SHA1 [20]byte + +// String returns a string representation of the SHA +func (s SHA1) String() string { + return hex.EncodeToString(s[:]) +} + +func (s SHA1) IsZero() bool { + var empty SHA1 + return s == empty +} + +// MustID always creates a new SHA1 from a [20]byte array with no validation of input. +func MustID(b []byte) SHA1 { + var id SHA1 + copy(id[:], b) + return id +} + +// NewID creates a new SHA1 from a [20]byte array. +func NewID(b []byte) (SHA1, error) { + if len(b) != 20 { + return SHA1{}, fmt.Errorf("Length must be 20: %v", b) + } + return MustID(b), nil +} + +// MustIDFromString always creates a new sha from a ID with no validation of input. +func MustIDFromString(s string) SHA1 { + b, _ := hex.DecodeString(s) + return MustID(b) +} + +// NewIDFromString creates a new SHA1 from a ID string of length 40. +func NewIDFromString(s string) (SHA1, error) { + var id SHA1 + s = strings.TrimSpace(s) + if len(s) != 40 { + return id, fmt.Errorf("Length must be 40: %s", s) + } + b, err := hex.DecodeString(s) + if err != nil { + return id, err + } + return NewID(b) +} + +// ComputeBlobHash compute the hash for a given blob content +func ComputeBlobHash(content []byte) SHA1 { + return ComputeHash(ObjectBlob, content) +} + +// ComputeHash compute the hash for a given ObjectType and content +func ComputeHash(t ObjectType, content []byte) SHA1 { + h := NewHasher(t, int64(len(content))) + h.Write(content) + return h.Sum() +} + +// Hasher is a struct that will generate a SHA1 +type Hasher struct { + hash.Hash +} + +// NewHasher takes an object type and size and creates a hasher to generate a SHA +func NewHasher(t ObjectType, size int64) Hasher { + h := Hasher{sha1.New()} + h.Write(t.Bytes()) + h.Write([]byte(" ")) + h.Write([]byte(strconv.FormatInt(size, 10))) + h.Write([]byte{0}) + return h +} + +// Sum generates a SHA1 for the provided hash +func (h Hasher) Sum() (sha1 SHA1) { + copy(sha1[:], h.Hash.Sum(nil)) + return +} diff --git a/modules/git/signature.go b/modules/git/signature.go index 4cb56b29f4458..0b6124eb61e59 100644 --- a/modules/git/signature.go +++ b/modules/git/signature.go @@ -3,6 +3,8 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. +// +build !nogogit + package git import ( diff --git a/modules/git/signature_nogogit.go b/modules/git/signature_nogogit.go new file mode 100644 index 0000000000000..c38821e9b3d63 --- /dev/null +++ b/modules/git/signature_nogogit.go @@ -0,0 +1,99 @@ +// Copyright 2015 The Gogs Authors. All rights reserved. +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build nogogit + +package git + +import ( + "bytes" + "fmt" + "strconv" + "time" +) + +// Signature represents the Author or Committer information. +type Signature struct { + // Name represents a person name. It is an arbitrary string. + Name string + // Email is an email, but it cannot be assumed to be well-formed. + Email string + // When is the timestamp of the signature. + When time.Time +} + +func (s *Signature) String() string { + return fmt.Sprintf("%s <%s>", s.Name, s.Email) +} + +func (s *Signature) Decode(b []byte) { + sig, _ := newSignatureFromCommitline(b) + s.Email = sig.Email + s.Name = sig.Name + s.When = sig.When +} + +const ( + // GitTimeLayout is the (default) time layout used by git. + GitTimeLayout = "Mon Jan _2 15:04:05 2006 -0700" +) + +// Helper to get a signature from the commit line, which looks like these: +// author Patrick Gundlach 1378823654 +0200 +// author Patrick Gundlach Thu, 07 Apr 2005 22:13:13 +0200 +// but without the "author " at the beginning (this method should) +// be used for author and committer. +func newSignatureFromCommitline(line []byte) (sig *Signature, err error) { + sig = new(Signature) + emailStart := bytes.LastIndexByte(line, '<') + emailEnd := bytes.LastIndexByte(line, '>') + if emailStart == -1 || emailEnd == -1 || emailEnd < emailStart { + return + } + + sig.Name = string(line[:emailStart-1]) + sig.Email = string(line[emailStart+1 : emailEnd]) + + hasTime := emailEnd+2 < len(line) + if !hasTime { + return + } + + // Check date format. + firstChar := line[emailEnd+2] + if firstChar >= 48 && firstChar <= 57 { + idx := bytes.IndexByte(line[emailEnd+2:], ' ') + if idx < 0 { + return + } + + timestring := string(line[emailEnd+2 : emailEnd+2+idx]) + seconds, _ := strconv.ParseInt(timestring, 10, 64) + sig.When = time.Unix(seconds, 0) + + idx += emailEnd + 3 + if idx >= len(line) || idx+5 > len(line) { + return + } + + timezone := string(line[idx : idx+5]) + tzhours, err1 := strconv.ParseInt(timezone[0:3], 10, 64) + tzmins, err2 := strconv.ParseInt(timezone[3:], 10, 64) + if err1 != nil || err2 != nil { + return + } + if tzhours < 0 { + tzmins *= -1 + } + tz := time.FixedZone("", int(tzhours*60*60+tzmins*60)) + sig.When = sig.When.In(tz) + } else { + sig.When, err = time.Parse(GitTimeLayout, string(line[emailEnd+2:])) + if err != nil { + return + } + } + return +} diff --git a/modules/git/tag.go b/modules/git/tag.go index c97f574fa62b9..d58a9a202dc0d 100644 --- a/modules/git/tag.go +++ b/modules/git/tag.go @@ -10,15 +10,19 @@ import ( "strings" ) +const beginpgp = "\n-----BEGIN PGP SIGNATURE-----\n" +const endpgp = "\n-----END PGP SIGNATURE-----" + // Tag represents a Git tag. type Tag struct { - Name string - ID SHA1 - repo *Repository - Object SHA1 // The id of this commit object - Type string - Tagger *Signature - Message string + Name string + ID SHA1 + repo *Repository + Object SHA1 // The id of this commit object + Type string + Tagger *Signature + Message string + Signature *CommitGPGSignature } // Commit return the commit of the tag reference @@ -60,12 +64,23 @@ l: } nextline += eol + 1 case eol == 0: - tag.Message = strings.TrimRight(string(data[nextline+1:]), "\n") + tag.Message = string(data[nextline+1 : len(data)-1]) break l default: break l } } + idx := strings.LastIndex(tag.Message, beginpgp) + if idx > 0 { + endSigIdx := strings.Index(tag.Message[idx:], endpgp) + if endSigIdx > 0 { + tag.Signature = &CommitGPGSignature{ + Signature: tag.Message[idx+1 : idx+endSigIdx+len(endpgp)], + Payload: string(data[:bytes.LastIndex(data, []byte(beginpgp))+1]), + } + tag.Message = tag.Message[:idx+1] + } + } return tag, nil } diff --git a/modules/git/tree.go b/modules/git/tree.go index 258b11aaac557..447c042e64058 100644 --- a/modules/git/tree.go +++ b/modules/git/tree.go @@ -3,6 +3,8 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. +// +build !nogogit + package git import ( diff --git a/modules/git/tree_blob.go b/modules/git/tree_blob.go index f9fc6db497cfd..bd62adbfdd307 100644 --- a/modules/git/tree_blob.go +++ b/modules/git/tree_blob.go @@ -3,6 +3,8 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. +// +build !nogogit + package git import ( diff --git a/modules/git/tree_blob_nogogit.go b/modules/git/tree_blob_nogogit.go new file mode 100644 index 0000000000000..88da499218d62 --- /dev/null +++ b/modules/git/tree_blob_nogogit.go @@ -0,0 +1,64 @@ +// Copyright 2015 The Gogs Authors. All rights reserved. +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build nogogit + +package git + +import ( + "path" + "strings" +) + +// GetTreeEntryByPath get the tree entries according the sub dir +func (t *Tree) GetTreeEntryByPath(relpath string) (*TreeEntry, error) { + if len(relpath) == 0 { + return &TreeEntry{ + ID: t.ID, + name: "", + fullName: "", + entryMode: EntryModeTree, + }, nil + } + + // FIXME: This should probably use git cat-file --batch to be a bit more efficient + relpath = path.Clean(relpath) + parts := strings.Split(relpath, "/") + var err error + tree := t + for i, name := range parts { + if i == len(parts)-1 { + entries, err := tree.ListEntries() + if err != nil { + return nil, err + } + for _, v := range entries { + if v.Name() == name { + return v, nil + } + } + } else { + tree, err = tree.SubTree(name) + if err != nil { + return nil, err + } + } + } + return nil, ErrNotExist{"", relpath} +} + +// GetBlobByPath get the blob object according the path +func (t *Tree) GetBlobByPath(relpath string) (*Blob, error) { + entry, err := t.GetTreeEntryByPath(relpath) + if err != nil { + return nil, err + } + + if !entry.IsDir() && !entry.IsSubModule() { + return entry.Blob(), nil + } + + return nil, ErrNotExist{"", relpath} +} diff --git a/modules/git/tree_entry.go b/modules/git/tree_entry.go index d9814120953f8..2b62ca4eb7a85 100644 --- a/modules/git/tree_entry.go +++ b/modules/git/tree_entry.go @@ -3,6 +3,8 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. +// +build !nogogit + package git import ( @@ -15,24 +17,6 @@ import ( "github.com/go-git/go-git/v5/plumbing/object" ) -// EntryMode the type of the object in the git tree -type EntryMode int - -// There are only a few file modes in Git. They look like unix file modes, but they can only be -// one of these. -const ( - // EntryModeBlob - EntryModeBlob EntryMode = 0100644 - // EntryModeExec - EntryModeExec EntryMode = 0100755 - // EntryModeSymlink - EntryModeSymlink EntryMode = 0120000 - // EntryModeCommit - EntryModeCommit EntryMode = 0160000 - // EntryModeTree - EntryModeTree EntryMode = 0040000 -) - // TreeEntry the leaf in the git tree type TreeEntry struct { ID SHA1 diff --git a/modules/git/tree_entry_mode.go b/modules/git/tree_entry_mode.go new file mode 100644 index 0000000000000..b029c6fc476f8 --- /dev/null +++ b/modules/git/tree_entry_mode.go @@ -0,0 +1,36 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package git + +import "strconv" + +// EntryMode the type of the object in the git tree +type EntryMode int + +// There are only a few file modes in Git. They look like unix file modes, but they can only be +// one of these. +const ( + // EntryModeBlob + EntryModeBlob EntryMode = 0100644 + // EntryModeExec + EntryModeExec EntryMode = 0100755 + // EntryModeSymlink + EntryModeSymlink EntryMode = 0120000 + // EntryModeCommit + EntryModeCommit EntryMode = 0160000 + // EntryModeTree + EntryModeTree EntryMode = 0040000 +) + +// String converts an EntryMode to a string +func (e EntryMode) String() string { + return strconv.FormatInt(int64(e), 8) +} + +// ToEntryMode converts a string to an EntryMode +func ToEntryMode(value string) EntryMode { + v, _ := strconv.ParseInt(value, 8, 32) + return EntryMode(v) +} diff --git a/modules/git/tree_entry_nogogit.go b/modules/git/tree_entry_nogogit.go new file mode 100644 index 0000000000000..f5a9e25917020 --- /dev/null +++ b/modules/git/tree_entry_nogogit.go @@ -0,0 +1,246 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build nogogit + +package git + +import ( + "io" + "sort" + "strconv" + "strings" +) + +// TreeEntry the leaf in the git tree +type TreeEntry struct { + ID SHA1 + + ptree *Tree + + entryMode EntryMode + name string + + size int64 + sized bool + fullName string +} + +// Name returns the name of the entry +func (te *TreeEntry) Name() string { + if te.fullName != "" { + return te.fullName + } + return te.name +} + +// Mode returns the mode of the entry +func (te *TreeEntry) Mode() EntryMode { + return te.entryMode +} + +// Type returns the type of the entry (commit, tree, blob) +func (te *TreeEntry) Type() string { + switch te.Mode() { + case EntryModeCommit: + return "commit" + case EntryModeTree: + return "tree" + default: + return "blob" + } +} + +// Size returns the size of the entry +func (te *TreeEntry) Size() int64 { + if te.IsDir() { + return 0 + } else if te.sized { + return te.size + } + + stdout, err := NewCommand("cat-file", "-s", te.ID.String()).RunInDir(te.ptree.repo.Path) + if err != nil { + return 0 + } + + te.sized = true + te.size, _ = strconv.ParseInt(strings.TrimSpace(stdout), 10, 64) + return te.size +} + +// IsSubModule if the entry is a sub module +func (te *TreeEntry) IsSubModule() bool { + return te.entryMode == EntryModeCommit +} + +// IsDir if the entry is a sub dir +func (te *TreeEntry) IsDir() bool { + return te.entryMode == EntryModeTree +} + +// IsLink if the entry is a symlink +func (te *TreeEntry) IsLink() bool { + return te.entryMode == EntryModeSymlink +} + +// IsRegular if the entry is a regular file +func (te *TreeEntry) IsRegular() bool { + return te.entryMode == EntryModeBlob +} + +// IsExecutable if the entry is an executable file (not necessarily binary) +func (te *TreeEntry) IsExecutable() bool { + return te.entryMode == EntryModeExec +} + +// Blob returns the blob object the entry +func (te *TreeEntry) Blob() *Blob { + return &Blob{ + ID: te.ID, + repoPath: te.ptree.repo.Path, + name: te.Name(), + } +} + +// FollowLink returns the entry pointed to by a symlink +func (te *TreeEntry) FollowLink() (*TreeEntry, error) { + if !te.IsLink() { + return nil, ErrBadLink{te.Name(), "not a symlink"} + } + + // read the link + r, err := te.Blob().DataAsync() + if err != nil { + return nil, err + } + defer r.Close() + buf := make([]byte, te.Size()) + _, err = io.ReadFull(r, buf) + if err != nil { + return nil, err + } + + lnk := string(buf) + t := te.ptree + + // traverse up directories + for ; t != nil && strings.HasPrefix(lnk, "../"); lnk = lnk[3:] { + t = t.ptree + } + + if t == nil { + return nil, ErrBadLink{te.Name(), "points outside of repo"} + } + + target, err := t.GetTreeEntryByPath(lnk) + if err != nil { + if IsErrNotExist(err) { + return nil, ErrBadLink{te.Name(), "broken link"} + } + return nil, err + } + return target, nil +} + +// FollowLinks returns the entry ultimately pointed to by a symlink +func (te *TreeEntry) FollowLinks() (*TreeEntry, error) { + if !te.IsLink() { + return nil, ErrBadLink{te.Name(), "not a symlink"} + } + entry := te + for i := 0; i < 999; i++ { + if entry.IsLink() { + next, err := entry.FollowLink() + if err != nil { + return nil, err + } + if next.ID == entry.ID { + return nil, ErrBadLink{ + entry.Name(), + "recursive link", + } + } + entry = next + } else { + break + } + } + if entry.IsLink() { + return nil, ErrBadLink{ + te.Name(), + "too many levels of symbolic links", + } + } + return entry, nil +} + +// GetSubJumpablePathName return the full path of subdirectory jumpable ( contains only one directory ) +func (te *TreeEntry) GetSubJumpablePathName() string { + if te.IsSubModule() || !te.IsDir() { + return "" + } + tree, err := te.ptree.SubTree(te.Name()) + if err != nil { + return te.Name() + } + entries, _ := tree.ListEntries() + if len(entries) == 1 && entries[0].IsDir() { + name := entries[0].GetSubJumpablePathName() + if name != "" { + return te.Name() + "/" + name + } + } + return te.Name() +} + +// Entries a list of entry +type Entries []*TreeEntry + +type customSortableEntries struct { + Comparer func(s1, s2 string) bool + Entries +} + +var sorter = []func(t1, t2 *TreeEntry, cmp func(s1, s2 string) bool) bool{ + func(t1, t2 *TreeEntry, cmp func(s1, s2 string) bool) bool { + return (t1.IsDir() || t1.IsSubModule()) && !t2.IsDir() && !t2.IsSubModule() + }, + func(t1, t2 *TreeEntry, cmp func(s1, s2 string) bool) bool { + return cmp(t1.Name(), t2.Name()) + }, +} + +func (ctes customSortableEntries) Len() int { return len(ctes.Entries) } + +func (ctes customSortableEntries) Swap(i, j int) { + ctes.Entries[i], ctes.Entries[j] = ctes.Entries[j], ctes.Entries[i] +} + +func (ctes customSortableEntries) Less(i, j int) bool { + t1, t2 := ctes.Entries[i], ctes.Entries[j] + var k int + for k = 0; k < len(sorter)-1; k++ { + s := sorter[k] + switch { + case s(t1, t2, ctes.Comparer): + return true + case s(t2, t1, ctes.Comparer): + return false + } + } + return sorter[k](t1, t2, ctes.Comparer) +} + +// Sort sort the list of entry +func (tes Entries) Sort() { + sort.Sort(customSortableEntries{func(s1, s2 string) bool { + return s1 < s2 + }, tes}) +} + +// CustomSort customizable string comparing sort entry list +func (tes Entries) CustomSort(cmp func(s1, s2 string) bool) { + sort.Sort(customSortableEntries{cmp, tes}) +} diff --git a/modules/git/tree_entry_test.go b/modules/git/tree_entry_test.go index 4878fce0b841b..69c17279f2894 100644 --- a/modules/git/tree_entry_test.go +++ b/modules/git/tree_entry_test.go @@ -2,6 +2,8 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. +// +build !nogogit + package git import ( diff --git a/modules/git/tree_nogogit.go b/modules/git/tree_nogogit.go new file mode 100644 index 0000000000000..8aff4ee7c919e --- /dev/null +++ b/modules/git/tree_nogogit.go @@ -0,0 +1,106 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build nogogit + +package git + +import ( + "strings" +) + +// Tree represents a flat directory listing. +type Tree struct { + ID SHA1 + ResolvedID SHA1 + repo *Repository + + // parent tree + ptree *Tree + + entries Entries + entriesParsed bool + + entriesRecursive Entries + entriesRecursiveParsed bool +} + +// NewTree create a new tree according the repository and tree id +func NewTree(repo *Repository, id SHA1) *Tree { + return &Tree{ + ID: id, + repo: repo, + } +} + +// SubTree get a sub tree by the sub dir path +func (t *Tree) SubTree(rpath string) (*Tree, error) { + if len(rpath) == 0 { + return t, nil + } + + paths := strings.Split(rpath, "/") + var ( + err error + g = t + p = t + te *TreeEntry + ) + for _, name := range paths { + te, err = p.GetTreeEntryByPath(name) + if err != nil { + return nil, err + } + + g, err = t.repo.getTree(te.ID) + if err != nil { + return nil, err + } + g.ptree = p + p = g + } + return g, nil +} + +// ListEntries returns all entries of current tree. +func (t *Tree) ListEntries() (Entries, error) { + if t.entriesParsed { + return t.entries, nil + } + + stdout, err := NewCommand("ls-tree", t.ID.String()).RunInDirBytes(t.repo.Path) + if err != nil { + if strings.Contains(err.Error(), "fatal: Not a valid object name") || strings.Contains(err.Error(), "fatal: not a tree object") { + return nil, ErrNotExist{ + ID: t.ID.String(), + } + } + return nil, err + } + + t.entries, err = parseTreeEntries(stdout, t) + if err == nil { + t.entriesParsed = true + } + + return t.entries, err +} + +// ListEntriesRecursive returns all entries of current tree recursively including all subtrees +func (t *Tree) ListEntriesRecursive() (Entries, error) { + if t.entriesRecursiveParsed { + return t.entriesRecursive, nil + } + stdout, err := NewCommand("ls-tree", "-t", "-r", t.ID.String()).RunInDirBytes(t.repo.Path) + if err != nil { + return nil, err + } + + t.entriesRecursive, err = parseTreeEntries(stdout, t) + if err == nil { + t.entriesRecursiveParsed = true + } + + return t.entriesRecursive, err +} diff --git a/modules/git/utils.go b/modules/git/utils.go index 83209924c8dbf..d95218941606b 100644 --- a/modules/git/utils.go +++ b/modules/git/utils.go @@ -6,6 +6,7 @@ package git import ( "fmt" + "io" "os" "strconv" "strings" @@ -68,11 +69,12 @@ func isExist(path string) bool { return err == nil || os.IsExist(err) } -func concatenateError(err error, stderr string) error { +// ConcatenateError concatenats an error with stderr string +func ConcatenateError(err error, stderr string) error { if len(stderr) == 0 { return err } - return fmt.Errorf("%v - %s", err, stderr) + return fmt.Errorf("%w - %s", err, stderr) } // RefEndName return the end name of a ref name @@ -140,3 +142,29 @@ func ParseBool(value string) (result bool, valid bool) { } return intValue != 0, true } + +// LimitedReaderCloser is a limited reader closer +type LimitedReaderCloser struct { + R io.Reader + C io.Closer + N int64 +} + +// Read implements io.Reader +func (l *LimitedReaderCloser) Read(p []byte) (n int, err error) { + if l.N <= 0 { + _ = l.C.Close() + return 0, io.EOF + } + if int64(len(p)) > l.N { + p = p[0:l.N] + } + n, err = l.R.Read(p) + l.N -= int64(n) + return +} + +// Close implements io.Closer +func (l *LimitedReaderCloser) Close() error { + return l.C.Close() +} diff --git a/modules/indexer/stats/db.go b/modules/indexer/stats/db.go index 6e10ee2052794..bc3fbc13d8932 100644 --- a/modules/indexer/stats/db.go +++ b/modules/indexer/stats/db.go @@ -7,6 +7,7 @@ package stats import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/log" ) // DBIndexer implements Indexer interface to use database's like search @@ -37,6 +38,7 @@ func (db *DBIndexer) Index(id int64) error { // Get latest commit for default branch commitID, err := gitRepo.GetBranchCommitID(repo.DefaultBranch) if err != nil { + log.Error("Unable to get commit ID for defaultbranch %s in %s", repo.DefaultBranch, repo.RepoPath()) return err } @@ -48,6 +50,7 @@ func (db *DBIndexer) Index(id int64) error { // Calculate and save language statistics to database stats, err := gitRepo.GetLanguageStats(commitID) if err != nil { + log.Error("Unable to get language stats for ID %s for defaultbranch %s in %s. Error: %v", commitID, repo.DefaultBranch, repo.RepoPath(), err) return err } return repo.UpdateLanguageStats(commitID, stats) From 764948fe91fd508de3af38a2b8a2a83664bd22c8 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sun, 22 Nov 2020 18:41:16 +0000 Subject: [PATCH 07/19] Submodule RefID Signed-off-by: Andrew Thornton --- templates/repo/view_list.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/repo/view_list.tmpl b/templates/repo/view_list.tmpl index 7549122dc5aab..d25ee2488afd6 100644 --- a/templates/repo/view_list.tmpl +++ b/templates/repo/view_list.tmpl @@ -50,7 +50,7 @@ {{svg "octicon-file-submodule"}} {{$refURL := $subModuleFile.RefURL AppUrl $.Repository.FullName $.SSHDomain}} {{if $refURL}} - {{$entry.Name}}@{{ShortSha $subModuleFile.RefID}} + {{$entry.Name}}@{{ShortSha $subModuleFile.RefID}} {{else}} {{$entry.Name}}@{{ShortSha $subModuleFile.RefID}} {{end}} From d8d498d763ce01f905611bcf9a293a4cfe77d4b0 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sun, 22 Nov 2020 18:42:51 +0000 Subject: [PATCH 08/19] fix issue with GetCommitsInfo Signed-off-by: Andrew Thornton --- modules/git/commit_info_nogogit.go | 69 ++++++++++-------------------- 1 file changed, 22 insertions(+), 47 deletions(-) diff --git a/modules/git/commit_info_nogogit.go b/modules/git/commit_info_nogogit.go index a82ca581e9025..aaf5ab51a038a 100644 --- a/modules/git/commit_info_nogogit.go +++ b/modules/git/commit_info_nogogit.go @@ -215,6 +215,10 @@ func getLastCommitForPathsByCache(commitID, treePath string, paths []string, cac // GetLastCommitForPaths returns last commit information func GetLastCommitForPaths(commit *Commit, treePath string, paths []string) ([]*Commit, error) { // We read backwards from the commit to obtain all of the commits + path2idx := make(map[string]int, len(paths)) + for i, path := range paths { + path2idx[path] = i + } // We'll do this by using rev-list to provide us with parent commits in order revListReader, revListWriter := io.Pipe() @@ -375,18 +379,17 @@ revListLoop: } n += int64(count) - // now fname and paths are both sorted and in order - // if fname == paths[i] => we're looking at the current path - if fname == paths[i] { + idx, ok := path2idx[fname] + if ok { // Now if this is the first time round set the initial Blob(ish) SHA ID and the commit - if ids[i] == "" { - ids[i] = sha - commits[i] = curCommit - } else if !found[i] { + if ids[idx] == "" { + ids[idx] = sha + commits[idx] = curCommit + } else if !found[idx] { // if we've not already found this path's commit - if ids[i] != sha { + if ids[idx] != sha { // if the SHA is different we've now found the commit for this path - found[i] = true + found[idx] = true // check if we've found all the paths done := true @@ -400,54 +403,26 @@ revListLoop: break revListLoop } } else { - commits[i] = curCommit + commits[idx] = curCommit } } - // Step the path index counter up - i++ } - done := false - // if fname > paths[i] well paths[i] must not be present in this set it to found - for i < len(paths) && (fname > paths[i] || n >= size) { - found[i] = true - done = true - i++ - } - - // check if we're really done - if done { - for _, find := range found { - if !find { - done = false - break + if n >= size { + done := true + for i := range paths { + if commits[i] != curCommit { + found[i] = true + } else { + done = done && found[i] } } + + // check if we're really done if done { break revListLoop } - } - - // Now if i >= len(paths) we're beyond the paths we're looking at - discard the rest of the tree if there is any left - if i >= len(paths) { - // set the commit as done - commitDone = true - if n < size { - discard := size - n - for discard > math.MaxInt32 { - _, err := batchReader.Discard(math.MaxInt32) - if err != nil { - return nil, err - } - discard -= math.MaxInt32 - } - _, err := batchReader.Discard(int(discard)) - if err != nil { - return nil, err - } - } - // break out of the commit reading loop break commitReadingLoop } } From abddc4e9405a99edc052b39dcd62b83b29833bbf Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sun, 22 Nov 2020 20:04:29 +0000 Subject: [PATCH 09/19] fix GetLastCommitForPaths Signed-off-by: Andrew Thornton --- modules/git/commit_info_nogogit.go | 115 +++++------------------------ 1 file changed, 17 insertions(+), 98 deletions(-) diff --git a/modules/git/commit_info_nogogit.go b/modules/git/commit_info_nogogit.go index aaf5ab51a038a..81197e1f6ec23 100644 --- a/modules/git/commit_info_nogogit.go +++ b/modules/git/commit_info_nogogit.go @@ -100,11 +100,17 @@ func ParseTreeLine(rd *bufio.Reader) (mode, fname, sha string, n int, err error) fname = fname[:len(fname)-1] shaBytes := make([]byte, 20) - read, err := rd.Read(shaBytes) - if err != nil { - return + idx := 0 + for idx < 20 { + read := 0 + read, err = rd.Read(shaBytes[idx:]) + if err != nil { + return + } + n += read + idx += read } - n += read + sha = MustID(shaBytes).String() return } @@ -265,8 +271,6 @@ func GetLastCommitForPaths(commit *Commit, treePath string, paths []string) ([]* commits := make([]*Commit, len(paths)) // ids are the blob/tree ids for the paths ids := make([]string, len(paths)) - // found is a shortcut to help break out of parsing early - found := make([]bool, len(paths)) // We'll use a scanner for the revList because it's simpler than a bufio.Reader scan := bufio.NewScanner(revListReader) @@ -289,10 +293,8 @@ revListLoop: currentPath := "" - i := 0 - commitDone := false commitReadingLoop: - for !commitDone { + for { _, typ, size, err := ReadBatchLine(batchReader) if err != nil { return nil, err @@ -321,41 +323,15 @@ revListLoop: id := curCommit.Tree.ID.String() // OK if the target tree path is "" and the "" is in the paths just set this now if treePath == "" && paths[0] == "" { - if i == 0 { - i++ - } // If this is the first time we see this set the id appropriate for this paths to this tree and set the last commit to curCommit if ids[0] == "" { - log("setting initial id to: %s on commit %s", string(id), curCommit.ID.String()) ids[0] = id commits[0] = curCommit - } else if ids[0] != id { - // Else if the last id doesn't match this we've found the last commit that added this - found[0] = true - - // check if that completes our list - done := true - for _, find := range found { - if !find { - done = false - break - } - } - if done { - break revListLoop - } - } else if !found[0] { - // Finally if we haven't found the commit set the curCommit to this + } else if ids[0] == id { commits[0] = curCommit } } - // in the unlikely event that we've now done all the paths - if i >= len(paths) { - commitDone = true - continue - } - // Finally add the tree back in to batch reader _, err = batchStdinWriter.Write([]byte(id + "\n")) if err != nil { @@ -378,56 +354,18 @@ revListLoop: return nil, err } n += int64(count) - idx, ok := path2idx[fname] if ok { // Now if this is the first time round set the initial Blob(ish) SHA ID and the commit if ids[idx] == "" { ids[idx] = sha commits[idx] = curCommit - } else if !found[idx] { - // if we've not already found this path's commit - if ids[idx] != sha { - // if the SHA is different we've now found the commit for this path - found[idx] = true - - // check if we've found all the paths - done := true - for _, find := range found { - if !find { - done = false - break - } - } - if done { - break revListLoop - } - } else { - commits[idx] = curCommit - } - } - } - - // if fname > paths[i] well paths[i] must not be present in this set it to found - if n >= size { - done := true - for i := range paths { - if commits[i] != curCommit { - found[i] = true - } else { - done = done && found[i] - } - } - - // check if we're really done - if done { - break revListLoop + } else if ids[idx] == sha { + commits[idx] = curCommit } - break commitReadingLoop } } - // We should not be able to get here... - break revListLoop + break commitReadingLoop } // We're in the wrong directory @@ -484,30 +422,11 @@ revListLoop: // if we've now found the curent path check its sha id and commit status if treePath == currentPath && paths[0] == "" { - if i == 0 { - i++ - } if ids[0] == "" { ids[0] = treeID commits[0] = curCommit - } else if !found[0] { - if ids[0] != treeID { - found[0] = true - - // check if that completes our list - done := true - for _, find := range found { - if !find { - done = false - break - } - } - if done { - break revListLoop - } - } else if !found[0] { - commits[0] = curCommit - } + } else if ids[0] == treeID { + commits[0] = curCommit } } _, err = batchStdinWriter.Write([]byte(treeID + "\n")) From 1e396b71c2ca57865020d0a3980ed252a4b89581 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Mon, 23 Nov 2020 18:11:56 +0000 Subject: [PATCH 10/19] Improve efficiency Signed-off-by: Andrew Thornton --- modules/git/commit_info_nogogit.go | 379 +++++++++++++++++----------- modules/git/pipeline/lfs_nogogit.go | 11 +- 2 files changed, 242 insertions(+), 148 deletions(-) diff --git a/modules/git/commit_info_nogogit.go b/modules/git/commit_info_nogogit.go index 81197e1f6ec23..ca943df7cf6e3 100644 --- a/modules/git/commit_info_nogogit.go +++ b/modules/git/commit_info_nogogit.go @@ -9,6 +9,7 @@ package git import ( "bufio" "bytes" + "fmt" "io" "math" "path" @@ -84,34 +85,119 @@ headerLoop: return id, err } -func ParseTreeLine(rd *bufio.Reader) (mode, fname, sha string, n int, err error) { - mode, err = rd.ReadString(' ') +func GetTreeID(rd *bufio.Reader, size int64) (string, error) { + id := "" + var n int64 +headerLoop: + for { + line, err := rd.ReadBytes('\n') + if err != nil { + return "", err + } + n += int64(len(line)) + idx := bytes.Index(line, []byte{' '}) + if idx < 0 { + continue + } + + if string(line[:idx]) == "tree" { + id = string(line[idx+1 : len(line)-1]) + break headerLoop + } + } + + // Discard the rest of the tag + discard := size - n + for discard > math.MaxInt32 { + _, err := rd.Discard(math.MaxInt32) + if err != nil { + return id, err + } + discard -= math.MaxInt32 + } + _, err := rd.Discard(int(discard)) + return id, err +} + +const hextable = "0123456789abcdef" + +func ParseTreeLineSkipMode(rd *bufio.Reader) (fname string, sha []byte, n int, err error) { + var readBytes []byte + readBytes, err = rd.ReadSlice(' ') if err != nil { return } - n += len(mode) - mode = mode[:len(mode)-1] + n += len(readBytes) - fname, err = rd.ReadString('\x00') + readBytes, err = rd.ReadSlice('\x00') + fname = string(readBytes) + for err == bufio.ErrBufferFull { + readBytes, err = rd.ReadSlice('\x00') + fname += string(readBytes) + } + n += len(fname) if err != nil { return } - n += len(fname) fname = fname[:len(fname)-1] - shaBytes := make([]byte, 20) + shaBytes := [40]byte{} idx := 0 for idx < 20 { read := 0 - read, err = rd.Read(shaBytes[idx:]) + read, err = rd.Read(shaBytes[idx:20]) + n += read if err != nil { return } - n += read idx += read } + for i := 19; i >= 0; i-- { + shaBytes[i*2+1] = hextable[shaBytes[i]&0x0f] + shaBytes[i*2] = hextable[shaBytes[i]>>4] // This must be after i*2+1 + } + sha = shaBytes[:] + return +} + +func ParseTreeLine(rd *bufio.Reader) (mode, fname string, sha []byte, n int, err error) { + var readBytes []byte + + readBytes, err = rd.ReadSlice(' ') + if err != nil { + return + } + n += len(readBytes) + mode = string(readBytes[:len(readBytes)-1]) + + readBytes, err = rd.ReadSlice('\x00') + fname = string(readBytes) + for err == bufio.ErrBufferFull { + readBytes, err = rd.ReadSlice('\x00') + fname += string(readBytes) + } + n += len(fname) + if err != nil { + return + } + fname = fname[:len(fname)-1] - sha = MustID(shaBytes).String() + shaBytes := [40]byte{} + idx := 0 + for idx < 20 { + read := 0 + read, err = rd.Read(shaBytes[idx:20]) + n += read + if err != nil { + return + } + idx += read + } + for i := 19; i >= 0; i-- { + shaBytes[i*2+1] = hextable[shaBytes[i]&0x0f] + shaBytes[i*2] = hextable[shaBytes[i]>>4] // This must be after i*2+1 + } + sha = shaBytes[:] return } @@ -221,10 +307,6 @@ func getLastCommitForPathsByCache(commitID, treePath string, paths []string, cac // GetLastCommitForPaths returns last commit information func GetLastCommitForPaths(commit *Commit, treePath string, paths []string) ([]*Commit, error) { // We read backwards from the commit to obtain all of the commits - path2idx := make(map[string]int, len(paths)) - for i, path := range paths { - path2idx[path] = i - } // We'll do this by using rev-list to provide us with parent commits in order revListReader, revListWriter := io.Pipe() @@ -235,7 +317,7 @@ func GetLastCommitForPaths(commit *Commit, treePath string, paths []string) ([]* go func() { stderr := strings.Builder{} - err := NewCommand("rev-list", commit.ID.String()).RunInDirPipeline(commit.repo.Path, revListWriter, &stderr) + err := NewCommand("rev-list", "--format=%T", commit.ID.String()).RunInDirPipeline(commit.repo.Path, revListWriter, &stderr) if err != nil { _ = revListWriter.CloseWithError(ConcatenateError(err, (&stderr).String())) } else { @@ -267,176 +349,183 @@ func GetLastCommitForPaths(commit *Commit, treePath string, paths []string) ([]* // For simplicities sake we'll us a buffered reader batchReader := bufio.NewReader(batchStdoutReader) + path2idx := make(map[string]int, len(paths)) + for i, path := range paths { + path2idx[path] = i + } // commits is the returnable commits matching the paths provided - commits := make([]*Commit, len(paths)) + commits := make([]string, len(paths)) // ids are the blob/tree ids for the paths - ids := make([]string, len(paths)) + ids := make([][]byte, len(paths)) // We'll use a scanner for the revList because it's simpler than a bufio.Reader scan := bufio.NewScanner(revListReader) revListLoop: for scan.Scan() { // Get the next parent commit ID - commitID := scan.Bytes() - - // push the commit to the cat-file --batch process - _, err := batchStdinWriter.Write(commitID) - if err != nil { - return nil, err + commitID := scan.Text() + if !scan.Scan() { + break revListLoop } - _, err = batchStdinWriter.Write([]byte{'\n'}) + commitID = commitID[7:] + rootTreeID := scan.Text() + + // push the tree to the cat-file --batch process + _, err := batchStdinWriter.Write([]byte(rootTreeID + "\n")) if err != nil { return nil, err } - var curCommit *Commit - currentPath := "" - commitReadingLoop: + // OK if the target tree path is "" and the "" is in the paths just set this now + if treePath == "" && paths[0] == "" { + // If this is the first time we see this set the id appropriate for this paths to this tree and set the last commit to curCommit + if len(ids[0]) == 0 { + ids[0] = []byte(rootTreeID) + commits[0] = string(commitID) + } else if bytes.Equal(ids[0], []byte(rootTreeID)) { + commits[0] = string(commitID) + } + } + + treeReadingLoop: for { - _, typ, size, err := ReadBatchLine(batchReader) + _, _, size, err := ReadBatchLine(batchReader) if err != nil { return nil, err } - switch typ { - case "tag": - // This shouldn't happen but if it does well just get the commit and try again - id, err := GetTagObjectID(batchReader, size) - if err != nil { - return nil, err - } - _, err = batchStdinWriter.Write([]byte(id + "\n")) - if err != nil { - return nil, err - } - continue - case "commit": - // Read in the commit to get its tree and in case this is one of the last used commits - curCommit, err = CommitFromReader(commit.repo, MustIDFromString(string(commitID)), io.LimitReader(batchReader, int64(size))) - if err != nil { - return nil, err - } - - // Get the tree for the commit - id := curCommit.Tree.ID.String() - // OK if the target tree path is "" and the "" is in the paths just set this now - if treePath == "" && paths[0] == "" { - // If this is the first time we see this set the id appropriate for this paths to this tree and set the last commit to curCommit - if ids[0] == "" { - ids[0] = id - commits[0] = curCommit - } else if ids[0] == id { - commits[0] = curCommit - } - } - - // Finally add the tree back in to batch reader - _, err = batchStdinWriter.Write([]byte(id + "\n")) - if err != nil { - return nil, err - } - continue - case "tree": - // Handle trees - - // n is counter for file position in the tree file - var n int64 - - // Two options: currentPath is the targetTreepath - if treePath == currentPath { - // We are in the right directory - // Parse each tree line in turn. (don't care about mode here.) - for n < size { - _, fname, sha, count, err := ParseTreeLine(batchReader) - if err != nil { - return nil, err - } - n += int64(count) - idx, ok := path2idx[fname] - if ok { - // Now if this is the first time round set the initial Blob(ish) SHA ID and the commit - if ids[idx] == "" { - ids[idx] = sha - commits[idx] = curCommit - } else if ids[idx] == sha { - commits[idx] = curCommit - } - } - } - break commitReadingLoop - } + // Handle trees - // We're in the wrong directory - // Find target directory in this directory - idx := len(currentPath) - if idx > 0 { - idx++ - } - target := strings.SplitN(treePath[idx:], "/", 2)[0] + // n is counter for file position in the tree file + var n int64 - treeID := "" + // Two options: currentPath is the targetTreepath + if treePath == currentPath { + // We are in the right directory + // Parse each tree line in turn. (don't care about mode here.) for n < size { - // Read each tree entry in turn - mode, fname, sha, count, err := ParseTreeLine(batchReader) + fname, sha, count, err := ParseTreeLineSkipMode(batchReader) if err != nil { return nil, err } n += int64(count) - - // if we have found the target directory - if fname == target && ToEntryMode(mode) == EntryModeTree { - treeID = sha - break - } else if fname > target { - break + idx, ok := path2idx[fname] + if ok { + // Now if this is the first time round set the initial Blob(ish) SHA ID and the commit + if len(ids[idx]) == 0 { + ids[idx] = sha + commits[idx] = string(commitID) + } else if bytes.Equal(ids[idx], sha) { + commits[idx] = string(commitID) + } } + // FIXME: is there any order to the way strings are emitted from cat-file? + // if there is - then we could skip once we've passed all of our data } - // if we haven't found a treeID for the target directory our search is over - if treeID == "" { - break revListLoop - } + break treeReadingLoop + } - if n < size { - // Discard any remaining entries in the current tree - discard := size - n - for discard > math.MaxInt32 { - _, err := batchReader.Discard(math.MaxInt32) - if err != nil { - return nil, err - } - discard -= math.MaxInt32 - } - _, err := batchReader.Discard(int(discard)) - if err != nil { - return nil, err - } + // We're in the wrong directory + // Find target directory in this directory + idx := len(currentPath) + if idx > 0 { + idx++ + } + target := strings.SplitN(treePath[idx:], "/", 2)[0] + + var treeID []byte + for n < size { + // Read each tree entry in turn + mode, fname, sha, count, err := ParseTreeLine(batchReader) + if err != nil { + return nil, err } + n += int64(count) - // add the target to the current path - if idx > 0 { - currentPath += "/" + // if we have found the target directory + if fname == target && ToEntryMode(mode) == EntryModeTree { + treeID = sha + break } - currentPath += target - - // if we've now found the curent path check its sha id and commit status - if treePath == currentPath && paths[0] == "" { - if ids[0] == "" { - ids[0] = treeID - commits[0] = curCommit - } else if ids[0] == treeID { - commits[0] = curCommit + } + + if n < size { + // Discard any remaining entries in the current tree + discard := size - n + for discard > math.MaxInt32 { + _, err := batchReader.Discard(math.MaxInt32) + if err != nil { + return nil, err } + discard -= math.MaxInt32 } - _, err = batchStdinWriter.Write([]byte(treeID + "\n")) + _, err := batchReader.Discard(int(discard)) if err != nil { return nil, err } - continue } + + // if we haven't found a treeID for the target directory our search is over + if len(treeID) == 0 { + break treeReadingLoop + } + + // add the target to the current path + if idx > 0 { + currentPath += "/" + } + currentPath += target + + // if we've now found the curent path check its sha id and commit status + if treePath == currentPath && paths[0] == "" { + if len(ids[0]) == 0 { + ids[0] = treeID + commits[0] = string(commitID) + } else if bytes.Equal(ids[0], treeID) { + commits[0] = string(commitID) + } + } + _, err = batchStdinWriter.Write(treeID) + if err != nil { + return nil, err + } + _, err = batchStdinWriter.Write([]byte("\n")) + if err != nil { + return nil, err + } + } + } + + commitsMap := make(map[string]*Commit, len(commits)) + commitsMap[commit.ID.String()] = commit + + commitCommits := make([]*Commit, len(commits)) + for i, commitID := range commits { + c, ok := commitsMap[commitID] + if ok { + commitCommits[i] = c + continue + } + + _, err := batchStdinWriter.Write([]byte(commitID + "\n")) + if err != nil { + return nil, err + } + _, typ, size, err := ReadBatchLine(batchReader) + if err != nil { + return nil, err + } + if typ != "commit" { + return nil, fmt.Errorf("unexpected type: %s for commit id: %s", typ, commitID) + } + c, err = CommitFromReader(commit.repo, MustIDFromString(string(commitID)), io.LimitReader(batchReader, int64(size))) + if err != nil { + return nil, err } + commitCommits[i] = c } - return commits, scan.Err() + return commitCommits, scan.Err() } diff --git a/modules/git/pipeline/lfs_nogogit.go b/modules/git/pipeline/lfs_nogogit.go index d054adb8f8261..c42d3a8846e0e 100644 --- a/modules/git/pipeline/lfs_nogogit.go +++ b/modules/git/pipeline/lfs_nogogit.go @@ -8,6 +8,7 @@ package pipeline import ( "bufio" + "bytes" "fmt" "io" "sort" @@ -87,7 +88,7 @@ func FindLFSFile(repo *git.Repository, hash git.SHA1) ([]*LFSResult, error) { // We'll use a scanner for the revList because it's simpler than a bufio.Reader scan := bufio.NewScanner(revListReader) - trees := []string{} + trees := [][]byte{} paths := []string{} for scan.Scan() { @@ -146,7 +147,7 @@ func FindLFSFile(repo *git.Repository, hash git.SHA1) ([]*LFSResult, error) { return nil, err } n += int64(count) - if sha == hashStr { + if bytes.Equal(sha, []byte(hashStr)) { result := LFSResult{ Name: curPath + fname, SHA: curCommit.ID.String(), @@ -161,7 +162,11 @@ func FindLFSFile(repo *git.Repository, hash git.SHA1) ([]*LFSResult, error) { } } if len(trees) > 0 { - _, err := batchStdinWriter.Write([]byte(trees[len(trees)-1] + "\n")) + _, err := batchStdinWriter.Write(trees[len(trees)-1]) + if err != nil { + return nil, err + } + _, err = batchStdinWriter.Write([]byte("\n")) if err != nil { return nil, err } From 4b35e94c1aa8d909fcf62fbcfeb6fcbb852255a8 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Tue, 24 Nov 2020 19:06:39 +0000 Subject: [PATCH 11/19] More efficiency Signed-off-by: Andrew Thornton --- modules/git/commit_info_nogogit.go | 105 ++++++++++++++++++++-------- modules/git/pipeline/lfs_nogogit.go | 14 ++-- 2 files changed, 83 insertions(+), 36 deletions(-) diff --git a/modules/git/commit_info_nogogit.go b/modules/git/commit_info_nogogit.go index ca943df7cf6e3..413ce9495b407 100644 --- a/modules/git/commit_info_nogogit.go +++ b/modules/git/commit_info_nogogit.go @@ -121,7 +121,7 @@ headerLoop: const hextable = "0123456789abcdef" -func ParseTreeLineSkipMode(rd *bufio.Reader) (fname string, sha []byte, n int, err error) { +func ParseTreeLineSkipMode(rd *bufio.Reader, fnameBuf, shaBuf []byte) (fname, sha []byte, n int, err error) { var readBytes []byte readBytes, err = rd.ReadSlice(' ') if err != nil { @@ -130,22 +130,27 @@ func ParseTreeLineSkipMode(rd *bufio.Reader) (fname string, sha []byte, n int, e n += len(readBytes) readBytes, err = rd.ReadSlice('\x00') - fname = string(readBytes) + copy(fnameBuf, readBytes) + if len(fnameBuf) > len(readBytes) { + fnameBuf = fnameBuf[:len(readBytes)] + } else { + fnameBuf = append(fnameBuf, readBytes[len(fnameBuf):]...) + } for err == bufio.ErrBufferFull { readBytes, err = rd.ReadSlice('\x00') - fname += string(readBytes) + fnameBuf = append(fnameBuf, readBytes...) } - n += len(fname) + n += len(fnameBuf) if err != nil { return } - fname = fname[:len(fname)-1] + fnameBuf = fnameBuf[:len(fnameBuf)-1] + fname = fnameBuf - shaBytes := [40]byte{} idx := 0 for idx < 20 { read := 0 - read, err = rd.Read(shaBytes[idx:20]) + read, err = rd.Read(shaBuf[idx:20]) n += read if err != nil { return @@ -153,14 +158,16 @@ func ParseTreeLineSkipMode(rd *bufio.Reader) (fname string, sha []byte, n int, e idx += read } for i := 19; i >= 0; i-- { - shaBytes[i*2+1] = hextable[shaBytes[i]&0x0f] - shaBytes[i*2] = hextable[shaBytes[i]>>4] // This must be after i*2+1 + v := shaBuf[i] + vhi, vlo := v>>4, v&0x0f + shi, slo := hextable[vhi], hextable[vlo] + shaBuf[i*2], shaBuf[i*2+1] = shi, slo } - sha = shaBytes[:] + sha = shaBuf return } -func ParseTreeLine(rd *bufio.Reader) (mode, fname string, sha []byte, n int, err error) { +func ParseTreeLine(rd *bufio.Reader, modeBuf, fnameBuf, shaBuf []byte) (mode, fname, sha []byte, n int, err error) { var readBytes []byte readBytes, err = rd.ReadSlice(' ') @@ -168,25 +175,37 @@ func ParseTreeLine(rd *bufio.Reader) (mode, fname string, sha []byte, n int, err return } n += len(readBytes) - mode = string(readBytes[:len(readBytes)-1]) + copy(modeBuf, readBytes) + if len(modeBuf) > len(readBytes) { + modeBuf = modeBuf[:len(readBytes)] + } else { + modeBuf = append(modeBuf, readBytes[len(modeBuf):]...) + + } + mode = modeBuf[:len(modeBuf)-1] readBytes, err = rd.ReadSlice('\x00') - fname = string(readBytes) + copy(fnameBuf, readBytes) + if len(fnameBuf) > len(readBytes) { + fnameBuf = fnameBuf[:len(readBytes)] + } else { + fnameBuf = append(fnameBuf, readBytes[len(fnameBuf):]...) + } for err == bufio.ErrBufferFull { readBytes, err = rd.ReadSlice('\x00') - fname += string(readBytes) + fnameBuf = append(fnameBuf, readBytes...) } - n += len(fname) + n += len(fnameBuf) if err != nil { return } - fname = fname[:len(fname)-1] + fnameBuf = fnameBuf[:len(fnameBuf)-1] + fname = fnameBuf - shaBytes := [40]byte{} idx := 0 for idx < 20 { read := 0 - read, err = rd.Read(shaBytes[idx:20]) + read, err = rd.Read(shaBuf[idx:20]) n += read if err != nil { return @@ -194,10 +213,12 @@ func ParseTreeLine(rd *bufio.Reader) (mode, fname string, sha []byte, n int, err idx += read } for i := 19; i >= 0; i-- { - shaBytes[i*2+1] = hextable[shaBytes[i]&0x0f] - shaBytes[i*2] = hextable[shaBytes[i]>>4] // This must be after i*2+1 + v := shaBuf[i] + vhi, vlo := v>>4, v&0x0f + shi, slo := hextable[vhi], hextable[vlo] + shaBuf[i*2], shaBuf[i*2+1] = shi, slo } - sha = shaBytes[:] + sha = shaBuf return } @@ -349,10 +370,23 @@ func GetLastCommitForPaths(commit *Commit, treePath string, paths []string) ([]* // For simplicities sake we'll us a buffered reader batchReader := bufio.NewReader(batchStdoutReader) - path2idx := make(map[string]int, len(paths)) + mapsize := 4096 + if len(paths) > mapsize { + mapsize = len(paths) + } + + path2idx := make(map[string]int, mapsize) for i, path := range paths { path2idx[path] = i } + + fnameBuf := make([]byte, 4096) + modeBuf := make([]byte, 40) + + allShaBuf := make([]byte, len(paths)*40) + shaBuf := make([]byte, 40) + tmpTreeID := make([]byte, 40) + // commits is the returnable commits matching the paths provided commits := make([]string, len(paths)) // ids are the blob/tree ids for the paths @@ -406,18 +440,20 @@ revListLoop: // We are in the right directory // Parse each tree line in turn. (don't care about mode here.) for n < size { - fname, sha, count, err := ParseTreeLineSkipMode(batchReader) + fname, sha, count, err := ParseTreeLineSkipMode(batchReader, fnameBuf, shaBuf) + shaBuf = sha if err != nil { return nil, err } n += int64(count) - idx, ok := path2idx[fname] + idx, ok := path2idx[string(fname)] if ok { // Now if this is the first time round set the initial Blob(ish) SHA ID and the commit if len(ids[idx]) == 0 { - ids[idx] = sha + copy(allShaBuf[40*idx:40*(idx+1)], shaBuf) + ids[idx] = allShaBuf[40*idx : 40*(idx+1)] commits[idx] = string(commitID) - } else if bytes.Equal(ids[idx], sha) { + } else if bytes.Equal(ids[idx], shaBuf) { commits[idx] = string(commitID) } } @@ -427,6 +463,8 @@ revListLoop: break treeReadingLoop } + var treeID []byte + // We're in the wrong directory // Find target directory in this directory idx := len(currentPath) @@ -435,18 +473,18 @@ revListLoop: } target := strings.SplitN(treePath[idx:], "/", 2)[0] - var treeID []byte for n < size { // Read each tree entry in turn - mode, fname, sha, count, err := ParseTreeLine(batchReader) + mode, fname, sha, count, err := ParseTreeLine(batchReader, modeBuf, fnameBuf, shaBuf) if err != nil { return nil, err } n += int64(count) // if we have found the target directory - if fname == target && ToEntryMode(mode) == EntryModeTree { - treeID = sha + if bytes.Equal(fname, []byte(target)) && bytes.Equal(mode, []byte("40000")) { + copy(tmpTreeID, sha) + treeID = tmpTreeID break } } @@ -481,7 +519,8 @@ revListLoop: // if we've now found the curent path check its sha id and commit status if treePath == currentPath && paths[0] == "" { if len(ids[0]) == 0 { - ids[0] = treeID + copy(allShaBuf[0:40], treeID) + ids[0] = allShaBuf[0:40] commits[0] = string(commitID) } else if bytes.Equal(ids[0], treeID) { commits[0] = string(commitID) @@ -509,6 +548,10 @@ revListLoop: continue } + if len(commitID) == 0 { + continue + } + _, err := batchStdinWriter.Write([]byte(commitID + "\n")) if err != nil { return nil, err diff --git a/modules/git/pipeline/lfs_nogogit.go b/modules/git/pipeline/lfs_nogogit.go index c42d3a8846e0e..eeb65e702681c 100644 --- a/modules/git/pipeline/lfs_nogogit.go +++ b/modules/git/pipeline/lfs_nogogit.go @@ -91,6 +91,10 @@ func FindLFSFile(repo *git.Repository, hash git.SHA1) ([]*LFSResult, error) { trees := [][]byte{} paths := []string{} + fnameBuf := make([]byte, 4096) + modeBuf := make([]byte, 40) + workingShaBuf := make([]byte, 40) + for scan.Scan() { // Get the next commit ID commitID := scan.Bytes() @@ -142,23 +146,23 @@ func FindLFSFile(repo *git.Repository, hash git.SHA1) ([]*LFSResult, error) { case "tree": var n int64 for n < size { - mode, fname, sha, count, err := git.ParseTreeLine(batchReader) + mode, fname, sha, count, err := git.ParseTreeLine(batchReader, modeBuf, fnameBuf, workingShaBuf) if err != nil { return nil, err } n += int64(count) if bytes.Equal(sha, []byte(hashStr)) { result := LFSResult{ - Name: curPath + fname, + Name: curPath + string(fname), SHA: curCommit.ID.String(), Summary: strings.Split(strings.TrimSpace(curCommit.CommitMessage), "\n")[0], When: curCommit.Author.When, ParentHashes: curCommit.Parents, } - resultsMap[curCommit.ID.String()+":"+curPath+fname] = &result - } else if mode == git.EntryModeTree.String() { + resultsMap[curCommit.ID.String()+":"+curPath+string(fname)] = &result + } else if string(mode) == git.EntryModeTree.String() { trees = append(trees, sha) - paths = append(paths, curPath+fname+"/") + paths = append(paths, curPath+string(fname)+"/") } } if len(trees) > 0 { From 95d6652865f333d9e43085ac47bd5c8bf4dd7cb2 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sat, 28 Nov 2020 11:35:24 +0000 Subject: [PATCH 12/19] even faster Signed-off-by: Andrew Thornton --- modules/git/commit_info_nogogit.go | 36 +++++++++++++++--------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/modules/git/commit_info_nogogit.go b/modules/git/commit_info_nogogit.go index 413ce9495b407..0c53a15a85420 100644 --- a/modules/git/commit_info_nogogit.go +++ b/modules/git/commit_info_nogogit.go @@ -121,6 +121,17 @@ headerLoop: const hextable = "0123456789abcdef" +// to40ByteSHA converts a 20byte SHA in a 40 byte slice into a 40 byte sha +func to40ByteSHA(sha []byte) []byte { + for i := 19; i >= 0; i-- { + v := sha[i] + vhi, vlo := v>>4, v&0x0f + shi, slo := hextable[vhi], hextable[vlo] + sha[i*2], sha[i*2+1] = shi, slo + } + return sha +} + func ParseTreeLineSkipMode(rd *bufio.Reader, fnameBuf, shaBuf []byte) (fname, sha []byte, n int, err error) { var readBytes []byte readBytes, err = rd.ReadSlice(' ') @@ -157,12 +168,6 @@ func ParseTreeLineSkipMode(rd *bufio.Reader, fnameBuf, shaBuf []byte) (fname, sh } idx += read } - for i := 19; i >= 0; i-- { - v := shaBuf[i] - vhi, vlo := v>>4, v&0x0f - shi, slo := hextable[vhi], hextable[vlo] - shaBuf[i*2], shaBuf[i*2+1] = shi, slo - } sha = shaBuf return } @@ -212,12 +217,6 @@ func ParseTreeLine(rd *bufio.Reader, modeBuf, fnameBuf, shaBuf []byte) (mode, fn } idx += read } - for i := 19; i >= 0; i-- { - v := shaBuf[i] - vhi, vlo := v>>4, v&0x0f - shi, slo := hextable[vhi], hextable[vlo] - shaBuf[i*2], shaBuf[i*2+1] = shi, slo - } sha = shaBuf return } @@ -383,8 +382,8 @@ func GetLastCommitForPaths(commit *Commit, treePath string, paths []string) ([]* fnameBuf := make([]byte, 4096) modeBuf := make([]byte, 40) - allShaBuf := make([]byte, len(paths)*40) - shaBuf := make([]byte, 40) + allShaBuf := make([]byte, (len(paths)+1)*20) + shaBuf := make([]byte, 20) tmpTreeID := make([]byte, 40) // commits is the returnable commits matching the paths provided @@ -450,8 +449,8 @@ revListLoop: if ok { // Now if this is the first time round set the initial Blob(ish) SHA ID and the commit if len(ids[idx]) == 0 { - copy(allShaBuf[40*idx:40*(idx+1)], shaBuf) - ids[idx] = allShaBuf[40*idx : 40*(idx+1)] + copy(allShaBuf[20*(idx+1):20*(idx+2)], shaBuf) + ids[idx] = allShaBuf[20*(idx+1) : 20*(idx+2)] commits[idx] = string(commitID) } else if bytes.Equal(ids[idx], shaBuf) { commits[idx] = string(commitID) @@ -519,13 +518,14 @@ revListLoop: // if we've now found the curent path check its sha id and commit status if treePath == currentPath && paths[0] == "" { if len(ids[0]) == 0 { - copy(allShaBuf[0:40], treeID) - ids[0] = allShaBuf[0:40] + copy(allShaBuf[0:20], treeID) + ids[0] = allShaBuf[0:20] commits[0] = string(commitID) } else if bytes.Equal(ids[0], treeID) { commits[0] = string(commitID) } } + treeID = to40ByteSHA(treeID) _, err = batchStdinWriter.Write(treeID) if err != nil { return nil, err From 20632e2156f2c3581abc0a3630d2ade81a69443d Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sat, 28 Nov 2020 17:39:17 +0000 Subject: [PATCH 13/19] Reduce duplication --- .drone.yml | 29 +- Makefile | 12 +- modules/git/batch_reader_nogogit.go | 243 +++++++++++ modules/git/blob.go | 23 +- modules/git/blob_gogit.go | 33 ++ modules/git/blob_nogogit.go | 72 --- ...ommit_gogit.go => commit_convert_gogit.go} | 0 modules/git/commit_info.go | 286 ------------ modules/git/commit_info_gogit.go | 291 +++++++++++++ modules/git/commit_info_nogogit.go | 204 --------- modules/git/commit_reader.go | 4 +- modules/git/last_commit_cache.go | 105 ----- modules/git/last_commit_cache_gogit.go | 113 +++++ modules/git/last_commit_cache_nogogit.go | 21 - modules/git/notes.go | 67 --- modules/git/notes_gogit.go | 72 +++ modules/git/notes_nogogit.go | 10 - modules/git/{parse.go => parse_gogit.go} | 0 .../{parse_test.go => parse_gogit_test.go} | 0 modules/git/pipeline/lfs_nogogit.go | 2 +- modules/git/repo.go | 66 --- modules/git/repo_base_gogit.go | 76 ++++ modules/git/repo_base_nogogit.go | 40 ++ modules/git/repo_blob.go | 20 +- modules/git/repo_blob_gogit.go | 23 + modules/git/repo_blob_nogogit.go | 9 - modules/git/repo_branch.go | 35 -- modules/git/repo_branch_gogit.go | 45 ++ modules/git/repo_branch_nogogit.go | 147 ------- modules/git/repo_commit.go | 100 ----- modules/git/repo_commit_gogit.go | 110 +++++ modules/git/repo_commit_nogogit.go | 412 +----------------- ...mmitgraph.go => repo_commitgraph_gogit.go} | 0 modules/git/repo_language_stats.go | 108 ----- modules/git/repo_language_stats_gogit.go | 113 +++++ modules/git/repo_language_stats_nogogit.go | 3 - modules/git/repo_nogogit.go | 403 ----------------- modules/git/repo_ref.go | 47 -- modules/git/repo_ref_gogit.go | 52 +++ modules/git/repo_ref_nogogit.go | 7 +- modules/git/repo_tag.go | 33 -- modules/git/repo_tag_gogit.go | 43 ++ modules/git/repo_tag_nogogit.go | 246 ----------- modules/git/repo_tree.go | 41 -- modules/git/repo_tree_gogit.go | 47 ++ modules/git/repo_tree_nogogit.go | 61 +-- modules/git/sha1.go | 12 - modules/git/sha1_gogit.go | 20 + modules/git/sha1_nogogit.go | 47 -- modules/git/signature.go | 48 -- modules/git/signature_gogit.go | 54 +++ modules/git/signature_nogogit.go | 5 - modules/git/tree.go | 85 ---- modules/git/tree_blob.go | 60 --- modules/git/tree_blob_gogit.go | 66 +++ modules/git/tree_blob_nogogit.go | 17 +- modules/git/tree_entry.go | 88 ---- modules/git/tree_entry_gogit.go | 96 ++++ modules/git/tree_entry_nogogit.go | 155 ------- modules/git/tree_gogit.go | 94 ++++ modules/git/tree_nogogit.go | 37 -- 61 files changed, 1674 insertions(+), 3084 deletions(-) create mode 100644 modules/git/batch_reader_nogogit.go create mode 100644 modules/git/blob_gogit.go rename modules/git/{commit_gogit.go => commit_convert_gogit.go} (100%) create mode 100644 modules/git/commit_info_gogit.go create mode 100644 modules/git/last_commit_cache_gogit.go create mode 100644 modules/git/notes_gogit.go rename modules/git/{parse.go => parse_gogit.go} (100%) rename modules/git/{parse_test.go => parse_gogit_test.go} (100%) create mode 100644 modules/git/repo_base_gogit.go create mode 100644 modules/git/repo_base_nogogit.go create mode 100644 modules/git/repo_blob_gogit.go create mode 100644 modules/git/repo_branch_gogit.go create mode 100644 modules/git/repo_commit_gogit.go rename modules/git/{repo_commitgraph.go => repo_commitgraph_gogit.go} (100%) create mode 100644 modules/git/repo_language_stats_gogit.go delete mode 100644 modules/git/repo_nogogit.go create mode 100644 modules/git/repo_ref_gogit.go create mode 100644 modules/git/repo_tag_gogit.go create mode 100644 modules/git/repo_tree_gogit.go create mode 100644 modules/git/sha1_gogit.go create mode 100644 modules/git/signature_gogit.go create mode 100644 modules/git/tree_blob_gogit.go create mode 100644 modules/git/tree_entry_gogit.go create mode 100644 modules/git/tree_gogit.go diff --git a/.drone.yml b/.drone.yml index 0f7f72b843388..81e51788ec1e9 100644 --- a/.drone.yml +++ b/.drone.yml @@ -33,6 +33,16 @@ steps: GOSUMDB: sum.golang.org TAGS: bindata sqlite sqlite_unlock_notify + - name: lint-backend-nogogit + pull: always + image: golang:1.15 + commands: + - make lint-backend + environment: + GOPROXY: https://goproxy.cn # proxy.golang.org is blocked in China, this proxy is not + GOSUMDB: sum.golang.org + TAGS: bindata nogogit sqlite sqlite_unlock_notify + - name: checks-frontend image: node:14 commands: @@ -69,7 +79,7 @@ steps: GOPROXY: off GOOS: linux GOARCH: arm64 - TAGS: bindata + TAGS: bindata nogogit commands: - make backend # test cross compile - rm ./gitea # clean @@ -173,6 +183,17 @@ steps: GITHUB_READ_TOKEN: from_secret: github_read_token + - name: unit-test-nogogit + pull: always + image: golang:1.15 + commands: + - make unit-test-coverage test-check + environment: + GOPROXY: off + TAGS: bindata nogogit sqlite sqlite_unlock_notify + GITHUB_READ_TOKEN: + from_secret: github_read_token + - name: test-mysql image: golang:1.15 commands: @@ -305,7 +326,8 @@ steps: - timeout -s ABRT 40m make test-sqlite-migration test-sqlite environment: GOPROXY: off - TAGS: bindata + TAGS: bindata nogogit + TEST_TAGS: nogogit sqlite sqlite_unlock_notify USE_REPO_TEST_DIR: 1 depends_on: - build @@ -318,7 +340,8 @@ steps: - timeout -s ABRT 40m make test-pgsql-migration test-pgsql environment: GOPROXY: off - TAGS: bindata + TAGS: bindata nogogit + TEST_TAGS: nogogit TEST_LDAP: 1 USE_REPO_TEST_DIR: 1 depends_on: diff --git a/Makefile b/Makefile index f74f1e67d48a6..3ebd77d627f55 100644 --- a/Makefile +++ b/Makefile @@ -110,6 +110,8 @@ TAGS ?= TAGS_SPLIT := $(subst $(COMMA), ,$(TAGS)) TAGS_EVIDENCE := $(MAKE_EVIDENCE_DIR)/tags +TEST_TAGS ?= "sqlite sqlite_unlock_notify" + GO_DIRS := cmd integrations models modules routers build services vendor GO_SOURCES := $(wildcard *.go) GO_SOURCES += $(shell find $(GO_DIRS) -type f -name "*.go" -not -path modules/options/bindata.go -not -path modules/public/bindata.go -not -path modules/templates/bindata.go) @@ -336,7 +338,7 @@ watch-backend: go-check .PHONY: test test: - $(GO) test $(GOTESTFLAGS) -mod=vendor -tags='sqlite sqlite_unlock_notify' $(GO_PACKAGES) + $(GO) test $(GOTESTFLAGS) -mod=vendor -tags='$(TEST_TAGS)' $(GO_PACKAGES) .PHONY: test-check test-check: @@ -352,7 +354,7 @@ test-check: .PHONY: test\#% test\#%: - $(GO) test -mod=vendor -tags='sqlite sqlite_unlock_notify' -run $(subst .,/,$*) $(GO_PACKAGES) + $(GO) test -mod=vendor -tags='$(TEST_TAGS)' -run $(subst .,/,$*) $(GO_PACKAGES) .PHONY: coverage coverage: @@ -360,7 +362,7 @@ coverage: .PHONY: unit-test-coverage unit-test-coverage: - $(GO) test $(GOTESTFLAGS) -mod=vendor -tags='sqlite sqlite_unlock_notify' -cover -coverprofile coverage.out $(GO_PACKAGES) && echo "\n==>\033[32m Ok\033[m\n" || exit 1 + $(GO) test $(GOTESTFLAGS) -mod=vendor -tags='$(TEST_TAGS)' -cover -coverprofile coverage.out $(GO_PACKAGES) && echo "\n==>\033[32m Ok\033[m\n" || exit 1 .PHONY: vendor vendor: @@ -505,7 +507,7 @@ integrations.mssql.test: git-check $(GO_SOURCES) $(GO) test $(GOTESTFLAGS) -mod=vendor -c code.gitea.io/gitea/integrations -o integrations.mssql.test integrations.sqlite.test: git-check $(GO_SOURCES) - $(GO) test $(GOTESTFLAGS) -mod=vendor -c code.gitea.io/gitea/integrations -o integrations.sqlite.test -tags 'sqlite sqlite_unlock_notify' + $(GO) test $(GOTESTFLAGS) -mod=vendor -c code.gitea.io/gitea/integrations -o integrations.sqlite.test -tags '$(TEST_TAGS)' integrations.cover.test: git-check $(GO_SOURCES) $(GO) test $(GOTESTFLAGS) -mod=vendor -c code.gitea.io/gitea/integrations -coverpkg $(shell echo $(GO_PACKAGES) | tr ' ' ',') -o integrations.cover.test @@ -528,7 +530,7 @@ migrations.mssql.test: $(GO_SOURCES) .PHONY: migrations.sqlite.test migrations.sqlite.test: $(GO_SOURCES) - $(GO) test $(GOTESTFLAGS) -c code.gitea.io/gitea/integrations/migration-test -o migrations.sqlite.test -tags 'sqlite sqlite_unlock_notify' + $(GO) test $(GOTESTFLAGS) -c code.gitea.io/gitea/integrations/migration-test -o migrations.sqlite.test -tags '$(TEST_TAGS)' .PHONY: check check: test diff --git a/modules/git/batch_reader_nogogit.go b/modules/git/batch_reader_nogogit.go new file mode 100644 index 0000000000000..952955297f537 --- /dev/null +++ b/modules/git/batch_reader_nogogit.go @@ -0,0 +1,243 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build nogogit + +package git + +import ( + "bufio" + "bytes" + "math" + "strconv" +) + +// ReadBatchLine reads the header line from cat-file --batch +// We expect: +// SP SP LF +func ReadBatchLine(rd *bufio.Reader) (sha []byte, typ string, size int64, err error) { + sha, err = rd.ReadBytes(' ') + if err != nil { + return + } + sha = sha[:len(sha)-1] + + typ, err = rd.ReadString(' ') + if err != nil { + return + } + typ = typ[:len(typ)-1] + + var sizeStr string + sizeStr, err = rd.ReadString('\n') + if err != nil { + return + } + + size, err = strconv.ParseInt(sizeStr[:len(sizeStr)-1], 10, 64) + return +} + +// ReadTagObjectID reads a tag object ID hash from a cat-file --batch stream, throwing away the rest of the stream. +func ReadTagObjectID(rd *bufio.Reader, size int64) (string, error) { + id := "" + var n int64 +headerLoop: + for { + line, err := rd.ReadBytes('\n') + if err != nil { + return "", err + } + n += int64(len(line)) + idx := bytes.Index(line, []byte{' '}) + if idx < 0 { + continue + } + + if string(line[:idx]) == "object" { + id = string(line[idx+1 : len(line)-1]) + break headerLoop + } + } + + // Discard the rest of the tag + discard := size - n + for discard > math.MaxInt32 { + _, err := rd.Discard(math.MaxInt32) + if err != nil { + return id, err + } + discard -= math.MaxInt32 + } + _, err := rd.Discard(int(discard)) + return id, err +} + +// ReadTreeID reads a tree ID from a cat-file --batch stream, throwing away the rest of the stream. +func ReadTreeID(rd *bufio.Reader, size int64) (string, error) { + id := "" + var n int64 +headerLoop: + for { + line, err := rd.ReadBytes('\n') + if err != nil { + return "", err + } + n += int64(len(line)) + idx := bytes.Index(line, []byte{' '}) + if idx < 0 { + continue + } + + if string(line[:idx]) == "tree" { + id = string(line[idx+1 : len(line)-1]) + break headerLoop + } + } + + // Discard the rest of the commit + discard := size - n + for discard > math.MaxInt32 { + _, err := rd.Discard(math.MaxInt32) + if err != nil { + return id, err + } + discard -= math.MaxInt32 + } + _, err := rd.Discard(int(discard)) + return id, err +} + +// git tree files are a list: +// SP NUL <20-byte SHA> +// +// Unfortunately this 20-byte notation is somewhat in conflict to all other git tools +// Therefore we need some method to convert these 20-byte SHAs to a 40-byte SHA + +// constant hextable to help quickly convert between 20byte and 40byte hashes +const hextable = "0123456789abcdef" + +// to40ByteSHA converts a 20-byte SHA in a 40-byte slice into a 40-byte sha in place +// without allocations. This is at least 100x quicker that hex.EncodeToString +// NB This requires that sha is a 40-byte slice +func to40ByteSHA(sha []byte) []byte { + for i := 19; i >= 0; i-- { + v := sha[i] + vhi, vlo := v>>4, v&0x0f + shi, slo := hextable[vhi], hextable[vlo] + sha[i*2], sha[i*2+1] = shi, slo + } + return sha +} + +// ParseTreeLineSkipMode reads an entry from a tree in a cat-file --batch stream +// This simply skips the mode - saving a substantial amount of time and carefully avoids allocations - except where fnameBuf is too small. +// It is recommended therefore to pass in an fnameBuf large enough to avoid almost all allocations +// +// Each line is composed of: +// SP NUL <20-byte SHA> +// +// We don't attempt to convert the 20-byte SHA to 40-byte SHA to save a lot of time +func ParseTreeLineSkipMode(rd *bufio.Reader, fnameBuf, shaBuf []byte) (fname, sha []byte, n int, err error) { + var readBytes []byte + // Skip the Mode + readBytes, err = rd.ReadSlice(' ') // NB: DOES NOT ALLOCATE SIMPLY RETURNS SLICE WITHIN READER BUFFER + if err != nil { + return + } + n += len(readBytes) + + // Deal with the fname + readBytes, err = rd.ReadSlice('\x00') + copy(fnameBuf, readBytes) + if len(fnameBuf) > len(readBytes) { + fnameBuf = fnameBuf[:len(readBytes)] // cut the buf the correct size + } else { + fnameBuf = append(fnameBuf, readBytes[len(fnameBuf):]...) // extend the buf and copy in the missing bits + } + for err == bufio.ErrBufferFull { // Then we need to read more + readBytes, err = rd.ReadSlice('\x00') + fnameBuf = append(fnameBuf, readBytes...) // there is little point attempting to avoid allocations here so just extend + } + n += len(fnameBuf) + if err != nil { + return + } + fnameBuf = fnameBuf[:len(fnameBuf)-1] // Drop the terminal NUL + fname = fnameBuf // set the returnable fname to the slice + + // Now deal with the 20-byte SHA + idx := 0 + for idx < 20 { + read := 0 + read, err = rd.Read(shaBuf[idx:20]) + n += read + if err != nil { + return + } + idx += read + } + sha = shaBuf + return +} + +// ParseTreeLine reads an entry from a tree in a cat-file --batch stream +// This carefully avoids allocations - except where fnameBuf is too small. +// It is recommended therefore to pass in an fnameBuf large enough to avoid almost all allocations +// +// Each line is composed of: +// SP NUL <20-byte SHA> +// +// We don't attempt to convert the 20-byte SHA to 40-byte SHA to save a lot of time +func ParseTreeLine(rd *bufio.Reader, modeBuf, fnameBuf, shaBuf []byte) (mode, fname, sha []byte, n int, err error) { + var readBytes []byte + + // Read the Mode + readBytes, err = rd.ReadSlice(' ') + if err != nil { + return + } + n += len(readBytes) + copy(modeBuf, readBytes) + if len(modeBuf) > len(readBytes) { + modeBuf = modeBuf[:len(readBytes)] + } else { + modeBuf = append(modeBuf, readBytes[len(modeBuf):]...) + + } + mode = modeBuf[:len(modeBuf)-1] // Drop the SP + + // Deal with the fname + readBytes, err = rd.ReadSlice('\x00') + copy(fnameBuf, readBytes) + if len(fnameBuf) > len(readBytes) { + fnameBuf = fnameBuf[:len(readBytes)] + } else { + fnameBuf = append(fnameBuf, readBytes[len(fnameBuf):]...) + } + for err == bufio.ErrBufferFull { + readBytes, err = rd.ReadSlice('\x00') + fnameBuf = append(fnameBuf, readBytes...) + } + n += len(fnameBuf) + if err != nil { + return + } + fnameBuf = fnameBuf[:len(fnameBuf)-1] + fname = fnameBuf + + // Deal with the 20-byte SHA + idx := 0 + for idx < 20 { + read := 0 + read, err = rd.Read(shaBuf[idx:20]) + n += read + if err != nil { + return + } + idx += read + } + sha = shaBuf + return +} diff --git a/modules/git/blob.go b/modules/git/blob.go index 9a23518522172..b1233d6ca95e0 100644 --- a/modules/git/blob.go +++ b/modules/git/blob.go @@ -3,8 +3,6 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -// +build !nogogit - package git import ( @@ -12,28 +10,9 @@ import ( "encoding/base64" "io" "io/ioutil" - - "github.com/go-git/go-git/v5/plumbing" ) -// Blob represents a Git object. -type Blob struct { - ID SHA1 - - gogitEncodedObj plumbing.EncodedObject - name string -} - -// DataAsync gets a ReadCloser for the contents of a blob without reading it all. -// Calling the Close function on the result will discard all unread output. -func (b *Blob) DataAsync() (io.ReadCloser, error) { - return b.gogitEncodedObj.Reader() -} - -// Size returns the uncompressed size of the blob -func (b *Blob) Size() int64 { - return b.gogitEncodedObj.Size() -} +// This file contains common functions between the gogit and nogogit variants for git Blobs // Name returns name of the tree entry this blob object was created from (or empty string) func (b *Blob) Name() string { diff --git a/modules/git/blob_gogit.go b/modules/git/blob_gogit.go new file mode 100644 index 0000000000000..9f8b5e50a26d9 --- /dev/null +++ b/modules/git/blob_gogit.go @@ -0,0 +1,33 @@ +// Copyright 2015 The Gogs Authors. All rights reserved. +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build !nogogit + +package git + +import ( + "io" + + "github.com/go-git/go-git/v5/plumbing" +) + +// Blob represents a Git object. +type Blob struct { + ID SHA1 + + gogitEncodedObj plumbing.EncodedObject + name string +} + +// DataAsync gets a ReadCloser for the contents of a blob without reading it all. +// Calling the Close function on the result will discard all unread output. +func (b *Blob) DataAsync() (io.ReadCloser, error) { + return b.gogitEncodedObj.Reader() +} + +// Size returns the uncompressed size of the blob +func (b *Blob) Size() int64 { + return b.gogitEncodedObj.Size() +} diff --git a/modules/git/blob_nogogit.go b/modules/git/blob_nogogit.go index 2eba974067a04..14ffc6c618c8a 100644 --- a/modules/git/blob_nogogit.go +++ b/modules/git/blob_nogogit.go @@ -8,10 +8,7 @@ package git import ( "bufio" - "bytes" - "encoding/base64" "io" - "io/ioutil" "strconv" "strings" ) @@ -78,72 +75,3 @@ func (b *Blob) Size() int64 { return b.size } - -// Name returns name of the tree entry this blob object was created from (or empty string) -func (b *Blob) Name() string { - return b.name -} - -// GetBlobContent Gets the content of the blob as raw text -func (b *Blob) GetBlobContent() (string, error) { - dataRc, err := b.DataAsync() - if err != nil { - return "", err - } - defer dataRc.Close() - buf := make([]byte, 1024) - n, _ := dataRc.Read(buf) - buf = buf[:n] - return string(buf), nil -} - -// GetBlobLineCount gets line count of lob as raw text -func (b *Blob) GetBlobLineCount() (int, error) { - reader, err := b.DataAsync() - if err != nil { - return 0, err - } - defer reader.Close() - buf := make([]byte, 32*1024) - count := 0 - lineSep := []byte{'\n'} - for { - c, err := reader.Read(buf) - count += bytes.Count(buf[:c], lineSep) - switch { - case err == io.EOF: - return count, nil - case err != nil: - return count, err - } - } -} - -// GetBlobContentBase64 Reads the content of the blob with a base64 encode and returns the encoded string -func (b *Blob) GetBlobContentBase64() (string, error) { - dataRc, err := b.DataAsync() - if err != nil { - return "", err - } - defer dataRc.Close() - - pr, pw := io.Pipe() - encoder := base64.NewEncoder(base64.StdEncoding, pw) - - go func() { - _, err := io.Copy(encoder, dataRc) - _ = encoder.Close() - - if err != nil { - _ = pw.CloseWithError(err) - } else { - _ = pw.Close() - } - }() - - out, err := ioutil.ReadAll(pr) - if err != nil { - return "", err - } - return string(out), nil -} diff --git a/modules/git/commit_gogit.go b/modules/git/commit_convert_gogit.go similarity index 100% rename from modules/git/commit_gogit.go rename to modules/git/commit_convert_gogit.go diff --git a/modules/git/commit_info.go b/modules/git/commit_info.go index e3224b24af2af..83e23545de526 100644 --- a/modules/git/commit_info.go +++ b/modules/git/commit_info.go @@ -2,297 +2,11 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -// +build !nogogit - package git -import ( - "path" - - "github.com/emirpasic/gods/trees/binaryheap" - "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/plumbing/object" - cgobject "github.com/go-git/go-git/v5/plumbing/object/commitgraph" -) - // CommitInfo describes the first commit with the provided entry type CommitInfo struct { Entry *TreeEntry Commit *Commit SubModuleFile *SubModuleFile } - -// GetCommitsInfo gets information of all commits that are corresponding to these entries -func (tes Entries) GetCommitsInfo(commit *Commit, treePath string, cache *LastCommitCache) ([]CommitInfo, *Commit, error) { - entryPaths := make([]string, len(tes)+1) - // Get the commit for the treePath itself - entryPaths[0] = "" - for i, entry := range tes { - entryPaths[i+1] = entry.Name() - } - - commitNodeIndex, commitGraphFile := commit.repo.CommitNodeIndex() - if commitGraphFile != nil { - defer commitGraphFile.Close() - } - - c, err := commitNodeIndex.Get(commit.ID) - if err != nil { - return nil, nil, err - } - - var revs map[string]*object.Commit - if cache != nil { - var unHitPaths []string - revs, unHitPaths, err = getLastCommitForPathsByCache(commit.ID.String(), treePath, entryPaths, cache) - if err != nil { - return nil, nil, err - } - if len(unHitPaths) > 0 { - revs2, err := GetLastCommitForPaths(c, treePath, unHitPaths) - if err != nil { - return nil, nil, err - } - - for k, v := range revs2 { - if err := cache.Put(commit.ID.String(), path.Join(treePath, k), v.ID().String()); err != nil { - return nil, nil, err - } - revs[k] = v - } - } - } else { - revs, err = GetLastCommitForPaths(c, treePath, entryPaths) - } - if err != nil { - return nil, nil, err - } - - commit.repo.gogitStorage.Close() - - commitsInfo := make([]CommitInfo, len(tes)) - for i, entry := range tes { - commitsInfo[i] = CommitInfo{ - Entry: entry, - } - if rev, ok := revs[entry.Name()]; ok { - entryCommit := convertCommit(rev) - commitsInfo[i].Commit = entryCommit - if entry.IsSubModule() { - subModuleURL := "" - var fullPath string - if len(treePath) > 0 { - fullPath = treePath + "/" + entry.Name() - } else { - fullPath = entry.Name() - } - if subModule, err := commit.GetSubModule(fullPath); err != nil { - return nil, nil, err - } else if subModule != nil { - subModuleURL = subModule.URL - } - subModuleFile := NewSubModuleFile(entryCommit, subModuleURL, entry.ID.String()) - commitsInfo[i].SubModuleFile = subModuleFile - } - } - } - - // Retrieve the commit for the treePath itself (see above). We basically - // get it for free during the tree traversal and it's used for listing - // pages to display information about newest commit for a given path. - var treeCommit *Commit - if treePath == "" { - treeCommit = commit - } else if rev, ok := revs[""]; ok { - treeCommit = convertCommit(rev) - treeCommit.repo = commit.repo - } - return commitsInfo, treeCommit, nil -} - -type commitAndPaths struct { - commit cgobject.CommitNode - // Paths that are still on the branch represented by commit - paths []string - // Set of hashes for the paths - hashes map[string]plumbing.Hash -} - -func getCommitTree(c cgobject.CommitNode, treePath string) (*object.Tree, error) { - tree, err := c.Tree() - if err != nil { - return nil, err - } - - // Optimize deep traversals by focusing only on the specific tree - if treePath != "" { - tree, err = tree.Tree(treePath) - if err != nil { - return nil, err - } - } - - return tree, nil -} - -func getFileHashes(c cgobject.CommitNode, treePath string, paths []string) (map[string]plumbing.Hash, error) { - tree, err := getCommitTree(c, treePath) - if err == object.ErrDirectoryNotFound { - // The whole tree didn't exist, so return empty map - return make(map[string]plumbing.Hash), nil - } - if err != nil { - return nil, err - } - - hashes := make(map[string]plumbing.Hash) - for _, path := range paths { - if path != "" { - entry, err := tree.FindEntry(path) - if err == nil { - hashes[path] = entry.Hash - } - } else { - hashes[path] = tree.Hash - } - } - - return hashes, nil -} - -func getLastCommitForPathsByCache(commitID, treePath string, paths []string, cache *LastCommitCache) (map[string]*object.Commit, []string, error) { - var unHitEntryPaths []string - var results = make(map[string]*object.Commit) - for _, p := range paths { - lastCommit, err := cache.Get(commitID, path.Join(treePath, p)) - if err != nil { - return nil, nil, err - } - if lastCommit != nil { - results[p] = lastCommit.(*object.Commit) - continue - } - - unHitEntryPaths = append(unHitEntryPaths, p) - } - - return results, unHitEntryPaths, nil -} - -// GetLastCommitForPaths returns last commit information -func GetLastCommitForPaths(c cgobject.CommitNode, treePath string, paths []string) (map[string]*object.Commit, error) { - // We do a tree traversal with nodes sorted by commit time - heap := binaryheap.NewWith(func(a, b interface{}) int { - if a.(*commitAndPaths).commit.CommitTime().Before(b.(*commitAndPaths).commit.CommitTime()) { - return 1 - } - return -1 - }) - - resultNodes := make(map[string]cgobject.CommitNode) - initialHashes, err := getFileHashes(c, treePath, paths) - if err != nil { - return nil, err - } - - // Start search from the root commit and with full set of paths - heap.Push(&commitAndPaths{c, paths, initialHashes}) - - for { - cIn, ok := heap.Pop() - if !ok { - break - } - current := cIn.(*commitAndPaths) - - // Load the parent commits for the one we are currently examining - numParents := current.commit.NumParents() - var parents []cgobject.CommitNode - for i := 0; i < numParents; i++ { - parent, err := current.commit.ParentNode(i) - if err != nil { - break - } - parents = append(parents, parent) - } - - // Examine the current commit and set of interesting paths - pathUnchanged := make([]bool, len(current.paths)) - parentHashes := make([]map[string]plumbing.Hash, len(parents)) - for j, parent := range parents { - parentHashes[j], err = getFileHashes(parent, treePath, current.paths) - if err != nil { - break - } - - for i, path := range current.paths { - if parentHashes[j][path] == current.hashes[path] { - pathUnchanged[i] = true - } - } - } - - var remainingPaths []string - for i, path := range current.paths { - // The results could already contain some newer change for the same path, - // so don't override that and bail out on the file early. - if resultNodes[path] == nil { - if pathUnchanged[i] { - // The path existed with the same hash in at least one parent so it could - // not have been changed in this commit directly. - remainingPaths = append(remainingPaths, path) - } else { - // There are few possible cases how can we get here: - // - The path didn't exist in any parent, so it must have been created by - // this commit. - // - The path did exist in the parent commit, but the hash of the file has - // changed. - // - We are looking at a merge commit and the hash of the file doesn't - // match any of the hashes being merged. This is more common for directories, - // but it can also happen if a file is changed through conflict resolution. - resultNodes[path] = current.commit - } - } - } - - if len(remainingPaths) > 0 { - // Add the parent nodes along with remaining paths to the heap for further - // processing. - for j, parent := range parents { - // Combine remainingPath with paths available on the parent branch - // and make union of them - remainingPathsForParent := make([]string, 0, len(remainingPaths)) - newRemainingPaths := make([]string, 0, len(remainingPaths)) - for _, path := range remainingPaths { - if parentHashes[j][path] == current.hashes[path] { - remainingPathsForParent = append(remainingPathsForParent, path) - } else { - newRemainingPaths = append(newRemainingPaths, path) - } - } - - if remainingPathsForParent != nil { - heap.Push(&commitAndPaths{parent, remainingPathsForParent, parentHashes[j]}) - } - - if len(newRemainingPaths) == 0 { - break - } else { - remainingPaths = newRemainingPaths - } - } - } - } - - // Post-processing - result := make(map[string]*object.Commit) - for path, commitNode := range resultNodes { - var err error - result[path], err = commitNode.Commit() - if err != nil { - return nil, err - } - } - - return result, nil -} diff --git a/modules/git/commit_info_gogit.go b/modules/git/commit_info_gogit.go new file mode 100644 index 0000000000000..6a7868be5cb19 --- /dev/null +++ b/modules/git/commit_info_gogit.go @@ -0,0 +1,291 @@ +// Copyright 2017 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build !nogogit + +package git + +import ( + "path" + + "github.com/emirpasic/gods/trees/binaryheap" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/object" + cgobject "github.com/go-git/go-git/v5/plumbing/object/commitgraph" +) + +// GetCommitsInfo gets information of all commits that are corresponding to these entries +func (tes Entries) GetCommitsInfo(commit *Commit, treePath string, cache *LastCommitCache) ([]CommitInfo, *Commit, error) { + entryPaths := make([]string, len(tes)+1) + // Get the commit for the treePath itself + entryPaths[0] = "" + for i, entry := range tes { + entryPaths[i+1] = entry.Name() + } + + commitNodeIndex, commitGraphFile := commit.repo.CommitNodeIndex() + if commitGraphFile != nil { + defer commitGraphFile.Close() + } + + c, err := commitNodeIndex.Get(commit.ID) + if err != nil { + return nil, nil, err + } + + var revs map[string]*object.Commit + if cache != nil { + var unHitPaths []string + revs, unHitPaths, err = getLastCommitForPathsByCache(commit.ID.String(), treePath, entryPaths, cache) + if err != nil { + return nil, nil, err + } + if len(unHitPaths) > 0 { + revs2, err := GetLastCommitForPaths(c, treePath, unHitPaths) + if err != nil { + return nil, nil, err + } + + for k, v := range revs2 { + if err := cache.Put(commit.ID.String(), path.Join(treePath, k), v.ID().String()); err != nil { + return nil, nil, err + } + revs[k] = v + } + } + } else { + revs, err = GetLastCommitForPaths(c, treePath, entryPaths) + } + if err != nil { + return nil, nil, err + } + + commit.repo.gogitStorage.Close() + + commitsInfo := make([]CommitInfo, len(tes)) + for i, entry := range tes { + commitsInfo[i] = CommitInfo{ + Entry: entry, + } + if rev, ok := revs[entry.Name()]; ok { + entryCommit := convertCommit(rev) + commitsInfo[i].Commit = entryCommit + if entry.IsSubModule() { + subModuleURL := "" + var fullPath string + if len(treePath) > 0 { + fullPath = treePath + "/" + entry.Name() + } else { + fullPath = entry.Name() + } + if subModule, err := commit.GetSubModule(fullPath); err != nil { + return nil, nil, err + } else if subModule != nil { + subModuleURL = subModule.URL + } + subModuleFile := NewSubModuleFile(entryCommit, subModuleURL, entry.ID.String()) + commitsInfo[i].SubModuleFile = subModuleFile + } + } + } + + // Retrieve the commit for the treePath itself (see above). We basically + // get it for free during the tree traversal and it's used for listing + // pages to display information about newest commit for a given path. + var treeCommit *Commit + if treePath == "" { + treeCommit = commit + } else if rev, ok := revs[""]; ok { + treeCommit = convertCommit(rev) + treeCommit.repo = commit.repo + } + return commitsInfo, treeCommit, nil +} + +type commitAndPaths struct { + commit cgobject.CommitNode + // Paths that are still on the branch represented by commit + paths []string + // Set of hashes for the paths + hashes map[string]plumbing.Hash +} + +func getCommitTree(c cgobject.CommitNode, treePath string) (*object.Tree, error) { + tree, err := c.Tree() + if err != nil { + return nil, err + } + + // Optimize deep traversals by focusing only on the specific tree + if treePath != "" { + tree, err = tree.Tree(treePath) + if err != nil { + return nil, err + } + } + + return tree, nil +} + +func getFileHashes(c cgobject.CommitNode, treePath string, paths []string) (map[string]plumbing.Hash, error) { + tree, err := getCommitTree(c, treePath) + if err == object.ErrDirectoryNotFound { + // The whole tree didn't exist, so return empty map + return make(map[string]plumbing.Hash), nil + } + if err != nil { + return nil, err + } + + hashes := make(map[string]plumbing.Hash) + for _, path := range paths { + if path != "" { + entry, err := tree.FindEntry(path) + if err == nil { + hashes[path] = entry.Hash + } + } else { + hashes[path] = tree.Hash + } + } + + return hashes, nil +} + +func getLastCommitForPathsByCache(commitID, treePath string, paths []string, cache *LastCommitCache) (map[string]*object.Commit, []string, error) { + var unHitEntryPaths []string + var results = make(map[string]*object.Commit) + for _, p := range paths { + lastCommit, err := cache.Get(commitID, path.Join(treePath, p)) + if err != nil { + return nil, nil, err + } + if lastCommit != nil { + results[p] = lastCommit.(*object.Commit) + continue + } + + unHitEntryPaths = append(unHitEntryPaths, p) + } + + return results, unHitEntryPaths, nil +} + +// GetLastCommitForPaths returns last commit information +func GetLastCommitForPaths(c cgobject.CommitNode, treePath string, paths []string) (map[string]*object.Commit, error) { + // We do a tree traversal with nodes sorted by commit time + heap := binaryheap.NewWith(func(a, b interface{}) int { + if a.(*commitAndPaths).commit.CommitTime().Before(b.(*commitAndPaths).commit.CommitTime()) { + return 1 + } + return -1 + }) + + resultNodes := make(map[string]cgobject.CommitNode) + initialHashes, err := getFileHashes(c, treePath, paths) + if err != nil { + return nil, err + } + + // Start search from the root commit and with full set of paths + heap.Push(&commitAndPaths{c, paths, initialHashes}) + + for { + cIn, ok := heap.Pop() + if !ok { + break + } + current := cIn.(*commitAndPaths) + + // Load the parent commits for the one we are currently examining + numParents := current.commit.NumParents() + var parents []cgobject.CommitNode + for i := 0; i < numParents; i++ { + parent, err := current.commit.ParentNode(i) + if err != nil { + break + } + parents = append(parents, parent) + } + + // Examine the current commit and set of interesting paths + pathUnchanged := make([]bool, len(current.paths)) + parentHashes := make([]map[string]plumbing.Hash, len(parents)) + for j, parent := range parents { + parentHashes[j], err = getFileHashes(parent, treePath, current.paths) + if err != nil { + break + } + + for i, path := range current.paths { + if parentHashes[j][path] == current.hashes[path] { + pathUnchanged[i] = true + } + } + } + + var remainingPaths []string + for i, path := range current.paths { + // The results could already contain some newer change for the same path, + // so don't override that and bail out on the file early. + if resultNodes[path] == nil { + if pathUnchanged[i] { + // The path existed with the same hash in at least one parent so it could + // not have been changed in this commit directly. + remainingPaths = append(remainingPaths, path) + } else { + // There are few possible cases how can we get here: + // - The path didn't exist in any parent, so it must have been created by + // this commit. + // - The path did exist in the parent commit, but the hash of the file has + // changed. + // - We are looking at a merge commit and the hash of the file doesn't + // match any of the hashes being merged. This is more common for directories, + // but it can also happen if a file is changed through conflict resolution. + resultNodes[path] = current.commit + } + } + } + + if len(remainingPaths) > 0 { + // Add the parent nodes along with remaining paths to the heap for further + // processing. + for j, parent := range parents { + // Combine remainingPath with paths available on the parent branch + // and make union of them + remainingPathsForParent := make([]string, 0, len(remainingPaths)) + newRemainingPaths := make([]string, 0, len(remainingPaths)) + for _, path := range remainingPaths { + if parentHashes[j][path] == current.hashes[path] { + remainingPathsForParent = append(remainingPathsForParent, path) + } else { + newRemainingPaths = append(newRemainingPaths, path) + } + } + + if remainingPathsForParent != nil { + heap.Push(&commitAndPaths{parent, remainingPathsForParent, parentHashes[j]}) + } + + if len(newRemainingPaths) == 0 { + break + } else { + remainingPaths = newRemainingPaths + } + } + } + } + + // Post-processing + result := make(map[string]*object.Commit) + for path, commitNode := range resultNodes { + var err error + result[path], err = commitNode.Commit() + if err != nil { + return nil, err + } + } + + return result, nil +} diff --git a/modules/git/commit_info_nogogit.go b/modules/git/commit_info_nogogit.go index 0c53a15a85420..8dd54a6ac577d 100644 --- a/modules/git/commit_info_nogogit.go +++ b/modules/git/commit_info_nogogit.go @@ -14,213 +14,9 @@ import ( "math" "path" "sort" - "strconv" "strings" ) -// CommitInfo describes the first commit with the provided entry -type CommitInfo struct { - Entry *TreeEntry - Commit *Commit - SubModuleFile *SubModuleFile -} - -// ReadBatchLine reads the header line from cat-file --batch -// We expect: -// SP SP LF -func ReadBatchLine(rd *bufio.Reader) (sha []byte, typ string, size int64, err error) { - sha, err = rd.ReadBytes(' ') - if err != nil { - return - } - sha = sha[:len(sha)-1] - - typ, err = rd.ReadString(' ') - if err != nil { - return - } - typ = typ[:len(typ)-1] - - var sizeStr string - sizeStr, err = rd.ReadString('\n') - if err != nil { - return - } - - size, err = strconv.ParseInt(sizeStr[:len(sizeStr)-1], 10, 64) - return -} - -func GetTagObjectID(rd *bufio.Reader, size int64) (string, error) { - id := "" - var n int64 -headerLoop: - for { - line, err := rd.ReadBytes('\n') - if err != nil { - return "", err - } - n += int64(len(line)) - idx := bytes.Index(line, []byte{' '}) - if idx < 0 { - continue - } - - if string(line[:idx]) == "object" { - id = string(line[idx+1 : len(line)-1]) - break headerLoop - } - } - - // Discard the rest of the tag - discard := size - n - for discard > math.MaxInt32 { - _, err := rd.Discard(math.MaxInt32) - if err != nil { - return id, err - } - discard -= math.MaxInt32 - } - _, err := rd.Discard(int(discard)) - return id, err -} - -func GetTreeID(rd *bufio.Reader, size int64) (string, error) { - id := "" - var n int64 -headerLoop: - for { - line, err := rd.ReadBytes('\n') - if err != nil { - return "", err - } - n += int64(len(line)) - idx := bytes.Index(line, []byte{' '}) - if idx < 0 { - continue - } - - if string(line[:idx]) == "tree" { - id = string(line[idx+1 : len(line)-1]) - break headerLoop - } - } - - // Discard the rest of the tag - discard := size - n - for discard > math.MaxInt32 { - _, err := rd.Discard(math.MaxInt32) - if err != nil { - return id, err - } - discard -= math.MaxInt32 - } - _, err := rd.Discard(int(discard)) - return id, err -} - -const hextable = "0123456789abcdef" - -// to40ByteSHA converts a 20byte SHA in a 40 byte slice into a 40 byte sha -func to40ByteSHA(sha []byte) []byte { - for i := 19; i >= 0; i-- { - v := sha[i] - vhi, vlo := v>>4, v&0x0f - shi, slo := hextable[vhi], hextable[vlo] - sha[i*2], sha[i*2+1] = shi, slo - } - return sha -} - -func ParseTreeLineSkipMode(rd *bufio.Reader, fnameBuf, shaBuf []byte) (fname, sha []byte, n int, err error) { - var readBytes []byte - readBytes, err = rd.ReadSlice(' ') - if err != nil { - return - } - n += len(readBytes) - - readBytes, err = rd.ReadSlice('\x00') - copy(fnameBuf, readBytes) - if len(fnameBuf) > len(readBytes) { - fnameBuf = fnameBuf[:len(readBytes)] - } else { - fnameBuf = append(fnameBuf, readBytes[len(fnameBuf):]...) - } - for err == bufio.ErrBufferFull { - readBytes, err = rd.ReadSlice('\x00') - fnameBuf = append(fnameBuf, readBytes...) - } - n += len(fnameBuf) - if err != nil { - return - } - fnameBuf = fnameBuf[:len(fnameBuf)-1] - fname = fnameBuf - - idx := 0 - for idx < 20 { - read := 0 - read, err = rd.Read(shaBuf[idx:20]) - n += read - if err != nil { - return - } - idx += read - } - sha = shaBuf - return -} - -func ParseTreeLine(rd *bufio.Reader, modeBuf, fnameBuf, shaBuf []byte) (mode, fname, sha []byte, n int, err error) { - var readBytes []byte - - readBytes, err = rd.ReadSlice(' ') - if err != nil { - return - } - n += len(readBytes) - copy(modeBuf, readBytes) - if len(modeBuf) > len(readBytes) { - modeBuf = modeBuf[:len(readBytes)] - } else { - modeBuf = append(modeBuf, readBytes[len(modeBuf):]...) - - } - mode = modeBuf[:len(modeBuf)-1] - - readBytes, err = rd.ReadSlice('\x00') - copy(fnameBuf, readBytes) - if len(fnameBuf) > len(readBytes) { - fnameBuf = fnameBuf[:len(readBytes)] - } else { - fnameBuf = append(fnameBuf, readBytes[len(fnameBuf):]...) - } - for err == bufio.ErrBufferFull { - readBytes, err = rd.ReadSlice('\x00') - fnameBuf = append(fnameBuf, readBytes...) - } - n += len(fnameBuf) - if err != nil { - return - } - fnameBuf = fnameBuf[:len(fnameBuf)-1] - fname = fnameBuf - - idx := 0 - for idx < 20 { - read := 0 - read, err = rd.Read(shaBuf[idx:20]) - n += read - if err != nil { - return - } - idx += read - } - sha = shaBuf - return -} - // GetCommitsInfo gets information of all commits that are corresponding to these entries func (tes Entries) GetCommitsInfo(commit *Commit, treePath string, cache *LastCommitCache) ([]CommitInfo, *Commit, error) { entryPaths := make([]string, len(tes)+1) diff --git a/modules/git/commit_reader.go b/modules/git/commit_reader.go index 9f95ca2a50d1c..4eb861040e62f 100644 --- a/modules/git/commit_reader.go +++ b/modules/git/commit_reader.go @@ -12,7 +12,9 @@ import ( ) // CommitFromReader will generate a Commit from a provided reader -// We will need this to interpret commits from cat-file +// We need this to interpret commits from cat-file or cat-file --batch +// +// If used as part of a cat-file --batch stream you need to limit the reader to the correct size func CommitFromReader(gitRepo *Repository, sha SHA1, reader io.Reader) (*Commit, error) { commit := &Commit{ ID: sha, diff --git a/modules/git/last_commit_cache.go b/modules/git/last_commit_cache.go index 9394df5b05266..7cca601226a1a 100644 --- a/modules/git/last_commit_cache.go +++ b/modules/git/last_commit_cache.go @@ -2,17 +2,11 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -// +build !nogogit - package git import ( "crypto/sha256" "fmt" - "path" - - "github.com/go-git/go-git/v5/plumbing/object" - cgobject "github.com/go-git/go-git/v5/plumbing/object/commitgraph" ) // Cache represents a caching interface @@ -23,112 +17,13 @@ type Cache interface { Get(key string) interface{} } -// LastCommitCache represents a cache to store last commit -type LastCommitCache struct { - repoPath string - ttl int64 - repo *Repository - commitCache map[string]*object.Commit - cache Cache -} - -// NewLastCommitCache creates a new last commit cache for repo -func NewLastCommitCache(repoPath string, gitRepo *Repository, ttl int64, cache Cache) *LastCommitCache { - if cache == nil { - return nil - } - return &LastCommitCache{ - repoPath: repoPath, - repo: gitRepo, - commitCache: make(map[string]*object.Commit), - ttl: ttl, - cache: cache, - } -} - func (c *LastCommitCache) getCacheKey(repoPath, ref, entryPath string) string { hashBytes := sha256.Sum256([]byte(fmt.Sprintf("%s:%s:%s", repoPath, ref, entryPath))) return fmt.Sprintf("last_commit:%x", hashBytes) } -// Get get the last commit information by commit id and entry path -func (c *LastCommitCache) Get(ref, entryPath string) (interface{}, error) { - v := c.cache.Get(c.getCacheKey(c.repoPath, ref, entryPath)) - if vs, ok := v.(string); ok { - log("LastCommitCache hit level 1: [%s:%s:%s]", ref, entryPath, vs) - if commit, ok := c.commitCache[vs]; ok { - log("LastCommitCache hit level 2: [%s:%s:%s]", ref, entryPath, vs) - return commit, nil - } - id, err := c.repo.ConvertToSHA1(vs) - if err != nil { - return nil, err - } - commit, err := c.repo.GoGitRepo().CommitObject(id) - if err != nil { - return nil, err - } - c.commitCache[vs] = commit - return commit, nil - } - return nil, nil -} - // Put put the last commit id with commit and entry path func (c *LastCommitCache) Put(ref, entryPath, commitID string) error { log("LastCommitCache save: [%s:%s:%s]", ref, entryPath, commitID) return c.cache.Put(c.getCacheKey(c.repoPath, ref, entryPath), commitID, c.ttl) } - -// CacheCommit will cache the commit from the gitRepository -func (c *LastCommitCache) CacheCommit(gitRepo *Repository, commit *Commit) error { - - commitNodeIndex, _ := gitRepo.CommitNodeIndex() - - index, err := commitNodeIndex.Get(commit.ID) - if err != nil { - return err - } - - return c.recursiveCache(gitRepo, index, &commit.Tree, "", 1) -} - -func (c *LastCommitCache) recursiveCache(gitRepo *Repository, index cgobject.CommitNode, tree *Tree, treePath string, level int) error { - if level == 0 { - return nil - } - - entries, err := tree.ListEntries() - if err != nil { - return err - } - - entryPaths := make([]string, len(entries)) - entryMap := make(map[string]*TreeEntry) - for i, entry := range entries { - entryPaths[i] = entry.Name() - entryMap[entry.Name()] = entry - } - - commits, err := GetLastCommitForPaths(index, treePath, entryPaths) - if err != nil { - return err - } - - for entry, cm := range commits { - if err := c.Put(index.ID().String(), path.Join(treePath, entry), cm.ID().String()); err != nil { - return err - } - if entryMap[entry].IsDir() { - subTree, err := tree.SubTree(entry) - if err != nil { - return err - } - if err := c.recursiveCache(gitRepo, index, subTree, entry, level-1); err != nil { - return err - } - } - } - - return nil -} diff --git a/modules/git/last_commit_cache_gogit.go b/modules/git/last_commit_cache_gogit.go new file mode 100644 index 0000000000000..ae1753e37680d --- /dev/null +++ b/modules/git/last_commit_cache_gogit.go @@ -0,0 +1,113 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build !nogogit + +package git + +import ( + "path" + + "github.com/go-git/go-git/v5/plumbing/object" + cgobject "github.com/go-git/go-git/v5/plumbing/object/commitgraph" +) + +// LastCommitCache represents a cache to store last commit +type LastCommitCache struct { + repoPath string + ttl int64 + repo *Repository + commitCache map[string]*object.Commit + cache Cache +} + +// NewLastCommitCache creates a new last commit cache for repo +func NewLastCommitCache(repoPath string, gitRepo *Repository, ttl int64, cache Cache) *LastCommitCache { + if cache == nil { + return nil + } + return &LastCommitCache{ + repoPath: repoPath, + repo: gitRepo, + commitCache: make(map[string]*object.Commit), + ttl: ttl, + cache: cache, + } +} + +// Get get the last commit information by commit id and entry path +func (c *LastCommitCache) Get(ref, entryPath string) (interface{}, error) { + v := c.cache.Get(c.getCacheKey(c.repoPath, ref, entryPath)) + if vs, ok := v.(string); ok { + log("LastCommitCache hit level 1: [%s:%s:%s]", ref, entryPath, vs) + if commit, ok := c.commitCache[vs]; ok { + log("LastCommitCache hit level 2: [%s:%s:%s]", ref, entryPath, vs) + return commit, nil + } + id, err := c.repo.ConvertToSHA1(vs) + if err != nil { + return nil, err + } + commit, err := c.repo.GoGitRepo().CommitObject(id) + if err != nil { + return nil, err + } + c.commitCache[vs] = commit + return commit, nil + } + return nil, nil +} + +// CacheCommit will cache the commit from the gitRepository +func (c *LastCommitCache) CacheCommit(gitRepo *Repository, commit *Commit) error { + + commitNodeIndex, _ := gitRepo.CommitNodeIndex() + + index, err := commitNodeIndex.Get(commit.ID) + if err != nil { + return err + } + + return c.recursiveCache(gitRepo, index, &commit.Tree, "", 1) +} + +func (c *LastCommitCache) recursiveCache(gitRepo *Repository, index cgobject.CommitNode, tree *Tree, treePath string, level int) error { + if level == 0 { + return nil + } + + entries, err := tree.ListEntries() + if err != nil { + return err + } + + entryPaths := make([]string, len(entries)) + entryMap := make(map[string]*TreeEntry) + for i, entry := range entries { + entryPaths[i] = entry.Name() + entryMap[entry.Name()] = entry + } + + commits, err := GetLastCommitForPaths(index, treePath, entryPaths) + if err != nil { + return err + } + + for entry, cm := range commits { + if err := c.Put(index.ID().String(), path.Join(treePath, entry), cm.ID().String()); err != nil { + return err + } + if entryMap[entry].IsDir() { + subTree, err := tree.SubTree(entry) + if err != nil { + return err + } + if err := c.recursiveCache(gitRepo, index, subTree, entry, level-1); err != nil { + return err + } + } + } + + return nil +} diff --git a/modules/git/last_commit_cache_nogogit.go b/modules/git/last_commit_cache_nogogit.go index b9d024c808a7a..d72695c7b9374 100644 --- a/modules/git/last_commit_cache_nogogit.go +++ b/modules/git/last_commit_cache_nogogit.go @@ -7,19 +7,9 @@ package git import ( - "crypto/sha256" - "fmt" "path" ) -// Cache represents a caching interface -type Cache interface { - // Put puts value into cache with key and expire time. - Put(key string, val interface{}, timeout int64) error - // Get gets cached value by given key. - Get(key string) interface{} -} - // LastCommitCache represents a cache to store last commit type LastCommitCache struct { repoPath string @@ -43,11 +33,6 @@ func NewLastCommitCache(repoPath string, gitRepo *Repository, ttl int64, cache C } } -func (c *LastCommitCache) getCacheKey(repoPath, ref, entryPath string) string { - hashBytes := sha256.Sum256([]byte(fmt.Sprintf("%s:%s:%s", repoPath, ref, entryPath))) - return fmt.Sprintf("last_commit:%x", hashBytes) -} - // Get get the last commit information by commit id and entry path func (c *LastCommitCache) Get(ref, entryPath string) (interface{}, error) { v := c.cache.Get(c.getCacheKey(c.repoPath, ref, entryPath)) @@ -71,12 +56,6 @@ func (c *LastCommitCache) Get(ref, entryPath string) (interface{}, error) { return nil, nil } -// Put put the last commit id with commit and entry path -func (c *LastCommitCache) Put(ref, entryPath, commitID string) error { - log("LastCommitCache save: [%s:%s:%s]", ref, entryPath, commitID) - return c.cache.Put(c.getCacheKey(c.repoPath, ref, entryPath), commitID, c.ttl) -} - // CacheCommit will cache the commit from the gitRepository func (c *LastCommitCache) CacheCommit(gitRepo *Repository, commit *Commit) error { return c.recursiveCache(gitRepo, commit, &commit.Tree, "", 1) diff --git a/modules/git/notes.go b/modules/git/notes.go index 4d8b9068f92b5..a8dd66df0bce4 100644 --- a/modules/git/notes.go +++ b/modules/git/notes.go @@ -2,16 +2,8 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -// +build !nogogit - package git -import ( - "io/ioutil" - - "github.com/go-git/go-git/v5/plumbing/object" -) - // NotesRef is the git ref where Gitea will look for git-notes data. // The value ("refs/notes/commits") is the default ref used by git-notes. const NotesRef = "refs/notes/commits" @@ -21,62 +13,3 @@ type Note struct { Message []byte Commit *Commit } - -// GetNote retrieves the git-notes data for a given commit. -func GetNote(repo *Repository, commitID string, note *Note) error { - notes, err := repo.GetCommit(NotesRef) - if err != nil { - return err - } - - remainingCommitID := commitID - path := "" - currentTree := notes.Tree.gogitTree - var file *object.File - for len(remainingCommitID) > 2 { - file, err = currentTree.File(remainingCommitID) - if err == nil { - path += remainingCommitID - break - } - if err == object.ErrFileNotFound { - currentTree, err = currentTree.Tree(remainingCommitID[0:2]) - path += remainingCommitID[0:2] + "/" - remainingCommitID = remainingCommitID[2:] - } - if err != nil { - return err - } - } - - blob := file.Blob - dataRc, err := blob.Reader() - if err != nil { - return err - } - - defer dataRc.Close() - d, err := ioutil.ReadAll(dataRc) - if err != nil { - return err - } - note.Message = d - - commitNodeIndex, commitGraphFile := repo.CommitNodeIndex() - if commitGraphFile != nil { - defer commitGraphFile.Close() - } - - commitNode, err := commitNodeIndex.Get(notes.ID) - if err != nil { - return err - } - - lastCommits, err := GetLastCommitForPaths(commitNode, "", []string{path}) - if err != nil { - return err - } - note.Commit = convertCommit(lastCommits[path]) - - return nil -} diff --git a/modules/git/notes_gogit.go b/modules/git/notes_gogit.go new file mode 100644 index 0000000000000..09371e212d850 --- /dev/null +++ b/modules/git/notes_gogit.go @@ -0,0 +1,72 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build !nogogit + +package git + +import ( + "io/ioutil" + + "github.com/go-git/go-git/v5/plumbing/object" +) + +// GetNote retrieves the git-notes data for a given commit. +func GetNote(repo *Repository, commitID string, note *Note) error { + notes, err := repo.GetCommit(NotesRef) + if err != nil { + return err + } + + remainingCommitID := commitID + path := "" + currentTree := notes.Tree.gogitTree + var file *object.File + for len(remainingCommitID) > 2 { + file, err = currentTree.File(remainingCommitID) + if err == nil { + path += remainingCommitID + break + } + if err == object.ErrFileNotFound { + currentTree, err = currentTree.Tree(remainingCommitID[0:2]) + path += remainingCommitID[0:2] + "/" + remainingCommitID = remainingCommitID[2:] + } + if err != nil { + return err + } + } + + blob := file.Blob + dataRc, err := blob.Reader() + if err != nil { + return err + } + + defer dataRc.Close() + d, err := ioutil.ReadAll(dataRc) + if err != nil { + return err + } + note.Message = d + + commitNodeIndex, commitGraphFile := repo.CommitNodeIndex() + if commitGraphFile != nil { + defer commitGraphFile.Close() + } + + commitNode, err := commitNodeIndex.Get(notes.ID) + if err != nil { + return err + } + + lastCommits, err := GetLastCommitForPaths(commitNode, "", []string{path}) + if err != nil { + return err + } + note.Commit = convertCommit(lastCommits[path]) + + return nil +} diff --git a/modules/git/notes_nogogit.go b/modules/git/notes_nogogit.go index 5dbfa12c23999..cbcc62cf4aa3a 100644 --- a/modules/git/notes_nogogit.go +++ b/modules/git/notes_nogogit.go @@ -10,16 +10,6 @@ import ( "io/ioutil" ) -// NotesRef is the git ref where Gitea will look for git-notes data. -// The value ("refs/notes/commits") is the default ref used by git-notes. -const NotesRef = "refs/notes/commits" - -// Note stores information about a note created using git-notes. -type Note struct { - Message []byte - Commit *Commit -} - // GetNote retrieves the git-notes data for a given commit. func GetNote(repo *Repository, commitID string, note *Note) error { notes, err := repo.GetCommit(NotesRef) diff --git a/modules/git/parse.go b/modules/git/parse_gogit.go similarity index 100% rename from modules/git/parse.go rename to modules/git/parse_gogit.go diff --git a/modules/git/parse_test.go b/modules/git/parse_gogit_test.go similarity index 100% rename from modules/git/parse_test.go rename to modules/git/parse_gogit_test.go diff --git a/modules/git/pipeline/lfs_nogogit.go b/modules/git/pipeline/lfs_nogogit.go index eeb65e702681c..516623e36e4d2 100644 --- a/modules/git/pipeline/lfs_nogogit.go +++ b/modules/git/pipeline/lfs_nogogit.go @@ -122,7 +122,7 @@ func FindLFSFile(repo *git.Repository, hash git.SHA1) ([]*LFSResult, error) { switch typ { case "tag": // This shouldn't happen but if it does well just get the commit and try again - id, err := git.GetTagObjectID(batchReader, size) + id, err := git.ReadTagObjectID(batchReader, size) if err != nil { return nil, err } diff --git a/modules/git/repo.go b/modules/git/repo.go index 8d6e9eca9135f..56bef5b45095e 100644 --- a/modules/git/repo.go +++ b/modules/git/repo.go @@ -3,41 +3,21 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -// +build !nogogit - package git import ( "bytes" "container/list" - "errors" "fmt" "os" "path" - "path/filepath" "strconv" "strings" "time" - gitealog "code.gitea.io/gitea/modules/log" - "github.com/go-git/go-billy/v5/osfs" - gogit "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/plumbing/cache" - "github.com/go-git/go-git/v5/storage/filesystem" "github.com/unknwon/com" ) -// Repository represents a Git repository. -type Repository struct { - Path string - - tagCache *ObjectCache - - gogitRepo *gogit.Repository - gogitStorage *filesystem.Storage - gpgSettings *GPGSettings -} - // GPGSettings represents the default GPG settings for this repository type GPGSettings struct { Sign bool @@ -94,52 +74,6 @@ func InitRepository(repoPath string, bare bool) error { return err } -// OpenRepository opens the repository at the given path. -func OpenRepository(repoPath string) (*Repository, error) { - repoPath, err := filepath.Abs(repoPath) - if err != nil { - return nil, err - } else if !isDir(repoPath) { - return nil, errors.New("no such file or directory") - } - - fs := osfs.New(repoPath) - _, err = fs.Stat(".git") - if err == nil { - fs, err = fs.Chroot(".git") - if err != nil { - return nil, err - } - } - storage := filesystem.NewStorageWithOptions(fs, cache.NewObjectLRUDefault(), filesystem.Options{KeepDescriptors: true}) - gogitRepo, err := gogit.Open(storage, fs) - if err != nil { - return nil, err - } - - return &Repository{ - Path: repoPath, - gogitRepo: gogitRepo, - gogitStorage: storage, - tagCache: newObjectCache(), - }, nil -} - -// Close this repository, in particular close the underlying gogitStorage if this is not nil -func (repo *Repository) Close() { - if repo == nil || repo.gogitStorage == nil { - return - } - if err := repo.gogitStorage.Close(); err != nil { - gitealog.Error("Error closing storage: %v", err) - } -} - -// GoGitRepo gets the go-git repo representation -func (repo *Repository) GoGitRepo() *gogit.Repository { - return repo.gogitRepo -} - // IsEmpty Check if repository is empty. func (repo *Repository) IsEmpty() (bool, error) { var errbuf strings.Builder diff --git a/modules/git/repo_base_gogit.go b/modules/git/repo_base_gogit.go new file mode 100644 index 0000000000000..3be0fce28dbd1 --- /dev/null +++ b/modules/git/repo_base_gogit.go @@ -0,0 +1,76 @@ +// Copyright 2015 The Gogs Authors. All rights reserved. +// Copyright 2017 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build !nogogit + +package git + +import ( + "errors" + "path/filepath" + + gitealog "code.gitea.io/gitea/modules/log" + "github.com/go-git/go-billy/v5/osfs" + gogit "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing/cache" + "github.com/go-git/go-git/v5/storage/filesystem" +) + +// Repository represents a Git repository. +type Repository struct { + Path string + + tagCache *ObjectCache + + gogitRepo *gogit.Repository + gogitStorage *filesystem.Storage + gpgSettings *GPGSettings +} + +// OpenRepository opens the repository at the given path. +func OpenRepository(repoPath string) (*Repository, error) { + repoPath, err := filepath.Abs(repoPath) + if err != nil { + return nil, err + } else if !isDir(repoPath) { + return nil, errors.New("no such file or directory") + } + + fs := osfs.New(repoPath) + _, err = fs.Stat(".git") + if err == nil { + fs, err = fs.Chroot(".git") + if err != nil { + return nil, err + } + } + storage := filesystem.NewStorageWithOptions(fs, cache.NewObjectLRUDefault(), filesystem.Options{KeepDescriptors: true}) + gogitRepo, err := gogit.Open(storage, fs) + if err != nil { + return nil, err + } + + return &Repository{ + Path: repoPath, + gogitRepo: gogitRepo, + gogitStorage: storage, + tagCache: newObjectCache(), + }, nil +} + +// Close this repository, in particular close the underlying gogitStorage if this is not nil +func (repo *Repository) Close() { + if repo == nil || repo.gogitStorage == nil { + return + } + if err := repo.gogitStorage.Close(); err != nil { + gitealog.Error("Error closing storage: %v", err) + } +} + +// GoGitRepo gets the go-git repo representation +func (repo *Repository) GoGitRepo() *gogit.Repository { + return repo.gogitRepo +} diff --git a/modules/git/repo_base_nogogit.go b/modules/git/repo_base_nogogit.go new file mode 100644 index 0000000000000..dc90d729b692e --- /dev/null +++ b/modules/git/repo_base_nogogit.go @@ -0,0 +1,40 @@ +// Copyright 2015 The Gogs Authors. All rights reserved. +// Copyright 2017 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build nogogit + +package git + +import ( + "errors" + "path/filepath" +) + +// Repository represents a Git repository. +type Repository struct { + Path string + + tagCache *ObjectCache + + gpgSettings *GPGSettings +} + +// OpenRepository opens the repository at the given path. +func OpenRepository(repoPath string) (*Repository, error) { + repoPath, err := filepath.Abs(repoPath) + if err != nil { + return nil, err + } else if !isDir(repoPath) { + return nil, errors.New("no such file or directory") + } + return &Repository{ + Path: repoPath, + tagCache: newObjectCache(), + }, nil +} + +// Close this repository, in particular close the underlying gogitStorage if this is not nil +func (repo *Repository) Close() { +} diff --git a/modules/git/repo_blob.go b/modules/git/repo_blob.go index 204f4a339fe02..5397f24cb6245 100644 --- a/modules/git/repo_blob.go +++ b/modules/git/repo_blob.go @@ -1,27 +1,9 @@ -// Copyright 2018 The Gitea Authors. All rights reserved. +// Copyright 2020 The Gitea Authors. All rights reserved. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -// +build !nogogit - package git -import ( - "github.com/go-git/go-git/v5/plumbing" -) - -func (repo *Repository) getBlob(id SHA1) (*Blob, error) { - encodedObj, err := repo.gogitRepo.Storer.EncodedObject(plumbing.AnyObject, id) - if err != nil { - return nil, ErrNotExist{id.String(), ""} - } - - return &Blob{ - ID: id, - gogitEncodedObj: encodedObj, - }, nil -} - // GetBlob finds the blob object in the repository. func (repo *Repository) GetBlob(idStr string) (*Blob, error) { id, err := NewIDFromString(idStr) diff --git a/modules/git/repo_blob_gogit.go b/modules/git/repo_blob_gogit.go new file mode 100644 index 0000000000000..be1efc949346d --- /dev/null +++ b/modules/git/repo_blob_gogit.go @@ -0,0 +1,23 @@ +// Copyright 2018 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build !nogogit + +package git + +import ( + "github.com/go-git/go-git/v5/plumbing" +) + +func (repo *Repository) getBlob(id SHA1) (*Blob, error) { + encodedObj, err := repo.gogitRepo.Storer.EncodedObject(plumbing.AnyObject, id) + if err != nil { + return nil, ErrNotExist{id.String(), ""} + } + + return &Blob{ + ID: id, + gogitEncodedObj: encodedObj, + }, nil +} diff --git a/modules/git/repo_blob_nogogit.go b/modules/git/repo_blob_nogogit.go index 958efc48f44fa..57df74007479c 100644 --- a/modules/git/repo_blob_nogogit.go +++ b/modules/git/repo_blob_nogogit.go @@ -15,12 +15,3 @@ func (repo *Repository) getBlob(id SHA1) (*Blob, error) { repoPath: repo.Path, }, nil } - -// GetBlob finds the blob object in the repository. -func (repo *Repository) GetBlob(idStr string) (*Blob, error) { - id, err := NewIDFromString(idStr) - if err != nil { - return nil, err - } - return repo.getBlob(id) -} diff --git a/modules/git/repo_branch.go b/modules/git/repo_branch.go index d6a215a9b3702..25438530f5530 100644 --- a/modules/git/repo_branch.go +++ b/modules/git/repo_branch.go @@ -3,15 +3,11 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -// +build !nogogit - package git import ( "fmt" "strings" - - "github.com/go-git/go-git/v5/plumbing" ) // BranchPrefix base dir of the branch information file store on git @@ -28,18 +24,6 @@ func IsBranchExist(repoPath, name string) bool { return IsReferenceExist(repoPath, BranchPrefix+name) } -// IsBranchExist returns true if given branch exists in current repository. -func (repo *Repository) IsBranchExist(name string) bool { - if name == "" { - return false - } - reference, err := repo.gogitRepo.Reference(plumbing.ReferenceName(BranchPrefix+name), true) - if err != nil { - return false - } - return reference.Type() != plumbing.InvalidReference -} - // Branch represents a Git branch. type Branch struct { Name string @@ -81,25 +65,6 @@ func (repo *Repository) GetDefaultBranch() (string, error) { return NewCommand("symbolic-ref", "HEAD").RunInDir(repo.Path) } -// GetBranches returns all branches of the repository. -func (repo *Repository) GetBranches() ([]string, error) { - var branchNames []string - - branches, err := repo.gogitRepo.Branches() - if err != nil { - return nil, err - } - - _ = branches.ForEach(func(branch *plumbing.Reference) error { - branchNames = append(branchNames, strings.TrimPrefix(branch.Name().String(), BranchPrefix)) - return nil - }) - - // TODO: Sort? - - return branchNames, nil -} - // GetBranch returns a branch by it's name func (repo *Repository) GetBranch(branch string) (*Branch, error) { if !repo.IsBranchExist(branch) { diff --git a/modules/git/repo_branch_gogit.go b/modules/git/repo_branch_gogit.go new file mode 100644 index 0000000000000..dccfa802cef5b --- /dev/null +++ b/modules/git/repo_branch_gogit.go @@ -0,0 +1,45 @@ +// Copyright 2015 The Gogs Authors. All rights reserved. +// Copyright 2018 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build !nogogit + +package git + +import ( + "strings" + + "github.com/go-git/go-git/v5/plumbing" +) + +// IsBranchExist returns true if given branch exists in current repository. +func (repo *Repository) IsBranchExist(name string) bool { + if name == "" { + return false + } + reference, err := repo.gogitRepo.Reference(plumbing.ReferenceName(BranchPrefix+name), true) + if err != nil { + return false + } + return reference.Type() != plumbing.InvalidReference +} + +// GetBranches returns all branches of the repository. +func (repo *Repository) GetBranches() ([]string, error) { + var branchNames []string + + branches, err := repo.gogitRepo.Branches() + if err != nil { + return nil, err + } + + _ = branches.ForEach(func(branch *plumbing.Reference) error { + branchNames = append(branchNames, strings.TrimPrefix(branch.Name().String(), BranchPrefix)) + return nil + }) + + // TODO: Sort? + + return branchNames, nil +} diff --git a/modules/git/repo_branch_nogogit.go b/modules/git/repo_branch_nogogit.go index a4c7f9c3efbcd..c11eb11e01181 100644 --- a/modules/git/repo_branch_nogogit.go +++ b/modules/git/repo_branch_nogogit.go @@ -9,25 +9,10 @@ package git import ( "bufio" - "fmt" "io" "strings" ) -// BranchPrefix base dir of the branch information file store on git -const BranchPrefix = "refs/heads/" - -// IsReferenceExist returns true if given reference exists in the repository. -func IsReferenceExist(repoPath, name string) bool { - _, err := NewCommand("show-ref", "--verify", "--", name).RunInDir(repoPath) - return err == nil -} - -// IsBranchExist returns true if given branch exists in the repository. -func IsBranchExist(repoPath, name string) bool { - return IsReferenceExist(repoPath, BranchPrefix+name) -} - // IsBranchExist returns true if given branch exists in current repository. func (repo *Repository) IsBranchExist(name string) bool { if name == "" { @@ -36,47 +21,6 @@ func (repo *Repository) IsBranchExist(name string) bool { return IsReferenceExist(repo.Path, BranchPrefix+name) } -// Branch represents a Git branch. -type Branch struct { - Name string - Path string - - gitRepo *Repository -} - -// GetHEADBranch returns corresponding branch of HEAD. -func (repo *Repository) GetHEADBranch() (*Branch, error) { - if repo == nil { - return nil, fmt.Errorf("nil repo") - } - stdout, err := NewCommand("symbolic-ref", "HEAD").RunInDir(repo.Path) - if err != nil { - return nil, err - } - stdout = strings.TrimSpace(stdout) - - if !strings.HasPrefix(stdout, BranchPrefix) { - return nil, fmt.Errorf("invalid HEAD branch: %v", stdout) - } - - return &Branch{ - Name: stdout[len(BranchPrefix):], - Path: stdout, - gitRepo: repo, - }, nil -} - -// SetDefaultBranch sets default branch of repository. -func (repo *Repository) SetDefaultBranch(name string) error { - _, err := NewCommand("symbolic-ref", "HEAD", BranchPrefix+name).RunInDir(repo.Path) - return err -} - -// GetDefaultBranch gets default branch of repository. -func (repo *Repository) GetDefaultBranch() (string, error) { - return NewCommand("symbolic-ref", "HEAD").RunInDir(repo.Path) -} - // GetBranches returns all branches of the repository. func (repo *Repository) GetBranches() ([]string, error) { return callShowRef(repo.Path, BranchPrefix, "--heads") @@ -136,94 +80,3 @@ func callShowRef(repoPath, prefix, arg string) ([]string, error) { branchNames = append(branchNames, branchName) } } - -// GetBranch returns a branch by it's name -func (repo *Repository) GetBranch(branch string) (*Branch, error) { - if !repo.IsBranchExist(branch) { - return nil, ErrBranchNotExist{branch} - } - return &Branch{ - Path: repo.Path, - Name: branch, - gitRepo: repo, - }, nil -} - -// GetBranchesByPath returns a branch by it's path -func GetBranchesByPath(path string) ([]*Branch, error) { - gitRepo, err := OpenRepository(path) - if err != nil { - return nil, err - } - defer gitRepo.Close() - - brs, err := gitRepo.GetBranches() - if err != nil { - return nil, err - } - - branches := make([]*Branch, len(brs)) - for i := range brs { - branches[i] = &Branch{ - Path: path, - Name: brs[i], - gitRepo: gitRepo, - } - } - - return branches, nil -} - -// DeleteBranchOptions Option(s) for delete branch -type DeleteBranchOptions struct { - Force bool -} - -// DeleteBranch delete a branch by name on repository. -func (repo *Repository) DeleteBranch(name string, opts DeleteBranchOptions) error { - cmd := NewCommand("branch") - - if opts.Force { - cmd.AddArguments("-D") - } else { - cmd.AddArguments("-d") - } - - cmd.AddArguments("--", name) - _, err := cmd.RunInDir(repo.Path) - - return err -} - -// CreateBranch create a new branch -func (repo *Repository) CreateBranch(branch, oldbranchOrCommit string) error { - cmd := NewCommand("branch") - cmd.AddArguments("--", branch, oldbranchOrCommit) - - _, err := cmd.RunInDir(repo.Path) - - return err -} - -// AddRemote adds a new remote to repository. -func (repo *Repository) AddRemote(name, url string, fetch bool) error { - cmd := NewCommand("remote", "add") - if fetch { - cmd.AddArguments("-f") - } - cmd.AddArguments(name, url) - - _, err := cmd.RunInDir(repo.Path) - return err -} - -// RemoveRemote removes a remote from repository. -func (repo *Repository) RemoveRemote(name string) error { - _, err := NewCommand("remote", "rm", name).RunInDir(repo.Path) - return err -} - -// GetCommit returns the head commit of a branch -func (branch *Branch) GetCommit() (*Commit, error) { - return branch.gitRepo.GetBranchCommit(branch.Name) -} diff --git a/modules/git/repo_commit.go b/modules/git/repo_commit.go index 3aa96dac175f5..c31f4166283bc 100644 --- a/modules/git/repo_commit.go +++ b/modules/git/repo_commit.go @@ -3,43 +3,15 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -// +build !nogogit - package git import ( "bytes" "container/list" - "fmt" "strconv" "strings" - - "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/plumbing/object" ) -// GetRefCommitID returns the last commit ID string of given reference (branch or tag). -func (repo *Repository) GetRefCommitID(name string) (string, error) { - ref, err := repo.gogitRepo.Reference(plumbing.ReferenceName(name), true) - if err != nil { - if err == plumbing.ErrReferenceNotFound { - return "", ErrNotExist{ - ID: name, - } - } - return "", err - } - - return ref.Hash().String(), nil -} - -// IsCommitExist returns true if given commit exists in current repository. -func (repo *Repository) IsCommitExist(name string) bool { - hash := plumbing.NewHash(name) - _, err := repo.gogitRepo.CommitObject(hash) - return err == nil -} - // GetBranchCommitID returns last commit ID string of given branch. func (repo *Repository) GetBranchCommitID(name string) (string, error) { return repo.GetRefCommitID(BranchPrefix + name) @@ -57,78 +29,6 @@ func (repo *Repository) GetTagCommitID(name string) (string, error) { return strings.TrimSpace(stdout), nil } -func convertPGPSignatureForTag(t *object.Tag) *CommitGPGSignature { - if t.PGPSignature == "" { - return nil - } - - var w strings.Builder - var err error - - if _, err = fmt.Fprintf(&w, - "object %s\ntype %s\ntag %s\ntagger ", - t.Target.String(), t.TargetType.Bytes(), t.Name); err != nil { - return nil - } - - if err = t.Tagger.Encode(&w); err != nil { - return nil - } - - if _, err = fmt.Fprintf(&w, "\n\n"); err != nil { - return nil - } - - if _, err = fmt.Fprintf(&w, t.Message); err != nil { - return nil - } - - return &CommitGPGSignature{ - Signature: t.PGPSignature, - Payload: strings.TrimSpace(w.String()) + "\n", - } -} - -func (repo *Repository) getCommit(id SHA1) (*Commit, error) { - var tagObject *object.Tag - - gogitCommit, err := repo.gogitRepo.CommitObject(id) - if err == plumbing.ErrObjectNotFound { - tagObject, err = repo.gogitRepo.TagObject(id) - if err == plumbing.ErrObjectNotFound { - return nil, ErrNotExist{ - ID: id.String(), - } - } - if err == nil { - gogitCommit, err = repo.gogitRepo.CommitObject(tagObject.Target) - } - // if we get a plumbing.ErrObjectNotFound here then the repository is broken and it should be 500 - } - if err != nil { - return nil, err - } - - commit := convertCommit(gogitCommit) - commit.repo = repo - - if tagObject != nil { - commit.CommitMessage = strings.TrimSpace(tagObject.Message) - commit.Author = &tagObject.Tagger - commit.Signature = convertPGPSignatureForTag(tagObject) - } - - tree, err := gogitCommit.Tree() - if err != nil { - return nil, err - } - - commit.Tree.ID = tree.Hash - commit.Tree.gogitTree = tree - - return commit, nil -} - // ConvertToSHA1 returns a Hash object from a potential ID string func (repo *Repository) ConvertToSHA1(commitID string) (SHA1, error) { if len(commitID) != 40 { diff --git a/modules/git/repo_commit_gogit.go b/modules/git/repo_commit_gogit.go new file mode 100644 index 0000000000000..89553e53112e2 --- /dev/null +++ b/modules/git/repo_commit_gogit.go @@ -0,0 +1,110 @@ +// Copyright 2015 The Gogs Authors. All rights reserved. +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build !nogogit + +package git + +import ( + "fmt" + "strings" + + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/object" +) + +// GetRefCommitID returns the last commit ID string of given reference (branch or tag). +func (repo *Repository) GetRefCommitID(name string) (string, error) { + ref, err := repo.gogitRepo.Reference(plumbing.ReferenceName(name), true) + if err != nil { + if err == plumbing.ErrReferenceNotFound { + return "", ErrNotExist{ + ID: name, + } + } + return "", err + } + + return ref.Hash().String(), nil +} + +// IsCommitExist returns true if given commit exists in current repository. +func (repo *Repository) IsCommitExist(name string) bool { + hash := plumbing.NewHash(name) + _, err := repo.gogitRepo.CommitObject(hash) + return err == nil +} + +func convertPGPSignatureForTag(t *object.Tag) *CommitGPGSignature { + if t.PGPSignature == "" { + return nil + } + + var w strings.Builder + var err error + + if _, err = fmt.Fprintf(&w, + "object %s\ntype %s\ntag %s\ntagger ", + t.Target.String(), t.TargetType.Bytes(), t.Name); err != nil { + return nil + } + + if err = t.Tagger.Encode(&w); err != nil { + return nil + } + + if _, err = fmt.Fprintf(&w, "\n\n"); err != nil { + return nil + } + + if _, err = fmt.Fprintf(&w, t.Message); err != nil { + return nil + } + + return &CommitGPGSignature{ + Signature: t.PGPSignature, + Payload: strings.TrimSpace(w.String()) + "\n", + } +} + +func (repo *Repository) getCommit(id SHA1) (*Commit, error) { + var tagObject *object.Tag + + gogitCommit, err := repo.gogitRepo.CommitObject(id) + if err == plumbing.ErrObjectNotFound { + tagObject, err = repo.gogitRepo.TagObject(id) + if err == plumbing.ErrObjectNotFound { + return nil, ErrNotExist{ + ID: id.String(), + } + } + if err == nil { + gogitCommit, err = repo.gogitRepo.CommitObject(tagObject.Target) + } + // if we get a plumbing.ErrObjectNotFound here then the repository is broken and it should be 500 + } + if err != nil { + return nil, err + } + + commit := convertCommit(gogitCommit) + commit.repo = repo + + if tagObject != nil { + commit.CommitMessage = strings.TrimSpace(tagObject.Message) + commit.Author = &tagObject.Tagger + commit.Signature = convertPGPSignatureForTag(tagObject) + } + + tree, err := gogitCommit.Tree() + if err != nil { + return nil, err + } + + commit.Tree.ID = tree.Hash + commit.Tree.gogitTree = tree + + return commit, nil +} diff --git a/modules/git/repo_commit_nogogit.go b/modules/git/repo_commit_nogogit.go index e3173918eff05..5184f3cbe4282 100644 --- a/modules/git/repo_commit_nogogit.go +++ b/modules/git/repo_commit_nogogit.go @@ -1,5 +1,4 @@ -// Copyright 2015 The Gogs Authors. All rights reserved. -// Copyright 2019 The Gitea Authors. All rights reserved. +// Copyright 2020 The Gitea Authors. All rights reserved. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. @@ -9,12 +8,9 @@ package git import ( "bufio" - "bytes" - "container/list" "fmt" "io" "io/ioutil" - "strconv" "strings" ) @@ -54,23 +50,6 @@ func (repo *Repository) IsCommitExist(name string) bool { return err == nil } -// GetBranchCommitID returns last commit ID string of given branch. -func (repo *Repository) GetBranchCommitID(name string) (string, error) { - return repo.GetRefCommitID(BranchPrefix + name) -} - -// GetTagCommitID returns last commit ID string of given tag. -func (repo *Repository) GetTagCommitID(name string) (string, error) { - stdout, err := NewCommand("rev-list", "-n", "1", TagPrefix+name).RunInDir(repo.Path) - if err != nil { - if strings.Contains(err.Error(), "unknown revision or path") { - return "", ErrNotExist{name, ""} - } - return "", err - } - return strings.TrimSpace(stdout), nil -} - func (repo *Repository) getCommit(id SHA1) (*Commit, error) { stdoutReader, stdoutWriter := io.Pipe() defer func() { @@ -128,392 +107,3 @@ func (repo *Repository) getCommit(id SHA1) (*Commit, error) { } } } - -// ConvertToSHA1 returns a Hash object from a potential ID string -func (repo *Repository) ConvertToSHA1(commitID string) (SHA1, error) { - if len(commitID) != 40 { - var err error - actualCommitID, err := NewCommand("rev-parse", "--verify", commitID).RunInDir(repo.Path) - if err != nil { - if strings.Contains(err.Error(), "unknown revision or path") || - strings.Contains(err.Error(), "fatal: Needed a single revision") { - return SHA1{}, ErrNotExist{commitID, ""} - } - return SHA1{}, err - } - commitID = actualCommitID - } - return NewIDFromString(commitID) -} - -// GetCommit returns commit object of by ID string. -func (repo *Repository) GetCommit(commitID string) (*Commit, error) { - id, err := repo.ConvertToSHA1(commitID) - if err != nil { - return nil, err - } - - return repo.getCommit(id) -} - -// GetBranchCommit returns the last commit of given branch. -func (repo *Repository) GetBranchCommit(name string) (*Commit, error) { - commitID, err := repo.GetBranchCommitID(name) - if err != nil { - return nil, err - } - return repo.GetCommit(commitID) -} - -// GetTagCommit get the commit of the specific tag via name -func (repo *Repository) GetTagCommit(name string) (*Commit, error) { - commitID, err := repo.GetTagCommitID(name) - if err != nil { - return nil, err - } - return repo.GetCommit(commitID) -} - -func (repo *Repository) getCommitByPathWithID(id SHA1, relpath string) (*Commit, error) { - // File name starts with ':' must be escaped. - if relpath[0] == ':' { - relpath = `\` + relpath - } - - stdout, err := NewCommand("log", "-1", prettyLogFormat, id.String(), "--", relpath).RunInDir(repo.Path) - if err != nil { - return nil, err - } - - id, err = NewIDFromString(stdout) - if err != nil { - return nil, err - } - - return repo.getCommit(id) -} - -// GetCommitByPath returns the last commit of relative path. -func (repo *Repository) GetCommitByPath(relpath string) (*Commit, error) { - stdout, err := NewCommand("log", "-1", prettyLogFormat, "--", relpath).RunInDirBytes(repo.Path) - if err != nil { - return nil, err - } - - commits, err := repo.parsePrettyFormatLogToList(stdout) - if err != nil { - return nil, err - } - return commits.Front().Value.(*Commit), nil -} - -// CommitsRangeSize the default commits range size -var CommitsRangeSize = 50 - -func (repo *Repository) commitsByRange(id SHA1, page, pageSize int) (*list.List, error) { - stdout, err := NewCommand("log", id.String(), "--skip="+strconv.Itoa((page-1)*pageSize), - "--max-count="+strconv.Itoa(pageSize), prettyLogFormat).RunInDirBytes(repo.Path) - - if err != nil { - return nil, err - } - return repo.parsePrettyFormatLogToList(stdout) -} - -func (repo *Repository) searchCommits(id SHA1, opts SearchCommitsOptions) (*list.List, error) { - // create new git log command with limit of 100 commis - cmd := NewCommand("log", id.String(), "-100", prettyLogFormat) - // ignore case - args := []string{"-i"} - - // add authors if present in search query - if len(opts.Authors) > 0 { - for _, v := range opts.Authors { - args = append(args, "--author="+v) - } - } - - // add commiters if present in search query - if len(opts.Committers) > 0 { - for _, v := range opts.Committers { - args = append(args, "--committer="+v) - } - } - - // add time constraints if present in search query - if len(opts.After) > 0 { - args = append(args, "--after="+opts.After) - } - if len(opts.Before) > 0 { - args = append(args, "--before="+opts.Before) - } - - // pretend that all refs along with HEAD were listed on command line as - // https://git-scm.com/docs/git-log#Documentation/git-log.txt---all - // note this is done only for command created above - if opts.All { - cmd.AddArguments("--all") - } - - // add remaining keywords from search string - // note this is done only for command created above - if len(opts.Keywords) > 0 { - for _, v := range opts.Keywords { - cmd.AddArguments("--grep=" + v) - } - } - - // search for commits matching given constraints and keywords in commit msg - cmd.AddArguments(args...) - stdout, err := cmd.RunInDirBytes(repo.Path) - if err != nil { - return nil, err - } - if len(stdout) != 0 { - stdout = append(stdout, '\n') - } - - // if there are any keywords (ie not commiter:, author:, time:) - // then let's iterate over them - if len(opts.Keywords) > 0 { - for _, v := range opts.Keywords { - // ignore anything below 4 characters as too unspecific - if len(v) >= 4 { - // create new git log command with 1 commit limit - hashCmd := NewCommand("log", "-1", prettyLogFormat) - // add previous arguments except for --grep and --all - hashCmd.AddArguments(args...) - // add keyword as - hashCmd.AddArguments(v) - - // search with given constraints for commit matching sha hash of v - hashMatching, err := hashCmd.RunInDirBytes(repo.Path) - if err != nil || bytes.Contains(stdout, hashMatching) { - continue - } - stdout = append(stdout, hashMatching...) - stdout = append(stdout, '\n') - } - } - } - - return repo.parsePrettyFormatLogToList(bytes.TrimSuffix(stdout, []byte{'\n'})) -} - -func (repo *Repository) getFilesChanged(id1, id2 string) ([]string, error) { - stdout, err := NewCommand("diff", "--name-only", id1, id2).RunInDirBytes(repo.Path) - if err != nil { - return nil, err - } - return strings.Split(string(stdout), "\n"), nil -} - -// FileChangedBetweenCommits Returns true if the file changed between commit IDs id1 and id2 -// You must ensure that id1 and id2 are valid commit ids. -func (repo *Repository) FileChangedBetweenCommits(filename, id1, id2 string) (bool, error) { - stdout, err := NewCommand("diff", "--name-only", "-z", id1, id2, "--", filename).RunInDirBytes(repo.Path) - if err != nil { - return false, err - } - return len(strings.TrimSpace(string(stdout))) > 0, nil -} - -// FileCommitsCount return the number of files at a revison -func (repo *Repository) FileCommitsCount(revision, file string) (int64, error) { - return CommitsCountFiles(repo.Path, []string{revision}, []string{file}) -} - -// CommitsByFileAndRange return the commits according revison file and the page -func (repo *Repository) CommitsByFileAndRange(revision, file string, page int) (*list.List, error) { - stdout, err := NewCommand("log", revision, "--follow", "--skip="+strconv.Itoa((page-1)*50), - "--max-count="+strconv.Itoa(CommitsRangeSize), prettyLogFormat, "--", file).RunInDirBytes(repo.Path) - if err != nil { - return nil, err - } - return repo.parsePrettyFormatLogToList(stdout) -} - -// CommitsByFileAndRangeNoFollow return the commits according revison file and the page -func (repo *Repository) CommitsByFileAndRangeNoFollow(revision, file string, page int) (*list.List, error) { - stdout, err := NewCommand("log", revision, "--skip="+strconv.Itoa((page-1)*50), - "--max-count="+strconv.Itoa(CommitsRangeSize), prettyLogFormat, "--", file).RunInDirBytes(repo.Path) - if err != nil { - return nil, err - } - return repo.parsePrettyFormatLogToList(stdout) -} - -// FilesCountBetween return the number of files changed between two commits -func (repo *Repository) FilesCountBetween(startCommitID, endCommitID string) (int, error) { - stdout, err := NewCommand("diff", "--name-only", startCommitID+"..."+endCommitID).RunInDir(repo.Path) - if err != nil && strings.Contains(err.Error(), "no merge base") { - // git >= 2.28 now returns an error if startCommitID and endCommitID have become unrelated. - // previously it would return the results of git diff --name-only startCommitID endCommitID so let's try that... - stdout, err = NewCommand("diff", "--name-only", startCommitID, endCommitID).RunInDir(repo.Path) - } - if err != nil { - return 0, err - } - return len(strings.Split(stdout, "\n")) - 1, nil -} - -// CommitsBetween returns a list that contains commits between [last, before). -func (repo *Repository) CommitsBetween(last *Commit, before *Commit) (*list.List, error) { - var stdout []byte - var err error - if before == nil { - stdout, err = NewCommand("rev-list", last.ID.String()).RunInDirBytes(repo.Path) - } else { - stdout, err = NewCommand("rev-list", before.ID.String()+"..."+last.ID.String()).RunInDirBytes(repo.Path) - if err != nil && strings.Contains(err.Error(), "no merge base") { - // future versions of git >= 2.28 are likely to return an error if before and last have become unrelated. - // previously it would return the results of git rev-list before last so let's try that... - stdout, err = NewCommand("rev-list", before.ID.String(), last.ID.String()).RunInDirBytes(repo.Path) - } - } - if err != nil { - return nil, err - } - return repo.parsePrettyFormatLogToList(bytes.TrimSpace(stdout)) -} - -// CommitsBetweenLimit returns a list that contains at most limit commits skipping the first skip commits between [last, before) -func (repo *Repository) CommitsBetweenLimit(last *Commit, before *Commit, limit, skip int) (*list.List, error) { - var stdout []byte - var err error - if before == nil { - stdout, err = NewCommand("rev-list", "--max-count", strconv.Itoa(limit), "--skip", strconv.Itoa(skip), last.ID.String()).RunInDirBytes(repo.Path) - } else { - stdout, err = NewCommand("rev-list", "--max-count", strconv.Itoa(limit), "--skip", strconv.Itoa(skip), before.ID.String()+"..."+last.ID.String()).RunInDirBytes(repo.Path) - if err != nil && strings.Contains(err.Error(), "no merge base") { - // future versions of git >= 2.28 are likely to return an error if before and last have become unrelated. - // previously it would return the results of git rev-list --max-count n before last so let's try that... - stdout, err = NewCommand("rev-list", "--max-count", strconv.Itoa(limit), "--skip", strconv.Itoa(skip), before.ID.String(), last.ID.String()).RunInDirBytes(repo.Path) - } - } - if err != nil { - return nil, err - } - return repo.parsePrettyFormatLogToList(bytes.TrimSpace(stdout)) -} - -// CommitsBetweenIDs return commits between twoe commits -func (repo *Repository) CommitsBetweenIDs(last, before string) (*list.List, error) { - lastCommit, err := repo.GetCommit(last) - if err != nil { - return nil, err - } - if before == "" { - return repo.CommitsBetween(lastCommit, nil) - } - beforeCommit, err := repo.GetCommit(before) - if err != nil { - return nil, err - } - return repo.CommitsBetween(lastCommit, beforeCommit) -} - -// CommitsCountBetween return numbers of commits between two commits -func (repo *Repository) CommitsCountBetween(start, end string) (int64, error) { - count, err := CommitsCountFiles(repo.Path, []string{start + "..." + end}, []string{}) - if err != nil && strings.Contains(err.Error(), "no merge base") { - // future versions of git >= 2.28 are likely to return an error if before and last have become unrelated. - // previously it would return the results of git rev-list before last so let's try that... - return CommitsCountFiles(repo.Path, []string{start, end}, []string{}) - } - - return count, err -} - -// commitsBefore the limit is depth, not total number of returned commits. -func (repo *Repository) commitsBefore(id SHA1, limit int) (*list.List, error) { - cmd := NewCommand("log") - if limit > 0 { - cmd.AddArguments("-"+strconv.Itoa(limit), prettyLogFormat, id.String()) - } else { - cmd.AddArguments(prettyLogFormat, id.String()) - } - - stdout, err := cmd.RunInDirBytes(repo.Path) - if err != nil { - return nil, err - } - - formattedLog, err := repo.parsePrettyFormatLogToList(bytes.TrimSpace(stdout)) - if err != nil { - return nil, err - } - - commits := list.New() - for logEntry := formattedLog.Front(); logEntry != nil; logEntry = logEntry.Next() { - commit := logEntry.Value.(*Commit) - branches, err := repo.getBranches(commit, 2) - if err != nil { - return nil, err - } - - if len(branches) > 1 { - break - } - - commits.PushBack(commit) - } - - return commits, nil -} - -func (repo *Repository) getCommitsBefore(id SHA1) (*list.List, error) { - return repo.commitsBefore(id, 0) -} - -func (repo *Repository) getCommitsBeforeLimit(id SHA1, num int) (*list.List, error) { - return repo.commitsBefore(id, num) -} - -func (repo *Repository) getBranches(commit *Commit, limit int) ([]string, error) { - if CheckGitVersionAtLeast("2.7.0") == nil { - stdout, err := NewCommand("for-each-ref", "--count="+strconv.Itoa(limit), "--format=%(refname:strip=2)", "--contains", commit.ID.String(), BranchPrefix).RunInDir(repo.Path) - if err != nil { - return nil, err - } - - branches := strings.Fields(stdout) - return branches, nil - } - - stdout, err := NewCommand("branch", "--contains", commit.ID.String()).RunInDir(repo.Path) - if err != nil { - return nil, err - } - - refs := strings.Split(stdout, "\n") - - var max int - if len(refs) > limit { - max = limit - } else { - max = len(refs) - 1 - } - - branches := make([]string, max) - for i, ref := range refs[:max] { - parts := strings.Fields(ref) - - branches[i] = parts[len(parts)-1] - } - return branches, nil -} - -// GetCommitsFromIDs get commits from commit IDs -func (repo *Repository) GetCommitsFromIDs(commitIDs []string) (commits *list.List) { - commits = list.New() - - for _, commitID := range commitIDs { - commit, err := repo.GetCommit(commitID) - if err == nil && commit != nil { - commits.PushBack(commit) - } - } - - return commits -} diff --git a/modules/git/repo_commitgraph.go b/modules/git/repo_commitgraph_gogit.go similarity index 100% rename from modules/git/repo_commitgraph.go rename to modules/git/repo_commitgraph_gogit.go diff --git a/modules/git/repo_language_stats.go b/modules/git/repo_language_stats.go index 2ab12de4fc83f..ac23caa0fc04b 100644 --- a/modules/git/repo_language_stats.go +++ b/modules/git/repo_language_stats.go @@ -2,115 +2,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -// +build !nogogit - package git -import ( - "bytes" - "io" - "io/ioutil" - - "code.gitea.io/gitea/modules/analyze" - - "github.com/go-enry/go-enry/v2" - "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/plumbing/object" -) - const fileSizeLimit int64 = 16 * 1024 // 16 KiB const bigFileSize int64 = 1024 * 1024 // 1 MiB - -// GetLanguageStats calculates language stats for git repository at specified commit -func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, error) { - r, err := git.PlainOpen(repo.Path) - if err != nil { - return nil, err - } - - rev, err := r.ResolveRevision(plumbing.Revision(commitID)) - if err != nil { - return nil, err - } - - commit, err := r.CommitObject(*rev) - if err != nil { - return nil, err - } - - tree, err := commit.Tree() - if err != nil { - return nil, err - } - - sizes := make(map[string]int64) - err = tree.Files().ForEach(func(f *object.File) error { - if f.Size == 0 || enry.IsVendor(f.Name) || enry.IsDotFile(f.Name) || - enry.IsDocumentation(f.Name) || enry.IsConfiguration(f.Name) { - return nil - } - - // If content can not be read or file is too big just do detection by filename - var content []byte - if f.Size <= bigFileSize { - content, _ = readFile(f, fileSizeLimit) - } - if enry.IsGenerated(f.Name, content) { - return nil - } - - // TODO: Use .gitattributes file for linguist overrides - - language := analyze.GetCodeLanguage(f.Name, content) - if language == enry.OtherLanguage || language == "" { - return nil - } - - // group languages, such as Pug -> HTML; SCSS -> CSS - group := enry.GetLanguageGroup(language) - if group != "" { - language = group - } - - sizes[language] += f.Size - - return nil - }) - if err != nil { - return nil, err - } - - // filter special languages unless they are the only language - if len(sizes) > 1 { - for language := range sizes { - langtype := enry.GetLanguageType(language) - if langtype != enry.Programming && langtype != enry.Markup { - delete(sizes, language) - } - } - } - - return sizes, nil -} - -func readFile(f *object.File, limit int64) ([]byte, error) { - r, err := f.Reader() - if err != nil { - return nil, err - } - defer r.Close() - - if limit <= 0 { - return ioutil.ReadAll(r) - } - - size := f.Size - if limit > 0 && size > limit { - size = limit - } - buf := bytes.NewBuffer(nil) - buf.Grow(int(size)) - _, err = io.Copy(buf, io.LimitReader(r, limit)) - return buf.Bytes(), err -} diff --git a/modules/git/repo_language_stats_gogit.go b/modules/git/repo_language_stats_gogit.go new file mode 100644 index 0000000000000..d802a7a1c0c45 --- /dev/null +++ b/modules/git/repo_language_stats_gogit.go @@ -0,0 +1,113 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build !nogogit + +package git + +import ( + "bytes" + "io" + "io/ioutil" + + "code.gitea.io/gitea/modules/analyze" + + "github.com/go-enry/go-enry/v2" + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/object" +) + +// GetLanguageStats calculates language stats for git repository at specified commit +func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, error) { + r, err := git.PlainOpen(repo.Path) + if err != nil { + return nil, err + } + + rev, err := r.ResolveRevision(plumbing.Revision(commitID)) + if err != nil { + return nil, err + } + + commit, err := r.CommitObject(*rev) + if err != nil { + return nil, err + } + + tree, err := commit.Tree() + if err != nil { + return nil, err + } + + sizes := make(map[string]int64) + err = tree.Files().ForEach(func(f *object.File) error { + if f.Size == 0 || enry.IsVendor(f.Name) || enry.IsDotFile(f.Name) || + enry.IsDocumentation(f.Name) || enry.IsConfiguration(f.Name) { + return nil + } + + // If content can not be read or file is too big just do detection by filename + var content []byte + if f.Size <= bigFileSize { + content, _ = readFile(f, fileSizeLimit) + } + if enry.IsGenerated(f.Name, content) { + return nil + } + + // TODO: Use .gitattributes file for linguist overrides + + language := analyze.GetCodeLanguage(f.Name, content) + if language == enry.OtherLanguage || language == "" { + return nil + } + + // group languages, such as Pug -> HTML; SCSS -> CSS + group := enry.GetLanguageGroup(language) + if group != "" { + language = group + } + + sizes[language] += f.Size + + return nil + }) + if err != nil { + return nil, err + } + + // filter special languages unless they are the only language + if len(sizes) > 1 { + for language := range sizes { + langtype := enry.GetLanguageType(language) + if langtype != enry.Programming && langtype != enry.Markup { + delete(sizes, language) + } + } + } + + return sizes, nil +} + +func readFile(f *object.File, limit int64) ([]byte, error) { + r, err := f.Reader() + if err != nil { + return nil, err + } + defer r.Close() + + if limit <= 0 { + return ioutil.ReadAll(r) + } + + size := f.Size + if limit > 0 && size > limit { + size = limit + } + buf := bytes.NewBuffer(nil) + buf.Grow(int(size)) + _, err = io.Copy(buf, io.LimitReader(r, limit)) + return buf.Bytes(), err +} diff --git a/modules/git/repo_language_stats_nogogit.go b/modules/git/repo_language_stats_nogogit.go index 1a2d45ca733ed..6be510ef22d2d 100644 --- a/modules/git/repo_language_stats_nogogit.go +++ b/modules/git/repo_language_stats_nogogit.go @@ -16,9 +16,6 @@ import ( "github.com/go-enry/go-enry/v2" ) -const fileSizeLimit int64 = 16 * 1024 // 16 KiB -const bigFileSize int64 = 1024 * 1024 // 1 MiB - // GetLanguageStats calculates language stats for git repository at specified commit func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, error) { // FIXME: We can be more efficient here... diff --git a/modules/git/repo_nogogit.go b/modules/git/repo_nogogit.go deleted file mode 100644 index efaa843ebf9bf..0000000000000 --- a/modules/git/repo_nogogit.go +++ /dev/null @@ -1,403 +0,0 @@ -// Copyright 2015 The Gogs Authors. All rights reserved. -// Copyright 2017 The Gitea Authors. All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. - -// +build nogogit - -package git - -import ( - "bytes" - "container/list" - "errors" - "fmt" - "os" - "path" - "path/filepath" - "strconv" - "strings" - "time" - - "github.com/unknwon/com" -) - -// Repository represents a Git repository. -type Repository struct { - Path string - - tagCache *ObjectCache - - gpgSettings *GPGSettings -} - -// GPGSettings represents the default GPG settings for this repository -type GPGSettings struct { - Sign bool - KeyID string - Email string - Name string - PublicKeyContent string -} - -const prettyLogFormat = `--pretty=format:%H` - -// GetAllCommitsCount returns count of all commits in repository -func (repo *Repository) GetAllCommitsCount() (int64, error) { - return AllCommitsCount(repo.Path, false) -} - -func (repo *Repository) parsePrettyFormatLogToList(logs []byte) (*list.List, error) { - l := list.New() - if len(logs) == 0 { - return l, nil - } - - parts := bytes.Split(logs, []byte{'\n'}) - - for _, commitID := range parts { - commit, err := repo.GetCommit(string(commitID)) - if err != nil { - return nil, err - } - l.PushBack(commit) - } - - return l, nil -} - -// IsRepoURLAccessible checks if given repository URL is accessible. -func IsRepoURLAccessible(url string) bool { - _, err := NewCommand("ls-remote", "-q", "-h", url, "HEAD").Run() - return err == nil -} - -// InitRepository initializes a new Git repository. -func InitRepository(repoPath string, bare bool) error { - err := os.MkdirAll(repoPath, os.ModePerm) - if err != nil { - return err - } - - cmd := NewCommand("init") - if bare { - cmd.AddArguments("--bare") - } - _, err = cmd.RunInDir(repoPath) - return err -} - -// OpenRepository opens the repository at the given path. -func OpenRepository(repoPath string) (*Repository, error) { - repoPath, err := filepath.Abs(repoPath) - if err != nil { - return nil, err - } else if !isDir(repoPath) { - return nil, errors.New("no such file or directory") - } - return &Repository{ - Path: repoPath, - tagCache: newObjectCache(), - }, nil -} - -// Close this repository, in particular close the underlying gogitStorage if this is not nil -func (repo *Repository) Close() { -} - -// IsEmpty Check if repository is empty. -func (repo *Repository) IsEmpty() (bool, error) { - var errbuf strings.Builder - if err := NewCommand("log", "-1").RunInDirPipeline(repo.Path, nil, &errbuf); err != nil { - if strings.Contains(errbuf.String(), "fatal: bad default revision 'HEAD'") || - strings.Contains(errbuf.String(), "fatal: your current branch 'master' does not have any commits yet") { - return true, nil - } - return true, fmt.Errorf("check empty: %v - %s", err, errbuf.String()) - } - - return false, nil -} - -// CloneRepoOptions options when clone a repository -type CloneRepoOptions struct { - Timeout time.Duration - Mirror bool - Bare bool - Quiet bool - Branch string - Shared bool - NoCheckout bool - Depth int -} - -// Clone clones original repository to target path. -func Clone(from, to string, opts CloneRepoOptions) (err error) { - cargs := make([]string, len(GlobalCommandArgs)) - copy(cargs, GlobalCommandArgs) - return CloneWithArgs(from, to, cargs, opts) -} - -// CloneWithArgs original repository to target path. -func CloneWithArgs(from, to string, args []string, opts CloneRepoOptions) (err error) { - toDir := path.Dir(to) - if err = os.MkdirAll(toDir, os.ModePerm); err != nil { - return err - } - - cmd := NewCommandNoGlobals(args...).AddArguments("clone") - if opts.Mirror { - cmd.AddArguments("--mirror") - } - if opts.Bare { - cmd.AddArguments("--bare") - } - if opts.Quiet { - cmd.AddArguments("--quiet") - } - if opts.Shared { - cmd.AddArguments("-s") - } - if opts.NoCheckout { - cmd.AddArguments("--no-checkout") - } - if opts.Depth > 0 { - cmd.AddArguments("--depth", strconv.Itoa(opts.Depth)) - } - - if len(opts.Branch) > 0 { - cmd.AddArguments("-b", opts.Branch) - } - cmd.AddArguments("--", from, to) - - if opts.Timeout <= 0 { - opts.Timeout = -1 - } - - _, err = cmd.RunTimeout(opts.Timeout) - return err -} - -// PullRemoteOptions options when pull from remote -type PullRemoteOptions struct { - Timeout time.Duration - All bool - Rebase bool - Remote string - Branch string -} - -// Pull pulls changes from remotes. -func Pull(repoPath string, opts PullRemoteOptions) error { - cmd := NewCommand("pull") - if opts.Rebase { - cmd.AddArguments("--rebase") - } - if opts.All { - cmd.AddArguments("--all") - } else { - cmd.AddArguments("--", opts.Remote, opts.Branch) - } - - if opts.Timeout <= 0 { - opts.Timeout = -1 - } - - _, err := cmd.RunInDirTimeout(opts.Timeout, repoPath) - return err -} - -// PushOptions options when push to remote -type PushOptions struct { - Remote string - Branch string - Force bool - Env []string -} - -// Push pushs local commits to given remote branch. -func Push(repoPath string, opts PushOptions) error { - cmd := NewCommand("push") - if opts.Force { - cmd.AddArguments("-f") - } - cmd.AddArguments("--", opts.Remote, opts.Branch) - var outbuf, errbuf strings.Builder - - err := cmd.RunInDirTimeoutEnvPipeline(opts.Env, -1, repoPath, &outbuf, &errbuf) - if err != nil { - if strings.Contains(errbuf.String(), "non-fast-forward") { - return &ErrPushOutOfDate{ - StdOut: outbuf.String(), - StdErr: errbuf.String(), - Err: err, - } - } else if strings.Contains(errbuf.String(), "! [remote rejected]") { - err := &ErrPushRejected{ - StdOut: outbuf.String(), - StdErr: errbuf.String(), - Err: err, - } - err.GenerateMessage() - return err - } - } - - if errbuf.Len() > 0 && err != nil { - return fmt.Errorf("%v - %s", err, errbuf.String()) - } - - return err -} - -// CheckoutOptions options when heck out some branch -type CheckoutOptions struct { - Timeout time.Duration - Branch string - OldBranch string -} - -// Checkout checkouts a branch -func Checkout(repoPath string, opts CheckoutOptions) error { - cmd := NewCommand("checkout") - if len(opts.OldBranch) > 0 { - cmd.AddArguments("-b") - } - - if opts.Timeout <= 0 { - opts.Timeout = -1 - } - - cmd.AddArguments(opts.Branch) - - if len(opts.OldBranch) > 0 { - cmd.AddArguments(opts.OldBranch) - } - - _, err := cmd.RunInDirTimeout(opts.Timeout, repoPath) - return err -} - -// ResetHEAD resets HEAD to given revision or head of branch. -func ResetHEAD(repoPath string, hard bool, revision string) error { - cmd := NewCommand("reset") - if hard { - cmd.AddArguments("--hard") - } - _, err := cmd.AddArguments(revision).RunInDir(repoPath) - return err -} - -// MoveFile moves a file to another file or directory. -func MoveFile(repoPath, oldTreeName, newTreeName string) error { - _, err := NewCommand("mv").AddArguments(oldTreeName, newTreeName).RunInDir(repoPath) - return err -} - -// CountObject represents repository count objects report -type CountObject struct { - Count int64 - Size int64 - InPack int64 - Packs int64 - SizePack int64 - PrunePack int64 - Garbage int64 - SizeGarbage int64 -} - -const ( - statCount = "count: " - statSize = "size: " - statInpack = "in-pack: " - statPacks = "packs: " - statSizePack = "size-pack: " - statPrunePackage = "prune-package: " - statGarbage = "garbage: " - statSizeGarbage = "size-garbage: " -) - -// CountObjects returns the results of git count-objects on the repoPath -func CountObjects(repoPath string) (*CountObject, error) { - cmd := NewCommand("count-objects", "-v") - stdout, err := cmd.RunInDir(repoPath) - if err != nil { - return nil, err - } - - return parseSize(stdout), nil -} - -// parseSize parses the output from count-objects and return a CountObject -func parseSize(objects string) *CountObject { - repoSize := new(CountObject) - for _, line := range strings.Split(objects, "\n") { - switch { - case strings.HasPrefix(line, statCount): - repoSize.Count = com.StrTo(line[7:]).MustInt64() - case strings.HasPrefix(line, statSize): - repoSize.Size = com.StrTo(line[6:]).MustInt64() * 1024 - case strings.HasPrefix(line, statInpack): - repoSize.InPack = com.StrTo(line[9:]).MustInt64() - case strings.HasPrefix(line, statPacks): - repoSize.Packs = com.StrTo(line[7:]).MustInt64() - case strings.HasPrefix(line, statSizePack): - repoSize.SizePack = com.StrTo(line[11:]).MustInt64() * 1024 - case strings.HasPrefix(line, statPrunePackage): - repoSize.PrunePack = com.StrTo(line[16:]).MustInt64() - case strings.HasPrefix(line, statGarbage): - repoSize.Garbage = com.StrTo(line[9:]).MustInt64() - case strings.HasPrefix(line, statSizeGarbage): - repoSize.SizeGarbage = com.StrTo(line[14:]).MustInt64() * 1024 - } - } - return repoSize -} - -// GetLatestCommitTime returns time for latest commit in repository (across all branches) -func GetLatestCommitTime(repoPath string) (time.Time, error) { - cmd := NewCommand("for-each-ref", "--sort=-committerdate", "refs/heads/", "--count", "1", "--format=%(committerdate)") - stdout, err := cmd.RunInDir(repoPath) - if err != nil { - return time.Time{}, err - } - commitTime := strings.TrimSpace(stdout) - return time.Parse(GitTimeLayout, commitTime) -} - -// DivergeObject represents commit count diverging commits -type DivergeObject struct { - Ahead int - Behind int -} - -func checkDivergence(repoPath string, baseBranch string, targetBranch string) (int, error) { - branches := fmt.Sprintf("%s..%s", baseBranch, targetBranch) - cmd := NewCommand("rev-list", "--count", branches) - stdout, err := cmd.RunInDir(repoPath) - if err != nil { - return -1, err - } - outInteger, errInteger := strconv.Atoi(strings.Trim(stdout, "\n")) - if errInteger != nil { - return -1, errInteger - } - return outInteger, nil -} - -// GetDivergingCommits returns the number of commits a targetBranch is ahead or behind a baseBranch -func GetDivergingCommits(repoPath string, baseBranch string, targetBranch string) (DivergeObject, error) { - // $(git rev-list --count master..feature) commits ahead of master - ahead, errorAhead := checkDivergence(repoPath, baseBranch, targetBranch) - if errorAhead != nil { - return DivergeObject{}, errorAhead - } - - // $(git rev-list --count feature..master) commits behind master - behind, errorBehind := checkDivergence(repoPath, targetBranch, baseBranch) - if errorBehind != nil { - return DivergeObject{}, errorBehind - } - - return DivergeObject{ahead, behind}, nil -} diff --git a/modules/git/repo_ref.go b/modules/git/repo_ref.go index ef4f73a1ebbfe..397434e12f2a6 100644 --- a/modules/git/repo_ref.go +++ b/modules/git/repo_ref.go @@ -2,56 +2,9 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -// +build !nogogit - package git -import ( - "strings" - - "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/plumbing" -) - // GetRefs returns all references of the repository. func (repo *Repository) GetRefs() ([]*Reference, error) { return repo.GetRefsFiltered("") } - -// GetRefsFiltered returns all references of the repository that matches patterm exactly or starting with. -func (repo *Repository) GetRefsFiltered(pattern string) ([]*Reference, error) { - r, err := git.PlainOpen(repo.Path) - if err != nil { - return nil, err - } - - refsIter, err := r.References() - if err != nil { - return nil, err - } - refs := make([]*Reference, 0) - if err = refsIter.ForEach(func(ref *plumbing.Reference) error { - if ref.Name() != plumbing.HEAD && !ref.Name().IsRemote() && - (pattern == "" || strings.HasPrefix(ref.Name().String(), pattern)) { - refType := string(ObjectCommit) - if ref.Name().IsTag() { - // tags can be of type `commit` (lightweight) or `tag` (annotated) - if tagType, _ := repo.GetTagType(ref.Hash()); err == nil { - refType = tagType - } - } - r := &Reference{ - Name: ref.Name().String(), - Object: ref.Hash(), - Type: refType, - repo: repo, - } - refs = append(refs, r) - } - return nil - }); err != nil { - return nil, err - } - - return refs, nil -} diff --git a/modules/git/repo_ref_gogit.go b/modules/git/repo_ref_gogit.go new file mode 100644 index 0000000000000..2b14c896e099b --- /dev/null +++ b/modules/git/repo_ref_gogit.go @@ -0,0 +1,52 @@ +// Copyright 2018 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build !nogogit + +package git + +import ( + "strings" + + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" +) + +// GetRefsFiltered returns all references of the repository that matches patterm exactly or starting with. +func (repo *Repository) GetRefsFiltered(pattern string) ([]*Reference, error) { + r, err := git.PlainOpen(repo.Path) + if err != nil { + return nil, err + } + + refsIter, err := r.References() + if err != nil { + return nil, err + } + refs := make([]*Reference, 0) + if err = refsIter.ForEach(func(ref *plumbing.Reference) error { + if ref.Name() != plumbing.HEAD && !ref.Name().IsRemote() && + (pattern == "" || strings.HasPrefix(ref.Name().String(), pattern)) { + refType := string(ObjectCommit) + if ref.Name().IsTag() { + // tags can be of type `commit` (lightweight) or `tag` (annotated) + if tagType, _ := repo.GetTagType(ref.Hash()); err == nil { + refType = tagType + } + } + r := &Reference{ + Name: ref.Name().String(), + Object: ref.Hash(), + Type: refType, + repo: repo, + } + refs = append(refs, r) + } + return nil + }); err != nil { + return nil, err + } + + return refs, nil +} diff --git a/modules/git/repo_ref_nogogit.go b/modules/git/repo_ref_nogogit.go index 74534046dd96e..15ca3c68fdc70 100644 --- a/modules/git/repo_ref_nogogit.go +++ b/modules/git/repo_ref_nogogit.go @@ -1,4 +1,4 @@ -// Copyright 2018 The Gitea Authors. All rights reserved. +// Copyright 2020 The Gitea Authors. All rights reserved. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. @@ -12,11 +12,6 @@ import ( "strings" ) -// GetRefs returns all references of the repository. -func (repo *Repository) GetRefs() ([]*Reference, error) { - return repo.GetRefsFiltered("") -} - // GetRefsFiltered returns all references of the repository that matches patterm exactly or starting with. func (repo *Repository) GetRefsFiltered(pattern string) ([]*Reference, error) { stdoutReader, stdoutWriter := io.Pipe() diff --git a/modules/git/repo_tag.go b/modules/git/repo_tag.go index 5bd85b8462e09..3e8f80fe82dd6 100644 --- a/modules/git/repo_tag.go +++ b/modules/git/repo_tag.go @@ -3,15 +3,11 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -// +build !nogogit - package git import ( "fmt" "strings" - - "github.com/go-git/go-git/v5/plumbing" ) // TagPrefix tags prefix path on the repository @@ -22,12 +18,6 @@ func IsTagExist(repoPath, name string) bool { return IsReferenceExist(repoPath, TagPrefix+name) } -// IsTagExist returns true if given tag exists in the repository. -func (repo *Repository) IsTagExist(name string) bool { - _, err := repo.gogitRepo.Reference(plumbing.ReferenceName(TagPrefix+name), true) - return err == nil -} - // CreateTag create one tag in the repository func (repo *Repository) CreateTag(name, revision string) error { _, err := NewCommand("tag", "--", name, revision).RunInDir(repo.Path) @@ -226,29 +216,6 @@ func (repo *Repository) GetTagInfos(page, pageSize int) ([]*Tag, error) { return tags, nil } -// GetTags returns all tags of the repository. -func (repo *Repository) GetTags() ([]string, error) { - var tagNames []string - - tags, err := repo.gogitRepo.Tags() - if err != nil { - return nil, err - } - - _ = tags.ForEach(func(tag *plumbing.Reference) error { - tagNames = append(tagNames, strings.TrimPrefix(tag.Name().String(), TagPrefix)) - return nil - }) - - // Reverse order - for i := 0; i < len(tagNames)/2; i++ { - j := len(tagNames) - i - 1 - tagNames[i], tagNames[j] = tagNames[j], tagNames[i] - } - - return tagNames, nil -} - // GetTagType gets the type of the tag, either commit (simple) or tag (annotated) func (repo *Repository) GetTagType(id SHA1) (string, error) { // Get tag type diff --git a/modules/git/repo_tag_gogit.go b/modules/git/repo_tag_gogit.go new file mode 100644 index 0000000000000..a53273a38e9b2 --- /dev/null +++ b/modules/git/repo_tag_gogit.go @@ -0,0 +1,43 @@ +// Copyright 2015 The Gogs Authors. All rights reserved. +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build !nogogit + +package git + +import ( + "strings" + + "github.com/go-git/go-git/v5/plumbing" +) + +// IsTagExist returns true if given tag exists in the repository. +func (repo *Repository) IsTagExist(name string) bool { + _, err := repo.gogitRepo.Reference(plumbing.ReferenceName(TagPrefix+name), true) + return err == nil +} + +// GetTags returns all tags of the repository. +func (repo *Repository) GetTags() ([]string, error) { + var tagNames []string + + tags, err := repo.gogitRepo.Tags() + if err != nil { + return nil, err + } + + _ = tags.ForEach(func(tag *plumbing.Reference) error { + tagNames = append(tagNames, strings.TrimPrefix(tag.Name().String(), TagPrefix)) + return nil + }) + + // Reverse order + for i := 0; i < len(tagNames)/2; i++ { + j := len(tagNames) - i - 1 + tagNames[i], tagNames[j] = tagNames[j], tagNames[i] + } + + return tagNames, nil +} diff --git a/modules/git/repo_tag_nogogit.go b/modules/git/repo_tag_nogogit.go index 52ed3da2eda9f..fba93e135e57a 100644 --- a/modules/git/repo_tag_nogogit.go +++ b/modules/git/repo_tag_nogogit.go @@ -7,258 +7,12 @@ package git -import ( - "fmt" - "strings" -) - -// TagPrefix tags prefix path on the repository -const TagPrefix = "refs/tags/" - -// IsTagExist returns true if given tag exists in the repository. -func IsTagExist(repoPath, name string) bool { - return IsReferenceExist(repoPath, TagPrefix+name) -} - // IsTagExist returns true if given tag exists in the repository. func (repo *Repository) IsTagExist(name string) bool { return IsReferenceExist(repo.Path, TagPrefix+name) } -// CreateTag create one tag in the repository -func (repo *Repository) CreateTag(name, revision string) error { - _, err := NewCommand("tag", "--", name, revision).RunInDir(repo.Path) - return err -} - -// CreateAnnotatedTag create one annotated tag in the repository -func (repo *Repository) CreateAnnotatedTag(name, message, revision string) error { - _, err := NewCommand("tag", "-a", "-m", message, "--", name, revision).RunInDir(repo.Path) - return err -} - -func (repo *Repository) getTag(id SHA1) (*Tag, error) { - t, ok := repo.tagCache.Get(id.String()) - if ok { - log("Hit cache: %s", id) - tagClone := *t.(*Tag) - return &tagClone, nil - } - - // Get tag name - name, err := repo.GetTagNameBySHA(id.String()) - if err != nil { - return nil, err - } - - tp, err := repo.GetTagType(id) - if err != nil { - return nil, err - } - - // Get the commit ID and tag ID (may be different for annotated tag) for the returned tag object - commitIDStr, err := repo.GetTagCommitID(name) - if err != nil { - // every tag should have a commit ID so return all errors - return nil, err - } - commitID, err := NewIDFromString(commitIDStr) - if err != nil { - return nil, err - } - - // tagID defaults to the commit ID as the tag ID and then tries to get a tag ID (only annotated tags) - tagID := commitID - if tagIDStr, err := repo.GetTagID(name); err != nil { - // if the err is NotExist then we can ignore and just keep tagID as ID (is lightweight tag) - // all other errors we return - if !IsErrNotExist(err) { - return nil, err - } - } else { - tagID, err = NewIDFromString(tagIDStr) - if err != nil { - return nil, err - } - } - - // If type is "commit, the tag is a lightweight tag - if ObjectType(tp) == ObjectCommit { - commit, err := repo.GetCommit(id.String()) - if err != nil { - return nil, err - } - tag := &Tag{ - Name: name, - ID: tagID, - Object: commitID, - Type: string(ObjectCommit), - Tagger: commit.Committer, - Message: commit.Message(), - repo: repo, - } - - repo.tagCache.Set(id.String(), tag) - return tag, nil - } - - // The tag is an annotated tag with a message. - data, err := NewCommand("cat-file", "-p", id.String()).RunInDirBytes(repo.Path) - if err != nil { - return nil, err - } - - tag, err := parseTagData(data) - if err != nil { - return nil, err - } - - tag.Name = name - tag.ID = id - tag.repo = repo - tag.Type = tp - - repo.tagCache.Set(id.String(), tag) - return tag, nil -} - -// GetTagNameBySHA returns the name of a tag from its tag object SHA or commit SHA -func (repo *Repository) GetTagNameBySHA(sha string) (string, error) { - if len(sha) < 5 { - return "", fmt.Errorf("SHA is too short: %s", sha) - } - - stdout, err := NewCommand("show-ref", "--tags", "-d").RunInDir(repo.Path) - if err != nil { - return "", err - } - - tagRefs := strings.Split(stdout, "\n") - for _, tagRef := range tagRefs { - if len(strings.TrimSpace(tagRef)) > 0 { - fields := strings.Fields(tagRef) - if strings.HasPrefix(fields[0], sha) && strings.HasPrefix(fields[1], TagPrefix) { - name := fields[1][len(TagPrefix):] - // annotated tags show up twice, we should only return if is not the ^{} ref - if !strings.HasSuffix(name, "^{}") { - return name, nil - } - } - } - } - return "", ErrNotExist{ID: sha} -} - -// GetTagID returns the object ID for a tag (annotated tags have both an object SHA AND a commit SHA) -func (repo *Repository) GetTagID(name string) (string, error) { - stdout, err := NewCommand("show-ref", "--tags", "--", name).RunInDir(repo.Path) - if err != nil { - return "", err - } - // Make sure exact match is used: "v1" != "release/v1" - for _, line := range strings.Split(stdout, "\n") { - fields := strings.Fields(line) - if len(fields) == 2 && fields[1] == "refs/tags/"+name { - return fields[0], nil - } - } - return "", ErrNotExist{ID: name} -} - -// GetTag returns a Git tag by given name. -func (repo *Repository) GetTag(name string) (*Tag, error) { - idStr, err := repo.GetTagID(name) - if err != nil { - return nil, err - } - - id, err := NewIDFromString(idStr) - if err != nil { - return nil, err - } - - tag, err := repo.getTag(id) - if err != nil { - return nil, err - } - return tag, nil -} - -// GetTagInfos returns all tag infos of the repository. -func (repo *Repository) GetTagInfos(page, pageSize int) ([]*Tag, error) { - // TODO this a slow implementation, makes one git command per tag - stdout, err := NewCommand("tag").RunInDir(repo.Path) - if err != nil { - return nil, err - } - - tagNames := strings.Split(strings.TrimRight(stdout, "\n"), "\n") - - if page != 0 { - skip := (page - 1) * pageSize - if skip >= len(tagNames) { - return nil, nil - } - if (len(tagNames) - skip) < pageSize { - pageSize = len(tagNames) - skip - } - tagNames = tagNames[skip : skip+pageSize] - } - - var tags = make([]*Tag, 0, len(tagNames)) - for _, tagName := range tagNames { - tagName = strings.TrimSpace(tagName) - if len(tagName) == 0 { - continue - } - - tag, err := repo.GetTag(tagName) - if err != nil { - return nil, err - } - tag.Name = tagName - tags = append(tags, tag) - } - sortTagsByTime(tags) - return tags, nil -} - // GetTags returns all tags of the repository. func (repo *Repository) GetTags() ([]string, error) { return callShowRef(repo.Path, TagPrefix, "--tags") } - -// GetTagType gets the type of the tag, either commit (simple) or tag (annotated) -func (repo *Repository) GetTagType(id SHA1) (string, error) { - // Get tag type - stdout, err := NewCommand("cat-file", "-t", id.String()).RunInDir(repo.Path) - if err != nil { - return "", err - } - if len(stdout) == 0 { - return "", ErrNotExist{ID: id.String()} - } - return strings.TrimSpace(stdout), nil -} - -// GetAnnotatedTag returns a Git tag by its SHA, must be an annotated tag -func (repo *Repository) GetAnnotatedTag(sha string) (*Tag, error) { - id, err := NewIDFromString(sha) - if err != nil { - return nil, err - } - - // Tag type must be "tag" (annotated) and not a "commit" (lightweight) tag - if tagType, err := repo.GetTagType(id); err != nil { - return nil, err - } else if ObjectType(tagType) != ObjectTag { - // not an annotated tag - return nil, ErrNotExist{ID: id.String()} - } - - tag, err := repo.getTag(id) - if err != nil { - return nil, err - } - return tag, nil -} diff --git a/modules/git/repo_tree.go b/modules/git/repo_tree.go index 562899bb12f9b..2053b6a1de17d 100644 --- a/modules/git/repo_tree.go +++ b/modules/git/repo_tree.go @@ -3,8 +3,6 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -// +build !nogogit - package git import ( @@ -15,45 +13,6 @@ import ( "time" ) -func (repo *Repository) getTree(id SHA1) (*Tree, error) { - gogitTree, err := repo.gogitRepo.TreeObject(id) - if err != nil { - return nil, err - } - - tree := NewTree(repo, id) - tree.gogitTree = gogitTree - return tree, nil -} - -// GetTree find the tree object in the repository. -func (repo *Repository) GetTree(idStr string) (*Tree, error) { - if len(idStr) != 40 { - res, err := NewCommand("rev-parse", "--verify", idStr).RunInDir(repo.Path) - if err != nil { - return nil, err - } - if len(res) > 0 { - idStr = res[:len(res)-1] - } - } - id, err := NewIDFromString(idStr) - if err != nil { - return nil, err - } - resolvedID := id - commitObject, err := repo.gogitRepo.CommitObject(id) - if err == nil { - id = SHA1(commitObject.TreeHash) - } - treeObject, err := repo.getTree(id) - if err != nil { - return nil, err - } - treeObject.ResolvedID = resolvedID - return treeObject, nil -} - // CommitTreeOpts represents the possible options to CommitTree type CommitTreeOpts struct { Parents []string diff --git a/modules/git/repo_tree_gogit.go b/modules/git/repo_tree_gogit.go new file mode 100644 index 0000000000000..c9f3ece07e782 --- /dev/null +++ b/modules/git/repo_tree_gogit.go @@ -0,0 +1,47 @@ +// Copyright 2015 The Gogs Authors. All rights reserved. +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build !nogogit + +package git + +func (repo *Repository) getTree(id SHA1) (*Tree, error) { + gogitTree, err := repo.gogitRepo.TreeObject(id) + if err != nil { + return nil, err + } + + tree := NewTree(repo, id) + tree.gogitTree = gogitTree + return tree, nil +} + +// GetTree find the tree object in the repository. +func (repo *Repository) GetTree(idStr string) (*Tree, error) { + if len(idStr) != 40 { + res, err := NewCommand("rev-parse", "--verify", idStr).RunInDir(repo.Path) + if err != nil { + return nil, err + } + if len(res) > 0 { + idStr = res[:len(res)-1] + } + } + id, err := NewIDFromString(idStr) + if err != nil { + return nil, err + } + resolvedID := id + commitObject, err := repo.gogitRepo.CommitObject(id) + if err == nil { + id = SHA1(commitObject.TreeHash) + } + treeObject, err := repo.getTree(id) + if err != nil { + return nil, err + } + treeObject.ResolvedID = resolvedID + return treeObject, nil +} diff --git a/modules/git/repo_tree_nogogit.go b/modules/git/repo_tree_nogogit.go index f2bccac6c27db..4935c0caa33fa 100644 --- a/modules/git/repo_tree_nogogit.go +++ b/modules/git/repo_tree_nogogit.go @@ -1,5 +1,4 @@ -// Copyright 2015 The Gogs Authors. All rights reserved. -// Copyright 2019 The Gitea Authors. All rights reserved. +// Copyright 2020 The Gitea Authors. All rights reserved. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. @@ -9,13 +8,10 @@ package git import ( "bufio" - "bytes" "fmt" "io" "io/ioutil" - "os" "strings" - "time" ) func (repo *Repository) getTree(id SHA1) (*Tree, error) { @@ -100,58 +96,3 @@ func (repo *Repository) GetTree(idStr string) (*Tree, error) { return repo.getTree(id) } - -// CommitTreeOpts represents the possible options to CommitTree -type CommitTreeOpts struct { - Parents []string - Message string - KeyID string - NoGPGSign bool - AlwaysSign bool -} - -// CommitTree creates a commit from a given tree id for the user with provided message -func (repo *Repository) CommitTree(author *Signature, committer *Signature, tree *Tree, opts CommitTreeOpts) (SHA1, error) { - err := LoadGitVersion() - if err != nil { - return SHA1{}, err - } - - commitTimeStr := time.Now().Format(time.RFC3339) - - // Because this may call hooks we should pass in the environment - env := append(os.Environ(), - "GIT_AUTHOR_NAME="+author.Name, - "GIT_AUTHOR_EMAIL="+author.Email, - "GIT_AUTHOR_DATE="+commitTimeStr, - "GIT_COMMITTER_NAME="+committer.Name, - "GIT_COMMITTER_EMAIL="+committer.Email, - "GIT_COMMITTER_DATE="+commitTimeStr, - ) - cmd := NewCommand("commit-tree", tree.ID.String()) - - for _, parent := range opts.Parents { - cmd.AddArguments("-p", parent) - } - - messageBytes := new(bytes.Buffer) - _, _ = messageBytes.WriteString(opts.Message) - _, _ = messageBytes.WriteString("\n") - - if CheckGitVersionAtLeast("1.7.9") == nil && (opts.KeyID != "" || opts.AlwaysSign) { - cmd.AddArguments(fmt.Sprintf("-S%s", opts.KeyID)) - } - - if CheckGitVersionAtLeast("2.0.0") == nil && opts.NoGPGSign { - cmd.AddArguments("--no-gpg-sign") - } - - stdout := new(bytes.Buffer) - stderr := new(bytes.Buffer) - err = cmd.RunInDirTimeoutEnvFullPipeline(env, -1, repo.Path, stdout, stderr, messageBytes) - - if err != nil { - return SHA1{}, ConcatenateError(err, stderr.String()) - } - return NewIDFromString(strings.TrimSpace(stdout.String())) -} diff --git a/modules/git/sha1.go b/modules/git/sha1.go index f6dc006cb2f79..2da74733df725 100644 --- a/modules/git/sha1.go +++ b/modules/git/sha1.go @@ -3,8 +3,6 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -// +build !nogogit - package git import ( @@ -12,8 +10,6 @@ import ( "fmt" "regexp" "strings" - - "github.com/go-git/go-git/v5/plumbing" ) // EmptySHA defines empty git SHA @@ -25,9 +21,6 @@ const EmptyTreeSHA = "4b825dc642cb6eb9a060e54bf8d69288fbee4904" // SHAPattern can be used to determine if a string is an valid sha var SHAPattern = regexp.MustCompile(`^[0-9a-f]{4,40}$`) -// SHA1 a git commit name -type SHA1 = plumbing.Hash - // MustID always creates a new SHA1 from a [20]byte array with no validation of input. func MustID(b []byte) SHA1 { var id SHA1 @@ -62,8 +55,3 @@ func NewIDFromString(s string) (SHA1, error) { } return NewID(b) } - -// ComputeBlobHash compute the hash for a given blob content -func ComputeBlobHash(content []byte) SHA1 { - return plumbing.ComputeHash(plumbing.BlobObject, content) -} diff --git a/modules/git/sha1_gogit.go b/modules/git/sha1_gogit.go new file mode 100644 index 0000000000000..6023ab94f2a35 --- /dev/null +++ b/modules/git/sha1_gogit.go @@ -0,0 +1,20 @@ +// Copyright 2015 The Gogs Authors. All rights reserved. +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build !nogogit + +package git + +import ( + "github.com/go-git/go-git/v5/plumbing" +) + +// SHA1 a git commit name +type SHA1 = plumbing.Hash + +// ComputeBlobHash compute the hash for a given blob content +func ComputeBlobHash(content []byte) SHA1 { + return plumbing.ComputeHash(plumbing.BlobObject, content) +} diff --git a/modules/git/sha1_nogogit.go b/modules/git/sha1_nogogit.go index 1de782bb152c9..4b4fea5060075 100644 --- a/modules/git/sha1_nogogit.go +++ b/modules/git/sha1_nogogit.go @@ -10,22 +10,10 @@ package git import ( "crypto/sha1" "encoding/hex" - "fmt" "hash" - "regexp" "strconv" - "strings" ) -// EmptySHA defines empty git SHA -const EmptySHA = "0000000000000000000000000000000000000000" - -// EmptyTreeSHA is the SHA of an empty tree -const EmptyTreeSHA = "4b825dc642cb6eb9a060e54bf8d69288fbee4904" - -// SHAPattern can be used to determine if a string is an valid sha -var SHAPattern = regexp.MustCompile(`^[0-9a-f]{4,40}$`) - // SHA1 a git commit name type SHA1 [20]byte @@ -39,41 +27,6 @@ func (s SHA1) IsZero() bool { return s == empty } -// MustID always creates a new SHA1 from a [20]byte array with no validation of input. -func MustID(b []byte) SHA1 { - var id SHA1 - copy(id[:], b) - return id -} - -// NewID creates a new SHA1 from a [20]byte array. -func NewID(b []byte) (SHA1, error) { - if len(b) != 20 { - return SHA1{}, fmt.Errorf("Length must be 20: %v", b) - } - return MustID(b), nil -} - -// MustIDFromString always creates a new sha from a ID with no validation of input. -func MustIDFromString(s string) SHA1 { - b, _ := hex.DecodeString(s) - return MustID(b) -} - -// NewIDFromString creates a new SHA1 from a ID string of length 40. -func NewIDFromString(s string) (SHA1, error) { - var id SHA1 - s = strings.TrimSpace(s) - if len(s) != 40 { - return id, fmt.Errorf("Length must be 40: %s", s) - } - b, err := hex.DecodeString(s) - if err != nil { - return id, err - } - return NewID(b) -} - // ComputeBlobHash compute the hash for a given blob content func ComputeBlobHash(content []byte) SHA1 { return ComputeHash(ObjectBlob, content) diff --git a/modules/git/signature.go b/modules/git/signature.go index 0b6124eb61e59..b59db8f490e70 100644 --- a/modules/git/signature.go +++ b/modules/git/signature.go @@ -3,57 +3,9 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -// +build !nogogit - package git -import ( - "bytes" - "strconv" - "time" - - "github.com/go-git/go-git/v5/plumbing/object" -) - -// Signature represents the Author or Committer information. -type Signature = object.Signature - const ( // GitTimeLayout is the (default) time layout used by git. GitTimeLayout = "Mon Jan _2 15:04:05 2006 -0700" ) - -// Helper to get a signature from the commit line, which looks like these: -// author Patrick Gundlach 1378823654 +0200 -// author Patrick Gundlach Thu, 07 Apr 2005 22:13:13 +0200 -// but without the "author " at the beginning (this method should) -// be used for author and committer. -// -// FIXME: include timezone for timestamp! -func newSignatureFromCommitline(line []byte) (_ *Signature, err error) { - sig := new(Signature) - emailStart := bytes.IndexByte(line, '<') - sig.Name = string(line[:emailStart-1]) - emailEnd := bytes.IndexByte(line, '>') - sig.Email = string(line[emailStart+1 : emailEnd]) - - // Check date format. - if len(line) > emailEnd+2 { - firstChar := line[emailEnd+2] - if firstChar >= 48 && firstChar <= 57 { - timestop := bytes.IndexByte(line[emailEnd+2:], ' ') - timestring := string(line[emailEnd+2 : emailEnd+2+timestop]) - seconds, _ := strconv.ParseInt(timestring, 10, 64) - sig.When = time.Unix(seconds, 0) - } else { - sig.When, err = time.Parse(GitTimeLayout, string(line[emailEnd+2:])) - if err != nil { - return nil, err - } - } - } else { - // Fall back to unix 0 time - sig.When = time.Unix(0, 0) - } - return sig, nil -} diff --git a/modules/git/signature_gogit.go b/modules/git/signature_gogit.go new file mode 100644 index 0000000000000..a4426e0958f3f --- /dev/null +++ b/modules/git/signature_gogit.go @@ -0,0 +1,54 @@ +// Copyright 2015 The Gogs Authors. All rights reserved. +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build !nogogit + +package git + +import ( + "bytes" + "strconv" + "time" + + "github.com/go-git/go-git/v5/plumbing/object" +) + +// Signature represents the Author or Committer information. +type Signature = object.Signature + +// Helper to get a signature from the commit line, which looks like these: +// author Patrick Gundlach 1378823654 +0200 +// author Patrick Gundlach Thu, 07 Apr 2005 22:13:13 +0200 +// but without the "author " at the beginning (this method should) +// be used for author and committer. +// +// FIXME: include timezone for timestamp! +func newSignatureFromCommitline(line []byte) (_ *Signature, err error) { + sig := new(Signature) + emailStart := bytes.IndexByte(line, '<') + sig.Name = string(line[:emailStart-1]) + emailEnd := bytes.IndexByte(line, '>') + sig.Email = string(line[emailStart+1 : emailEnd]) + + // Check date format. + if len(line) > emailEnd+2 { + firstChar := line[emailEnd+2] + if firstChar >= 48 && firstChar <= 57 { + timestop := bytes.IndexByte(line[emailEnd+2:], ' ') + timestring := string(line[emailEnd+2 : emailEnd+2+timestop]) + seconds, _ := strconv.ParseInt(timestring, 10, 64) + sig.When = time.Unix(seconds, 0) + } else { + sig.When, err = time.Parse(GitTimeLayout, string(line[emailEnd+2:])) + if err != nil { + return nil, err + } + } + } else { + // Fall back to unix 0 time + sig.When = time.Unix(0, 0) + } + return sig, nil +} diff --git a/modules/git/signature_nogogit.go b/modules/git/signature_nogogit.go index c38821e9b3d63..6b498c7fd0396 100644 --- a/modules/git/signature_nogogit.go +++ b/modules/git/signature_nogogit.go @@ -35,11 +35,6 @@ func (s *Signature) Decode(b []byte) { s.When = sig.When } -const ( - // GitTimeLayout is the (default) time layout used by git. - GitTimeLayout = "Mon Jan _2 15:04:05 2006 -0700" -) - // Helper to get a signature from the commit line, which looks like these: // author Patrick Gundlach 1378823654 +0200 // author Patrick Gundlach Thu, 07 Apr 2005 22:13:13 +0200 diff --git a/modules/git/tree.go b/modules/git/tree.go index 447c042e64058..059f0a8287d2a 100644 --- a/modules/git/tree.go +++ b/modules/git/tree.go @@ -3,30 +3,12 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -// +build !nogogit - package git import ( - "io" "strings" - - "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/plumbing/object" ) -// Tree represents a flat directory listing. -type Tree struct { - ID SHA1 - ResolvedID SHA1 - repo *Repository - - gogitTree *object.Tree - - // parent tree - ptree *Tree -} - // NewTree create a new tree according the repository and tree id func NewTree(repo *Repository, id SHA1) *Tree { return &Tree{ @@ -63,70 +45,3 @@ func (t *Tree) SubTree(rpath string) (*Tree, error) { } return g, nil } - -func (t *Tree) loadTreeObject() error { - gogitTree, err := t.repo.gogitRepo.TreeObject(t.ID) - if err != nil { - return err - } - - t.gogitTree = gogitTree - return nil -} - -// ListEntries returns all entries of current tree. -func (t *Tree) ListEntries() (Entries, error) { - if t.gogitTree == nil { - err := t.loadTreeObject() - if err != nil { - return nil, err - } - } - - entries := make([]*TreeEntry, len(t.gogitTree.Entries)) - for i, entry := range t.gogitTree.Entries { - entries[i] = &TreeEntry{ - ID: entry.Hash, - gogitTreeEntry: &t.gogitTree.Entries[i], - ptree: t, - } - } - - return entries, nil -} - -// ListEntriesRecursive returns all entries of current tree recursively including all subtrees -func (t *Tree) ListEntriesRecursive() (Entries, error) { - if t.gogitTree == nil { - err := t.loadTreeObject() - if err != nil { - return nil, err - } - } - - var entries []*TreeEntry - seen := map[plumbing.Hash]bool{} - walker := object.NewTreeWalker(t.gogitTree, true, seen) - for { - fullName, entry, err := walker.Next() - if err == io.EOF { - break - } - if err != nil { - return nil, err - } - if seen[entry.Hash] { - continue - } - - convertedEntry := &TreeEntry{ - ID: entry.Hash, - gogitTreeEntry: &entry, - ptree: t, - fullName: fullName, - } - entries = append(entries, convertedEntry) - } - - return entries, nil -} diff --git a/modules/git/tree_blob.go b/modules/git/tree_blob.go index bd62adbfdd307..19edcf4c6cbaf 100644 --- a/modules/git/tree_blob.go +++ b/modules/git/tree_blob.go @@ -3,68 +3,8 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -// +build !nogogit - package git -import ( - "path" - "strings" - - "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/plumbing/filemode" - "github.com/go-git/go-git/v5/plumbing/object" -) - -// GetTreeEntryByPath get the tree entries according the sub dir -func (t *Tree) GetTreeEntryByPath(relpath string) (*TreeEntry, error) { - if len(relpath) == 0 { - return &TreeEntry{ - ID: t.ID, - //Type: ObjectTree, - gogitTreeEntry: &object.TreeEntry{ - Name: "", - Mode: filemode.Dir, - Hash: t.ID, - }, - }, nil - } - - relpath = path.Clean(relpath) - parts := strings.Split(relpath, "/") - var err error - tree := t - for i, name := range parts { - if i == len(parts)-1 { - entries, err := tree.ListEntries() - if err != nil { - if err == plumbing.ErrObjectNotFound { - return nil, ErrNotExist{ - RelPath: relpath, - } - } - return nil, err - } - for _, v := range entries { - if v.Name() == name { - return v, nil - } - } - } else { - tree, err = tree.SubTree(name) - if err != nil { - if err == plumbing.ErrObjectNotFound { - return nil, ErrNotExist{ - RelPath: relpath, - } - } - return nil, err - } - } - } - return nil, ErrNotExist{"", relpath} -} - // GetBlobByPath get the blob object according the path func (t *Tree) GetBlobByPath(relpath string) (*Blob, error) { entry, err := t.GetTreeEntryByPath(relpath) diff --git a/modules/git/tree_blob_gogit.go b/modules/git/tree_blob_gogit.go new file mode 100644 index 0000000000000..3d762a9c6abc6 --- /dev/null +++ b/modules/git/tree_blob_gogit.go @@ -0,0 +1,66 @@ +// Copyright 2015 The Gogs Authors. All rights reserved. +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build !nogogit + +package git + +import ( + "path" + "strings" + + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/filemode" + "github.com/go-git/go-git/v5/plumbing/object" +) + +// GetTreeEntryByPath get the tree entries according the sub dir +func (t *Tree) GetTreeEntryByPath(relpath string) (*TreeEntry, error) { + if len(relpath) == 0 { + return &TreeEntry{ + ID: t.ID, + //Type: ObjectTree, + gogitTreeEntry: &object.TreeEntry{ + Name: "", + Mode: filemode.Dir, + Hash: t.ID, + }, + }, nil + } + + relpath = path.Clean(relpath) + parts := strings.Split(relpath, "/") + var err error + tree := t + for i, name := range parts { + if i == len(parts)-1 { + entries, err := tree.ListEntries() + if err != nil { + if err == plumbing.ErrObjectNotFound { + return nil, ErrNotExist{ + RelPath: relpath, + } + } + return nil, err + } + for _, v := range entries { + if v.Name() == name { + return v, nil + } + } + } else { + tree, err = tree.SubTree(name) + if err != nil { + if err == plumbing.ErrObjectNotFound { + return nil, ErrNotExist{ + RelPath: relpath, + } + } + return nil, err + } + } + } + return nil, ErrNotExist{"", relpath} +} diff --git a/modules/git/tree_blob_nogogit.go b/modules/git/tree_blob_nogogit.go index 88da499218d62..ce2221ebd427e 100644 --- a/modules/git/tree_blob_nogogit.go +++ b/modules/git/tree_blob_nogogit.go @@ -1,5 +1,4 @@ -// Copyright 2015 The Gogs Authors. All rights reserved. -// Copyright 2019 The Gitea Authors. All rights reserved. +// Copyright 2020 The Gitea Authors. All rights reserved. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. @@ -48,17 +47,3 @@ func (t *Tree) GetTreeEntryByPath(relpath string) (*TreeEntry, error) { } return nil, ErrNotExist{"", relpath} } - -// GetBlobByPath get the blob object according the path -func (t *Tree) GetBlobByPath(relpath string) (*Blob, error) { - entry, err := t.GetTreeEntryByPath(relpath) - if err != nil { - return nil, err - } - - if !entry.IsDir() && !entry.IsSubModule() { - return entry.Blob(), nil - } - - return nil, ErrNotExist{"", relpath} -} diff --git a/modules/git/tree_entry.go b/modules/git/tree_entry.go index 2b62ca4eb7a85..498767a63eb0f 100644 --- a/modules/git/tree_entry.go +++ b/modules/git/tree_entry.go @@ -3,45 +3,14 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -// +build !nogogit - package git import ( "io" "sort" "strings" - - "github.com/go-git/go-git/v5/plumbing" - "github.com/go-git/go-git/v5/plumbing/filemode" - "github.com/go-git/go-git/v5/plumbing/object" ) -// TreeEntry the leaf in the git tree -type TreeEntry struct { - ID SHA1 - - gogitTreeEntry *object.TreeEntry - ptree *Tree - - size int64 - sized bool - fullName string -} - -// Name returns the name of the entry -func (te *TreeEntry) Name() string { - if te.fullName != "" { - return te.fullName - } - return te.gogitTreeEntry.Name -} - -// Mode returns the mode of the entry -func (te *TreeEntry) Mode() EntryMode { - return EntryMode(te.gogitTreeEntry.Mode) -} - // Type returns the type of the entry (commit, tree, blob) func (te *TreeEntry) Type() string { switch te.Mode() { @@ -54,63 +23,6 @@ func (te *TreeEntry) Type() string { } } -// Size returns the size of the entry -func (te *TreeEntry) Size() int64 { - if te.IsDir() { - return 0 - } else if te.sized { - return te.size - } - - file, err := te.ptree.gogitTree.TreeEntryFile(te.gogitTreeEntry) - if err != nil { - return 0 - } - - te.sized = true - te.size = file.Size - return te.size -} - -// IsSubModule if the entry is a sub module -func (te *TreeEntry) IsSubModule() bool { - return te.gogitTreeEntry.Mode == filemode.Submodule -} - -// IsDir if the entry is a sub dir -func (te *TreeEntry) IsDir() bool { - return te.gogitTreeEntry.Mode == filemode.Dir -} - -// IsLink if the entry is a symlink -func (te *TreeEntry) IsLink() bool { - return te.gogitTreeEntry.Mode == filemode.Symlink -} - -// IsRegular if the entry is a regular file -func (te *TreeEntry) IsRegular() bool { - return te.gogitTreeEntry.Mode == filemode.Regular -} - -// IsExecutable if the entry is an executable file (not necessarily binary) -func (te *TreeEntry) IsExecutable() bool { - return te.gogitTreeEntry.Mode == filemode.Executable -} - -// Blob returns the blob object the entry -func (te *TreeEntry) Blob() *Blob { - encodedObj, err := te.ptree.repo.gogitRepo.Storer.EncodedObject(plumbing.AnyObject, te.gogitTreeEntry.Hash) - if err != nil { - return nil - } - - return &Blob{ - ID: te.gogitTreeEntry.Hash, - gogitEncodedObj: encodedObj, - name: te.Name(), - } -} - // FollowLink returns the entry pointed to by a symlink func (te *TreeEntry) FollowLink() (*TreeEntry, error) { if !te.IsLink() { diff --git a/modules/git/tree_entry_gogit.go b/modules/git/tree_entry_gogit.go new file mode 100644 index 0000000000000..41c534994c3c6 --- /dev/null +++ b/modules/git/tree_entry_gogit.go @@ -0,0 +1,96 @@ +// Copyright 2015 The Gogs Authors. All rights reserved. +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build !nogogit + +package git + +import ( + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/filemode" + "github.com/go-git/go-git/v5/plumbing/object" +) + +// TreeEntry the leaf in the git tree +type TreeEntry struct { + ID SHA1 + + gogitTreeEntry *object.TreeEntry + ptree *Tree + + size int64 + sized bool + fullName string +} + +// Name returns the name of the entry +func (te *TreeEntry) Name() string { + if te.fullName != "" { + return te.fullName + } + return te.gogitTreeEntry.Name +} + +// Mode returns the mode of the entry +func (te *TreeEntry) Mode() EntryMode { + return EntryMode(te.gogitTreeEntry.Mode) +} + +// Size returns the size of the entry +func (te *TreeEntry) Size() int64 { + if te.IsDir() { + return 0 + } else if te.sized { + return te.size + } + + file, err := te.ptree.gogitTree.TreeEntryFile(te.gogitTreeEntry) + if err != nil { + return 0 + } + + te.sized = true + te.size = file.Size + return te.size +} + +// IsSubModule if the entry is a sub module +func (te *TreeEntry) IsSubModule() bool { + return te.gogitTreeEntry.Mode == filemode.Submodule +} + +// IsDir if the entry is a sub dir +func (te *TreeEntry) IsDir() bool { + return te.gogitTreeEntry.Mode == filemode.Dir +} + +// IsLink if the entry is a symlink +func (te *TreeEntry) IsLink() bool { + return te.gogitTreeEntry.Mode == filemode.Symlink +} + +// IsRegular if the entry is a regular file +func (te *TreeEntry) IsRegular() bool { + return te.gogitTreeEntry.Mode == filemode.Regular +} + +// IsExecutable if the entry is an executable file (not necessarily binary) +func (te *TreeEntry) IsExecutable() bool { + return te.gogitTreeEntry.Mode == filemode.Executable +} + +// Blob returns the blob object the entry +func (te *TreeEntry) Blob() *Blob { + encodedObj, err := te.ptree.repo.gogitRepo.Storer.EncodedObject(plumbing.AnyObject, te.gogitTreeEntry.Hash) + if err != nil { + return nil + } + + return &Blob{ + ID: te.gogitTreeEntry.Hash, + gogitEncodedObj: encodedObj, + name: te.Name(), + } +} diff --git a/modules/git/tree_entry_nogogit.go b/modules/git/tree_entry_nogogit.go index f5a9e25917020..c7d67ee9d6b77 100644 --- a/modules/git/tree_entry_nogogit.go +++ b/modules/git/tree_entry_nogogit.go @@ -7,8 +7,6 @@ package git import ( - "io" - "sort" "strconv" "strings" ) @@ -40,18 +38,6 @@ func (te *TreeEntry) Mode() EntryMode { return te.entryMode } -// Type returns the type of the entry (commit, tree, blob) -func (te *TreeEntry) Type() string { - switch te.Mode() { - case EntryModeCommit: - return "commit" - case EntryModeTree: - return "tree" - default: - return "blob" - } -} - // Size returns the size of the entry func (te *TreeEntry) Size() int64 { if te.IsDir() { @@ -103,144 +89,3 @@ func (te *TreeEntry) Blob() *Blob { name: te.Name(), } } - -// FollowLink returns the entry pointed to by a symlink -func (te *TreeEntry) FollowLink() (*TreeEntry, error) { - if !te.IsLink() { - return nil, ErrBadLink{te.Name(), "not a symlink"} - } - - // read the link - r, err := te.Blob().DataAsync() - if err != nil { - return nil, err - } - defer r.Close() - buf := make([]byte, te.Size()) - _, err = io.ReadFull(r, buf) - if err != nil { - return nil, err - } - - lnk := string(buf) - t := te.ptree - - // traverse up directories - for ; t != nil && strings.HasPrefix(lnk, "../"); lnk = lnk[3:] { - t = t.ptree - } - - if t == nil { - return nil, ErrBadLink{te.Name(), "points outside of repo"} - } - - target, err := t.GetTreeEntryByPath(lnk) - if err != nil { - if IsErrNotExist(err) { - return nil, ErrBadLink{te.Name(), "broken link"} - } - return nil, err - } - return target, nil -} - -// FollowLinks returns the entry ultimately pointed to by a symlink -func (te *TreeEntry) FollowLinks() (*TreeEntry, error) { - if !te.IsLink() { - return nil, ErrBadLink{te.Name(), "not a symlink"} - } - entry := te - for i := 0; i < 999; i++ { - if entry.IsLink() { - next, err := entry.FollowLink() - if err != nil { - return nil, err - } - if next.ID == entry.ID { - return nil, ErrBadLink{ - entry.Name(), - "recursive link", - } - } - entry = next - } else { - break - } - } - if entry.IsLink() { - return nil, ErrBadLink{ - te.Name(), - "too many levels of symbolic links", - } - } - return entry, nil -} - -// GetSubJumpablePathName return the full path of subdirectory jumpable ( contains only one directory ) -func (te *TreeEntry) GetSubJumpablePathName() string { - if te.IsSubModule() || !te.IsDir() { - return "" - } - tree, err := te.ptree.SubTree(te.Name()) - if err != nil { - return te.Name() - } - entries, _ := tree.ListEntries() - if len(entries) == 1 && entries[0].IsDir() { - name := entries[0].GetSubJumpablePathName() - if name != "" { - return te.Name() + "/" + name - } - } - return te.Name() -} - -// Entries a list of entry -type Entries []*TreeEntry - -type customSortableEntries struct { - Comparer func(s1, s2 string) bool - Entries -} - -var sorter = []func(t1, t2 *TreeEntry, cmp func(s1, s2 string) bool) bool{ - func(t1, t2 *TreeEntry, cmp func(s1, s2 string) bool) bool { - return (t1.IsDir() || t1.IsSubModule()) && !t2.IsDir() && !t2.IsSubModule() - }, - func(t1, t2 *TreeEntry, cmp func(s1, s2 string) bool) bool { - return cmp(t1.Name(), t2.Name()) - }, -} - -func (ctes customSortableEntries) Len() int { return len(ctes.Entries) } - -func (ctes customSortableEntries) Swap(i, j int) { - ctes.Entries[i], ctes.Entries[j] = ctes.Entries[j], ctes.Entries[i] -} - -func (ctes customSortableEntries) Less(i, j int) bool { - t1, t2 := ctes.Entries[i], ctes.Entries[j] - var k int - for k = 0; k < len(sorter)-1; k++ { - s := sorter[k] - switch { - case s(t1, t2, ctes.Comparer): - return true - case s(t2, t1, ctes.Comparer): - return false - } - } - return sorter[k](t1, t2, ctes.Comparer) -} - -// Sort sort the list of entry -func (tes Entries) Sort() { - sort.Sort(customSortableEntries{func(s1, s2 string) bool { - return s1 < s2 - }, tes}) -} - -// CustomSort customizable string comparing sort entry list -func (tes Entries) CustomSort(cmp func(s1, s2 string) bool) { - sort.Sort(customSortableEntries{cmp, tes}) -} diff --git a/modules/git/tree_gogit.go b/modules/git/tree_gogit.go new file mode 100644 index 0000000000000..01aa94ca1d8dd --- /dev/null +++ b/modules/git/tree_gogit.go @@ -0,0 +1,94 @@ +// Copyright 2015 The Gogs Authors. All rights reserved. +// Copyright 2019 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +// +build !nogogit + +package git + +import ( + "io" + + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/object" +) + +// Tree represents a flat directory listing. +type Tree struct { + ID SHA1 + ResolvedID SHA1 + repo *Repository + + gogitTree *object.Tree + + // parent tree + ptree *Tree +} + +func (t *Tree) loadTreeObject() error { + gogitTree, err := t.repo.gogitRepo.TreeObject(t.ID) + if err != nil { + return err + } + + t.gogitTree = gogitTree + return nil +} + +// ListEntries returns all entries of current tree. +func (t *Tree) ListEntries() (Entries, error) { + if t.gogitTree == nil { + err := t.loadTreeObject() + if err != nil { + return nil, err + } + } + + entries := make([]*TreeEntry, len(t.gogitTree.Entries)) + for i, entry := range t.gogitTree.Entries { + entries[i] = &TreeEntry{ + ID: entry.Hash, + gogitTreeEntry: &t.gogitTree.Entries[i], + ptree: t, + } + } + + return entries, nil +} + +// ListEntriesRecursive returns all entries of current tree recursively including all subtrees +func (t *Tree) ListEntriesRecursive() (Entries, error) { + if t.gogitTree == nil { + err := t.loadTreeObject() + if err != nil { + return nil, err + } + } + + var entries []*TreeEntry + seen := map[plumbing.Hash]bool{} + walker := object.NewTreeWalker(t.gogitTree, true, seen) + for { + fullName, entry, err := walker.Next() + if err == io.EOF { + break + } + if err != nil { + return nil, err + } + if seen[entry.Hash] { + continue + } + + convertedEntry := &TreeEntry{ + ID: entry.Hash, + gogitTreeEntry: &entry, + ptree: t, + fullName: fullName, + } + entries = append(entries, convertedEntry) + } + + return entries, nil +} diff --git a/modules/git/tree_nogogit.go b/modules/git/tree_nogogit.go index 8aff4ee7c919e..576a6d95201de 100644 --- a/modules/git/tree_nogogit.go +++ b/modules/git/tree_nogogit.go @@ -26,43 +26,6 @@ type Tree struct { entriesRecursiveParsed bool } -// NewTree create a new tree according the repository and tree id -func NewTree(repo *Repository, id SHA1) *Tree { - return &Tree{ - ID: id, - repo: repo, - } -} - -// SubTree get a sub tree by the sub dir path -func (t *Tree) SubTree(rpath string) (*Tree, error) { - if len(rpath) == 0 { - return t, nil - } - - paths := strings.Split(rpath, "/") - var ( - err error - g = t - p = t - te *TreeEntry - ) - for _, name := range paths { - te, err = p.GetTreeEntryByPath(name) - if err != nil { - return nil, err - } - - g, err = t.repo.getTree(te.ID) - if err != nil { - return nil, err - } - g.ptree = p - p = g - } - return g, nil -} - // ListEntries returns all entries of current tree. func (t *Tree) ListEntries() (Entries, error) { if t.entriesParsed { From a35cf1874319aaf818d98c74f3966b4700a3c7f0 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sat, 28 Nov 2020 17:43:29 +0000 Subject: [PATCH 14/19] As per @lunny Signed-off-by: Andrew Thornton --- modules/git/last_commit_cache_gogit.go | 10 +++++----- modules/git/last_commit_cache_nogogit.go | 8 ++++---- modules/repository/cache.go | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/modules/git/last_commit_cache_gogit.go b/modules/git/last_commit_cache_gogit.go index ae1753e37680d..cf33414fef138 100644 --- a/modules/git/last_commit_cache_gogit.go +++ b/modules/git/last_commit_cache_gogit.go @@ -60,19 +60,19 @@ func (c *LastCommitCache) Get(ref, entryPath string) (interface{}, error) { } // CacheCommit will cache the commit from the gitRepository -func (c *LastCommitCache) CacheCommit(gitRepo *Repository, commit *Commit) error { +func (c *LastCommitCache) CacheCommit(commit *Commit) error { - commitNodeIndex, _ := gitRepo.CommitNodeIndex() + commitNodeIndex, _ := commit.repo.CommitNodeIndex() index, err := commitNodeIndex.Get(commit.ID) if err != nil { return err } - return c.recursiveCache(gitRepo, index, &commit.Tree, "", 1) + return c.recursiveCache(index, &commit.Tree, "", 1) } -func (c *LastCommitCache) recursiveCache(gitRepo *Repository, index cgobject.CommitNode, tree *Tree, treePath string, level int) error { +func (c *LastCommitCache) recursiveCache(index cgobject.CommitNode, tree *Tree, treePath string, level int) error { if level == 0 { return nil } @@ -103,7 +103,7 @@ func (c *LastCommitCache) recursiveCache(gitRepo *Repository, index cgobject.Com if err != nil { return err } - if err := c.recursiveCache(gitRepo, index, subTree, entry, level-1); err != nil { + if err := c.recursiveCache(index, subTree, entry, level-1); err != nil { return err } } diff --git a/modules/git/last_commit_cache_nogogit.go b/modules/git/last_commit_cache_nogogit.go index d72695c7b9374..965760556582b 100644 --- a/modules/git/last_commit_cache_nogogit.go +++ b/modules/git/last_commit_cache_nogogit.go @@ -57,11 +57,11 @@ func (c *LastCommitCache) Get(ref, entryPath string) (interface{}, error) { } // CacheCommit will cache the commit from the gitRepository -func (c *LastCommitCache) CacheCommit(gitRepo *Repository, commit *Commit) error { - return c.recursiveCache(gitRepo, commit, &commit.Tree, "", 1) +func (c *LastCommitCache) CacheCommit(commit *Commit) error { + return c.recursiveCache(commit, &commit.Tree, "", 1) } -func (c *LastCommitCache) recursiveCache(gitRepo *Repository, commit *Commit, tree *Tree, treePath string, level int) error { +func (c *LastCommitCache) recursiveCache(commit *Commit, tree *Tree, treePath string, level int) error { if level == 0 { return nil } @@ -93,7 +93,7 @@ func (c *LastCommitCache) recursiveCache(gitRepo *Repository, commit *Commit, tr if err != nil { return err } - if err := c.recursiveCache(gitRepo, commit, subTree, entry, level-1); err != nil { + if err := c.recursiveCache(commit, subTree, entry, level-1); err != nil { return err } } diff --git a/modules/repository/cache.go b/modules/repository/cache.go index 1ac1bdd277eed..0852771a55994 100644 --- a/modules/repository/cache.go +++ b/modules/repository/cache.go @@ -43,5 +43,5 @@ func CacheRef(repo *models.Repository, gitRepo *git.Repository, fullRefName stri commitCache := git.NewLastCommitCache(repo.FullName(), gitRepo, int64(setting.CacheService.LastCommit.TTL.Seconds()), cache.GetCache()) - return commitCache.CacheCommit(gitRepo, commit) + return commitCache.CacheCommit(commit) } From b047f9325a73b8285a5ab7f69c365a9e54f1aed0 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Mon, 30 Nov 2020 20:18:39 +0000 Subject: [PATCH 15/19] attempt to fix drone Signed-off-by: Andrew Thornton --- .drone.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.drone.yml b/.drone.yml index 81e51788ec1e9..0620969fedbc8 100644 --- a/.drone.yml +++ b/.drone.yml @@ -316,7 +316,7 @@ steps: environment: GOPROXY: https://goproxy.cn # proxy.golang.org is blocked in China, this proxy is not GOSUMDB: sum.golang.org - TAGS: bindata sqlite sqlite_unlock_notify + TAGS: bindata nogogit sqlite sqlite_unlock_notify - name: test-sqlite image: golang:1.15 @@ -326,7 +326,7 @@ steps: - timeout -s ABRT 40m make test-sqlite-migration test-sqlite environment: GOPROXY: off - TAGS: bindata nogogit + TAGS: bindata nogogit sqlite sqlite_unlock_notify TEST_TAGS: nogogit sqlite sqlite_unlock_notify USE_REPO_TEST_DIR: 1 depends_on: From 55f4e03ab0e01b69f84589e294f093900efc7faa Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Tue, 8 Dec 2020 19:39:37 +0000 Subject: [PATCH 16/19] fix test-tags Signed-off-by: Andrew Thornton --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 6a4f9e124d1ac..7f2a3feb8b73d 100644 --- a/Makefile +++ b/Makefile @@ -110,7 +110,7 @@ TAGS ?= TAGS_SPLIT := $(subst $(COMMA), ,$(TAGS)) TAGS_EVIDENCE := $(MAKE_EVIDENCE_DIR)/tags -TEST_TAGS ?= "sqlite sqlite_unlock_notify" +TEST_TAGS ?= sqlite sqlite_unlock_notify GO_DIRS := cmd integrations models modules routers build services vendor GO_SOURCES := $(wildcard *.go) From 8c887fcdff65d9fd9c8c73576ac2fe8fa1455db4 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Thu, 10 Dec 2020 14:04:17 +0000 Subject: [PATCH 17/19] default to use no-go-git variants and add gogit build tag Signed-off-by: Andrew Thornton --- .drone.yml | 20 +++++++++---------- .../doc/installation/from-source.en-us.md | 1 + modules/git/batch_reader_nogogit.go | 2 +- modules/git/blob.go | 2 +- modules/git/blob_gogit.go | 2 +- modules/git/blob_nogogit.go | 2 +- modules/git/commit_convert_gogit.go | 2 +- modules/git/commit_info_gogit.go | 2 +- modules/git/commit_info_nogogit.go | 2 +- modules/git/last_commit_cache_gogit.go | 2 +- modules/git/last_commit_cache_nogogit.go | 2 +- modules/git/notes_gogit.go | 2 +- modules/git/notes_nogogit.go | 2 +- modules/git/parse_gogit.go | 2 +- modules/git/parse_gogit_test.go | 2 +- modules/git/parse_nogogit.go | 2 +- modules/git/pipeline/lfs.go | 2 +- modules/git/pipeline/lfs_nogogit.go | 2 +- modules/git/repo_base_gogit.go | 2 +- modules/git/repo_base_nogogit.go | 2 +- modules/git/repo_blob_gogit.go | 2 +- modules/git/repo_blob_nogogit.go | 2 +- modules/git/repo_branch_gogit.go | 2 +- modules/git/repo_branch_nogogit.go | 2 +- modules/git/repo_commit_gogit.go | 2 +- modules/git/repo_commit_nogogit.go | 2 +- modules/git/repo_commitgraph_gogit.go | 2 +- modules/git/repo_language_stats_gogit.go | 2 +- modules/git/repo_language_stats_nogogit.go | 2 +- modules/git/repo_ref_gogit.go | 2 +- modules/git/repo_ref_nogogit.go | 2 +- modules/git/repo_tag_gogit.go | 2 +- modules/git/repo_tag_nogogit.go | 2 +- modules/git/repo_tree_gogit.go | 2 +- modules/git/repo_tree_nogogit.go | 2 +- modules/git/sha1_gogit.go | 2 +- modules/git/sha1_nogogit.go | 2 +- modules/git/signature_gogit.go | 2 +- modules/git/signature_nogogit.go | 2 +- modules/git/tree_blob_gogit.go | 2 +- modules/git/tree_blob_nogogit.go | 2 +- modules/git/tree_entry_gogit.go | 2 +- modules/git/tree_entry_nogogit.go | 2 +- modules/git/tree_entry_test.go | 2 +- modules/git/tree_gogit.go | 2 +- modules/git/tree_nogogit.go | 2 +- 46 files changed, 55 insertions(+), 54 deletions(-) diff --git a/.drone.yml b/.drone.yml index 0620969fedbc8..e97d65e5db94d 100644 --- a/.drone.yml +++ b/.drone.yml @@ -33,7 +33,7 @@ steps: GOSUMDB: sum.golang.org TAGS: bindata sqlite sqlite_unlock_notify - - name: lint-backend-nogogit + - name: lint-backend-gogit pull: always image: golang:1.15 commands: @@ -41,7 +41,7 @@ steps: environment: GOPROXY: https://goproxy.cn # proxy.golang.org is blocked in China, this proxy is not GOSUMDB: sum.golang.org - TAGS: bindata nogogit sqlite sqlite_unlock_notify + TAGS: bindata gogit sqlite sqlite_unlock_notify - name: checks-frontend image: node:14 @@ -79,7 +79,7 @@ steps: GOPROXY: off GOOS: linux GOARCH: arm64 - TAGS: bindata nogogit + TAGS: bindata gogit commands: - make backend # test cross compile - rm ./gitea # clean @@ -183,14 +183,14 @@ steps: GITHUB_READ_TOKEN: from_secret: github_read_token - - name: unit-test-nogogit + - name: unit-test-gogit pull: always image: golang:1.15 commands: - make unit-test-coverage test-check environment: GOPROXY: off - TAGS: bindata nogogit sqlite sqlite_unlock_notify + TAGS: bindata gogit sqlite sqlite_unlock_notify GITHUB_READ_TOKEN: from_secret: github_read_token @@ -316,7 +316,7 @@ steps: environment: GOPROXY: https://goproxy.cn # proxy.golang.org is blocked in China, this proxy is not GOSUMDB: sum.golang.org - TAGS: bindata nogogit sqlite sqlite_unlock_notify + TAGS: bindata sqlite sqlite_unlock_notify - name: test-sqlite image: golang:1.15 @@ -326,8 +326,8 @@ steps: - timeout -s ABRT 40m make test-sqlite-migration test-sqlite environment: GOPROXY: off - TAGS: bindata nogogit sqlite sqlite_unlock_notify - TEST_TAGS: nogogit sqlite sqlite_unlock_notify + TAGS: bindata gogit sqlite sqlite_unlock_notify + TEST_TAGS: gogit sqlite sqlite_unlock_notify USE_REPO_TEST_DIR: 1 depends_on: - build @@ -340,8 +340,8 @@ steps: - timeout -s ABRT 40m make test-pgsql-migration test-pgsql environment: GOPROXY: off - TAGS: bindata nogogit - TEST_TAGS: nogogit + TAGS: bindata gogit + TEST_TAGS: gogit TEST_LDAP: 1 USE_REPO_TEST_DIR: 1 depends_on: diff --git a/docs/content/doc/installation/from-source.en-us.md b/docs/content/doc/installation/from-source.en-us.md index 5fc521204d98f..a1f99a771c682 100644 --- a/docs/content/doc/installation/from-source.en-us.md +++ b/docs/content/doc/installation/from-source.en-us.md @@ -99,6 +99,7 @@ Depending on requirements, the following build tags can be included. * `pam`: Enable support for PAM (Linux Pluggable Authentication Modules). Can be used to authenticate local users or extend authentication to methods available to PAM. +* `gogit`: (EXPERIMENTAL) Use go-git variants of git commands. Bundling assets into the binary using the `bindata` build tag is recommended for production deployments. It is possible to serve the static assets directly via a reverse proxy, diff --git a/modules/git/batch_reader_nogogit.go b/modules/git/batch_reader_nogogit.go index 952955297f537..6a236e50027b2 100644 --- a/modules/git/batch_reader_nogogit.go +++ b/modules/git/batch_reader_nogogit.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -// +build nogogit +// +build !gogit package git diff --git a/modules/git/blob.go b/modules/git/blob.go index b1233d6ca95e0..674a6a9592778 100644 --- a/modules/git/blob.go +++ b/modules/git/blob.go @@ -12,7 +12,7 @@ import ( "io/ioutil" ) -// This file contains common functions between the gogit and nogogit variants for git Blobs +// This file contains common functions between the gogit and !gogit variants for git Blobs // Name returns name of the tree entry this blob object was created from (or empty string) func (b *Blob) Name() string { diff --git a/modules/git/blob_gogit.go b/modules/git/blob_gogit.go index 9f8b5e50a26d9..7a82eb5c3706f 100644 --- a/modules/git/blob_gogit.go +++ b/modules/git/blob_gogit.go @@ -3,7 +3,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -// +build !nogogit +// +build gogit package git diff --git a/modules/git/blob_nogogit.go b/modules/git/blob_nogogit.go index 14ffc6c618c8a..ef04e368fc490 100644 --- a/modules/git/blob_nogogit.go +++ b/modules/git/blob_nogogit.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -// +build nogogit +// +build !gogit package git diff --git a/modules/git/commit_convert_gogit.go b/modules/git/commit_convert_gogit.go index 51152b58c2049..be2b948b363f3 100644 --- a/modules/git/commit_convert_gogit.go +++ b/modules/git/commit_convert_gogit.go @@ -3,7 +3,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -// +build !nogogit +// +build gogit package git diff --git a/modules/git/commit_info_gogit.go b/modules/git/commit_info_gogit.go index 6a7868be5cb19..6d95e22d0c7c8 100644 --- a/modules/git/commit_info_gogit.go +++ b/modules/git/commit_info_gogit.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -// +build !nogogit +// +build gogit package git diff --git a/modules/git/commit_info_nogogit.go b/modules/git/commit_info_nogogit.go index 8dd54a6ac577d..d09a6c3f52f8f 100644 --- a/modules/git/commit_info_nogogit.go +++ b/modules/git/commit_info_nogogit.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -// +build nogogit +// +build !gogit package git diff --git a/modules/git/last_commit_cache_gogit.go b/modules/git/last_commit_cache_gogit.go index cf33414fef138..76c97a4cc04c8 100644 --- a/modules/git/last_commit_cache_gogit.go +++ b/modules/git/last_commit_cache_gogit.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -// +build !nogogit +// +build gogit package git diff --git a/modules/git/last_commit_cache_nogogit.go b/modules/git/last_commit_cache_nogogit.go index 965760556582b..b9c50b5cfb49e 100644 --- a/modules/git/last_commit_cache_nogogit.go +++ b/modules/git/last_commit_cache_nogogit.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -// +build nogogit +// +build !gogit package git diff --git a/modules/git/notes_gogit.go b/modules/git/notes_gogit.go index 09371e212d850..173d29cee6b44 100644 --- a/modules/git/notes_gogit.go +++ b/modules/git/notes_gogit.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -// +build !nogogit +// +build gogit package git diff --git a/modules/git/notes_nogogit.go b/modules/git/notes_nogogit.go index cbcc62cf4aa3a..613efd2e0e06a 100644 --- a/modules/git/notes_nogogit.go +++ b/modules/git/notes_nogogit.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -// +build nogogit +// +build !gogit package git diff --git a/modules/git/parse_gogit.go b/modules/git/parse_gogit.go index 99cb1306cccef..434fb4160f75b 100644 --- a/modules/git/parse_gogit.go +++ b/modules/git/parse_gogit.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -// +build !nogogit +// +build gogit package git diff --git a/modules/git/parse_gogit_test.go b/modules/git/parse_gogit_test.go index 550754ba6a799..cf38c29932f5a 100644 --- a/modules/git/parse_gogit_test.go +++ b/modules/git/parse_gogit_test.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -// +build !nogogit +// +build gogit package git diff --git a/modules/git/parse_nogogit.go b/modules/git/parse_nogogit.go index 867de4bacdbbb..26dd700af732a 100644 --- a/modules/git/parse_nogogit.go +++ b/modules/git/parse_nogogit.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -// +build nogogit +// +build !gogit package git diff --git a/modules/git/pipeline/lfs.go b/modules/git/pipeline/lfs.go index 36480174d953b..d47b7d91ea86c 100644 --- a/modules/git/pipeline/lfs.go +++ b/modules/git/pipeline/lfs.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -// +build !nogogit +// +build gogit package pipeline diff --git a/modules/git/pipeline/lfs_nogogit.go b/modules/git/pipeline/lfs_nogogit.go index 516623e36e4d2..30d33e27e0378 100644 --- a/modules/git/pipeline/lfs_nogogit.go +++ b/modules/git/pipeline/lfs_nogogit.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -// +build nogogit +// +build !gogit package pipeline diff --git a/modules/git/repo_base_gogit.go b/modules/git/repo_base_gogit.go index 3be0fce28dbd1..19a3f84571fb6 100644 --- a/modules/git/repo_base_gogit.go +++ b/modules/git/repo_base_gogit.go @@ -3,7 +3,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -// +build !nogogit +// +build gogit package git diff --git a/modules/git/repo_base_nogogit.go b/modules/git/repo_base_nogogit.go index dc90d729b692e..e05219a4e705e 100644 --- a/modules/git/repo_base_nogogit.go +++ b/modules/git/repo_base_nogogit.go @@ -3,7 +3,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -// +build nogogit +// +build !gogit package git diff --git a/modules/git/repo_blob_gogit.go b/modules/git/repo_blob_gogit.go index be1efc949346d..485c233ff8914 100644 --- a/modules/git/repo_blob_gogit.go +++ b/modules/git/repo_blob_gogit.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -// +build !nogogit +// +build gogit package git diff --git a/modules/git/repo_blob_nogogit.go b/modules/git/repo_blob_nogogit.go index 57df74007479c..9959420df4e98 100644 --- a/modules/git/repo_blob_nogogit.go +++ b/modules/git/repo_blob_nogogit.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -// +build nogogit +// +build !gogit package git diff --git a/modules/git/repo_branch_gogit.go b/modules/git/repo_branch_gogit.go index dccfa802cef5b..65cb77a8b55ff 100644 --- a/modules/git/repo_branch_gogit.go +++ b/modules/git/repo_branch_gogit.go @@ -3,7 +3,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -// +build !nogogit +// +build gogit package git diff --git a/modules/git/repo_branch_nogogit.go b/modules/git/repo_branch_nogogit.go index c11eb11e01181..5ec46d725e4b4 100644 --- a/modules/git/repo_branch_nogogit.go +++ b/modules/git/repo_branch_nogogit.go @@ -3,7 +3,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -// +build nogogit +// +build !gogit package git diff --git a/modules/git/repo_commit_gogit.go b/modules/git/repo_commit_gogit.go index 89553e53112e2..48b0cfe19d063 100644 --- a/modules/git/repo_commit_gogit.go +++ b/modules/git/repo_commit_gogit.go @@ -3,7 +3,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -// +build !nogogit +// +build gogit package git diff --git a/modules/git/repo_commit_nogogit.go b/modules/git/repo_commit_nogogit.go index 5184f3cbe4282..896fc7ea18c45 100644 --- a/modules/git/repo_commit_nogogit.go +++ b/modules/git/repo_commit_nogogit.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -// +build nogogit +// +build !gogit package git diff --git a/modules/git/repo_commitgraph_gogit.go b/modules/git/repo_commitgraph_gogit.go index 4e5f8a37786f5..67731094513cc 100644 --- a/modules/git/repo_commitgraph_gogit.go +++ b/modules/git/repo_commitgraph_gogit.go @@ -3,7 +3,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -// +build !nogogit +// +build gogit package git diff --git a/modules/git/repo_language_stats_gogit.go b/modules/git/repo_language_stats_gogit.go index d802a7a1c0c45..b5a235921c8ae 100644 --- a/modules/git/repo_language_stats_gogit.go +++ b/modules/git/repo_language_stats_gogit.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -// +build !nogogit +// +build gogit package git diff --git a/modules/git/repo_language_stats_nogogit.go b/modules/git/repo_language_stats_nogogit.go index 6be510ef22d2d..5607e4591ad0e 100644 --- a/modules/git/repo_language_stats_nogogit.go +++ b/modules/git/repo_language_stats_nogogit.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -// +build nogogit +// +build !gogit package git diff --git a/modules/git/repo_ref_gogit.go b/modules/git/repo_ref_gogit.go index 2b14c896e099b..2e83e6c4627be 100644 --- a/modules/git/repo_ref_gogit.go +++ b/modules/git/repo_ref_gogit.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -// +build !nogogit +// +build gogit package git diff --git a/modules/git/repo_ref_nogogit.go b/modules/git/repo_ref_nogogit.go index 15ca3c68fdc70..540961592b4c0 100644 --- a/modules/git/repo_ref_nogogit.go +++ b/modules/git/repo_ref_nogogit.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -// +build nogogit +// +build !gogit package git diff --git a/modules/git/repo_tag_gogit.go b/modules/git/repo_tag_gogit.go index a53273a38e9b2..3ac097c9a8c2c 100644 --- a/modules/git/repo_tag_gogit.go +++ b/modules/git/repo_tag_gogit.go @@ -3,7 +3,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -// +build !nogogit +// +build gogit package git diff --git a/modules/git/repo_tag_nogogit.go b/modules/git/repo_tag_nogogit.go index fba93e135e57a..83cbc58e342ee 100644 --- a/modules/git/repo_tag_nogogit.go +++ b/modules/git/repo_tag_nogogit.go @@ -3,7 +3,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -// +build nogogit +// +build !gogit package git diff --git a/modules/git/repo_tree_gogit.go b/modules/git/repo_tree_gogit.go index c9f3ece07e782..d878f5e7a7258 100644 --- a/modules/git/repo_tree_gogit.go +++ b/modules/git/repo_tree_gogit.go @@ -3,7 +3,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -// +build !nogogit +// +build gogit package git diff --git a/modules/git/repo_tree_nogogit.go b/modules/git/repo_tree_nogogit.go index 4935c0caa33fa..fff0e192bfb71 100644 --- a/modules/git/repo_tree_nogogit.go +++ b/modules/git/repo_tree_nogogit.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -// +build nogogit +// +build !gogit package git diff --git a/modules/git/sha1_gogit.go b/modules/git/sha1_gogit.go index 6023ab94f2a35..5953af58bfcc3 100644 --- a/modules/git/sha1_gogit.go +++ b/modules/git/sha1_gogit.go @@ -3,7 +3,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -// +build !nogogit +// +build gogit package git diff --git a/modules/git/sha1_nogogit.go b/modules/git/sha1_nogogit.go index 4b4fea5060075..c954cea8e210c 100644 --- a/modules/git/sha1_nogogit.go +++ b/modules/git/sha1_nogogit.go @@ -3,7 +3,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -// +build nogogit +// +build !gogit package git diff --git a/modules/git/signature_gogit.go b/modules/git/signature_gogit.go index a4426e0958f3f..804c0074d351c 100644 --- a/modules/git/signature_gogit.go +++ b/modules/git/signature_gogit.go @@ -3,7 +3,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -// +build !nogogit +// +build gogit package git diff --git a/modules/git/signature_nogogit.go b/modules/git/signature_nogogit.go index 6b498c7fd0396..50e8045f59485 100644 --- a/modules/git/signature_nogogit.go +++ b/modules/git/signature_nogogit.go @@ -3,7 +3,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -// +build nogogit +// +build !gogit package git diff --git a/modules/git/tree_blob_gogit.go b/modules/git/tree_blob_gogit.go index 3d762a9c6abc6..93ebc8a367da4 100644 --- a/modules/git/tree_blob_gogit.go +++ b/modules/git/tree_blob_gogit.go @@ -3,7 +3,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -// +build !nogogit +// +build gogit package git diff --git a/modules/git/tree_blob_nogogit.go b/modules/git/tree_blob_nogogit.go index ce2221ebd427e..6da0ccfe8e56e 100644 --- a/modules/git/tree_blob_nogogit.go +++ b/modules/git/tree_blob_nogogit.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -// +build nogogit +// +build !gogit package git diff --git a/modules/git/tree_entry_gogit.go b/modules/git/tree_entry_gogit.go index 41c534994c3c6..219251a77e42c 100644 --- a/modules/git/tree_entry_gogit.go +++ b/modules/git/tree_entry_gogit.go @@ -3,7 +3,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -// +build !nogogit +// +build gogit package git diff --git a/modules/git/tree_entry_nogogit.go b/modules/git/tree_entry_nogogit.go index c7d67ee9d6b77..f18daee7788f8 100644 --- a/modules/git/tree_entry_nogogit.go +++ b/modules/git/tree_entry_nogogit.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -// +build nogogit +// +build !gogit package git diff --git a/modules/git/tree_entry_test.go b/modules/git/tree_entry_test.go index 69c17279f2894..16cfbc4fc3adf 100644 --- a/modules/git/tree_entry_test.go +++ b/modules/git/tree_entry_test.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -// +build !nogogit +// +build gogit package git diff --git a/modules/git/tree_gogit.go b/modules/git/tree_gogit.go index 01aa94ca1d8dd..79132c5548a92 100644 --- a/modules/git/tree_gogit.go +++ b/modules/git/tree_gogit.go @@ -3,7 +3,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -// +build !nogogit +// +build gogit package git diff --git a/modules/git/tree_nogogit.go b/modules/git/tree_nogogit.go index 576a6d95201de..e78115b77765f 100644 --- a/modules/git/tree_nogogit.go +++ b/modules/git/tree_nogogit.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. -// +build nogogit +// +build !gogit package git From 1897dade340f06cd6da057097de2193a22c18655 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Thu, 10 Dec 2020 14:18:14 +0000 Subject: [PATCH 18/19] placate lint Signed-off-by: Andrew Thornton --- modules/git/blob_nogogit.go | 4 ++-- modules/git/commit_info_nogogit.go | 2 +- modules/git/repo_commit_nogogit.go | 4 ++-- modules/git/repo_tree_nogogit.go | 8 ++++---- modules/git/sha1_nogogit.go | 11 ++++++----- modules/git/signature_nogogit.go | 1 + 6 files changed, 16 insertions(+), 14 deletions(-) diff --git a/modules/git/blob_nogogit.go b/modules/git/blob_nogogit.go index ef04e368fc490..401b172860ee6 100644 --- a/modules/git/blob_nogogit.go +++ b/modules/git/blob_nogogit.go @@ -34,9 +34,9 @@ func (b *Blob) DataAsync() (io.ReadCloser, error) { err = NewCommand("cat-file", "--batch").RunInDirFullPipeline(b.repoPath, stdoutWriter, stderr, strings.NewReader(b.ID.String()+"\n")) if err != nil { err = ConcatenateError(err, stderr.String()) - stdoutWriter.CloseWithError(err) + _ = stdoutWriter.CloseWithError(err) } else { - stdoutWriter.Close() + _ = stdoutWriter.Close() } }() diff --git a/modules/git/commit_info_nogogit.go b/modules/git/commit_info_nogogit.go index d09a6c3f52f8f..ac0c7cff5dcaf 100644 --- a/modules/git/commit_info_nogogit.go +++ b/modules/git/commit_info_nogogit.go @@ -311,7 +311,7 @@ revListLoop: } currentPath += target - // if we've now found the curent path check its sha id and commit status + // if we've now found the current path check its sha id and commit status if treePath == currentPath && paths[0] == "" { if len(ids[0]) == 0 { copy(allShaBuf[0:20], treeID) diff --git a/modules/git/repo_commit_nogogit.go b/modules/git/repo_commit_nogogit.go index 896fc7ea18c45..a43fe4b3349f9 100644 --- a/modules/git/repo_commit_nogogit.go +++ b/modules/git/repo_commit_nogogit.go @@ -14,7 +14,7 @@ import ( "strings" ) -// GetRefCommitID returns the last commit ID string of given reference (branch or tag). +// ResolveReference resolves a name to a reference func (repo *Repository) ResolveReference(name string) (string, error) { stdout, err := NewCommand("show-ref", "--hash", name).RunInDir(repo.Path) if err != nil { @@ -100,7 +100,7 @@ func (repo *Repository) getCommit(id SHA1) (*Commit, error) { case "commit": return CommitFromReader(repo, id, io.LimitReader(bufReader, size)) default: - stdoutReader.CloseWithError(fmt.Errorf("unknown typ: %s", typ)) + _ = stdoutReader.CloseWithError(fmt.Errorf("unknown typ: %s", typ)) log("Unknown typ: %s", typ) return nil, ErrNotExist{ ID: id.String(), diff --git a/modules/git/repo_tree_nogogit.go b/modules/git/repo_tree_nogogit.go index fff0e192bfb71..416205d8a0967 100644 --- a/modules/git/repo_tree_nogogit.go +++ b/modules/git/repo_tree_nogogit.go @@ -25,9 +25,9 @@ func (repo *Repository) getTree(id SHA1) (*Tree, error) { stderr := &strings.Builder{} err := NewCommand("cat-file", "--batch").RunInDirFullPipeline(repo.Path, stdoutWriter, stderr, strings.NewReader(id.String()+"\n")) if err != nil { - stdoutWriter.CloseWithError(ConcatenateError(err, stderr.String())) + _ = stdoutWriter.CloseWithError(ConcatenateError(err, stderr.String())) } else { - stdoutWriter.Close() + _ = stdoutWriter.Close() } }() @@ -59,7 +59,7 @@ func (repo *Repository) getTree(id SHA1) (*Tree, error) { case "commit": commit, err := CommitFromReader(repo, id, bufReader) if err != nil { - stdoutReader.CloseWithError(err) + _ = stdoutReader.CloseWithError(err) return nil, err } commit.Tree.ResolvedID = commit.ID @@ -71,7 +71,7 @@ func (repo *Repository) getTree(id SHA1) (*Tree, error) { tree.ResolvedID = id return tree, nil default: - stdoutReader.CloseWithError(fmt.Errorf("unknown typ: %s", typ)) + _ = stdoutReader.CloseWithError(fmt.Errorf("unknown typ: %s", typ)) return nil, ErrNotExist{ ID: id.String(), } diff --git a/modules/git/sha1_nogogit.go b/modules/git/sha1_nogogit.go index c954cea8e210c..09b5baacd5c98 100644 --- a/modules/git/sha1_nogogit.go +++ b/modules/git/sha1_nogogit.go @@ -22,6 +22,7 @@ func (s SHA1) String() string { return hex.EncodeToString(s[:]) } +// IsZero returns whether this SHA1 is all zeroes func (s SHA1) IsZero() bool { var empty SHA1 return s == empty @@ -35,7 +36,7 @@ func ComputeBlobHash(content []byte) SHA1 { // ComputeHash compute the hash for a given ObjectType and content func ComputeHash(t ObjectType, content []byte) SHA1 { h := NewHasher(t, int64(len(content))) - h.Write(content) + _, _ = h.Write(content) return h.Sum() } @@ -47,10 +48,10 @@ type Hasher struct { // NewHasher takes an object type and size and creates a hasher to generate a SHA func NewHasher(t ObjectType, size int64) Hasher { h := Hasher{sha1.New()} - h.Write(t.Bytes()) - h.Write([]byte(" ")) - h.Write([]byte(strconv.FormatInt(size, 10))) - h.Write([]byte{0}) + _, _ = h.Write(t.Bytes()) + _, _ = h.Write([]byte(" ")) + _, _ = h.Write([]byte(strconv.FormatInt(size, 10))) + _, _ = h.Write([]byte{0}) return h } diff --git a/modules/git/signature_nogogit.go b/modules/git/signature_nogogit.go index 50e8045f59485..753d87b6059f1 100644 --- a/modules/git/signature_nogogit.go +++ b/modules/git/signature_nogogit.go @@ -28,6 +28,7 @@ func (s *Signature) String() string { return fmt.Sprintf("%s <%s>", s.Name, s.Email) } +// Decode decodes a byte array representing a signature to signature func (s *Signature) Decode(b []byte) { sig, _ := newSignatureFromCommitline(b) s.Email = sig.Email From 320adb4437fbc5cec00b2e48be0acafc70a75b28 Mon Sep 17 00:00:00 2001 From: Andrew Thornton Date: Sat, 12 Dec 2020 17:05:32 +0000 Subject: [PATCH 19/19] as per @6543 Signed-off-by: Andrew Thornton --- Makefile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index f4fbccc5bccc9..fe26a413bd082 100644 --- a/Makefile +++ b/Makefile @@ -342,8 +342,8 @@ watch-backend: go-check .PHONY: test test: - @echo "Running go test..." - $(GO) test $(GOTESTFLAGS) -mod=vendor -tags='$(TEST_TAGS)' $(GO_PACKAGES) + @echo "Running go test with -tags '$(TEST_TAGS)'..." + @$(GO) test $(GOTESTFLAGS) -mod=vendor -tags='$(TEST_TAGS)' $(GO_PACKAGES) .PHONY: test-check test-check: @@ -359,8 +359,8 @@ test-check: .PHONY: test\#% test\#%: - @echo "Running go test..." - $(GO) test -mod=vendor -tags='$(TEST_TAGS)' -run $(subst .,/,$*) $(GO_PACKAGES) + @echo "Running go test with -tags '$(TEST_TAGS)'..." + @$(GO) test -mod=vendor -tags='$(TEST_TAGS)' -run $(subst .,/,$*) $(GO_PACKAGES) .PHONY: coverage coverage: @@ -368,8 +368,8 @@ coverage: .PHONY: unit-test-coverage unit-test-coverage: - @echo "Running unit-test-coverage..." - $(GO) test $(GOTESTFLAGS) -mod=vendor -tags='$(TEST_TAGS)' -cover -coverprofile coverage.out $(GO_PACKAGES) && echo "\n==>\033[32m Ok\033[m\n" || exit 1 + @echo "Running unit-test-coverage -tags '$(TEST_TAGS)'..." + @$(GO) test $(GOTESTFLAGS) -mod=vendor -tags='$(TEST_TAGS)' -cover -coverprofile coverage.out $(GO_PACKAGES) && echo "\n==>\033[32m Ok\033[m\n" || exit 1 .PHONY: vendor vendor: