From c5f1fa8ec298677049a423e27c92492fdd287345 Mon Sep 17 00:00:00 2001 From: Nick Guenther Date: Sun, 8 May 2022 01:51:34 -0400 Subject: [PATCH 1/4] Support downloading from "dumb http" with git-annex. I don't think git-annex supports uploading over http, so I didn't try to. Strictly download only. To support private repos, I had to hunt down and patch a secret extra corner of security that Gitea only applies to HTTP Basic Auth for some reason (in services/auth/basic.go) Ref: https://git-annex.branchable.com/tips/setup_a_public_repository_on_a_web_site/ Fixes https://github.com/neuropoly/gitea/issues/3 --- routers/web/repo/http.go | 27 +++++++++++++++++++++++++++ routers/web/web.go | 2 ++ services/auth/auth.go | 14 ++++++++++++++ services/auth/basic.go | 4 ++-- 4 files changed, 45 insertions(+), 2 deletions(-) diff --git a/routers/web/repo/http.go b/routers/web/repo/http.go index 5aa2bcd13471b..61a7dfdf20b16 100644 --- a/routers/web/repo/http.go +++ b/routers/web/repo/http.go @@ -611,3 +611,30 @@ func GetIdxFile(ctx *context.Context) { h.sendFile("application/x-git-packed-objects-toc", "objects/pack/pack-"+ctx.Params("file")+".idx") } } + +// GetAnnexObject implements git-annex dumb HTTP +func GetAnnexObject(ctx *context.Context) { + //if !setting.Annex.Enabled { // TODO + if false { + ctx.PlainText(http.StatusNotFound, "Not found") + return + } + h := httpBase(ctx) + if h != nil { + // git-annex objects are stored in .git/annex/objects/{hash1}/{hash2}/{key}/{key} + // where key is a string containing the size and (usually SHA256) checksum of the file, + // and hash1+hash2 are the first few bits of the md5sum of key itself. + // ({hash1}/{hash2}/ is just there to avoid putting too many files in one directory) + // ref: https://git-annex.branchable.com/internals/hashing/ + + // keyDir should = key, but we don't enforce that + object := ctx.Params("hash1") + "/" + ctx.Params("hash2") + "/" + ctx.Params("keyDir") + "/" + ctx.Params("key") + + // use path.Clean() to sanitize the input but otherwise trust it + // the router code disallows, so this should be redundant, but it's harmless extra safety even if it is. + object = path.Clean("/" + object)[1:] // path.Clean() removes all directory traversals *if* given an absolute path to begin with, so make object an absolute path and then a relative path again + + h.setHeaderCacheForever() + h.sendFile("application/octet-stream", "annex/objects/"+object) + } +} diff --git a/routers/web/web.go b/routers/web/web.go index 1852ecc2e24fd..e27d14473842e 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -1255,6 +1255,7 @@ func RegisterRoutes(m *web.Route) { m.PostOptions("/git-receive-pack", repo.ServiceReceivePack) m.GetOptions("/info/refs", repo.GetInfoRefs) m.GetOptions("/HEAD", repo.GetTextFile("HEAD")) + m.GetOptions("/config", repo.GetTextFile("config")) // needed by git-annex's dumb http mode m.GetOptions("/objects/info/alternates", repo.GetTextFile("objects/info/alternates")) m.GetOptions("/objects/info/http-alternates", repo.GetTextFile("objects/info/http-alternates")) m.GetOptions("/objects/info/packs", repo.GetInfoPacks) @@ -1262,6 +1263,7 @@ func RegisterRoutes(m *web.Route) { m.GetOptions("/objects/{head:[0-9a-f]{2}}/{hash:[0-9a-f]{38}}", repo.GetLooseObject) m.GetOptions("/objects/pack/pack-{file:[0-9a-f]{40}}.pack", repo.GetPackFile) m.GetOptions("/objects/pack/pack-{file:[0-9a-f]{40}}.idx", repo.GetIdxFile) + m.GetOptions("/annex/objects/{hash1}/{hash2}/{keyDir}/{key}", repo.GetAnnexObject) // for git-annex }, ignSignInAndCsrf, context_service.UserAssignmentWeb()) }) }) diff --git a/services/auth/auth.go b/services/auth/auth.go index 3a5bb9d27e65b..5b77ced077c12 100644 --- a/services/auth/auth.go +++ b/services/auth/auth.go @@ -51,6 +51,20 @@ func isGitRawReleaseOrLFSPath(req *http.Request) bool { return false } +var ( + annexPathRe = regexp.MustCompile(`^/[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+/annex/`) +) + +func isAnnexPath(req *http.Request) bool { + //if setting.Annex.Enabled { // TODO + if true { + // "/config" is git's config, not specifically git-annex's; but the only current + // user of it is when git-annex downloads the annex.uuid during 'git annex init'. + return strings.HasSuffix(req.URL.Path, "/config") || annexPathRe.MatchString(req.URL.Path) + } + return false +} + // handleSignIn clears existing session variables and stores new ones for the specified user object func handleSignIn(resp http.ResponseWriter, req *http.Request, sess SessionStore, user *user_model.User) { // We need to regenerate the session... diff --git a/services/auth/basic.go b/services/auth/basic.go index 9b32ad29af8bd..ba61c3dd91fd4 100644 --- a/services/auth/basic.go +++ b/services/auth/basic.go @@ -42,8 +42,8 @@ func (b *Basic) Name() string { // name/token on successful validation. // Returns nil if header is empty or validation fails. func (b *Basic) Verify(req *http.Request, w http.ResponseWriter, store DataStore, sess SessionStore) *user_model.User { - // Basic authentication should only fire on API, Download or on Git or LFSPaths - if !middleware.IsAPIPath(req) && !isContainerPath(req) && !isAttachmentDownload(req) && !isGitRawReleaseOrLFSPath(req) { + // Basic authentication should only fire on API, Download or on Git, LFSPaths or Git-Annex paths + if !middleware.IsAPIPath(req) && !isContainerPath(req) && !isAttachmentDownload(req) && !isGitRawReleaseOrLFSPath(req) && !isAnnexPath(req) { return nil } From 3f9028bea2ad3b208fe09d09163e5e643dfde049 Mon Sep 17 00:00:00 2001 From: Nick Guenther Date: Mon, 19 Sep 2022 11:59:44 -0400 Subject: [PATCH 2/4] git-annex: enforce setting.Annex.Enabled with a middleware Fixes https://github.com/neuropoly/gitea/pull/6#discussion_r868488809 + https://github.com/neuropoly/gitea/pull/6#discussion_r868484761 Co-authored-by: Mathieu Guay-Paquet --- routers/web/web.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/routers/web/web.go b/routers/web/web.go index e27d14473842e..608e58c151919 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -282,6 +282,13 @@ func RegisterRoutes(m *web.Route) { } } + annexEnabled := func(ctx *context.Context) { + if !setting.Annex.Enabled { + ctx.Error(http.StatusNotFound) + return + } + } + federationEnabled := func(ctx *context.Context) { if !setting.Federation.Enabled { ctx.Error(http.StatusNotFound) @@ -1250,12 +1257,17 @@ func RegisterRoutes(m *web.Route) { }) }, ignSignInAndCsrf, lfsServerEnabled) + m.Group("", func() { + // for git-annex + m.GetOptions("/config", repo.GetTextFile("config")) // needed by clients reading annex.uuid during `git annex initremote` + m.GetOptions("/annex/objects/{hash1}/{hash2}/{keyDir}/{key}", repo.GetAnnexObject) + }, ignSignInAndCsrf, annexEnabled, context_service.UserAssignmentWeb()) + m.Group("", func() { m.PostOptions("/git-upload-pack", repo.ServiceUploadPack) m.PostOptions("/git-receive-pack", repo.ServiceReceivePack) m.GetOptions("/info/refs", repo.GetInfoRefs) m.GetOptions("/HEAD", repo.GetTextFile("HEAD")) - m.GetOptions("/config", repo.GetTextFile("config")) // needed by git-annex's dumb http mode m.GetOptions("/objects/info/alternates", repo.GetTextFile("objects/info/alternates")) m.GetOptions("/objects/info/http-alternates", repo.GetTextFile("objects/info/http-alternates")) m.GetOptions("/objects/info/packs", repo.GetInfoPacks) @@ -1263,7 +1275,6 @@ func RegisterRoutes(m *web.Route) { m.GetOptions("/objects/{head:[0-9a-f]{2}}/{hash:[0-9a-f]{38}}", repo.GetLooseObject) m.GetOptions("/objects/pack/pack-{file:[0-9a-f]{40}}.pack", repo.GetPackFile) m.GetOptions("/objects/pack/pack-{file:[0-9a-f]{40}}.idx", repo.GetIdxFile) - m.GetOptions("/annex/objects/{hash1}/{hash2}/{keyDir}/{key}", repo.GetAnnexObject) // for git-annex }, ignSignInAndCsrf, context_service.UserAssignmentWeb()) }) }) From b5f8933e85dfd1df3ae8b8ab4567ae5570861dcc Mon Sep 17 00:00:00 2001 From: Nick Guenther Date: Mon, 19 Sep 2022 12:04:13 -0400 Subject: [PATCH 3/4] git-annex: use path.Join() instead of appending "/" This is a bit more conventional/safe. Fixes https://github.com/neuropoly/gitea/pull/6#discussion_r868505765 Co-authored-by: Mathieu Guay-Paquet --- routers/web/repo/http.go | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/routers/web/repo/http.go b/routers/web/repo/http.go index 61a7dfdf20b16..b10a651a83cb5 100644 --- a/routers/web/repo/http.go +++ b/routers/web/repo/http.go @@ -628,11 +628,19 @@ func GetAnnexObject(ctx *context.Context) { // ref: https://git-annex.branchable.com/internals/hashing/ // keyDir should = key, but we don't enforce that - object := ctx.Params("hash1") + "/" + ctx.Params("hash2") + "/" + ctx.Params("keyDir") + "/" + ctx.Params("key") - - // use path.Clean() to sanitize the input but otherwise trust it - // the router code disallows, so this should be redundant, but it's harmless extra safety even if it is. - object = path.Clean("/" + object)[1:] // path.Clean() removes all directory traversals *if* given an absolute path to begin with, so make object an absolute path and then a relative path again + object := path.Join(ctx.Params("hash1"), ctx.Params("hash2"), ctx.Params("keyDir"), ctx.Params("key")) + + // Sanitize the input against directory traversals. + // + // This works because, if a path starts rooted, + // path.Clean() will remove all excess '..'. So + // this pretends the path is rooted ("/"), then + // path.Join() calls path.Clean() internally, + // then this unroots the path it ([1:]) (and + // + // The router code also disallows "..", so this should be) + // redundant, but it's defensive to have it here. + object = path.Join("/", object)[1:] h.setHeaderCacheForever() h.sendFile("application/octet-stream", "annex/objects/"+object) From 65d8c81e537da1bc6919e91c04ae6a12e6045260 Mon Sep 17 00:00:00 2001 From: Nick Guenther Date: Wed, 7 Sep 2022 15:35:43 -0400 Subject: [PATCH 4/4] git-annex tests: HTTP This distinguishes between anonymous and passworded HTTP downloads, a distinction which does not exist for SSH. But it only tests downloading, because I haven't figured if/how git-annex supports uploading over HTTP (I suspect it doesn't). --- modules/git/command.go | 3 +- tests/integration/git_annex_test.go | 360 +++++++++++++++++- .../git_helper_for_declarative_test.go | 7 + 3 files changed, 355 insertions(+), 15 deletions(-) diff --git a/modules/git/command.go b/modules/git/command.go index b24d32dbe8743..ae2cd250336ed 100644 --- a/modules/git/command.go +++ b/modules/git/command.go @@ -292,12 +292,13 @@ func (c *Command) RunStdBytes(opts *RunOpts) (stdout, stderr []byte, runErr RunS } // AllowLFSFiltersArgs return globalCommandArgs with lfs filter, it should only be used for tests +// It also re-enables git-credential(1), which is used to test git-annex's HTTP support func AllowLFSFiltersArgs() []string { // Now here we should explicitly allow lfs filters to run filteredLFSGlobalArgs := make([]string, len(globalCommandArgs)) j := 0 for _, arg := range globalCommandArgs { - if strings.Contains(arg, "lfs") { + if strings.Contains(arg, "lfs") || strings.Contains(arg, "credential") { j-- } else { filteredLFSGlobalArgs[j] = arg diff --git a/tests/integration/git_annex_test.go b/tests/integration/git_annex_test.go index dee8e4a30522d..433057ae569b1 100644 --- a/tests/integration/git_annex_test.go +++ b/tests/integration/git_annex_test.go @@ -51,7 +51,8 @@ func doCreateRemoteAnnexRepository(t *testing.T, u *url.URL, ctx APITestContext, /* Test that permissions are enforced on git-annex-shell commands. - Along the way, test that uploading, downloading, and deleting all work. + Along the way, this also tests that uploading, downloading, and deleting all work, + so we haven't written separate tests for those. */ func TestGitAnnexPermissions(t *testing.T) { if !setting.Annex.Enabled { @@ -68,6 +69,16 @@ func TestGitAnnexPermissions(t *testing.T) { onGiteaRun(t, func(t *testing.T, u *url.URL) { + // Tell git-annex to allow http://127.0.0.1, http://localhost and http://::1. Without + // this, all `git annex` commands will silently fail when run against http:// remotes + // without explaining what's wrong. + // + // Note: onGiteaRun() sets up an alternate HOME so this actually edits + // tests/integration/gitea-integration-*/data/home/.gitconfig and + // if you're debugging you need to remember to match that. + _, _, err := git.NewCommandNoGlobals("config", "--global", "annex.security.allowed-ip-addresses", "all").RunStdString(&git.RunOpts{}) + require.NoError(t, err) + t.Run("Public", func(t *testing.T) { defer tests.PrintCurrentTest(t)() @@ -80,8 +91,6 @@ func TestGitAnnexPermissions(t *testing.T) { require.NoError(t, err) require.False(t, repo.IsPrivate) - // Remote addresses of the repo - repoURL := createSSHUrl(ownerCtx.GitPath(), u) // remote git URL remoteRepoPath := path.Join(setting.RepoRootPath, ownerCtx.GitPath()) // path on disk -- which can be examined directly because we're testing from localhost // Different sessions, so we can test different permissions. @@ -103,6 +112,8 @@ func TestGitAnnexPermissions(t *testing.T) { t.Run("SSH", func(t *testing.T) { defer tests.PrintCurrentTest(t)() + repoURL := createSSHUrl(ownerCtx.GitPath(), u) + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions @@ -130,6 +141,31 @@ func TestGitAnnexPermissions(t *testing.T) { }) }) }) + + t.Run("HTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + }) }) t.Run("Writer", func(t *testing.T) { @@ -138,6 +174,8 @@ func TestGitAnnexPermissions(t *testing.T) { t.Run("SSH", func(t *testing.T) { defer tests.PrintCurrentTest(t)() + repoURL := createSSHUrl(ownerCtx.GitPath(), u) + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions @@ -165,6 +203,31 @@ func TestGitAnnexPermissions(t *testing.T) { }) }) }) + + t.Run("HTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + }) }) t.Run("Reader", func(t *testing.T) { @@ -173,6 +236,8 @@ func TestGitAnnexPermissions(t *testing.T) { t.Run("SSH", func(t *testing.T) { defer tests.PrintCurrentTest(t)() + repoURL := createSSHUrl(ownerCtx.GitPath(), u) + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions @@ -200,6 +265,31 @@ func TestGitAnnexPermissions(t *testing.T) { }) }) }) + + t.Run("HTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + }) }) t.Run("Outsider", func(t *testing.T) { @@ -208,6 +298,8 @@ func TestGitAnnexPermissions(t *testing.T) { t.Run("SSH", func(t *testing.T) { defer tests.PrintCurrentTest(t)() + repoURL := createSSHUrl(ownerCtx.GitPath(), u) + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions @@ -235,6 +327,61 @@ func TestGitAnnexPermissions(t *testing.T) { }) }) }) + + t.Run("HTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + }) + }) + + t.Run("Anonymous", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + // Only HTTP has an anonymous mode + t.Run("HTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + // unlike the other tests, at this step we *do not* define credentials: + + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) }) t.Run("Delete", func(t *testing.T) { @@ -259,8 +406,6 @@ func TestGitAnnexPermissions(t *testing.T) { require.NoError(t, err) require.True(t, repo.IsPrivate) - // Remote addresses of the repo - repoURL := createSSHUrl(ownerCtx.GitPath(), u) // remote git URL remoteRepoPath := path.Join(setting.RepoRootPath, ownerCtx.GitPath()) // path on disk -- which can be examined directly because we're testing from localhost // Different sessions, so we can test different permissions. @@ -284,6 +429,8 @@ func TestGitAnnexPermissions(t *testing.T) { t.Run("SSH", func(t *testing.T) { defer tests.PrintCurrentTest(t)() + repoURL := createSSHUrl(ownerCtx.GitPath(), u) + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions @@ -311,6 +458,31 @@ func TestGitAnnexPermissions(t *testing.T) { }) }) }) + + t.Run("HTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + }) }) t.Run("Writer", func(t *testing.T) { @@ -319,6 +491,8 @@ func TestGitAnnexPermissions(t *testing.T) { t.Run("SSH", func(t *testing.T) { defer tests.PrintCurrentTest(t)() + repoURL := createSSHUrl(ownerCtx.GitPath(), u) + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions @@ -346,6 +520,31 @@ func TestGitAnnexPermissions(t *testing.T) { }) }) }) + + t.Run("HTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + }) }) t.Run("Reader", func(t *testing.T) { @@ -354,6 +553,8 @@ func TestGitAnnexPermissions(t *testing.T) { t.Run("SSH", func(t *testing.T) { defer tests.PrintCurrentTest(t)() + repoURL := createSSHUrl(ownerCtx.GitPath(), u) + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions @@ -381,6 +582,31 @@ func TestGitAnnexPermissions(t *testing.T) { }) }) }) + + t.Run("HTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + }) }) t.Run("Outsider", func(t *testing.T) { @@ -389,6 +615,8 @@ func TestGitAnnexPermissions(t *testing.T) { t.Run("SSH", func(t *testing.T) { defer tests.PrintCurrentTest(t)() + repoURL := createSSHUrl(ownerCtx.GitPath(), u) + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions @@ -416,6 +644,61 @@ func TestGitAnnexPermissions(t *testing.T) { }) }) }) + + t.Run("HTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + }) + }) + + t.Run("Anonymous", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + // Only HTTP has an anonymous mode + t.Run("HTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + // unlike the other tests, at this step we *do not* define credentials: + + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) }) t.Run("Delete", func(t *testing.T) { @@ -437,7 +720,7 @@ precondition: repoPath contains a pre-cloned git repo with an annex: a valid git */ func doAnnexInitTest(remoteRepoPath string, repoPath string) (err error) { - _, _, err = git.NewCommand(git.DefaultContext, "annex", "init").RunStdString(&git.RunOpts{Dir: repoPath}) + _, _, err = git.NewCommandNoGlobals("annex", "init").RunStdString(&git.RunOpts{Dir: repoPath}) if err != nil { return fmt.Errorf("Couldn't `git annex init`: %w", err) } @@ -445,7 +728,7 @@ func doAnnexInitTest(remoteRepoPath string, repoPath string) (err error) { // - method 0: 'git config remote.origin.annex-uuid'. // Demonstrates that 'git annex init' successfully contacted // the remote git-annex and was able to learn its ID number. - readAnnexUUID, _, err := git.NewCommand(git.DefaultContext, "config", "remote.origin.annex-uuid").RunStdString(&git.RunOpts{Dir: repoPath}) + readAnnexUUID, _, err := git.NewCommandNoGlobals("config", "remote.origin.annex-uuid").RunStdString(&git.RunOpts{Dir: repoPath}) if err != nil { return fmt.Errorf("Couldn't read remote `git config remote.origin.annex-uuid`: %w", err) } @@ -456,7 +739,7 @@ func doAnnexInitTest(remoteRepoPath string, repoPath string) (err error) { return errors.New(fmt.Sprintf("'git config remote.origin.annex-uuid' should have been able to download the remote's uuid; but instead read '%s'.", readAnnexUUID)) } - remoteAnnexUUID, _, err := git.NewCommand(git.DefaultContext, "config", "annex.uuid").RunStdString(&git.RunOpts{Dir: remoteRepoPath}) + remoteAnnexUUID, _, err := git.NewCommandNoGlobals("config", "annex.uuid").RunStdString(&git.RunOpts{Dir: remoteRepoPath}) if err != nil { return fmt.Errorf("Couldn't read local `git config annex.uuid`: %w", err) } @@ -473,7 +756,7 @@ func doAnnexInitTest(remoteRepoPath string, repoPath string) (err error) { // - method 1: 'git annex whereis'. // Demonstrates that git-annex understands the annexed file can be found in the remote annex. - annexWhereis, _, err := git.NewCommand(git.DefaultContext, "annex", "whereis", "large.bin").RunStdString(&git.RunOpts{Dir: repoPath}) + annexWhereis, _, err := git.NewCommandNoGlobals("annex", "whereis", "large.bin").RunStdString(&git.RunOpts{Dir: repoPath}) if err != nil { return fmt.Errorf("Couldn't `git annex whereis large.bin`: %w", err) } @@ -492,7 +775,7 @@ func doAnnexDownloadTest(remoteRepoPath string, repoPath string) (err error) { // "git annex copy" will notice and run "git annex init", silently. // This shouldn't change any results, but be aware in case it does. - _, _, err = git.NewCommand(git.DefaultContext, "annex", "copy", "--from", "origin").RunStdString(&git.RunOpts{Dir: repoPath}) + _, _, err = git.NewCommandNoGlobals("annex", "copy", "--from", "origin").RunStdString(&git.RunOpts{Dir: repoPath}) if err != nil { return err } @@ -540,12 +823,12 @@ func doAnnexUploadTest(remoteRepoPath string, repoPath string) (err error) { return err } - _, _, err = git.NewCommand(git.DefaultContext, "annex", "copy", "--to", "origin").RunStdString(&git.RunOpts{Dir: repoPath}) + _, _, err = git.NewCommandNoGlobals("annex", "copy", "--to", "origin").RunStdString(&git.RunOpts{Dir: repoPath}) if err != nil { return err } - _, _, err = git.NewCommand(git.DefaultContext, "annex", "sync", "--no-content").RunStdString(&git.RunOpts{Dir: repoPath}) + _, _, err = git.NewCommandNoGlobals("annex", "sync", "--no-content").RunStdString(&git.RunOpts{Dir: repoPath}) if err != nil { return err } @@ -654,7 +937,7 @@ func doInitAnnexRepository(repoPath string) error { // 'git annex init' // 'gitea-annex-test' is there to avoid the nuisance comment getting stored. - err = git.NewCommand(git.DefaultContext, "annex", "init", "gitea-annex-test").Run(&git.RunOpts{Dir: repoPath}) + err = git.NewCommandNoGlobals("annex", "init", "gitea-annex-test").Run(&git.RunOpts{Dir: repoPath}) if err != nil { return err } @@ -700,7 +983,7 @@ func doInitRemoteAnnexRepository(t *testing.T, repoURL *url.URL) error { return err } - _, _, err = git.NewCommand(git.DefaultContext, "annex", "sync", "--content").RunStdString(&git.RunOpts{Dir: repoPath}) + _, _, err = git.NewCommandNoGlobals("annex", "sync", "--content").RunStdString(&git.RunOpts{Dir: repoPath}) if err != nil { return err } @@ -749,3 +1032,52 @@ func withAnnexCtxKeyFile(t *testing.T, ctx APITestContext, callback func()) { withCtxKeyFile(t, ctx, callback) } + +/* like withKeyFile(), but sets HTTP credentials instead of SSH credentials. + + It does this by temporarily arranging for through `git config --global` + to use git-credential-store(1) with the password written to a tempfile. + + This is the only reliable way to pass HTTP credentials non-interactively + to git-annex. See https://git-annex.branchable.com/bugs/http_remotes_ignore_annex.web-options_--netrc/#comment-b5a299e9826b322f2d85c96d4929a430 + for joeyh's proclamation on the subject. + + This **is only effective** when used around git.NewCommandNoGlobals() calls. + git.NewCommand() disables credential.helper as a precaution (see modules/git/git.go). + + In contrast, the tests in git_test.go put the password in the remote's URL like + `git config remote.origin.url http://user2:password@localhost:3003/user2/repo-name.git`, + writing the password in repoPath+"/.git/config". That would be equally good, except + that git-annex ignores it! +*/ +func withAnnexCtxHTTPPassword(t *testing.T, u *url.URL, ctx APITestContext, callback func()) { + + credentialedURL := *u + credentialedURL.User = url.UserPassword(ctx.Username, userPassword) // NB: all test users use the same password + + creds := path.Join(t.TempDir(), "creds") + require.NoError(t, os.WriteFile(creds, []byte(credentialedURL.String()), 0600)) + + originalCredentialHelper, _, err := git.NewCommandNoGlobals("config", "--global", "credential.helper").RunStdString(&git.RunOpts{}) + if err != nil && !err.IsExitCode(1) { + // ignore the 'error' thrown when credential.helper is unset (when git config returns 1) + // but catch all others + require.NoError(t, err) + } + hasOriginalCredentialHelper := (err == nil) + + _, _, err = git.NewCommandNoGlobals("config", "--global", "credential.helper", fmt.Sprintf("store --file=%s", creds)).RunStdString(&git.RunOpts{}) + require.NoError(t, err) + + defer (func() { + // reset + if hasOriginalCredentialHelper { + _, _, err = git.NewCommandNoGlobals("config", "--global", "credential.helper", originalCredentialHelper).RunStdString(&git.RunOpts{}) + } else { + _, _, err = git.NewCommandNoGlobals("config", "--global", "--unset", "credential.helper").RunStdString(&git.RunOpts{}) + } + require.NoError(t, err) + })() + + callback() +} diff --git a/tests/integration/git_helper_for_declarative_test.go b/tests/integration/git_helper_for_declarative_test.go index 71abad695dc3b..2d747d3bdbecd 100644 --- a/tests/integration/git_helper_for_declarative_test.go +++ b/tests/integration/git_helper_for_declarative_test.go @@ -71,6 +71,13 @@ func withKeyFile(t *testing.T, keyname string, callback func(string)) { callback(keyFile) } +func createHTTPUrl(gitPath string, u *url.URL) *url.URL { + // this assumes u contains the HTTP base URL that Gitea is running on + u2 := *u + u2.Path = gitPath + return &u2 +} + func createSSHUrl(gitPath string, u *url.URL) *url.URL { u2 := *u u2.Scheme = "ssh"