diff --git a/.github/workflows/pull-db-tests.yml b/.github/workflows/pull-db-tests.yml index 97446e6cd3b2f..f19f066b29e5a 100644 --- a/.github/workflows/pull-db-tests.yml +++ b/.github/workflows/pull-db-tests.yml @@ -46,10 +46,11 @@ jobs: - name: Add hosts to /etc/hosts run: '[ -e "/.dockerenv" ] || [ -e "/run/.containerenv" ] || echo "127.0.0.1 pgsql ldap minio" | sudo tee -a /etc/hosts' - run: make deps-backend + - run: sudo apt update && sudo DEBIAN_FRONTEND=noninteractive apt install -y git-annex - run: make backend env: TAGS: bindata - - run: make test-pgsql-migration test-pgsql + - run: make test-pgsql-migration test-pgsql#TestGitAnnex timeout-minutes: 50 env: TAGS: bindata gogit @@ -69,10 +70,11 @@ jobs: go-version-file: go.mod check-latest: true - run: make deps-backend + - run: sudo apt update && sudo DEBIAN_FRONTEND=noninteractive apt install -y git-annex - run: make backend env: TAGS: bindata gogit sqlite sqlite_unlock_notify - - run: make test-sqlite-migration test-sqlite + - run: make test-sqlite-migration test-sqlite#TestGitAnnex timeout-minutes: 50 env: TAGS: bindata gogit sqlite sqlite_unlock_notify @@ -172,11 +174,12 @@ jobs: - name: Add hosts to /etc/hosts run: '[ -e "/.dockerenv" ] || [ -e "/run/.containerenv" ] || echo "127.0.0.1 mysql elasticsearch smtpimap" | sudo tee -a /etc/hosts' - run: make deps-backend + - run: sudo apt update && sudo DEBIAN_FRONTEND=noninteractive apt install -y git-annex - run: make backend env: TAGS: bindata - name: run tests - run: make test-mysql-migration integration-test-coverage + run: make test-mysql-migration test-mysql#TestGitAnnex env: TAGS: bindata RACE_ENABLED: true @@ -205,10 +208,11 @@ jobs: - name: Add hosts to /etc/hosts run: '[ -e "/.dockerenv" ] || [ -e "/run/.containerenv" ] || echo "127.0.0.1 mssql" | sudo tee -a /etc/hosts' - run: make deps-backend + - run: sudo apt update && sudo DEBIAN_FRONTEND=noninteractive apt install -y git-annex - run: make backend env: TAGS: bindata - - run: make test-mssql-migration test-mssql + - run: make test-mssql-migration test-mssql#TestGitAnnex timeout-minutes: 50 env: TAGS: bindata diff --git a/Makefile b/Makefile index c38e88a8e0ad9..19c6f719c6151 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ self := $(location) @tmpdir=`mktemp --tmpdir -d` ; \ echo Using temporary directory $$tmpdir for test repositories ; \ USE_REPO_TEST_DIR= $(MAKE) -f $(self) --no-print-directory REPO_TEST_DIR=$$tmpdir/ $@ ; \ - STATUS=$$? ; rm -r "$$tmpdir" ; exit $$STATUS + STATUS=$$? ; chmod -R +w "$$tmpdir" && rm -r "$$tmpdir" ; exit $$STATUS else @@ -114,8 +114,9 @@ LDFLAGS := $(LDFLAGS) -X "main.MakeVersion=$(MAKE_VERSION)" -X "main.Version=$(G LINUX_ARCHS ?= linux/amd64,linux/386,linux/arm-5,linux/arm-6,linux/arm64 GO_PACKAGES ?= $(filter-out code.gitea.io/gitea/tests/integration/migration-test code.gitea.io/gitea/tests code.gitea.io/gitea/tests/integration code.gitea.io/gitea/tests/e2e,$(shell $(GO) list ./... | grep -v /vendor/)) -GO_TEST_PACKAGES ?= $(filter-out $(shell $(GO) list code.gitea.io/gitea/models/migrations/...) code.gitea.io/gitea/tests/integration/migration-test code.gitea.io/gitea/tests code.gitea.io/gitea/tests/integration code.gitea.io/gitea/tests/e2e,$(shell $(GO) list ./... | grep -v /vendor/)) - +# Only test code modified in the git-annex feature branch; upstream can handle testing the full suite. +# This list was generated by `git diff --stat --name-only main.. -- '*.go' | xargs dirname | sort | uniq` +GO_TEST_PACKAGES ?= code.gitea.io/gitea/modules/annex code.gitea.io/gitea/modules/base code.gitea.io/gitea/modules/git code.gitea.io/gitea/modules/private code.gitea.io/gitea/modules/setting code.gitea.io/gitea/modules/util code.gitea.io/gitea/routers/private code.gitea.io/gitea/routers/web code.gitea.io/gitea/services/auth FOMANTIC_WORK_DIR := web_src/fomantic WEBPACK_SOURCES := $(shell find web_src/js web_src/css -type f) diff --git a/cmd/serv.go b/cmd/serv.go index 26fc91a3b7a30..48489501e13b2 100644 --- a/cmd/serv.go +++ b/cmd/serv.go @@ -37,6 +37,7 @@ import ( const ( lfsAuthenticateVerb = "git-lfs-authenticate" + gitAnnexShellVerb = "git-annex-shell" ) // CmdServ represents the available serv sub-command. @@ -89,6 +90,7 @@ var ( "git-upload-archive": perm.AccessModeRead, "git-receive-pack": perm.AccessModeWrite, lfsAuthenticateVerb: perm.AccessModeNone, + gitAnnexShellVerb: perm.AccessModeNone, // annex permissions are enforced by GIT_ANNEX_SHELL_READONLY, rather than the Gitea API } alphaDashDotPattern = regexp.MustCompile(`[^\w-\.]`) ) @@ -201,6 +203,7 @@ func runServ(c *cli.Context) error { verb := words[0] repoPath := words[1] + if repoPath[0] == '/' { repoPath = repoPath[1:] } @@ -216,9 +219,43 @@ func runServ(c *cli.Context) error { } } + if verb == gitAnnexShellVerb { + if !setting.Annex.Enabled { + return fail(ctx, "Unknown git command", "git-annex request over SSH denied, git-annex support is disabled") + } + + if len(words) < 3 { + return fail(ctx, "Too few arguments", "Too few arguments in cmd: %s", cmd) + } + + // git-annex always puts the repo in words[2], unlike most other + // git subcommands; and it sometimes names repos like /~/, as if + // $HOME should get expanded while also being rooted. e.g.: + // git-annex-shell 'configlist' '/~/user/repo' + // git-annex-shell 'sendkey' '/user/repo 'key' + repoPath = words[2] + repoPath = strings.TrimPrefix(repoPath, "/") + repoPath = strings.TrimPrefix(repoPath, "~/") + } + // LowerCase and trim the repoPath as that's how they are stored. repoPath = strings.ToLower(strings.TrimSpace(repoPath)) + // prevent directory traversal attacks + repoPath = filepath.Clean("/" + repoPath)[1:] + + // put the sanitized repoPath back into the argument list for later + if verb == gitAnnexShellVerb { + // git-annex-shell demands an absolute path + absRepoPath, err := filepath.Abs(filepath.Join(setting.RepoRootPath, repoPath)) + if err != nil { + return fail(ctx, "Error locating repoPath", "%v", err) + } + words[2] = absRepoPath + } else { + words[1] = repoPath + } + rr := strings.SplitN(repoPath, "/", 2) if len(rr) != 2 { return fail(ctx, "Invalid repository path", "Invalid repository path: %v", repoPath) @@ -305,21 +342,46 @@ func runServ(c *cli.Context) error { return nil } - var gitcmd *exec.Cmd gitBinPath := filepath.Dir(git.GitExecutable) // e.g. /usr/bin gitBinVerb := filepath.Join(gitBinPath, verb) // e.g. /usr/bin/git-upload-pack if _, err := os.Stat(gitBinVerb); err != nil { // if the command "git-upload-pack" doesn't exist, try to split "git-upload-pack" to use the sub-command with git // ps: Windows only has "git.exe" in the bin path, so Windows always uses this way + // ps: git-annex-shell and other extensions may not necessarily be in gitBinPath, + // but '{gitBinPath}/git annex-shell' should be able to find them on $PATH. verbFields := strings.SplitN(verb, "-", 2) if len(verbFields) == 2 { // use git binary with the sub-command part: "C:\...\bin\git.exe", "upload-pack", ... - gitcmd = exec.CommandContext(ctx, git.GitExecutable, verbFields[1], repoPath) + gitBinVerb = git.GitExecutable + words = append([]string{verbFields[1]}, words...) } } - if gitcmd == nil { - // by default, use the verb (it has been checked above by allowedCommands) - gitcmd = exec.CommandContext(ctx, gitBinVerb, repoPath) + + // by default, use the verb (it has been checked above by allowedCommands) + gitcmd := exec.CommandContext(ctx, gitBinVerb, words[1:]...) + + if verb == gitAnnexShellVerb { + // This doesn't get its own isolated section like LFS does, because LFS + // is handled by internal Gitea routines, but git-annex has to be shelled out + // to like other git subcommands, so we need to build up gitcmd. + + // TODO: does this work on Windows? + gitcmd.Env = append(gitcmd.Env, + // "If set, disallows running git-shell to handle unknown commands." + // - git-annex-shell(1) + "GIT_ANNEX_SHELL_LIMITED=True", + // "If set, git-annex-shell will refuse to run commands + // that do not operate on the specified directory." + // - git-annex-shell(1) + fmt.Sprintf("GIT_ANNEX_SHELL_DIRECTORY=%s", words[2]), + ) + if results.UserMode < perm.AccessModeWrite { + // "If set, disallows any action that could modify the git-annex repository." + // - git-annex-shell(1) + // We set this when the backend API has told us that we don't have write permission to this repo. + log.Debug("Setting GIT_ANNEX_SHELL_READONLY=True") + gitcmd.Env = append(gitcmd.Env, "GIT_ANNEX_SHELL_READONLY=True") + } } process.SetSysProcAttribute(gitcmd) diff --git a/cmd/web.go b/cmd/web.go index 01386251becfa..ce22d708acccb 100644 --- a/cmd/web.go +++ b/cmd/web.go @@ -311,6 +311,10 @@ func listen(m http.Handler, handleRedirector bool) error { log.Info("LFS server enabled") } + if setting.Annex.Enabled { + log.Info("git-annex enabled") + } + var err error switch setting.Protocol { case setting.HTTP: diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index 2209822ff0869..6e5351cc75214 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -2517,6 +2517,15 @@ LEVEL = Info ;; override the minio base path if storage type is minio ;MINIO_BASE_PATH = lfs/ +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;[annex] +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; +;; Whether git-annex is enabled; defaults to false +;ENABLED = false + ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; settings for packages, will override storage setting diff --git a/modules/annex/annex.go b/modules/annex/annex.go new file mode 100644 index 0000000000000..af7fb7182dc97 --- /dev/null +++ b/modules/annex/annex.go @@ -0,0 +1,160 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +// Unlike modules/lfs, which operates mainly on git.Blobs, this operates on git.TreeEntrys. +// The motivation for this is that TreeEntrys have an easy pointer to the on-disk repo path, +// while blobs do not (in fact, if building with TAGS=gogit, blobs might exist only in a mock +// filesystem, living only in process RAM). We must have the on-disk path to do anything +// useful with git-annex because all of its interesting data is on-disk under .git/annex/. + +package annex + +import ( + "errors" + "fmt" + "os" + "path" + "strings" + + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" +) + +const ( + // > The maximum size of a pointer file is 32 kb. + // - https://git-annex.branchable.com/internals/pointer_file/ + // It's unclear if that's kilobytes or kibibytes; assuming kibibytes: + blobSizeCutoff = 32 * 1024 +) + +// ErrInvalidPointer occurs if the pointer's value doesn't parse +var ErrInvalidPointer = errors.New("Not a git-annex pointer") + +// Gets the content of the blob as raw text, up to n bytes. +// (the pre-existing blob.GetBlobContent() has a hardcoded 1024-byte limit) +func getBlobContent(b *git.Blob, n int) (string, error) { + dataRc, err := b.DataAsync() + if err != nil { + return "", err + } + defer dataRc.Close() + buf := make([]byte, n) + n, _ = util.ReadAtMost(dataRc, buf) + buf = buf[:n] + return string(buf), nil +} + +func Pointer(blob *git.Blob) (string, error) { + // git-annex doesn't seem fully spec what its pointer are, but + // the fullest description is here: + // https://git-annex.branchable.com/internals/pointer_file/ + + // a pointer can be: + // the original format, generated by `git annex add`: a symlink to '.git/annex/objects/$HASHDIR/$HASHDIR2/$KEY/$KEY' + // the newer, git-lfs influenced, format, generated by `git annex smudge`: a text file containing '/annex/objects/$KEY' + // + // in either case we can extract the $KEY the same way, and we need not actually know if it's a symlink or not because + // git.Blob.DataAsync() works like open() + readlink(), handling both cases in one. + + if blob.Size() > blobSizeCutoff { + // > The maximum size of a pointer file is 32 kb. If it is any longer, it is not considered to be a valid pointer file. + // https://git-annex.branchable.com/internals/pointer_file/ + + // It's unclear to me whether the same size limit applies to symlink-pointers, but it seems sensible to limit them too. + return "", ErrInvalidPointer + } + + pointer, err := getBlobContent(blob, blobSizeCutoff) + if err != nil { + return "", fmt.Errorf("error reading %s: %w", blob.Name(), err) + } + + // the spec says a pointer file can contain multiple lines each with a pointer in them + // but that makes no sense to me, so I'm just ignoring all but the first + lines := strings.Split(pointer, "\n") + if len(lines) < 1 { + return "", ErrInvalidPointer + } + pointer = lines[0] + + // in both the symlink and pointer-file formats, the pointer must have "/annex/" somewhere in it + if !strings.Contains(pointer, "/annex/") { + return "", ErrInvalidPointer + } + + // extract $KEY + pointer = path.Base(strings.TrimSpace(pointer)) + + // ask git-annex's opinion on $KEY + // XXX: this is probably a bit slow, especially if this operation gets run often + // and examinekey is not that strict: + // - it doesn't enforce that the "BACKEND" tag is one it knows, + // - it doesn't enforce that the fields and their format fit the "BACKEND" tag + // so maybe this is a wasteful step + _, examineStderr, err := git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "examinekey").AddDynamicArguments(pointer).RunStdString(&git.RunOpts{Dir: blob.Repo().Path}) + if err != nil { + // TODO: make ErrInvalidPointer into a type capable of wrapping err + if strings.TrimSpace(examineStderr) == "git-annex: bad key" { + return "", ErrInvalidPointer + } + return "", err + } + + return pointer, nil +} + +// return the absolute path of the content pointed to by the annex pointer stored in the git object +// errors if the content is not found in this repo +func ContentLocation(blob *git.Blob) (string, error) { + pointer, err := Pointer(blob) + if err != nil { + return "", err + } + + contentLocation, _, err := git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "contentlocation").AddDynamicArguments(pointer).RunStdString(&git.RunOpts{Dir: blob.Repo().Path}) + if err != nil { + return "", fmt.Errorf("in %s: %s does not seem to be a valid annexed file: %w", blob.Repo().Path, pointer, err) + } + contentLocation = strings.TrimSpace(contentLocation) + contentLocation = path.Clean("/" + contentLocation)[1:] // prevent directory traversals + contentLocation = path.Join(blob.Repo().Path, contentLocation) + + return contentLocation, nil +} + +// returns a stream open to the annex content +func Content(blob *git.Blob) (*os.File, error) { + contentLocation, err := ContentLocation(blob) + if err != nil { + return nil, err + } + + return os.Open(contentLocation) +} + +// whether the object appears to be a valid annex pointer +// does *not* verify if the content is actually in this repo; +// for that, use ContentLocation() +func IsAnnexed(blob *git.Blob) (bool, error) { + if !setting.Annex.Enabled { + return false, nil + } + + // Pointer() is written to only return well-formed pointers + // so the test is just to see if it errors + _, err := Pointer(blob) + if err != nil { + if errors.Is(err, ErrInvalidPointer) { + return false, nil + } + return false, err + } + return true, nil +} + +// IsAnnexRepo determines if repo is a git-annex enabled repository +func IsAnnexRepo(repo *git.Repository) bool { + _, _, err := git.NewCommand(repo.Ctx, "annex", "info").RunStdString(&git.RunOpts{Dir: repo.Path}) + return err == nil +} diff --git a/modules/annex/annex_archive.go b/modules/annex/annex_archive.go new file mode 100644 index 0000000000000..6c6661fea8505 --- /dev/null +++ b/modules/annex/annex_archive.go @@ -0,0 +1,258 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package annex + +import ( + "archive/tar" + "archive/zip" + "bytes" + "compress/gzip" + "context" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "strings" + + "code.gitea.io/gitea/modules/git" +) + +func createArchiveTargz(ctx context.Context, repo *git.Repository, target io.Writer, prefix, commitID string) error { + // create a plain git tar archive + var tarB bytes.Buffer + err := repo.CreateArchive(ctx, git.TAR, &tarB, prefix != "", commitID) + if err != nil { + return err + } + + gitFilesR := tar.NewReader(&tarB) + + gzipW := gzip.NewWriter(target) + defer gzipW.Close() + tarW := tar.NewWriter(gzipW) + defer tarW.Close() + + tree, err := repo.GetTree(commitID) + if err != nil { + return err + } + + for { + oldHeader, err := gitFilesR.Next() + // TODO: handle non-local names in tar archives? + if err != nil { + if err == io.EOF { + break + } + return err + } + + // default to copying the current file from the archive created by repo.CreateArchive + header := oldHeader + dataR := io.Reader(gitFilesR) + + // if we can get a annex content location for the file we use that instead + te, err := tree.GetTreeEntryByPath(strings.TrimPrefix(oldHeader.Name, prefix)) + if err == nil && (te.IsRegular() || te.IsExecutable() || te.IsLink()) { + blob := te.Blob() + annexPath, err := ContentLocation(blob) + if err == nil { + // blob corresponds to an annexed file + + // build a tar header for the annexed file + file, err := os.Open(annexPath) + if err != nil { + return fmt.Errorf("opening %s failed: %w", annexPath, err) + } + stat, err := file.Stat() + if err != nil { + return fmt.Errorf("getting FileInfo for %s failed: %w", file.Name(), err) + } + + // https://pkg.go.dev/archive/tar#Header: + // > For forward compatibility, users that retrieve a Header from Reader.Next, + // > mutate it in some ways, and then pass it back to Writer.WriteHeader should + // > do so by creating a new Header and copying the fields that they are interested in preserving. + header, err = tar.FileInfoHeader(stat, "") + if err != nil { + return fmt.Errorf("creating header failed: %w", err) + } + + header.Size = stat.Size() + if te.IsExecutable() || (te.IsLink() && (stat.Mode().Perm()&0o100) != 0) { + // If the file is executable in git, or is a symlink to an annexed executable + // file, archive it with permissions 0775, as `git archive` would do for + // executables as well. + header.Mode = int64(fs.FileMode(0o775)) + } else if te.IsRegular() || (te.IsLink() && (stat.Mode().Perm()&0o100) == 0) { + // If the file is not executable in git (i.e. a regular file), or a symlink to + // an annexed file that is not executable, archive it with permissions 0664. + header.Mode = int64(fs.FileMode(0o664)) + } + + // preserve these + header.Name = oldHeader.Name + header.Linkname = oldHeader.Linkname + header.Uid = oldHeader.Uid + header.Gid = oldHeader.Gid + header.Uname = oldHeader.Uname + header.Gname = oldHeader.Gname + header.ModTime = oldHeader.ModTime + header.AccessTime = oldHeader.AccessTime + header.ChangeTime = oldHeader.ChangeTime + header.PAXRecords = oldHeader.PAXRecords + header.Format = oldHeader.Format + + // set the data reader + dataR = file + } + } + + // write header + err = tarW.WriteHeader(header) + if err != nil { + return fmt.Errorf("writing header for %s failed: %w", header.Name, err) + } + + // write data + _, err = io.Copy(tarW, dataR) + if err != nil { + return fmt.Errorf("writing data for %s failed: %w", header.Name, err) + } + } + + return nil +} + +func createArchiveZip(ctx context.Context, repo *git.Repository, target io.Writer, prefix, commitID string) error { + // create a plain git zip archive + var zipB bytes.Buffer + err := repo.CreateArchive(ctx, git.ZIP, &zipB, prefix != "", commitID) + if err != nil { + return err + } + + gitFilesR, err := zip.NewReader(bytes.NewReader(zipB.Bytes()), int64(zipB.Len())) + if err != nil { + return err + } + + tree, err := repo.GetTree(commitID) + if err != nil { + return err + } + + zipW := zip.NewWriter(target) + defer zipW.Close() + + err = zipW.SetComment(gitFilesR.Comment) + if err != nil { + return fmt.Errorf("setting archive comment field failed: %w", err) + } + + for _, f := range gitFilesR.File { + oldHeader := f.FileHeader + + // default to copying the current file from the archive created by repo.CreateArchive + // dataR is set later to avoid unnecessarily opening a file here + header := &oldHeader + dataR := io.Reader(nil) + + te, err := tree.GetTreeEntryByPath(strings.TrimPrefix(oldHeader.Name, prefix)) + if err == nil && (te.IsRegular() || te.IsExecutable() || te.IsLink()) { + blob := te.Blob() + annexPath, err := ContentLocation(blob) + if err == nil { + // blob corresponds to an annexed file + + // build a zip header for the file + file, err := os.Open(annexPath) + if err != nil { + return fmt.Errorf("opening %s failed: %w", annexPath, err) + } + stat, err := file.Stat() + if err != nil { + return fmt.Errorf("getting FileInfo for %s failed: %w", file.Name(), err) + } + header, err = zip.FileInfoHeader(stat) + if err != nil { + return fmt.Errorf("creating header failed: %w", err) + } + header.Name = oldHeader.Name + header.Method = zip.Deflate + + if te.IsExecutable() || (te.IsLink() && (stat.Mode().Perm()&0o100) != 0) { + // If the file is executable in git, or is a symlink to an annexed executable + // file, archive it with permissions 0775, as `git archive` would do for + // executables as well. + header.SetMode(fs.FileMode(0o755)) + } else if te.IsRegular() || (te.IsLink() && (stat.Mode().Perm()&0o100) == 0) { + // If the file is not executable in git (i.e. a regular file), or a symlink to + // an annexed file that is not executable, archive it with the "FAT creator" + // in zip and set rw permissions through the external attrs. + // `git archive` does the same for regular files. + header.CreatorVersion = 0 + header.ExternalAttrs = 0 + } + + // set the data reader + dataR = file + } + } + + if dataR == nil { + // data reader was not yet set, take the data from the archive created by repo.CreateArchive + file, err := f.Open() + if err != nil { + return fmt.Errorf("opening %s failed: %w", f.Name, err) + } + dataR = file + } + + // write header + fileW, err := zipW.CreateHeader(header) + if err != nil { + return fmt.Errorf("writing header for %s failed: %w", header.Name, err) + } + + // write data + _, err = io.Copy(fileW, dataR) + if err != nil { + return fmt.Errorf("writing data for %s failed: %w", header.Name, err) + } + } + + return nil +} + +// CreateArchive creates an archive of format from repo at commitID and writes it to target. +// Files in the archive are prefixed with the repositories name if usePrefix is true. +// It is an annex-aware alternative to Repository.CreateArchive in the git package. +func CreateArchive(ctx context.Context, repo *git.Repository, format git.ArchiveType, target io.Writer, usePrefix bool, commitID string) error { + if format.String() == "unknown" { + return fmt.Errorf("unknown format: %v", format) + } + + var prefix string + if usePrefix { + prefix = filepath.Base(strings.TrimSuffix(repo.Path, ".git")) + "/" + } else { + prefix = "" + } + + var err error + if format == git.TARGZ { + err = createArchiveTargz(ctx, repo, target, prefix, commitID) + } else if format == git.ZIP { + err = createArchiveZip(ctx, repo, target, prefix, commitID) + } else { + return fmt.Errorf("unsupported format: %v", format) + } + if err != nil { + return fmt.Errorf("failed to create archive: %w", err) + } + + return nil +} diff --git a/modules/base/tool.go b/modules/base/tool.go index 71dcb83fb4850..da0df40757822 100644 --- a/modules/base/tool.go +++ b/modules/base/tool.go @@ -19,6 +19,7 @@ import ( "unicode" "unicode/utf8" + "code.gitea.io/gitea/modules/annex" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" @@ -204,6 +205,12 @@ func IsLetter(ch rune) bool { func EntryIcon(entry *git.TreeEntry) string { switch { case entry.IsLink(): + isAnnexed, _ := annex.IsAnnexed(entry.Blob()) + if isAnnexed { + // git-annex files are sometimes stored as symlinks; + // short-circuit that so like LFS they are displayed as regular files + return "file" + } te, err := entry.FollowLink() if err != nil { log.Debug(err.Error()) diff --git a/modules/git/blob.go b/modules/git/blob.go index bcecb42e16ebb..34224f6c085b0 100644 --- a/modules/git/blob.go +++ b/modules/git/blob.go @@ -15,6 +15,10 @@ import ( // This file contains common functions between the gogit and !gogit variants for git Blobs +func (b *Blob) Repo() *Repository { + return b.repo +} + // Name returns name of the tree entry this blob object was created from (or empty string) func (b *Blob) Name() string { return b.name diff --git a/modules/git/blob_gogit.go b/modules/git/blob_gogit.go index aa206409d0b6f..f98a9d9084f99 100644 --- a/modules/git/blob_gogit.go +++ b/modules/git/blob_gogit.go @@ -14,7 +14,8 @@ import ( // Blob represents a Git object. type Blob struct { - ID SHA1 + ID SHA1 + repo *Repository gogitEncodedObj plumbing.EncodedObject name string diff --git a/modules/git/command.go b/modules/git/command.go index f095bb18bef75..8550f7759451a 100644 --- a/modules/git/command.go +++ b/modules/git/command.go @@ -439,12 +439,13 @@ func (c *Command) RunStdBytes(opts *RunOpts) (stdout, stderr []byte, runErr RunS } // AllowLFSFiltersArgs return globalCommandArgs with lfs filter, it should only be used for tests +// It also re-enables git-credential(1), which is used to test git-annex's HTTP support func AllowLFSFiltersArgs() TrustedCmdArgs { // Now here we should explicitly allow lfs filters to run filteredLFSGlobalArgs := make(TrustedCmdArgs, len(globalCommandArgs)) j := 0 for _, arg := range globalCommandArgs { - if strings.Contains(string(arg), "lfs") { + if strings.Contains(string(arg), "lfs") || strings.Contains(string(arg), "credential") { j-- } else { filteredLFSGlobalArgs[j] = arg diff --git a/modules/git/repo_archive.go b/modules/git/repo_archive.go index 2b45a50f191b6..0f27d989dde36 100644 --- a/modules/git/repo_archive.go +++ b/modules/git/repo_archive.go @@ -18,6 +18,8 @@ type ArchiveType int const ( // ZIP zip archive type ZIP ArchiveType = iota + 1 + // TAR tar archive type + TAR // TARGZ tar gz archive type TARGZ // BUNDLE bundle archive type @@ -29,6 +31,8 @@ func (a ArchiveType) String() string { switch a { case ZIP: return "zip" + case TAR: + return "tar" case TARGZ: return "tar.gz" case BUNDLE: @@ -41,6 +45,8 @@ func ToArchiveType(s string) ArchiveType { switch s { case "zip": return ZIP + case "tar": + return TAR case "tar.gz": return TARGZ case "bundle": diff --git a/modules/git/repo_blob_gogit.go b/modules/git/repo_blob_gogit.go index 7f0892f6f5e91..605c05072b771 100644 --- a/modules/git/repo_blob_gogit.go +++ b/modules/git/repo_blob_gogit.go @@ -17,6 +17,7 @@ func (repo *Repository) getBlob(id SHA1) (*Blob, error) { return &Blob{ ID: id, + repo: repo, gogitEncodedObj: encodedObj, }, nil } diff --git a/modules/git/tree_entry_gogit.go b/modules/git/tree_entry_gogit.go index 194dd12f7dbb1..0c08a766d8229 100644 --- a/modules/git/tree_entry_gogit.go +++ b/modules/git/tree_entry_gogit.go @@ -89,6 +89,7 @@ func (te *TreeEntry) Blob() *Blob { return &Blob{ ID: te.gogitTreeEntry.Hash, + repo: te.ptree.repo, gogitEncodedObj: encodedObj, name: te.Name(), } diff --git a/modules/private/serv.go b/modules/private/serv.go index 480a44695496d..6c7c753cf09ca 100644 --- a/modules/private/serv.go +++ b/modules/private/serv.go @@ -40,6 +40,7 @@ type ServCommandResults struct { UserName string UserEmail string UserID int64 + UserMode perm.AccessMode OwnerName string RepoName string RepoID int64 diff --git a/modules/setting/annex.go b/modules/setting/annex.go new file mode 100644 index 0000000000000..a0eeac9bb8a1c --- /dev/null +++ b/modules/setting/annex.go @@ -0,0 +1,20 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package setting + +import ( + "code.gitea.io/gitea/modules/log" +) + +// Annex represents the configuration for git-annex +var Annex = struct { + Enabled bool `ini:"ENABLED"` +}{} + +func loadAnnexFrom(rootCfg ConfigProvider) { + sec := rootCfg.Section("annex") + if err := sec.MapTo(&Annex); err != nil { + log.Fatal("Failed to map Annex settings: %v", err) + } +} diff --git a/modules/setting/setting.go b/modules/setting/setting.go index d444d9a0175c6..2b201bb51c01b 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -149,6 +149,7 @@ func loadCommonSettingsFrom(cfg ConfigProvider) error { loadCamoFrom(cfg) loadI18nFrom(cfg) loadGitFrom(cfg) + loadAnnexFrom(cfg) loadMirrorFrom(cfg) loadMarkupFrom(cfg) loadOtherFrom(cfg) diff --git a/modules/util/filecmp.go b/modules/util/filecmp.go new file mode 100644 index 0000000000000..76e7705cc1b56 --- /dev/null +++ b/modules/util/filecmp.go @@ -0,0 +1,87 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package util + +import ( + "bytes" + "io" + "os" +) + +// Decide if two files have the same contents or not. +// chunkSize is the size of the blocks to scan by; pass 0 to get a sensible default. +// *Follows* symlinks. +// +// May return an error if something else goes wrong; in this case, you should ignore the value of 'same'. +// +// derived from https://stackoverflow.com/a/30038571 +// under CC-BY-SA-4.0 by several contributors +func FileCmp(file1, file2 string, chunkSize int) (same bool, err error) { + if chunkSize == 0 { + chunkSize = 4 * 1024 + } + + // shortcuts: check file metadata + stat1, err := os.Stat(file1) + if err != nil { + return false, err + } + + stat2, err := os.Stat(file2) + if err != nil { + return false, err + } + + // are inputs are literally the same file? + if os.SameFile(stat1, stat2) { + return true, nil + } + + // do inputs at least have the same size? + if stat1.Size() != stat2.Size() { + return false, nil + } + + // long way: compare contents + f1, err := os.Open(file1) + if err != nil { + return false, err + } + defer f1.Close() + + f2, err := os.Open(file2) + if err != nil { + return false, err + } + defer f2.Close() + + b1 := make([]byte, chunkSize) + b2 := make([]byte, chunkSize) + for { + n1, err1 := io.ReadFull(f1, b1) + n2, err2 := io.ReadFull(f2, b2) + + // https://pkg.go.dev/io#Reader + // > Callers should always process the n > 0 bytes returned + // > before considering the error err. Doing so correctly + // > handles I/O errors that happen after reading some bytes + // > and also both of the allowed EOF behaviors. + + if !bytes.Equal(b1[:n1], b2[:n2]) { + return false, nil + } + + if (err1 == io.EOF && err2 == io.EOF) || (err1 == io.ErrUnexpectedEOF && err2 == io.ErrUnexpectedEOF) { + return true, nil + } + + // some other error, like a dropped network connection or a bad transfer + if err1 != nil { + return false, err1 + } + if err2 != nil { + return false, err2 + } + } +} diff --git a/modules/util/remove.go b/modules/util/remove.go index d1e38faf5f1fb..0f41471fcc374 100644 --- a/modules/util/remove.go +++ b/modules/util/remove.go @@ -4,7 +4,9 @@ package util import ( + "io/fs" "os" + "path/filepath" "runtime" "syscall" "time" @@ -41,10 +43,40 @@ func Remove(name string) error { return err } -// RemoveAll removes the named file or (empty) directory with at most 5 attempts. +// RemoveAll removes the named file or directory with at most 5 attempts. func RemoveAll(name string) error { var err error + for i := 0; i < 5; i++ { + // Do chmod -R +w to help ensure the removal succeeds. + // In particular, in the git-annex case, this handles + // https://git-annex.branchable.com/internals/lockdown/ : + // + // > (The only bad consequence of this is that rm -rf .git + // > doesn't work unless you first run chmod -R +w .git) + + err = filepath.WalkDir(name, func(path string, d fs.DirEntry, err error) error { + // NB: this is called WalkDir but it works on a single file too + if err == nil { + info, err := d.Info() + if err != nil { + return err + } + + // 0200 == u+w, in octal unix permission notation + err = os.Chmod(path, info.Mode()|0o200) + if err != nil { + return err + } + } + return nil + }) + if err != nil { + // try again + <-time.After(100 * time.Millisecond) + continue + } + err = os.RemoveAll(name) if err == nil { break diff --git a/options/locale/locale_cs-CZ.ini b/options/locale/locale_cs-CZ.ini index 602fe0bce83f7..4115aac6e4410 100644 --- a/options/locale/locale_cs-CZ.ini +++ b/options/locale/locale_cs-CZ.ini @@ -1094,6 +1094,7 @@ view_git_blame=Zobrazit Git Blame video_not_supported_in_browser=Váš prohlížeč nepodporuje značku pro HTML5 video. audio_not_supported_in_browser=Váš prohlížeč nepodporuje značku pro HTML5 audio. stored_lfs=Uloženo pomocí Git LFS +stored_annex=Uloženo pomocí Git Annex symbolic_link=Symbolický odkaz commit_graph=Graf commitů commit_graph.select=Vybrat větve @@ -1113,6 +1114,7 @@ editor.upload_file=Nahrát soubor editor.edit_file=Upravit soubor editor.preview_changes=Náhled změn editor.cannot_edit_lfs_files=LFS soubory nemohou být upravovány přes webové rozhraní. +editor.cannot_edit_annex_files=Annex soubory nemohou být upravovány přes webové rozhraní. editor.cannot_edit_non_text_files=Binární soubory nemohou být upravovány přes webové rozhraní. editor.edit_this_file=Upravit soubor editor.this_file_locked=Soubor je uzamčen diff --git a/options/locale/locale_de-DE.ini b/options/locale/locale_de-DE.ini index f970fdb666846..cb76dddeaeaa1 100644 --- a/options/locale/locale_de-DE.ini +++ b/options/locale/locale_de-DE.ini @@ -1182,6 +1182,7 @@ view_git_blame=Git Blame ansehen video_not_supported_in_browser=Dein Browser unterstützt das HTML5 'video'-Tag nicht. audio_not_supported_in_browser=Dein Browser unterstützt den HTML5 'audio'-Tag nicht. stored_lfs=Gespeichert mit Git LFS +stored_annex=Gespeichert mit Git Annex symbolic_link=Softlink executable_file=Ausführbare Datei commit_graph=Commit graph @@ -1205,6 +1206,7 @@ editor.upload_file=Datei hochladen editor.edit_file=Datei bearbeiten editor.preview_changes=Vorschau der Änderungen editor.cannot_edit_lfs_files=LFS-Dateien können im Webinterface nicht bearbeitet werden. +editor.cannot_edit_annex_files=Annex-Dateien können im Webinterface nicht bearbeitet werden. editor.cannot_edit_non_text_files=Binärdateien können nicht im Webinterface bearbeitet werden. editor.edit_this_file=Datei bearbeiten editor.this_file_locked=Datei ist gesperrt diff --git a/options/locale/locale_el-GR.ini b/options/locale/locale_el-GR.ini index 2dc615a1b336f..ee8f303d00490 100644 --- a/options/locale/locale_el-GR.ini +++ b/options/locale/locale_el-GR.ini @@ -1111,6 +1111,7 @@ view_git_blame=Προβολή Git Blame video_not_supported_in_browser=Το πρόγραμμα περιήγησής σας δεν υποστηρίζει την ετικέτα HTML5 'video'. audio_not_supported_in_browser=Το πρόγραμμα περιήγησής σας δεν υποστηρίζει την ετικέτα HTML5 'audio'. stored_lfs=Αποθηκεύτηκε με το Git LFS +stored_annex=Αποθηκεύτηκε με το Git Annex symbolic_link=Symbolic link commit_graph=Γράφημα Υποβολών commit_graph.select=Επιλογή κλάδων @@ -1130,6 +1131,7 @@ editor.upload_file=Ανέβασμα Αρχείου editor.edit_file=Επεξεργασία Αρχείου editor.preview_changes=Προεπισκόπηση Αλλαγών editor.cannot_edit_lfs_files=Τα αρχεία LFS δεν μπορούν να επεξεργαστούν στη διεπαφή web. +editor.cannot_edit_annex_files=Τα αρχεία Annex δεν μπορούν να επεξεργαστούν στη διεπαφή web. editor.cannot_edit_non_text_files=Τα δυαδικά αρχεία δεν μπορούν να επεξεργαστούν στη διεπαφή web. editor.edit_this_file=Επεξεργασία Αρχείου editor.this_file_locked=Το αρχείο είναι κλειδωμένο diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index a7a7a4f4c50f9..6c344c745a451 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -1179,6 +1179,7 @@ view_git_blame = View Git Blame video_not_supported_in_browser = Your browser does not support the HTML5 'video' tag. audio_not_supported_in_browser = Your browser does not support the HTML5 'audio' tag. stored_lfs = Stored with Git LFS +stored_annex = Stored with Git Annex symbolic_link = Symbolic link executable_file = Executable File commit_graph = Commit Graph @@ -1202,6 +1203,7 @@ editor.upload_file = Upload File editor.edit_file = Edit File editor.preview_changes = Preview Changes editor.cannot_edit_lfs_files = LFS files cannot be edited in the web interface. +editor.cannot_edit_annex_files = Annex files cannot be edited in the web interface. editor.cannot_edit_non_text_files = Binary files cannot be edited in the web interface. editor.edit_this_file = Edit File editor.this_file_locked = File is locked diff --git a/options/locale/locale_es-ES.ini b/options/locale/locale_es-ES.ini index 9eb9e856cef14..4f46f5671c160 100644 --- a/options/locale/locale_es-ES.ini +++ b/options/locale/locale_es-ES.ini @@ -1182,6 +1182,7 @@ view_git_blame=Ver la culpa de Git video_not_supported_in_browser=Su navegador no soporta el tag video de HTML5. audio_not_supported_in_browser=Su navegador no soporta el tag audio de HTML5. stored_lfs=Almacenados con Git LFS +stored_annex=Almacenados con Git Annex symbolic_link=Enlace simbólico executable_file=Archivo Ejecutable commit_graph=Gráfico de commits @@ -1205,6 +1206,7 @@ editor.upload_file=Subir archivo editor.edit_file=Editar Archivo editor.preview_changes=Vista previa de los cambios editor.cannot_edit_lfs_files=Los archivos LFS no se pueden editar en la interfaz web. +editor.cannot_edit_annex_files=Los archivos Annex no se pueden editar en la interfaz web. editor.cannot_edit_non_text_files=Los archivos binarios no se pueden editar en la interfaz web. editor.edit_this_file=Editar Archivo editor.this_file_locked=El archivo está bloqueado diff --git a/options/locale/locale_fa-IR.ini b/options/locale/locale_fa-IR.ini index b9194c48a55a8..b94436f594b88 100644 --- a/options/locale/locale_fa-IR.ini +++ b/options/locale/locale_fa-IR.ini @@ -923,6 +923,7 @@ file_copy_permalink=پرمالینک را کپی کنید video_not_supported_in_browser=مرورگر شما از تگ video که در HTML5 تعریف شده است، پشتیبانی نمی کند. audio_not_supported_in_browser=مرورگر شما از تگ audio که در HTML5 تعریف شده است، پشتیبانی نمی کند. stored_lfs=ذخیره شده با GIT LFS +stored_annex=ذخیره شده با GIT Annex symbolic_link=پیوند نمادین commit_graph=نمودار کامیت commit_graph.select=انتخاب برنچها @@ -940,6 +941,7 @@ editor.upload_file=بارگذاری پرونده editor.edit_file=ویرایش پرونده editor.preview_changes=پیش نمایش تغییرات editor.cannot_edit_lfs_files=پرونده های LFS در صحفه وب قابل تغییر نیست. +editor.cannot_edit_annex_files=پرونده های Annex در صحفه وب قابل تغییر نیست. editor.cannot_edit_non_text_files=پرونده‎های دودویی در صفحه وب قابل تغییر نیست. editor.edit_this_file=ویرایش پرونده editor.this_file_locked=پرونده قفل شده است diff --git a/options/locale/locale_fr-FR.ini b/options/locale/locale_fr-FR.ini index ebde7e5baeba7..de6bc5e787396 100644 --- a/options/locale/locale_fr-FR.ini +++ b/options/locale/locale_fr-FR.ini @@ -1182,6 +1182,7 @@ view_git_blame=Voir Git Blâme video_not_supported_in_browser=Votre navigateur ne supporte pas la balise « vidéo » HTML5. audio_not_supported_in_browser=Votre navigateur ne supporte pas la balise « audio » HTML5. stored_lfs=Stocké avec Git LFS +stored_annex=Stocké avec Git Annex symbolic_link=Lien symbolique executable_file=Fichiers exécutables commit_graph=Graphe des révisions @@ -1205,6 +1206,7 @@ editor.upload_file=Téléverser un fichier editor.edit_file=Modifier le fichier editor.preview_changes=Aperçu des modifications editor.cannot_edit_lfs_files=Les fichiers LFS ne peuvent pas être modifiés dans l'interface web. +editor.cannot_edit_annex_files=Les fichiers Annex ne peuvent pas être modifiés dans l'interface web. editor.cannot_edit_non_text_files=Les fichiers binaires ne peuvent pas être édités dans l'interface web. editor.edit_this_file=Modifier le fichier editor.this_file_locked=Le fichier est verrouillé diff --git a/options/locale/locale_hu-HU.ini b/options/locale/locale_hu-HU.ini index c22a049817671..e5a50face2188 100644 --- a/options/locale/locale_hu-HU.ini +++ b/options/locale/locale_hu-HU.ini @@ -692,6 +692,7 @@ file_too_large=Ez a fájl túl nagy ahhoz, hogy megjelenítsük. video_not_supported_in_browser=A böngésző nem támogatja a HTML5 video tag-et. audio_not_supported_in_browser=A böngésző nem támogatja a HTML5 audio tag-et. stored_lfs=Git LFS-el eltárolva +stored_annex=Git Annex-el eltárolva symbolic_link=Szimbolikus hivatkozás commit_graph=Commit gráf commit_graph.hide_pr_refs=Pull request-ek elrejtése @@ -704,6 +705,7 @@ editor.upload_file=Fájl feltöltése editor.edit_file=Fájl szerkesztése editor.preview_changes=Változások előnézete editor.cannot_edit_lfs_files=LFS fájlok nem szerkeszthetőek a webes felületen. +editor.cannot_edit_annex_files=Annex fájlok nem szerkeszthetőek a webes felületen. editor.cannot_edit_non_text_files=Bináris fájlok nem szerkeszthetőek a webes felületen. editor.edit_this_file=Fájl szerkesztése editor.this_file_locked=Zárolt állomány diff --git a/options/locale/locale_id-ID.ini b/options/locale/locale_id-ID.ini index a5efde6d07b82..60f2d5ce53f64 100644 --- a/options/locale/locale_id-ID.ini +++ b/options/locale/locale_id-ID.ini @@ -598,6 +598,7 @@ file_permalink=Permalink file_too_large=Berkas terlalu besar untuk ditampilkan. stored_lfs=Tersimpan dengan GIT LFS +stored_annex=Tersimpan dengan GIT Annex commit_graph=Grafik Komit blame=Salahkan normal_view=Pandangan Normal @@ -609,6 +610,7 @@ editor.upload_file=Unggah Berkas editor.edit_file=Sunting Berkas editor.preview_changes=Tinjau Perubahan editor.cannot_edit_lfs_files=Berkas LFS tidak dapat disunting dalam antarmuka web. +editor.cannot_edit_annex_files=Berkas Annex tidak dapat disunting dalam antarmuka web. editor.cannot_edit_non_text_files=Berkas biner tidak dapat disunting dalam antarmuka web. editor.edit_this_file=Sunting Berkas editor.this_file_locked=Berkas terkunci diff --git a/options/locale/locale_is-IS.ini b/options/locale/locale_is-IS.ini index 28ce5ae7f7ac7..57471c0b1432c 100644 --- a/options/locale/locale_is-IS.ini +++ b/options/locale/locale_is-IS.ini @@ -683,6 +683,7 @@ file_view_rendered=Skoða Unnið file_copy_permalink=Afrita Varanlega Slóð stored_lfs=Geymt með Git LFS +stored_annex=Geymt með Git Annex commit_graph.hide_pr_refs=Fela Sameiningarbeiðnir commit_graph.monochrome=Einlitað commit_graph.color=Litað diff --git a/options/locale/locale_it-IT.ini b/options/locale/locale_it-IT.ini index e73d063613707..1d575c5e63f93 100644 --- a/options/locale/locale_it-IT.ini +++ b/options/locale/locale_it-IT.ini @@ -994,6 +994,7 @@ view_git_blame=Visualizza Git Blame video_not_supported_in_browser=Il tuo browser non supporta i tag "video" di HTML5. audio_not_supported_in_browser=Il tuo browser non supporta il tag "video" di HTML5. stored_lfs=Memorizzati con Git LFS +stored_annex=Memorizzati con Git Annex symbolic_link=Link Simbolico commit_graph=Grafico dei commit commit_graph.select=Seleziona rami @@ -1012,6 +1013,7 @@ editor.upload_file=Carica File editor.edit_file=Modifica File editor.preview_changes=Anteprima modifiche editor.cannot_edit_lfs_files=I file LFS non possono essere modificati nell'interfaccia web. +editor.cannot_edit_annex_files=I file Annex non possono essere modificati nell'interfaccia web. editor.cannot_edit_non_text_files=I file binari non possono essere modificati tramite interfaccia web. editor.edit_this_file=Modifica file editor.this_file_locked=Il file è bloccato diff --git a/options/locale/locale_ja-JP.ini b/options/locale/locale_ja-JP.ini index df1cd0a540e75..7a28031c9b4c9 100644 --- a/options/locale/locale_ja-JP.ini +++ b/options/locale/locale_ja-JP.ini @@ -1182,6 +1182,7 @@ view_git_blame=Git Blameを表示 video_not_supported_in_browser=このブラウザはHTML5のvideoタグをサポートしていません。 audio_not_supported_in_browser=このブラウザーはHTML5のaudioタグをサポートしていません。 stored_lfs=Git LFSで保管されています +stored_annex=Git Annexで保管されています symbolic_link=シンボリック リンク executable_file=実行ファイル commit_graph=コミットグラフ @@ -1205,6 +1206,7 @@ editor.upload_file=ファイルをアップロード editor.edit_file=ファイルを編集 editor.preview_changes=変更をプレビュー editor.cannot_edit_lfs_files=LFSのファイルはWebインターフェースで編集できません。 +editor.cannot_edit_annex_files=AnnexのファイルはWebインターフェースで編集できません。 editor.cannot_edit_non_text_files=バイナリファイルはWebインターフェースで編集できません。 editor.edit_this_file=ファイルを編集 editor.this_file_locked=ファイルはロックされています diff --git a/options/locale/locale_ko-KR.ini b/options/locale/locale_ko-KR.ini index d33bf0f850bc3..52cbaf18c2d50 100644 --- a/options/locale/locale_ko-KR.ini +++ b/options/locale/locale_ko-KR.ini @@ -638,6 +638,7 @@ file_too_large=보여주기에는 파일이 너무 큽니다. video_not_supported_in_browser=당신의 브라우저가 HTML5 'video' 태그를 지원하지 않습니다. audio_not_supported_in_browser=당신의 브라우저가 HTML5 'audio' 태그를 지원하지 않습니다. stored_lfs=Git LFS에 저장되어 있습니다 +stored_annex=Git Annex에 저장되어 있습니다 commit_graph=커밋 그래프 editor.new_file=새 파일 diff --git a/options/locale/locale_lv-LV.ini b/options/locale/locale_lv-LV.ini index 1af1cc368564c..a7d4ee5164d52 100644 --- a/options/locale/locale_lv-LV.ini +++ b/options/locale/locale_lv-LV.ini @@ -1110,6 +1110,7 @@ view_git_blame=Aplūkot Git vainīgos video_not_supported_in_browser=Jūsu pārlūks neatbalsta HTML5 video. audio_not_supported_in_browser=Jūsu pārlūks neatbalsta HTML5 audio. stored_lfs=Saglabāts Git LFS +stored_annex=Saglabāts Git Annex symbolic_link=Simboliska saite commit_graph=Revīziju grafs commit_graph.select=Izvēlieties atzarus @@ -1129,6 +1130,7 @@ editor.upload_file=Augšupielādēt failu editor.edit_file=Labot failu editor.preview_changes=Priekšskatīt izmaiņas editor.cannot_edit_lfs_files=LFS failus nevar labot no tīmekļa saskarnes. +editor.cannot_edit_annex_files=Annex failus nevar labot no tīmekļa saskarnes. editor.cannot_edit_non_text_files=Nav iespējams labot bināros failus no pārlūka saskarnes. editor.edit_this_file=Labot failu editor.this_file_locked=Fails ir bloķēts diff --git a/options/locale/locale_nl-NL.ini b/options/locale/locale_nl-NL.ini index 2440002241354..4ef7787b4316f 100644 --- a/options/locale/locale_nl-NL.ini +++ b/options/locale/locale_nl-NL.ini @@ -992,6 +992,7 @@ view_git_blame=Bekijk Git Blame video_not_supported_in_browser=Je browser ondersteunt de HTML5 'video'-tag niet. audio_not_supported_in_browser=Je browser ondersteunt de HTML5 'audio'-tag niet. stored_lfs=Opgeslagen met Git LFS +stored_annex=Opgeslagen met Git Annex symbolic_link=Symbolic link commit_graph=Commit grafiek commit_graph.select=Selecteer branches @@ -1010,6 +1011,7 @@ editor.upload_file=Upload bestand editor.edit_file=Bewerk bestand editor.preview_changes=Voorbeeld tonen editor.cannot_edit_lfs_files=LFS-bestanden kunnen niet worden bewerkt in de webinterface. +editor.cannot_edit_annex_files=Annex-bestanden kunnen niet worden bewerkt in de webinterface. editor.cannot_edit_non_text_files=Binaire bestanden kunnen niet worden bewerkt in de webinterface. editor.edit_this_file=Bewerk bestand editor.this_file_locked=Bestand is vergrendeld diff --git a/options/locale/locale_pl-PL.ini b/options/locale/locale_pl-PL.ini index 8811b218927a6..a2e4f12996110 100644 --- a/options/locale/locale_pl-PL.ini +++ b/options/locale/locale_pl-PL.ini @@ -927,6 +927,7 @@ file_copy_permalink=Kopiuj bezpośredni odnośnik video_not_supported_in_browser=Twoja przeglądarka nie obsługuje znacznika HTML5 "video". audio_not_supported_in_browser=Twoja przeglądarka nie obsługuje znacznika HTML5 "audio". stored_lfs=Przechowane za pomocą Git LFS +stored_annex=Przechowane za pomocą Git Annex symbolic_link=Dowiązanie symboliczne commit_graph=Wykres commitów commit_graph.select=Wybierz gałęzie @@ -944,6 +945,7 @@ editor.upload_file=Wyślij plik editor.edit_file=Edytuj plik editor.preview_changes=Podgląd zmian editor.cannot_edit_lfs_files=Pliki LFS nie mogą być edytowane poprzez interfejs przeglądarkowy. +editor.cannot_edit_annex_files=Pliki Annex nie mogą być edytowane poprzez interfejs przeglądarkowy. editor.cannot_edit_non_text_files=Pliki binarne nie mogą być edytowane poprzez interfejs przeglądarkowy. editor.edit_this_file=Edytuj plik editor.this_file_locked=Plik jest zablokowany diff --git a/options/locale/locale_pt-BR.ini b/options/locale/locale_pt-BR.ini index 302ff934667f4..62cc4e46549fd 100644 --- a/options/locale/locale_pt-BR.ini +++ b/options/locale/locale_pt-BR.ini @@ -1171,6 +1171,7 @@ view_git_blame=Ver Git Blame video_not_supported_in_browser=Seu navegador não suporta a tag 'video' do HTML5. audio_not_supported_in_browser=Seu navegador não suporta a tag 'audio' do HTML5. stored_lfs=Armazenado com Git LFS +stored_annex=Armazenado com Git Annex symbolic_link=Link simbólico executable_file=Arquivo executável commit_graph=Gráfico de commits @@ -1194,6 +1195,7 @@ editor.upload_file=Enviar arquivo editor.edit_file=Editar arquivo editor.preview_changes=Visualizar alterações editor.cannot_edit_lfs_files=Arquivos LFS não podem ser editados na interface web. +editor.cannot_edit_annex_files=Arquivos Annex não podem ser editados na interface web. editor.cannot_edit_non_text_files=Arquivos binários não podem ser editados na interface web. editor.edit_this_file=Editar arquivo editor.this_file_locked=Arquivo está bloqueado diff --git a/options/locale/locale_pt-PT.ini b/options/locale/locale_pt-PT.ini index 9d9c5eada1c07..776e4d04c061e 100644 --- a/options/locale/locale_pt-PT.ini +++ b/options/locale/locale_pt-PT.ini @@ -1182,6 +1182,7 @@ view_git_blame=Ver Git Blame video_not_supported_in_browser=O seu navegador não suporta a etiqueta 'video' do HTML5. audio_not_supported_in_browser=O seu navegador não suporta a etiqueta 'audio' do HTML5. stored_lfs=Armazenado com Git LFS +stored_annex=Armazenado com Git Annex symbolic_link=Ligação simbólica executable_file=Ficheiro executável commit_graph=Gráfico de cometimentos @@ -1205,6 +1206,7 @@ editor.upload_file=Carregar ficheiro editor.edit_file=Editar ficheiro editor.preview_changes=Pré-visualizar modificações editor.cannot_edit_lfs_files=Ficheiros LFS não podem ser editados na interface web. +editor.cannot_edit_annex_files=Ficheiros Annex não podem ser editados na interface web. editor.cannot_edit_non_text_files=Ficheiros binários não podem ser editados na interface da web. editor.edit_this_file=Editar ficheiro editor.this_file_locked=Ficheiro bloqueado diff --git a/options/locale/locale_ru-RU.ini b/options/locale/locale_ru-RU.ini index 3a560eba1d19d..e6c4cad266c0b 100644 --- a/options/locale/locale_ru-RU.ini +++ b/options/locale/locale_ru-RU.ini @@ -1156,6 +1156,7 @@ view_git_blame=Показать git blame video_not_supported_in_browser=Ваш браузер не поддерживает HTML5 'video' тэг. audio_not_supported_in_browser=Ваш браузер не поддерживает HTML5 'audio' тэг. stored_lfs=Хранится Git LFS +stored_annex=Хранится Git Annex symbolic_link=Символическая ссылка executable_file=Исполняемый файл commit_graph=Граф коммитов @@ -1179,6 +1180,7 @@ editor.upload_file=Загрузить файл editor.edit_file=Редактировать файл editor.preview_changes=Просмотр изменений editor.cannot_edit_lfs_files=LFS файлы невозможно редактировать в веб-интерфейсе. +editor.cannot_edit_annex_files=Annex файлы невозможно редактировать в веб-интерфейсе. editor.cannot_edit_non_text_files=Двоичные файлы нельзя редактировать в веб-интерфейсе. editor.edit_this_file=Редактировать файл editor.this_file_locked=Файл заблокирован diff --git a/options/locale/locale_si-LK.ini b/options/locale/locale_si-LK.ini index 6fdcb3422856b..603619653d436 100644 --- a/options/locale/locale_si-LK.ini +++ b/options/locale/locale_si-LK.ini @@ -892,6 +892,7 @@ file_copy_permalink=පිටපත් මාමලින්ක් video_not_supported_in_browser=ඔබගේ බ්රව්සරය HTML5 'වීඩියෝ' ටැගය සඳහා සහය නොදක්වයි. audio_not_supported_in_browser=ඔබගේ බ්රව්සරය HTML5 'ශ්රව්ය' ටැගය සඳහා සහය නොදක්වයි. stored_lfs=Git LFS සමඟ ගබඩා +stored_annex=Git Annex සමඟ ගබඩා symbolic_link=සංකේතාත්මක සබැඳිය commit_graph=ප්රස්තාරය කැප commit_graph.select=ශාඛා තෝරන්න @@ -909,6 +910,7 @@ editor.upload_file=ගොනුව උඩුගත කරන්න editor.edit_file=ගොනුව සංස්කරණය editor.preview_changes=වෙනස්කම් පෙරදසුන editor.cannot_edit_lfs_files=LFS ගොනු වෙබ් අතුරු මුහුණත තුළ සංස්කරණය කළ නොහැක. +editor.cannot_edit_annex_files=Annex ගොනු වෙබ් අතුරු මුහුණත තුළ සංස්කරණය කළ නොහැක. editor.cannot_edit_non_text_files=ද්විමය ගොනු වෙබ් අතුරු මුහුණත තුළ සංස්කරණය කළ නොහැක. editor.edit_this_file=ගොනුව සංස්කරණය editor.this_file_locked=ගොනුවට අගුළු ලා ඇත diff --git a/options/locale/locale_sk-SK.ini b/options/locale/locale_sk-SK.ini index d137a6d2beda6..0ba495cda4c84 100644 --- a/options/locale/locale_sk-SK.ini +++ b/options/locale/locale_sk-SK.ini @@ -914,6 +914,7 @@ view_git_blame=Zobraziť Git Blame video_not_supported_in_browser=Váš prehliadač nepodporuje HTML5 tag 'video'. audio_not_supported_in_browser=Váš prehliadač nepodporuje HTML5 tag 'audio'. stored_lfs=Uložené pomocou Git LFS +stored_annex=Uložené pomocou Git Annex symbolic_link=Symbolický odkaz commit_graph=Graf commitov diff --git a/options/locale/locale_sv-SE.ini b/options/locale/locale_sv-SE.ini index ce3f7d5d9bec4..d988e3b8c39b2 100644 --- a/options/locale/locale_sv-SE.ini +++ b/options/locale/locale_sv-SE.ini @@ -754,6 +754,7 @@ file_too_large=Filen är för stor för att visas. video_not_supported_in_browser=Din webbläsare stödjer ej HTML5-taggen 'video'. audio_not_supported_in_browser=Din webbläsare stöder inte taggen 'audio' i HTML5. stored_lfs=Sparad med Git LFS +stored_annex=Sparad med Git Annex symbolic_link=Symbolisk länk commit_graph=Commit-Graf commit_graph.monochrome=Mono @@ -767,6 +768,7 @@ editor.upload_file=Ladda Upp Fil editor.edit_file=Redigera Fil editor.preview_changes=Förhandsgranska ändringar editor.cannot_edit_lfs_files=LFS-filer kan inte redigeras i webbgränssnittet. +editor.cannot_edit_annex_files=Annex-filer kan inte redigeras i webbgränssnittet. editor.cannot_edit_non_text_files=Binära filer kan inte redigeras genom webbgränssnittet. editor.edit_this_file=Redigera Fil editor.this_file_locked=Filen är låst diff --git a/options/locale/locale_tr-TR.ini b/options/locale/locale_tr-TR.ini index 4c0fc95c0a26b..229dd4336105f 100644 --- a/options/locale/locale_tr-TR.ini +++ b/options/locale/locale_tr-TR.ini @@ -1169,6 +1169,7 @@ view_git_blame=Git Suç Görüntüle video_not_supported_in_browser=Tarayıcınız HTML5 'video' etiketini desteklemiyor. audio_not_supported_in_browser=Tarayıcınız HTML5 'audio' etiketini desteklemiyor. stored_lfs=Git LFS ile depolandı +stored_annex=Git Annex ile depolandı symbolic_link=Sembolik Bağlantı executable_file=Çalıştırılabilir Dosya commit_graph=İşleme Grafiği @@ -1192,6 +1193,7 @@ editor.upload_file=Dosya Yükle editor.edit_file=Dosyayı Düzenle editor.preview_changes=Değişiklikleri Önizle editor.cannot_edit_lfs_files=LFS dosyaları web arayüzünde düzenlenemez. +editor.cannot_edit_annex_files=Annex dosyaları web arayüzünde düzenlenemez. editor.cannot_edit_non_text_files=Bu tür dosyalar web arayüzünden düzenlenemez. editor.edit_this_file=Dosyayı Düzenle editor.this_file_locked=Dosya kilitlendi diff --git a/options/locale/locale_uk-UA.ini b/options/locale/locale_uk-UA.ini index 85adc61d33911..9d4671cb4d1c3 100644 --- a/options/locale/locale_uk-UA.ini +++ b/options/locale/locale_uk-UA.ini @@ -931,6 +931,7 @@ file_copy_permalink=Копіювати постійне посилання video_not_supported_in_browser=Ваш браузер не підтримує тег 'video' HTML5. audio_not_supported_in_browser=Ваш браузер не підтримує тег HTML5 'audio'. stored_lfs=Збережено з Git LFS +stored_annex=Збережено з Git Annex symbolic_link=Символічне посилання commit_graph=Графік комітів commit_graph.select=Виберіть гілки @@ -948,6 +949,7 @@ editor.upload_file=Завантажити файл editor.edit_file=Редагування файлу editor.preview_changes=Попередній перегляд змін editor.cannot_edit_lfs_files=Файли LFS не можна редагувати в веб-інтерфейсі. +editor.cannot_edit_annex_files=Файли Annex не можна редагувати в веб-інтерфейсі. editor.cannot_edit_non_text_files=Бінарні файли не можливо редагувати у веб-інтерфейсі. editor.edit_this_file=Редагувати файл editor.this_file_locked=Файл заблоковано diff --git a/options/locale/locale_zh-CN.ini b/options/locale/locale_zh-CN.ini index 8ebf6c509c3f9..4c565ac1c8688 100644 --- a/options/locale/locale_zh-CN.ini +++ b/options/locale/locale_zh-CN.ini @@ -1182,6 +1182,7 @@ view_git_blame=查看 Git Blame video_not_supported_in_browser=您的浏览器不支持使用 HTML5 'video' 标签。 audio_not_supported_in_browser=您的浏览器不支持使用 HTML5 'video' 标签。 stored_lfs=存储到Git LFS +stored_annex=存储到Git Annex symbolic_link=符号链接 executable_file=可执行文件 commit_graph=提交图 @@ -1205,6 +1206,7 @@ editor.upload_file=上传文件 editor.edit_file=编辑文件 editor.preview_changes=预览变更 editor.cannot_edit_lfs_files=无法在 web 界面中编辑 lfs 文件。 +editor.cannot_edit_annex_files=无法在 web 界面中编辑 lfs 文件。 editor.cannot_edit_non_text_files=网页不能编辑二进制文件。 editor.edit_this_file=编辑文件 editor.this_file_locked=文件已锁定 diff --git a/options/locale/locale_zh-HK.ini b/options/locale/locale_zh-HK.ini index dfa048c5ef0b9..fe71ddd9c1ac7 100644 --- a/options/locale/locale_zh-HK.ini +++ b/options/locale/locale_zh-HK.ini @@ -365,6 +365,7 @@ file_view_raw=查看原始文件 file_permalink=永久連結 stored_lfs=儲存到到 Git LFS +stored_annex=儲存到到 Git Annex editor.preview_changes=預覽更改 editor.or=或 diff --git a/options/locale/locale_zh-TW.ini b/options/locale/locale_zh-TW.ini index e058ca71919fe..a2460857aa434 100644 --- a/options/locale/locale_zh-TW.ini +++ b/options/locale/locale_zh-TW.ini @@ -1081,6 +1081,7 @@ view_git_blame=檢視 Git Blame video_not_supported_in_browser=您的瀏覽器不支援使用 HTML5 播放影片。 audio_not_supported_in_browser=您的瀏覽器不支援 HTML5 的「audio」標籤 stored_lfs=已使用 Git LFS 儲存 +stored_annex=已使用 Git Annex 儲存 symbolic_link=符號連結 commit_graph=提交線圖 commit_graph.select=選擇分支 @@ -1100,6 +1101,7 @@ editor.upload_file=上傳檔案 editor.edit_file=編輯檔案 editor.preview_changes=預覽更改 editor.cannot_edit_lfs_files=無法在 web 介面中編輯 LFS 檔。 +editor.cannot_edit_annex_files=無法在 web 介面中編輯 Annex 檔。 editor.cannot_edit_non_text_files=網站介面不能編輯二進位檔案 editor.edit_this_file=編輯檔案 editor.this_file_locked=檔案已被鎖定 diff --git a/routers/private/serv.go b/routers/private/serv.go index 00731947a55a8..df56553eeac63 100644 --- a/routers/private/serv.go +++ b/routers/private/serv.go @@ -81,12 +81,14 @@ func ServCommand(ctx *context.PrivateContext) { ownerName := ctx.Params(":owner") repoName := ctx.Params(":repo") mode := perm.AccessMode(ctx.FormInt("mode")) + verbs := ctx.FormStrings("verb") // Set the basic parts of the results to return results := private.ServCommandResults{ RepoName: repoName, OwnerName: ownerName, KeyID: keyID, + UserMode: perm.AccessModeNone, } // Now because we're not translating things properly let's just default some English strings here @@ -287,8 +289,10 @@ func ServCommand(ctx *context.PrivateContext) { repo.IsPrivate || owner.Visibility.IsPrivate() || (user != nil && user.IsRestricted) || // user will be nil if the key is a deploykey + (setting.Annex.Enabled && len(verbs) > 0 && verbs[0] == "git-annex-shell") || // git-annex has its own permission enforcement, for which we expose results.UserMode setting.Service.RequireSignInView) { if key.Type == asymkey_model.KeyTypeDeploy { + results.UserMode = deployKey.Mode if deployKey.Mode < mode { ctx.JSON(http.StatusUnauthorized, private.Response{ UserMsg: fmt.Sprintf("Deploy Key: %d:%s is not authorized to %s %s/%s.", key.ID, key.Name, modeString, results.OwnerName, results.RepoName), @@ -310,9 +314,9 @@ func ServCommand(ctx *context.PrivateContext) { return } - userMode := perm.UnitAccessMode(unitType) + results.UserMode = perm.UnitAccessMode(unitType) - if userMode < mode { + if results.UserMode < mode { log.Warn("Failed authentication attempt for %s with key %s (not authorized to %s %s/%s) from %s", user.Name, key.Name, modeString, ownerName, repoName, ctx.RemoteAddr()) ctx.JSON(http.StatusUnauthorized, private.Response{ UserMsg: fmt.Sprintf("User: %d:%s with Key: %d:%s is not authorized to %s %s/%s.", user.ID, user.Name, key.ID, key.Name, modeString, ownerName, repoName), @@ -353,6 +357,7 @@ func ServCommand(ctx *context.PrivateContext) { }) return } + results.UserMode = perm.AccessModeWrite results.RepoID = repo.ID } @@ -381,13 +386,14 @@ func ServCommand(ctx *context.PrivateContext) { return } } - log.Debug("Serv Results:\nIsWiki: %t\nDeployKeyID: %d\nKeyID: %d\tKeyName: %s\nUserName: %s\nUserID: %d\nOwnerName: %s\nRepoName: %s\nRepoID: %d", + log.Debug("Serv Results:\nIsWiki: %t\nDeployKeyID: %d\nKeyID: %d\tKeyName: %s\nUserName: %s\nUserID: %d\nUserMode: %d\nOwnerName: %s\nRepoName: %s\nRepoID: %d", results.IsWiki, results.DeployKeyID, results.KeyID, results.KeyName, results.UserName, results.UserID, + results.UserMode, results.OwnerName, results.RepoName, results.RepoID) diff --git a/routers/web/repo/download.go b/routers/web/repo/download.go index a9e2e2b2fad77..45b975537a2f4 100644 --- a/routers/web/repo/download.go +++ b/routers/web/repo/download.go @@ -9,6 +9,7 @@ import ( "time" git_model "code.gitea.io/gitea/models/git" + "code.gitea.io/gitea/modules/annex" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/httpcache" @@ -79,6 +80,26 @@ func ServeBlobOrLFS(ctx *context.Context, blob *git.Blob, lastModified *time.Tim } closed = true + // check for git-annex files + // (this code is weirdly redundant because I'm trying not to delete any lines in order to make merges easier) + isAnnexed, err := annex.IsAnnexed(blob) + if err != nil { + ctx.ServerError("annex.IsAnnexed", err) + return err + } + if isAnnexed { + content, err := annex.Content(blob) + if err != nil { + // XXX are there any other possible failure cases here? + // there are, there could be unrelated io errors; those should be ctx.ServerError()s + ctx.NotFound("annex.Content", err) + return err + } + defer content.Close() + common.ServeContentByReadSeeker(ctx.Base, ctx.Repo.TreePath, lastModified, content) + return nil + } + return common.ServeBlob(ctx.Base, ctx.Repo.TreePath, blob, lastModified) } diff --git a/routers/web/repo/githttp.go b/routers/web/repo/githttp.go index 6ff385f989050..41dcc6f6b95e9 100644 --- a/routers/web/repo/githttp.go +++ b/routers/web/repo/githttp.go @@ -611,3 +611,34 @@ func GetIdxFile(ctx *context.Context) { h.sendFile("application/x-git-packed-objects-toc", "objects/pack/pack-"+ctx.Params("file")+".idx") } } + +// GetAnnexObject implements git-annex dumb HTTP +func GetAnnexObject(ctx *context.Context) { + h := httpBase(ctx) + if h != nil { + // git-annex objects are stored in .git/annex/objects/{hash1}/{hash2}/{key}/{key} + // where key is a string containing the size and (usually SHA256) checksum of the file, + // and hash1+hash2 are the first few bits of the md5sum of key itself. + // ({hash1}/{hash2}/ is just there to avoid putting too many files in one directory) + // ref: https://git-annex.branchable.com/internals/hashing/ + + // keyDir should = key, but we don't enforce that + object := path.Join(ctx.Params("hash1"), ctx.Params("hash2"), ctx.Params("keyDir"), ctx.Params("key")) + + // Sanitize the input against directory traversals. + // + // This works because at the filesystem root, "/.." = "/"; + // So if a path starts rooted ("/"), path.Clean(), which + // path.Join() calls internally, removes all '..' prefixes. + // After, this unroots the path unconditionally ([1:]), which + // works because we know the input is never supposed to be rooted. + // + // The router code probably also disallows "..", so this + // should be redundant, but it's defensive to keep it + // whenever touching filesystem paths with user input. + object = path.Join("/", object)[1:] + + h.setHeaderCacheForever() + h.sendFile("application/octet-stream", "annex/objects/"+object) + } +} diff --git a/routers/web/repo/view.go b/routers/web/repo/view.go index 4dfa01d8e2a73..3da88a0ffdcbc 100644 --- a/routers/web/repo/view.go +++ b/routers/web/repo/view.go @@ -32,6 +32,7 @@ import ( unit_model "code.gitea.io/gitea/models/unit" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/actions" + "code.gitea.io/gitea/modules/annex" "code.gitea.io/gitea/modules/base" "code.gitea.io/gitea/modules/charset" "code.gitea.io/gitea/modules/container" @@ -200,14 +201,47 @@ func localizedExtensions(ext, languageCode string) (localizedExts []string) { } type fileInfo struct { - isTextFile bool - isLFSFile bool - fileSize int64 - lfsMeta *lfs.Pointer - st typesniffer.SniffedType + isTextFile bool + isLFSFile bool + isAnnexFile bool + fileSize int64 + lfsMeta *lfs.Pointer + st typesniffer.SniffedType } func getFileReader(ctx gocontext.Context, repoID int64, blob *git.Blob) ([]byte, io.ReadCloser, *fileInfo, error) { + isAnnexed, err := annex.IsAnnexed(blob) + if err != nil { + return nil, nil, nil, err + } + if isAnnexed { + // TODO: this code could be merged with the LFS case, especially the redundant type sniffer, + // but it is *currently* written this way to make merging with the non-annex upstream easier: + // this way, the git-annex patch is (mostly) pure additions. + + annexContent, err := annex.Content(blob) + if err != nil { + // in the case where annex content is missing, what should happen? + // do we render the page with an error message? + // actually that's not a bad idea, there's some sort of error message situation + // TODO: display an error to the user explaining that their data is missing + return nil, nil, nil, err + } + + stat, err := annexContent.Stat() + if err != nil { + return nil, nil, nil, err + } + + buf := make([]byte, 1024) + n, _ := util.ReadAtMost(annexContent, buf) + buf = buf[:n] + + st := typesniffer.DetectContentType(buf) + + return buf, annexContent, &fileInfo{st.IsText(), false, true, stat.Size(), nil, st}, nil + } + dataRc, err := blob.DataAsync() if err != nil { return nil, nil, nil, err @@ -222,17 +256,17 @@ func getFileReader(ctx gocontext.Context, repoID int64, blob *git.Blob) ([]byte, // FIXME: what happens when README file is an image? if !isTextFile || !setting.LFS.StartServer { - return buf, dataRc, &fileInfo{isTextFile, false, blob.Size(), nil, st}, nil + return buf, dataRc, &fileInfo{isTextFile, false, false, blob.Size(), nil, st}, nil } pointer, _ := lfs.ReadPointerFromBuffer(buf) if !pointer.IsValid() { // fallback to plain file - return buf, dataRc, &fileInfo{isTextFile, false, blob.Size(), nil, st}, nil + return buf, dataRc, &fileInfo{isTextFile, false, false, blob.Size(), nil, st}, nil } meta, err := git_model.GetLFSMetaObjectByOid(ctx, repoID, pointer.Oid) if err != nil && err != git_model.ErrLFSObjectNotExist { // fallback to plain file - return buf, dataRc, &fileInfo{isTextFile, false, blob.Size(), nil, st}, nil + return buf, dataRc, &fileInfo{isTextFile, false, false, blob.Size(), nil, st}, nil } dataRc.Close() @@ -255,7 +289,7 @@ func getFileReader(ctx gocontext.Context, repoID int64, blob *git.Blob) ([]byte, st = typesniffer.DetectContentType(buf) - return buf, dataRc, &fileInfo{st.IsText(), true, meta.Size, &meta.Pointer, st}, nil + return buf, dataRc, &fileInfo{st.IsText(), true, false, meta.Size, &meta.Pointer, st}, nil } func renderReadmeFile(ctx *context.Context, subfolder string, readmeFile *git.TreeEntry, readmeTreelink string) { @@ -383,10 +417,17 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st isDisplayingSource := ctx.FormString("display") == "source" isDisplayingRendered := !isDisplayingSource - if fInfo.isLFSFile { + if fInfo.isLFSFile || fInfo.isAnnexFile { ctx.Data["RawFileLink"] = ctx.Repo.RepoLink + "/media/" + ctx.Repo.BranchNameSubURL() + "/" + util.PathEscapeSegments(ctx.Repo.TreePath) } + if fInfo.isAnnexFile { + // pre-git-annex v7, all annexed files were represented in-repo as symlinks; + // but we pretend they aren't, since that's a distracting quirk of git-annex + // and not a meaningful choice on the user's part + ctx.Data["FileIsSymlink"] = false + } + isRepresentableAsText := fInfo.st.IsRepresentableAsText() if !isRepresentableAsText { // If we can't show plain text, always try to render. @@ -394,6 +435,7 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st isDisplayingRendered = true } ctx.Data["IsLFSFile"] = fInfo.isLFSFile + ctx.Data["IsAnnexFile"] = fInfo.isAnnexFile ctx.Data["FileSize"] = fInfo.fileSize ctx.Data["IsTextFile"] = fInfo.isTextFile ctx.Data["IsRepresentableAsText"] = isRepresentableAsText @@ -428,6 +470,8 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st // Assume file is not editable first. if fInfo.isLFSFile { ctx.Data["EditFileTooltip"] = ctx.Tr("repo.editor.cannot_edit_lfs_files") + } else if fInfo.isAnnexFile { + ctx.Data["EditFileTooltip"] = ctx.Tr("repo.editor.cannot_edit_annex_files") } else if !isRepresentableAsText { ctx.Data["EditFileTooltip"] = ctx.Tr("repo.editor.cannot_edit_non_text_files") } @@ -543,7 +587,7 @@ func renderFile(ctx *context.Context, entry *git.TreeEntry, treeLink, rawLink st ctx.Data["FileContent"] = fileContent ctx.Data["LineEscapeStatus"] = statuses } - if !fInfo.isLFSFile { + if !fInfo.isLFSFile && !fInfo.isAnnexFile { if ctx.Repo.CanEnableEditor(ctx, ctx.Doer) { if lfsLock != nil && lfsLock.OwnerID != ctx.Doer.ID { ctx.Data["CanEditFile"] = false diff --git a/routers/web/web.go b/routers/web/web.go index f8b745fb10b55..d7bcdef290492 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -331,6 +331,13 @@ func registerRoutes(m *web.Route) { } } + annexEnabled := func(ctx *context.Context) { + if !setting.Annex.Enabled { + ctx.Error(http.StatusNotFound) + return + } + } + federationEnabled := func(ctx *context.Context) { if !setting.Federation.Enabled { ctx.Error(http.StatusNotFound) @@ -1514,6 +1521,12 @@ func registerRoutes(m *web.Route) { }) }, ignSignInAndCsrf, lfsServerEnabled) + m.Group("", func() { + // for git-annex + m.GetOptions("/config", repo.GetTextFile("config")) // needed by clients reading annex.uuid during `git annex initremote` + m.GetOptions("/annex/objects/{hash1}/{hash2}/{keyDir}/{key}", repo.GetAnnexObject) + }, ignSignInAndCsrf, annexEnabled, context_service.UserAssignmentWeb()) + gitHTTPRouters(m) }) }) diff --git a/services/auth/auth.go b/services/auth/auth.go index 713463a3d47ed..b391486dd5033 100644 --- a/services/auth/auth.go +++ b/services/auth/auth.go @@ -54,6 +54,17 @@ func isGitRawOrAttachOrLFSPath(req *http.Request) bool { return false } +var annexPathRe = regexp.MustCompile(`^/[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+/annex/`) + +func isAnnexPath(req *http.Request) bool { + if setting.Annex.Enabled { + // "/config" is git's config, not specifically git-annex's; but the only current + // user of it is when git-annex downloads the annex.uuid during 'git annex init'. + return strings.HasSuffix(req.URL.Path, "/config") || annexPathRe.MatchString(req.URL.Path) + } + return false +} + // handleSignIn clears existing session variables and stores new ones for the specified user object func handleSignIn(resp http.ResponseWriter, req *http.Request, sess SessionStore, user *user_model.User) { // We need to regenerate the session... diff --git a/services/auth/basic.go b/services/auth/basic.go index 6c3fbf595e44c..8987f4cf199a8 100644 --- a/services/auth/basic.go +++ b/services/auth/basic.go @@ -41,8 +41,8 @@ func (b *Basic) Name() string { // name/token on successful validation. // Returns nil if header is empty or validation fails. func (b *Basic) Verify(req *http.Request, w http.ResponseWriter, store DataStore, sess SessionStore) (*user_model.User, error) { - // Basic authentication should only fire on API, Download or on Git or LFSPaths - if !middleware.IsAPIPath(req) && !isContainerPath(req) && !isAttachmentDownload(req) && !isGitRawOrAttachOrLFSPath(req) { + // Basic authentication should only fire on API, Download or on Git or LFSPaths or Git-Annex paths + if !middleware.IsAPIPath(req) && !isContainerPath(req) && !isAttachmentDownload(req) && !isGitRawOrAttachOrLFSPath(req) && !isAnnexPath(req) { return nil, nil } diff --git a/services/repository/archiver/archiver.go b/services/repository/archiver/archiver.go index 9f1ea48dca400..053c52d9771d1 100644 --- a/services/repository/archiver/archiver.go +++ b/services/repository/archiver/archiver.go @@ -15,6 +15,7 @@ import ( "code.gitea.io/gitea/models/db" repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/modules/annex" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/log" @@ -251,13 +252,24 @@ func doArchive(ctx context.Context, r *ArchiveRequest) (*repo_model.RepoArchiver w, ) } else { - err = gitRepo.CreateArchive( - ctx, - archiver.Type, - w, - setting.Repository.PrefixArchiveFiles, - archiver.CommitID, - ) + if annex.IsAnnexRepo(gitRepo) { + err = annex.CreateArchive( + ctx, + gitRepo, + archiver.Type, + w, + setting.Repository.PrefixArchiveFiles, + archiver.CommitID, + ) + } else { + err = gitRepo.CreateArchive( + ctx, + archiver.Type, + w, + setting.Repository.PrefixArchiveFiles, + archiver.CommitID, + ) + } } _ = w.CloseWithError(err) done <- err @@ -266,6 +278,13 @@ func doArchive(ctx context.Context, r *ArchiveRequest) (*repo_model.RepoArchiver // TODO: add lfs data to zip // TODO: add submodule data to zip + // commit and close here to avoid blocking the database for the entirety of the archive generation process (might only be an issue with sqlite) + err = committer.Commit() + if err != nil { + return nil, err + } + committer.Close() + if _, err := storage.RepoArchives.Save(rPath, rd, -1); err != nil { return nil, fmt.Errorf("unable to write archive: %w", err) } @@ -275,6 +294,14 @@ func doArchive(ctx context.Context, r *ArchiveRequest) (*repo_model.RepoArchiver return nil, err } + txCtx, committer, err = db.TxContext(db.DefaultContext) + if err != nil { + return nil, err + } + defer committer.Close() + ctx, _, finished = process.GetManager().AddContext(txCtx, fmt.Sprintf("ArchiveRequest[%d]: %s", r.RepoID, r.GetArchiveName())) + defer finished() + if archiver.Status == repo_model.ArchiverGenerating { archiver.Status = repo_model.ArchiverReady if err = repo_model.UpdateRepoArchiverStatus(ctx, archiver); err != nil { diff --git a/templates/repo/file_info.tmpl b/templates/repo/file_info.tmpl index 3003fbbdb6df5..16d8dcb9e8f64 100644 --- a/templates/repo/file_info.tmpl +++ b/templates/repo/file_info.tmpl @@ -12,6 +12,7 @@ {{if .FileSize}}
{{FileSize .FileSize}}{{if .IsLFSFile}} ({{ctx.Locale.Tr "repo.stored_lfs"}}){{end}} + {{if .IsAnnexFile}} ({{ctx.Locale.Tr "repo.stored_annex"}}){{end}}
{{end}} {{if .LFSLock}} diff --git a/tests/integration/api_helper_for_declarative_test.go b/tests/integration/api_helper_for_declarative_test.go index 3524ce9834add..58f4d63f30bac 100644 --- a/tests/integration/api_helper_for_declarative_test.go +++ b/tests/integration/api_helper_for_declarative_test.go @@ -21,6 +21,7 @@ import ( api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/services/forms" + "github.com/google/uuid" "github.com/stretchr/testify/assert" ) @@ -462,3 +463,27 @@ func doAPIAddRepoToOrganizationTeam(ctx APITestContext, teamID int64, orgName, r ctx.Session.MakeRequest(t, req, http.StatusNoContent) } } + +// generate and activate an ssh key for the user attached to the APITestContext +// TODO: pick a better name; golang doesn't do method overriding. +func withCtxKeyFile(t *testing.T, ctx APITestContext, callback func()) { + // we need to have write:public_key to do this step + // the easiest way is to create a throwaway ctx that is identical but only has that permission + ctxKeyWriter := ctx + ctxKeyWriter.Token = getTokenForLoggedInUser(t, ctx.Session, auth.AccessTokenScopeWriteUser) + + keyName := "One of " + ctx.Username + "'s keys: #" + uuid.New().String() + withKeyFile(t, keyName, func(keyFile string) { + var key api.PublicKey + + doAPICreateUserKey(ctxKeyWriter, keyName, keyFile, + func(t *testing.T, _key api.PublicKey) { + // save the key ID so we can delete it at the end + key = _key + })(t) + + defer doAPIDeleteUserKey(ctxKeyWriter, key.ID)(t) + + callback() + }) +} diff --git a/tests/integration/git_annex_archive_test.go b/tests/integration/git_annex_archive_test.go new file mode 100644 index 0000000000000..01141306479fa --- /dev/null +++ b/tests/integration/git_annex_archive_test.go @@ -0,0 +1,292 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package integration + +import ( + "archive/tar" + "archive/zip" + "bytes" + "compress/gzip" + "fmt" + "io" + "io/fs" + "net/http" + "net/url" + "path" + "sort" + "strings" + "sync" + "testing" + + auth_model "code.gitea.io/gitea/models/auth" + "code.gitea.io/gitea/modules/annex" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/setting" + + "github.com/stretchr/testify/require" +) + +type record struct { + name string + isAnnexed bool + annexedMode fs.FileMode + gitMode git.EntryMode + content []byte +} + +func TestGitAnnexArchive(t *testing.T) { + onGiteaRun(t, func(t *testing.T, u *url.URL) { + ctx := NewAPITestContext(t, "user2", "annex-archive-test", auth_model.AccessTokenScopeWriteRepository) + require.NoError(t, doCreateRemoteAnnexRepository(t, u, ctx, false)) + req := NewRequest(t, "GET", u.String()) + _ = ctx.Session.MakeRequest(t, req, http.StatusOK) + + remoteRepoPath := path.Join(setting.RepoRootPath, ctx.GitPath()) + + // get the commitID of the master branch + repo, err := git.OpenRepository(git.DefaultContext, remoteRepoPath) + require.NoError(t, err) + commitID, err := repo.GetBranchCommitID("master") + require.NoError(t, err) + tree, err := repo.GetTree("master") + require.NoError(t, err) + entries, err := tree.ListEntriesRecursiveFast() + require.NoError(t, err) + filesInGit := make(map[string]record, len(entries)) + var annexedMode fs.FileMode + for _, entry := range entries { + if !entry.IsDir() { + name := entry.Name() + blob := entry.Blob() + isAnnexed, err := annex.IsAnnexed(blob) + require.NoError(t, err) + var r io.Reader + if isAnnexed { + fa, err := annex.Content(blob) + require.NoError(t, err) + defer fa.Close() + r = fa + stat, err := fa.Stat() + require.NoError(t, err) + annexedMode = stat.Mode() + } else { + // standard git file + br, err := blob.DataAsync() + require.NoError(t, err) + defer br.Close() + r = br + } + expectedContent, err := io.ReadAll(r) + require.NoError(t, err) + filesInGit[name] = record{ + name: name, + isAnnexed: isAnnexed, + annexedMode: annexedMode, + gitMode: entry.Mode(), + content: expectedContent, + } + } + } + + t.Run("api-v1", func(t *testing.T) { + urlBase := fmt.Sprintf("/api/v1/repos/%s/%s/archive/master", ctx.Username, ctx.Reponame) + doTestArchive(t, ctx, urlBase, commitID, filesInGit) + }) + + t.Run("web", func(t *testing.T) { + urlBase := fmt.Sprintf("/%s/%s/archive/master", ctx.Username, ctx.Reponame) + doTestArchive(t, ctx, urlBase, commitID, filesInGit) + }) + }) +} + +func doTestArchive(t *testing.T, ctx APITestContext, urlBase, commitID string, filesInGit map[string]record) { + // cleanup previously generated archives + adminSession := loginUser(t, "user1") + adminToken := getTokenForLoggedInUser(t, adminSession, auth_model.AccessTokenScopeWriteAdmin) + link, _ := url.Parse("/api/v1/admin/cron/delete_repo_archives") + link.RawQuery = url.Values{"token": {adminToken}}.Encode() + resp := adminSession.MakeRequest(t, NewRequest(t, "POST", link.String()), http.StatusNoContent) + bs, err := io.ReadAll(resp.Body) + require.NoError(t, err) + require.Len(t, bs, 0) + + t.Run("TARGZ", func(t *testing.T) { + var wg sync.WaitGroup + for i := 0; i < 100; i++ { + wg.Add(1) + go func() { + defer wg.Done() + doTestTarGzArchive(t, urlBase+".tar.gz", ctx, commitID, filesInGit) + }() + } + wg.Wait() + }) + t.Run("ZIP", func(t *testing.T) { + var wg sync.WaitGroup + for i := 0; i < 100; i++ { + wg.Add(1) + go func() { + defer wg.Done() + doTestZipArchive(t, urlBase+".zip", ctx, commitID, filesInGit) + }() + } + wg.Wait() + }) +} + +func getArchiveFromEndpoint(t *testing.T, ctx APITestContext, endpointURL string) []byte { + link, _ := url.Parse(endpointURL) + resp := ctx.Session.MakeRequest(t, NewRequest(t, "GET", link.String()), http.StatusOK) + bs, err := io.ReadAll(resp.Body) + require.NoError(t, err) + return bs +} + +func doTestTarGzArchive(t *testing.T, endpointURL string, ctx APITestContext, commitID string, filesInGit map[string]record) { + // request a tar.gz archive of the repo + bs := getArchiveFromEndpoint(t, ctx, endpointURL) + + // open the archive for reading + gzrd, err := gzip.NewReader(bytes.NewReader(bs)) + require.NoError(t, err) + defer gzrd.Close() + rd := tar.NewReader(gzrd) + + var filesInArchive []string + for { + header, err := rd.Next() + if err == io.EOF { + break + } + require.NoError(t, err) + + // skip directories + if header.Typeflag == tar.TypeDir { + continue + } + + // check that the pax_global_header comment field is correctly set + if path.Base(header.Name) == "pax_global_header" { + require.Equal(t, commitID, header.PAXRecords["comment"]) + continue // skip the remaining checks since this file does not exist in git + } + + name := strings.TrimPrefix(header.Name, ctx.Reponame+"/") + filesInArchive = append(filesInArchive, name) + + // make sure all files are the same as in the repo itself + actualContent, err := io.ReadAll(rd) + require.NoError(t, err) + actualFileMode := header.FileInfo().Mode() + compareToGitRecord(t, git.TARGZ, filesInGit[name], actualContent, actualFileMode) + } + // check that all files that are in git are also present in the archive + compareListOfStrings(t, mapKeys(filesInGit), filesInArchive) +} + +func doTestZipArchive(t *testing.T, endpointURL string, ctx APITestContext, commitID string, filesInGit map[string]record) { + // request a zip archive of the repo + bs := getArchiveFromEndpoint(t, ctx, endpointURL) + + // open the archive for reading + r, err := zip.NewReader(bytes.NewReader(bs), int64(len(bs))) + require.NoError(t, err) + + // check that the comment field is correctly set + require.Equal(t, commitID, r.Comment) + + var filesInArchive []string + for _, f := range r.File { + // skip directories + if f.FileInfo().IsDir() { + continue + } + + name := strings.TrimPrefix(f.Name, ctx.Reponame+"/") + filesInArchive = append(filesInArchive, name) + + // make sure all files are the same as in the repo itself + frd, err := f.Open() + require.NoError(t, err) + defer frd.Close() + actualContent, err := io.ReadAll(frd) + require.NoError(t, err) + actualFileMode := f.Mode() + compareToGitRecord(t, git.ZIP, filesInGit[name], actualContent, actualFileMode) + } + // check that all files that are in git are also present in the archive + compareListOfStrings(t, mapKeys(filesInGit), filesInArchive) +} + +func compareToGitRecord(t *testing.T, archiveType git.ArchiveType, gitRecord record, actualContent []byte, actualFileMode fs.FileMode) { + expectedContent := gitRecord.content + var expectedFileMode fs.FileMode + // The expected file mode depends on the archive type and the type of file in git + // (regular, executable, symlink, annex pointer, annex symlink) + switch archiveType { + case git.TARGZ: + if gitRecord.gitMode == git.EntryModeExec || + (gitRecord.gitMode == git.EntryModeSymlink && gitRecord.isAnnexed && (gitRecord.annexedMode&0o100) != 0) { + // If the file is a regular executable git file (plain-git or annex pointer file) + // or a symlink to an executable annexed file expect file mode 0775, just like + // what `git archive` would do for executable files. + expectedFileMode = fs.FileMode(0o775) + } else if gitRecord.gitMode == git.EntryModeBlob || + (gitRecord.gitMode == git.EntryModeSymlink && gitRecord.isAnnexed && (gitRecord.annexedMode&0o100) == 0) { + // If the file is a regular non-executable git file (plain-git or annex pointer + // file) or a symlink to a non-executable annexed file expect file mode 0664, like + // what `git archive` would do. + expectedFileMode = fs.FileMode(0o664) + } else if gitRecord.gitMode == git.EntryModeSymlink { + // If the file is a plain-git symlink expect a normal symlink. + expectedFileMode = fs.FileMode(0o777) | fs.ModeSymlink + // gitRecord.content contains the link target, but in tar.gz there is no content + // for symlinks, so set the expectedContent to an empty byte slice. + expectedContent = []byte{} + } + case git.ZIP: + if gitRecord.gitMode == git.EntryModeExec || + (gitRecord.gitMode == git.EntryModeSymlink && gitRecord.isAnnexed && (gitRecord.annexedMode&0o100) != 0) { + // If the file is a regular executable git file (plain-git or annex pointer file) + // or a symlink to an executable annexed file expect file mode 0755. + expectedFileMode = fs.FileMode(0o755) + } else if gitRecord.gitMode == git.EntryModeBlob || + (gitRecord.gitMode == git.EntryModeSymlink && gitRecord.isAnnexed && (gitRecord.annexedMode&0o100) == 0) { + // If the file is a regular non-executable git file (plain-git or annex pointer + // file) or a symlink to a non-executable annexed file it should be archived with + // creatorFAT and have rw permissions, like what `git archive` would do. + // This means go should read it in with 0666 permissions. + expectedFileMode = fs.FileMode(0o666) + } else if gitRecord.gitMode == git.EntryModeSymlink { + // If the file is a plain-git symlink expect a normal symlink. + expectedFileMode = fs.FileMode(0o777) | fs.ModeSymlink + } + } + + // check that the file modes (type and permissions) are equal + require.Equal(t, expectedFileMode.String(), actualFileMode.String()) + + // check that the contents are equal + require.Equal(t, expectedContent, actualContent) +} + +func compareListOfStrings(t *testing.T, l1, l2 []string) { + sort.Strings(l1) + sort.Strings(l2) + require.Equal(t, l1, l2) +} + +func mapKeys[K comparable, V any](m map[K]V) []K { + keys := make([]K, len(m)) + + i := 0 + for k := range m { + keys[i] = k + i++ + } + + return keys +} diff --git a/tests/integration/git_annex_test.go b/tests/integration/git_annex_test.go new file mode 100644 index 0000000000000..83d5dbceb36d1 --- /dev/null +++ b/tests/integration/git_annex_test.go @@ -0,0 +1,1359 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package integration + +import ( + "errors" + "fmt" + "io" + "math/rand" + "net/http" + "net/url" + "os" + "path" + "regexp" + "strings" + "testing" + + auth_model "code.gitea.io/gitea/models/auth" + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/models/perm" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/modules/annex" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/setting" + api "code.gitea.io/gitea/modules/structs" + "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/tests" + + "github.com/stretchr/testify/require" +) + +// Some guidelines: +// +// * a APITestContext is an awkward union of session credential + username + target repo +// which is assumed to be owned by that username; if you want to target a different +// repo, you need to edit its .Reponame or just ignore it and write "username/reponame.git" + +func doCreateRemoteAnnexRepository(t *testing.T, u *url.URL, ctx APITestContext, private bool) (err error) { + // creating a repo counts as editing the user's profile (is done by POSTing + // to /api/v1/user/repos/) -- which means it needs a User-scoped token and + // both that and editing need a Repo-scoped token because they edit repositories. + rescopedCtx := ctx + rescopedCtx.Token = getTokenForLoggedInUser(t, ctx.Session, auth_model.AccessTokenScopeWriteUser, auth_model.AccessTokenScopeWriteRepository) + doAPICreateRepository(rescopedCtx, false)(t) + doAPIEditRepository(rescopedCtx, &api.EditRepoOption{Private: &private})(t) + + repoURL := createSSHUrl(ctx.GitPath(), u) + + // Fill in fixture data + withAnnexCtxKeyFile(t, ctx, func() { + err = doInitRemoteAnnexRepository(t, repoURL) + }) + if err != nil { + return fmt.Errorf("Unable to initialize remote repo with git-annex fixture: %w", err) + } + return nil +} + +func TestGitAnnexMedia(t *testing.T) { + if !setting.Annex.Enabled { + t.Skip("Skipping since annex support is disabled.") + } + + onGiteaRun(t, func(t *testing.T, u *url.URL) { + ctx := NewAPITestContext(t, "user2", "annex-media-test", auth_model.AccessTokenScopeWriteRepository) + + // create a public repo + require.NoError(t, doCreateRemoteAnnexRepository(t, u, ctx, false)) + + // the filenames here correspond to specific cases defined in doInitAnnexRepository() + t.Run("AnnexSymlink", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + doAnnexMediaTest(t, ctx, "annexed.tiff") + }) + t.Run("AnnexPointer", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + doAnnexMediaTest(t, ctx, "annexed.bin") + }) + }) +} + +func doAnnexMediaTest(t *testing.T, ctx APITestContext, file string) { + // Make sure that downloading via /media on the website recognizes it should give the annexed content + + // TODO: + // - [ ] roll this into TestGitAnnexPermissions to ensure that permission enforcement works correctly even on /media? + + session := loginUser(t, ctx.Username) // logs in to the http:// site/API, storing a cookie; + // this is a different auth method than the git+ssh:// or git+http:// protocols TestGitAnnexPermissions uses! + + // compute server-side path of the annexed file + remoteRepoPath := path.Join(setting.RepoRootPath, ctx.GitPath()) + remoteObjectPath, err := contentLocation(remoteRepoPath, file) + require.NoError(t, err) + + // download annexed file + localObjectPath := path.Join(t.TempDir(), file) + fd, err := os.OpenFile(localObjectPath, os.O_CREATE|os.O_WRONLY, 0o777) + defer fd.Close() + require.NoError(t, err) + + mediaLink := path.Join("/", ctx.Username, ctx.Reponame, "/media/branch/master", file) + req := NewRequest(t, "GET", mediaLink) + resp := session.MakeRequest(t, req, http.StatusOK) + + _, err = io.Copy(fd, resp.Body) + require.NoError(t, err) + fd.Close() + + // verify the download + match, err := util.FileCmp(localObjectPath, remoteObjectPath, 0) + require.NoError(t, err) + require.True(t, match, "Annexed files should be the same") +} + +func TestGitAnnexViews(t *testing.T) { + if !setting.Annex.Enabled { + t.Skip("Skipping since annex support is disabled.") + } + + onGiteaRun(t, func(t *testing.T, u *url.URL) { + ctx := NewAPITestContext(t, "user2", "annex-template-render-test", auth_model.AccessTokenScopeWriteRepository) + + // create a public repo + require.NoError(t, doCreateRemoteAnnexRepository(t, u, ctx, false)) + + session := loginUser(t, ctx.Username) + + t.Run("Index", func(t *testing.T) { + // test that annex symlinks renders with the _file icon_ on the main list + defer tests.PrintCurrentTest(t)() + + repoLink := path.Join("/", ctx.Username, ctx.Reponame) + req := NewRequest(t, "GET", repoLink) + resp := session.MakeRequest(t, req, http.StatusOK) + + htmlDoc := NewHTMLParser(t, resp.Body) + isFileIcon := htmlDoc.Find("tr[data-entryname='annexed.tiff'] > td.name svg").HasClass("octicon-file") + require.True(t, isFileIcon, "annexed files should render a plain file icon, even when stored via annex symlink") + }) + + t.Run("View", func(t *testing.T) { + // test how routers/web/repo/view.go + templates/repo/view_file.tmpl handle annexed files + defer tests.PrintCurrentTest(t)() + + doViewTest := func(file string) (htmlDoc *HTMLDoc, viewLink, mediaLink string) { + viewLink = path.Join("/", ctx.Username, ctx.Reponame, "/src/branch/master", file) + // rawLink := strings.Replace(viewLink, "/src/", "/raw/", 1) // TODO: do something with this? + mediaLink = strings.Replace(viewLink, "/src/", "/media/", 1) + + req := NewRequest(t, "GET", viewLink) + resp := session.MakeRequest(t, req, http.StatusOK) + + htmlDoc = NewHTMLParser(t, resp.Body) + // the first button on the toolbar on the view template is the "Raw" button + // this CSS selector is the most precise I can think to use + buttonLink, exists := htmlDoc.Find(".file-header").Find("a[download]").Attr("href") + require.True(t, exists, "Download button should exist on the file header") + require.EqualValues(t, mediaLink, buttonLink, "Download link should use /media URL for annex files") + + return htmlDoc, viewLink, mediaLink + } + + t.Run("Binary", func(t *testing.T) { + // test that annexing a file renders the /media link in /src and NOT the /raw link + defer tests.PrintCurrentTest(t)() + + doBinaryViewTest := func(file string) { + htmlDoc, _, mediaLink := doViewTest(file) + + rawLink, exists := htmlDoc.Find("div.file-view > div.view-raw > a").Attr("href") + require.True(t, exists, "Download link should render instead of content because this is a binary file") + require.EqualValues(t, mediaLink, rawLink) + } + + t.Run("AnnexSymlink", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + doBinaryViewTest("annexed.tiff") + }) + t.Run("AnnexPointer", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + doBinaryViewTest("annexed.bin") + }) + }) + + t.Run("Text", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + doTextViewTest := func(file string) { + htmlDoc, _, _ := doViewTest(file) + require.True(t, htmlDoc.Find("div.file-view").Is(".code-view"), "should render as code") + } + + t.Run("AnnexSymlink", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + doTextViewTest("annexed.txt") + + t.Run("Markdown", func(t *testing.T) { + // special case: check that markdown can be pulled out of the annex and rendered, too + defer tests.PrintCurrentTest(t)() + htmlDoc, _, _ := doViewTest("annexed.md") + require.True(t, htmlDoc.Find("div.file-view").Is(".markdown"), "should render as markdown") + }) + }) + t.Run("AnnexPointer", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + doTextViewTest("annexed.rst") + + t.Run("Markdown", func(t *testing.T) { + // special case: check that markdown can be pulled out of the annex and rendered, too + defer tests.PrintCurrentTest(t)() + htmlDoc, _, _ := doViewTest("annexed.markdown") + require.True(t, htmlDoc.Find("div.file-view").Is(".markdown"), "should render as markdown") + }) + }) + }) + }) + }) +} + +/* +Test that permissions are enforced on git-annex-shell commands. + + Along the way, this also tests that uploading, downloading, and deleting all work, + so we haven't written separate tests for those. +*/ +func TestGitAnnexPermissions(t *testing.T) { + if !setting.Annex.Enabled { + t.Skip("Skipping since annex support is disabled.") + } + + // Each case below is split so that 'clone' is done as + // the repo owner, but 'copy' as the user under test. + // + // Otherwise, in cases where permissions block the + // initial 'clone', the test would simply end there + // and never verify if permissions apply properly to + // 'annex copy' -- potentially leaving a security gap. + + onGiteaRun(t, func(t *testing.T, u *url.URL) { + // Tell git-annex to allow http://127.0.0.1, http://localhost and http://::1. Without + // this, all `git annex` commands will silently fail when run against http:// remotes + // without explaining what's wrong. + // + // Note: onGiteaRun() sets up an alternate HOME so this actually edits + // tests/integration/gitea-integration-*/data/home/.gitconfig and + // if you're debugging you need to remember to match that. + _, _, err := git.NewCommandContextNoGlobals(git.DefaultContext, "config").AddOptionValues("--global").AddArguments("annex.security.allowed-ip-addresses", "all").RunStdString(&git.RunOpts{}) + require.NoError(t, err) + + t.Run("Public", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + ownerCtx := NewAPITestContext(t, "user2", "annex-public", auth_model.AccessTokenScopeWriteRepository) + + // create a public repo + require.NoError(t, doCreateRemoteAnnexRepository(t, u, ownerCtx, false)) + + // double-check it's public + repo, err := repo_model.GetRepositoryByOwnerAndName(db.DefaultContext, ownerCtx.Username, ownerCtx.Reponame) + require.NoError(t, err) + require.False(t, repo.IsPrivate) + + remoteRepoPath := path.Join(setting.RepoRootPath, ownerCtx.GitPath()) // path on disk -- which can be examined directly because we're testing from localhost + + // Different sessions, so we can test different permissions. + // We leave Reponame blank because we don't actually then later add it according to each case if needed + // + // NB: these usernames need to match appropriate entries in models/fixtures/user.yml + writerCtx := NewAPITestContext(t, "user5", "", auth_model.AccessTokenScopeWriteRepository) + readerCtx := NewAPITestContext(t, "user4", "", auth_model.AccessTokenScopeReadRepository) + outsiderCtx := NewAPITestContext(t, "user8", "", auth_model.AccessTokenScopeReadRepository) // a user with no specific access + + // set up collaborators + doAPIAddCollaborator(ownerCtx, readerCtx.Username, perm.AccessModeRead)(t) + doAPIAddCollaborator(ownerCtx, writerCtx.Username, perm.AccessModeWrite)(t) + + // tests + t.Run("Owner", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + t.Run("SSH", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createSSHUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxKeyFile(t, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxKeyFile(t, ownerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + }) + }) + + t.Run("HTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + }) + }) + + t.Run("Writer", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + t.Run("SSH", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createSSHUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxKeyFile(t, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxKeyFile(t, writerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + }) + }) + + t.Run("HTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + }) + }) + + t.Run("Reader", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + t.Run("SSH", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createSSHUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxKeyFile(t, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxKeyFile(t, readerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath), "Uploading should fail due to permissions") + }) + }) + }) + + t.Run("HTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + }) + }) + + t.Run("Outsider", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + t.Run("SSH", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createSSHUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxKeyFile(t, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxKeyFile(t, outsiderCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath), "Uploading should fail due to permissions") + }) + }) + }) + + t.Run("HTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + }) + }) + + t.Run("Anonymous", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + // Only HTTP has an anonymous mode + t.Run("HTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + // unlike the other tests, at this step we *do not* define credentials: + + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + }) + + t.Run("Delete", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + // Delete the repo, make sure it's fully gone + doAPIDeleteRepository(ownerCtx)(t) + _, statErr := os.Stat(remoteRepoPath) + require.True(t, os.IsNotExist(statErr), "Remote annex repo should be removed from disk") + }) + }) + + t.Run("Private", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + ownerCtx := NewAPITestContext(t, "user2", "annex-private", auth_model.AccessTokenScopeWriteRepository) + + // create a private repo + require.NoError(t, doCreateRemoteAnnexRepository(t, u, ownerCtx, true)) + + // double-check it's private + repo, err := repo_model.GetRepositoryByOwnerAndName(db.DefaultContext, ownerCtx.Username, ownerCtx.Reponame) + require.NoError(t, err) + require.True(t, repo.IsPrivate) + + remoteRepoPath := path.Join(setting.RepoRootPath, ownerCtx.GitPath()) // path on disk -- which can be examined directly because we're testing from localhost + + // Different sessions, so we can test different permissions. + // We leave Reponame blank because we don't actually then later add it according to each case if needed + // + // NB: these usernames need to match appropriate entries in models/fixtures/user.yml + writerCtx := NewAPITestContext(t, "user5", "", auth_model.AccessTokenScopeWriteRepository) + readerCtx := NewAPITestContext(t, "user4", "", auth_model.AccessTokenScopeReadRepository) + outsiderCtx := NewAPITestContext(t, "user8", "", auth_model.AccessTokenScopeReadRepository) // a user with no specific access + // Note: there's also full anonymous access, which is only available for public HTTP repos; + // it should behave the same as 'outsider' but we (will) test it separately below anyway + + // set up collaborators + doAPIAddCollaborator(ownerCtx, readerCtx.Username, perm.AccessModeRead)(t) + doAPIAddCollaborator(ownerCtx, writerCtx.Username, perm.AccessModeWrite)(t) + + // tests + t.Run("Owner", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + t.Run("SSH", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createSSHUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxKeyFile(t, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxKeyFile(t, ownerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + }) + }) + + t.Run("HTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + }) + }) + + t.Run("Writer", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + t.Run("SSH", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createSSHUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxKeyFile(t, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxKeyFile(t, writerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexUploadTest(remoteRepoPath, repoPath)) + }) + }) + }) + + t.Run("HTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxHTTPPassword(t, u, writerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + }) + }) + + t.Run("Reader", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + t.Run("SSH", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createSSHUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxKeyFile(t, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxKeyFile(t, readerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath), "Uploading should fail due to permissions") + }) + }) + }) + + t.Run("HTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxHTTPPassword(t, u, readerCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.NoError(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + }) + }) + + t.Run("Outsider", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + t.Run("SSH", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createSSHUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxKeyFile(t, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxKeyFile(t, outsiderCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.Error(t, doAnnexInitTest(remoteRepoPath, repoPath), "annex init should fail due to permissions") + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath), "annex copy --from should fail due to permissions") + }) + + t.Run("Upload", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + require.Error(t, doAnnexUploadTest(remoteRepoPath, repoPath), "annex copy --to should fail due to permissions") + }) + }) + }) + + t.Run("HTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + withAnnexCtxHTTPPassword(t, u, outsiderCtx, func() { + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + }) + }) + + t.Run("Anonymous", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + // Only HTTP has an anonymous mode + t.Run("HTTP", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + repoURL := createHTTPUrl(ownerCtx.GitPath(), u) + + repoPath := path.Join(t.TempDir(), ownerCtx.Reponame) + defer util.RemoveAll(repoPath) // cleans out git-annex lockdown permissions + + withAnnexCtxHTTPPassword(t, u, ownerCtx, func() { + doGitClone(repoPath, repoURL)(t) + }) + + // unlike the other tests, at this step we *do not* define credentials: + + t.Run("Init", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexInitTest(remoteRepoPath, repoPath)) + }) + + t.Run("Download", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + require.Error(t, doAnnexDownloadTest(remoteRepoPath, repoPath)) + }) + }) + }) + + t.Run("Delete", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + + // Delete the repo, make sure it's fully gone + doAPIDeleteRepository(ownerCtx)(t) + _, statErr := os.Stat(remoteRepoPath) + require.True(t, os.IsNotExist(statErr), "Remote annex repo should be removed from disk") + }) + }) + }) +} + +/* +Test that 'git annex init' works. + + precondition: repoPath contains a pre-cloned repo set up by doInitAnnexRepository(). +*/ +func doAnnexInitTest(remoteRepoPath, repoPath string) (err error) { + _, _, err = git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "init", "cloned-repo").RunStdString(&git.RunOpts{Dir: repoPath}) + if err != nil { + return fmt.Errorf("Couldn't `git annex init`: %w", err) + } + + // - method 0: 'git config remote.origin.annex-uuid'. + // Demonstrates that 'git annex init' successfully contacted + // the remote git-annex and was able to learn its ID number. + readAnnexUUID, _, err := git.NewCommandContextNoGlobals(git.DefaultContext, "config", "remote.origin.annex-uuid").RunStdString(&git.RunOpts{Dir: repoPath}) + if err != nil { + return fmt.Errorf("Couldn't read remote `git config remote.origin.annex-uuid`: %w", err) + } + readAnnexUUID = strings.TrimSpace(readAnnexUUID) + + match := regexp.MustCompile("^[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}$").MatchString(readAnnexUUID) + if !match { + return fmt.Errorf("'git config remote.origin.annex-uuid' should have been able to download the remote's uuid; but instead read '%s'", readAnnexUUID) + } + + remoteAnnexUUID, _, err := git.NewCommandContextNoGlobals(git.DefaultContext, "config", "annex.uuid").RunStdString(&git.RunOpts{Dir: remoteRepoPath}) + if err != nil { + return fmt.Errorf("Couldn't read local `git config annex.uuid`: %w", err) + } + + remoteAnnexUUID = strings.TrimSpace(remoteAnnexUUID) + match = regexp.MustCompile("^[0-9a-f]{8}-([0-9a-f]{4}-){3}[0-9a-f]{12}$").MatchString(remoteAnnexUUID) + if !match { + return fmt.Errorf("'git annex init' should have been able to download the remote's uuid; but instead read '%s'", remoteAnnexUUID) + } + + if readAnnexUUID != remoteAnnexUUID { + return fmt.Errorf("'git annex init' should have read the expected annex UUID '%s', but instead got '%s'", remoteAnnexUUID, readAnnexUUID) + } + + // - method 1: 'git annex whereis'. + // Demonstrates that git-annex understands annexed files can be found in the remote annex. + annexWhereis, _, err := git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "whereis", "annexed.bin").RunStdString(&git.RunOpts{Dir: repoPath}) + if err != nil { + return fmt.Errorf("Couldn't `git annex whereis`: %w", err) + } + // Note: this regex is unanchored because 'whereis' outputs multiple lines containing + // headers and 1+ remotes and we just want to find one of them. + match = regexp.MustCompile(regexp.QuoteMeta(remoteAnnexUUID) + " -- .* \\[origin\\]\n").MatchString(annexWhereis) + if !match { + return errors.New("'git annex whereis' should report files are known to be in [origin]") + } + + return nil +} + +func doAnnexDownloadTest(remoteRepoPath, repoPath string) (err error) { + // NB: this test does something slightly different if run separately from "doAnnexInitTest()": + // "git annex copy" will notice and run "git annex init", silently. + // This shouldn't change any results, but be aware in case it does. + + _, _, err = git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "copy", "--from", "origin").RunStdString(&git.RunOpts{Dir: repoPath}) + if err != nil { + return err + } + + // verify the files downloaded + + cmp := func(filename string) error { + localObjectPath, err := contentLocation(repoPath, filename) + if err != nil { + return err + } + // localObjectPath := path.Join(repoPath, filename) // or, just compare against the checked-out file + + remoteObjectPath, err := contentLocation(remoteRepoPath, filename) + if err != nil { + return err + } + + match, err := util.FileCmp(localObjectPath, remoteObjectPath, 0) + if err != nil { + return err + } + if !match { + return errors.New("Annexed files should be the same") + } + + return nil + } + + // this is the annex-symlink file + stat, err := os.Lstat(path.Join(repoPath, "annexed.tiff")) + if err != nil { + return fmt.Errorf("Lstat: %w", err) + } + if !((stat.Mode() & os.ModeSymlink) != 0) { + // this line is really just double-checking that the text fixture is set up correctly + return errors.New("*.tiff should be a symlink") + } + if err = cmp("annexed.tiff"); err != nil { + return err + } + + // this is the annex-pointer file + stat, err = os.Lstat(path.Join(repoPath, "annexed.bin")) + if err != nil { + return fmt.Errorf("Lstat: %w", err) + } + if !((stat.Mode() & os.ModeSymlink) == 0) { + // this line is really just double-checking that the text fixture is set up correctly + return errors.New("*.bin should not be a symlink") + } + err = cmp("annexed.bin") + + return err +} + +func doAnnexUploadTest(remoteRepoPath, repoPath string) (err error) { + // NB: this test does something slightly different if run separately from "Init": + // it first runs "git annex init" silently in the background. + // This shouldn't change any results, but be aware in case it does. + + err = generateRandomFile(1024*1024/4, path.Join(repoPath, "contribution.bin")) + if err != nil { + return err + } + + err = git.AddChanges(repoPath, false, ".") + if err != nil { + return err + } + + err = git.CommitChanges(repoPath, git.CommitChangesOptions{Message: "Annex another file"}) + if err != nil { + return err + } + + _, _, err = git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "copy", "--to", "origin").RunStdString(&git.RunOpts{Dir: repoPath}) + if err != nil { + return err + } + + _, _, err = git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "sync", "--no-content").RunStdString(&git.RunOpts{Dir: repoPath}) + if err != nil { + return err + } + + // verify the file was uploaded + localObjectPath, err := contentLocation(repoPath, "contribution.bin") + if err != nil { + return err + } + // localObjectPath := path.Join(repoPath, "contribution.bin") // or, just compare against the checked-out file + + remoteObjectPath, err := contentLocation(remoteRepoPath, "contribution.bin") + if err != nil { + return err + } + + match, err := util.FileCmp(localObjectPath, remoteObjectPath, 0) + if err != nil { + return err + } + if !match { + return errors.New("Annexed files should be the same") + } + + return nil +} + +// ---- Helpers ---- + +func generateRandomFile(size int, path string) (err error) { + // Generate random file + + // XXX TODO: maybe this should not be random, but instead a predictable pattern, so that the test is deterministic + bufSize := 4 * 1024 + if bufSize > size { + bufSize = size + } + + buffer := make([]byte, bufSize) + + f, err := os.Create(path) + if err != nil { + return err + } + defer f.Close() + + written := 0 + for written < size { + n := size - written + if n > bufSize { + n = bufSize + } + _, err := rand.Read(buffer[:n]) + if err != nil { + return err + } + n, err = f.Write(buffer[:n]) + if err != nil { + return err + } + written += n + } + if err != nil { + return err + } + + return nil +} + +// ---- Annex-specific helpers ---- + +/* +Initialize a repo with some baseline annexed and non-annexed files. + + TODO: perhaps this generator could be replaced with a fixture (see + integrations/gitea-repositories-meta/ and models/fixtures/repository.yml). + However we reuse this template for -different- repos, so maybe not. +*/ +func doInitAnnexRepository(repoPath string) error { + // set up what files should be annexed + // in this case, all *.bin files will be annexed + // without this, git-annex's default config annexes every file larger than some number of megabytes + f, err := os.Create(path.Join(repoPath, ".gitattributes")) + if err != nil { + return err + } + defer f.Close() + + // set up git-annex to store certain filetypes via *annex* pointers + // (https://git-annex.branchable.com/internals/pointer_file/). + // but only when run via 'git add' (see git-annex-smudge(1)) + _, err = f.WriteString("* annex.largefiles=anything\n") + if err != nil { + return err + } + _, err = f.WriteString("*.bin filter=annex\n") + if err != nil { + return err + } + _, err = f.WriteString("*.rst filter=annex\n") + if err != nil { + return err + } + _, err = f.WriteString("*.markdown filter=annex\n") + if err != nil { + return err + } + f.Close() + + err = git.AddChanges(repoPath, false, ".") + if err != nil { + return err + } + err = git.CommitChanges(repoPath, git.CommitChangesOptions{Message: "Configure git-annex settings"}) + if err != nil { + return err + } + + // 'git annex init' + err = git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "init", "test-repo").Run(&git.RunOpts{Dir: repoPath}) + if err != nil { + return err + } + + // add files to the annex, stored via annex symlinks + // // a binary file + err = generateRandomFile(1024*1024/4, path.Join(repoPath, "annexed.tiff")) + if err != nil { + return err + } + + // // a file in a subdirectory + err = os.Mkdir(path.Join(repoPath, "subdir"), 0o755) + if err != nil { + return err + } + err = generateRandomFile(1024*1024/4, path.Join(repoPath, "subdir/subdir-annexed.tiff")) + if err != nil { + return err + } + + // // a text file + err = os.WriteFile(path.Join(repoPath, "annexed.md"), []byte("Overview\n=====\n\n1. Profit\n2. ???\n3. Review Life Activations\n"), 0o777) + if err != nil { + return err + } + + // // a markdown file + err = os.WriteFile(path.Join(repoPath, "annexed.txt"), []byte("We're going to see the wizard\nThe wonderful\nMonkey of\nBoz\n"), 0o777) + if err != nil { + return err + } + + err = git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "add", ".").Run(&git.RunOpts{Dir: repoPath}) + if err != nil { + return err + } + + // add files to the annex, stored via git-annex-smudge + // // a binary file + err = generateRandomFile(1024*1024/4, path.Join(repoPath, "annexed.bin")) + if err != nil { + return err + } + + // // a text file + err = os.WriteFile(path.Join(repoPath, "annexed.rst"), []byte("Title\n=====\n\n- this is to test annexing a text file\n- lists are fun\n"), 0o777) + if err != nil { + return err + } + + // // a markdown file + err = os.WriteFile(path.Join(repoPath, "annexed.markdown"), []byte("Overview\n=====\n\n1. Profit\n2. ???\n3. Review Life Activations\n"), 0o777) + if err != nil { + return err + } + + err = git.AddChanges(repoPath, false, ".") + if err != nil { + return err + } + + // add standard symlinks to git + err = os.Symlink(".gitattributes", path.Join(repoPath, "link-to-.gitattributes")) + if err != nil { + return err + } + err = os.Symlink("missing-file", path.Join(repoPath, "link-to-missing-file")) + if err != nil { + return err + } + + err = git.AddChanges(repoPath, false, ".") + if err != nil { + return err + } + + // save everything + err = git.CommitChanges(repoPath, git.CommitChangesOptions{Message: "Annex files"}) + if err != nil { + return err + } + + return nil +} + +/* +Initialize a remote repo with some baseline annexed and non-annexed files. +*/ +func doInitRemoteAnnexRepository(t *testing.T, repoURL *url.URL) error { + repoPath := path.Join(t.TempDir(), path.Base(repoURL.Path)) + // This clone is immediately thrown away, which + // helps force the tests to be end-to-end. + defer util.RemoveAll(repoPath) + + doGitClone(repoPath, repoURL)(t) // TODO: this call is the only reason for the testing.T; can it be removed? + + err := doInitAnnexRepository(repoPath) + if err != nil { + return err + } + + _, _, err = git.NewCommandContextNoGlobals(git.DefaultContext, "annex", "sync", "--content").RunStdString(&git.RunOpts{Dir: repoPath}) + if err != nil { + return err + } + + return nil +} + +/* +Find the path in .git/annex/objects/ of the contents for a given annexed file. + + repoPath: the git repository to examine + file: the path (in the repo's current HEAD) of the annex pointer + + TODO: pass a parameter to allow examining non-HEAD branches +*/ +func contentLocation(repoPath, file string) (path string, err error) { + path = "" + + repo, err := git.OpenRepository(git.DefaultContext, repoPath) + if err != nil { + return path, nil + } + + commitID, err := repo.GetRefCommitID("HEAD") // NB: to examine a *branch*, prefix with "refs/branch/", or call repo.GetBranchCommitID(); ditto for tags + if err != nil { + return path, nil + } + + commit, err := repo.GetCommit(commitID) + if err != nil { + return path, nil + } + + treeEntry, err := commit.GetTreeEntryByPath(file) + if err != nil { + return path, nil + } + + return annex.ContentLocation(treeEntry.Blob()) +} + +/* like withKeyFile(), but automatically sets it the account given in ctx for use by git-annex */ +func withAnnexCtxKeyFile(t *testing.T, ctx APITestContext, callback func()) { + _gitAnnexUseGitSSH, gitAnnexUseGitSSHExists := os.LookupEnv("GIT_ANNEX_USE_GIT_SSH") + defer func() { + // reset + if gitAnnexUseGitSSHExists { + os.Setenv("GIT_ANNEX_USE_GIT_SSH", _gitAnnexUseGitSSH) + } + }() + + os.Setenv("GIT_ANNEX_USE_GIT_SSH", "1") // withKeyFile works by setting GIT_SSH_COMMAND, but git-annex only respects that if this is set + + withCtxKeyFile(t, ctx, callback) +} + +/* +Like withKeyFile(), but sets HTTP credentials instead of SSH credentials. + + It does this by temporarily arranging through `git config --global` + to use git-credential-store(1) with the password written to a tempfile. + + This is the only reliable way to pass HTTP credentials non-interactively + to git-annex. See https://git-annex.branchable.com/bugs/http_remotes_ignore_annex.web-options_--netrc/#comment-b5a299e9826b322f2d85c96d4929a430 + for joeyh's proclamation on the subject. + + This **is only effective** when used around git.NewCommandContextNoGlobals() calls. + git.NewCommand() disables credential.helper as a precaution (see modules/git/git.go). + + In contrast, the tests in git_test.go put the password in the remote's URL like + `git config remote.origin.url http://user2:password@localhost:3003/user2/repo-name.git`, + writing the password in repoPath+"/.git/config". That would be equally good, except + that git-annex ignores it! +*/ +func withAnnexCtxHTTPPassword(t *testing.T, u *url.URL, ctx APITestContext, callback func()) { + credentialedURL := *u + credentialedURL.User = url.UserPassword(ctx.Username, userPassword) // NB: all test users use the same password + + creds := path.Join(t.TempDir(), "creds") + require.NoError(t, os.WriteFile(creds, []byte(credentialedURL.String()), 0o600)) + + originalCredentialHelper, _, err := git.NewCommandContextNoGlobals(git.DefaultContext, "config").AddOptionValues("--global", "credential.helper").RunStdString(&git.RunOpts{}) + if err != nil && !err.IsExitCode(1) { + // ignore the 'error' thrown when credential.helper is unset (when git config returns 1) + // but catch all others + require.NoError(t, err) + } + hasOriginalCredentialHelper := (err == nil) + + _, _, err = git.NewCommandContextNoGlobals(git.DefaultContext, "config").AddOptionValues("--global", "credential.helper", fmt.Sprintf("store --file=%s", creds)).RunStdString(&git.RunOpts{}) + require.NoError(t, err) + + defer (func() { + // reset + if hasOriginalCredentialHelper { + _, _, err = git.NewCommandContextNoGlobals(git.DefaultContext, "config").AddOptionValues("--global").AddArguments("credential.helper").AddDynamicArguments(originalCredentialHelper).RunStdString(&git.RunOpts{}) + } else { + _, _, err = git.NewCommandContextNoGlobals(git.DefaultContext, "config").AddOptionValues("--global").AddOptionValues("--unset").AddArguments("credential.helper").RunStdString(&git.RunOpts{}) + } + require.NoError(t, err) + })() + + callback() +} diff --git a/tests/integration/git_helper_for_declarative_test.go b/tests/integration/git_helper_for_declarative_test.go index 10cf79b9fd8b2..4d91c4d78b0c2 100644 --- a/tests/integration/git_helper_for_declarative_test.go +++ b/tests/integration/git_helper_for_declarative_test.go @@ -39,6 +39,28 @@ func withKeyFile(t *testing.T, keyname string, callback func(string)) { "ssh -o \"UserKnownHostsFile=/dev/null\" -o \"StrictHostKeyChecking=no\" -o \"IdentitiesOnly=yes\" -i \""+keyFile+"\" \"$@\""), 0o700) assert.NoError(t, err) + // reset ssh wrapper afterwards + _gitSSH, gitSSHExists := os.LookupEnv("GIT_SSH") + defer func() { + if gitSSHExists { + os.Setenv("GIT_SSH", _gitSSH) + } + }() + + _gitSSHCommand, gitSSHCommandExists := os.LookupEnv("GIT_SSH_COMMAND") + defer func() { + if gitSSHCommandExists { + os.Setenv("GIT_SSH_COMMAND", _gitSSHCommand) + } + }() + + _gitSSHVariant, gitSSHVariantExists := os.LookupEnv("GIT_SSH_VARIANT") + defer func() { + if gitSSHVariantExists { + os.Setenv("GIT_SSH_VARIANT", _gitSSHVariant) + } + }() + // Setup ssh wrapper os.Setenv("GIT_SSH", path.Join(tmpDir, "ssh")) os.Setenv("GIT_SSH_COMMAND", @@ -48,6 +70,13 @@ func withKeyFile(t *testing.T, keyname string, callback func(string)) { callback(keyFile) } +func createHTTPUrl(gitPath string, u *url.URL) *url.URL { + // this assumes u contains the HTTP base URL that Gitea is running on + u2 := *u + u2.Path = gitPath + return &u2 +} + func createSSHUrl(gitPath string, u *url.URL) *url.URL { u2 := *u u2.Scheme = "ssh" diff --git a/tests/mssql.ini.tmpl b/tests/mssql.ini.tmpl index 3cd64ec5cb8ca..5933e865aba7f 100644 --- a/tests/mssql.ini.tmpl +++ b/tests/mssql.ini.tmpl @@ -104,6 +104,9 @@ INTERNAL_TOKEN = eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYmYiOjE0OTU1NTE2MTh9.h [lfs] PATH = tests/{{TEST_TYPE}}/gitea-{{TEST_TYPE}}-mssql/data/lfs +[annex] +ENABLED = true + [packages] ENABLED = true diff --git a/tests/mysql.ini.tmpl b/tests/mysql.ini.tmpl index 2f890e67eb926..b9b83cc4f654a 100644 --- a/tests/mysql.ini.tmpl +++ b/tests/mysql.ini.tmpl @@ -102,6 +102,9 @@ INTERNAL_TOKEN = eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYmYiOjE0OTU1NTE2MTh9.h [lfs] PATH = tests/{{TEST_TYPE}}/gitea-{{TEST_TYPE}}-mysql/data/lfs +[annex] +ENABLED = true + [packages] ENABLED = true diff --git a/tests/pgsql.ini.tmpl b/tests/pgsql.ini.tmpl index a1679cad6a6e9..ad4c7b646c61c 100644 --- a/tests/pgsql.ini.tmpl +++ b/tests/pgsql.ini.tmpl @@ -105,6 +105,9 @@ INTERNAL_TOKEN = eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJuYmYiOjE0OTU1NTE2MTh9.h [lfs] MINIO_BASE_PATH = lfs/ +[annex] +ENABLED = true + [attachment] MINIO_BASE_PATH = attachments/ diff --git a/tests/sqlite.ini.tmpl b/tests/sqlite.ini.tmpl index 74e1957113150..3f7a40f72c7dc 100644 --- a/tests/sqlite.ini.tmpl +++ b/tests/sqlite.ini.tmpl @@ -104,6 +104,9 @@ JWT_SECRET = KZb_QLUd4fYVyxetjxC4eZkrBgWM2SndOOWDNtgUUko [lfs] PATH = tests/{{TEST_TYPE}}/gitea-{{TEST_TYPE}}-sqlite/data/lfs +[annex] +ENABLED = true + [packages] ENABLED = true