LFS support to be stored on minio (#12518)

* LFS support to be stored on minio

* Fix test

* Fix lint

* Fix lint

* Fix check

* Fix test

* Update documents and add migration for LFS

* Fix some bugs

Author: lunny

cmd/migrate_storage.go

@@ -83,6 +83,13 @@ func migrateAttachments(dstStorage storage.ObjectStorage) error {
 	})
 }
 
+func migrateLFS(dstStorage storage.ObjectStorage) error {
+	return models.IterateLFS(func(mo *models.LFSMetaObject) error {
+		_, err := storage.Copy(dstStorage, mo.RelativePath(), storage.LFS, mo.RelativePath())
+		return err
+	})
+}
+
 func runMigrateStorage(ctx *cli.Context) error {
 	if err := initDB(); err != nil {
 		return err
@@ -103,45 +110,50 @@ func runMigrateStorage(ctx *cli.Context) error {
 		return err
 	}
 
+	var dstStorage storage.ObjectStorage
+	var err error
+	switch ctx.String("store") {
+	case "local":
+		p := ctx.String("path")
+		if p == "" {
+			log.Fatal("Path must be given when store is loal")
+			return nil
+		}
+		dstStorage, err = storage.NewLocalStorage(p)
+	case "minio":
+		dstStorage, err = storage.NewMinioStorage(
+			context.Background(),
+			ctx.String("minio-endpoint"),
+			ctx.String("minio-access-key-id"),
+			ctx.String("minio-secret-access-key"),
+			ctx.String("minio-bucket"),
+			ctx.String("minio-location"),
+			ctx.String("minio-base-path"),
+			ctx.Bool("minio-use-ssl"),
+		)
+	default:
+		return fmt.Errorf("Unsupported attachments store type: %s", ctx.String("store"))
+	}
+
+	if err != nil {
+		return err
+	}
+
 	tp := ctx.String("type")
 	switch tp {
 	case "attachments":
-		var dstStorage storage.ObjectStorage
-		var err error
-		switch ctx.String("store") {
-		case "local":
-			p := ctx.String("path")
-			if p == "" {
-				log.Fatal("Path must be given when store is loal")
-				return nil
-			}
-			dstStorage, err = storage.NewLocalStorage(p)
-		case "minio":
-			dstStorage, err = storage.NewMinioStorage(
-				context.Background(),
-				ctx.String("minio-endpoint"),
-				ctx.String("minio-access-key-id"),
-				ctx.String("minio-secret-access-key"),
-				ctx.String("minio-bucket"),
-				ctx.String("minio-location"),
-				ctx.String("minio-base-path"),
-				ctx.Bool("minio-use-ssl"),
-			)
-		default:
-			return fmt.Errorf("Unsupported attachments store type: %s", ctx.String("store"))
-		}
-
-		if err != nil {
+		if err := migrateAttachments(dstStorage); err != nil {
 			return err
 		}
-		if err := migrateAttachments(dstStorage); err != nil {
+	case "lfs":
+		if err := migrateLFS(dstStorage); err != nil {
 			return err
 		}
-
-		log.Warn("All files have been copied to the new placement but old files are still on the orignial placement.")
-
-		return nil
+	default:
+		return fmt.Errorf("Unsupported storage: %s", ctx.String("type"))
 	}
 
+	log.Warn("All files have been copied to the new placement but old files are still on the orignial placement.")
+
 	return nil
 }

docs/content/doc/advanced/config-cheat-sheet.en-us.md

@@ -206,12 +206,23 @@ Values containing `#` or `;` must be quoted using `` ` `` or `"""`.
 - `STATIC_CACHE_TIME`: **6h**: Web browser cache time for static resources on `custom/`, `public/` and all uploaded avatars.
 - `ENABLE_GZIP`: **false**: Enables application-level GZIP support.
 - `LANDING_PAGE`: **home**: Landing page for unauthenticated users \[home, explore, organizations, login\].
+
 - `LFS_START_SERVER`: **false**: Enables git-lfs support.
-- `LFS_CONTENT_PATH`: **./data/lfs**: Where to store LFS files.
+- `LFS_STORE_TYPE`: **local**: Storage type for lfs, `local` for local disk or `minio` for s3 compatible object storage service.
+- `LFS_SERVE_DIRECT`: **false**: Allows the storage driver to redirect to authenticated URLs to serve files directly. Currently, only Minio/S3 is supported via signed URLs, local does nothing.
+- `LFS_CONTENT_PATH`: **./data/lfs**: Where to store LFS files, only available when `LFS_STORE_TYPE` is `local`.
+- `LFS_MINIO_ENDPOINT`: **localhost:9000**: Minio endpoint to connect only available when `LFS_STORE_TYPE` is `minio`
+- `LFS_MINIO_ACCESS_KEY_ID`: Minio accessKeyID to connect only available when `LFS_STORE_TYPE` is `minio`
+- `LFS_MINIO_SECRET_ACCESS_KEY`: Minio secretAccessKey to connect only available when `LFS_STORE_TYPE is` `minio`
+- `LFS_MINIO_BUCKET`: **gitea**: Minio bucket to store the lfs only available when `LFS_STORE_TYPE` is `minio`
+- `LFS_MINIO_LOCATION`: **us-east-1**: Minio location to create bucket only available when `LFS_STORE_TYPE` is `minio`
+- `LFS_MINIO_BASE_PATH`: **lfs/**: Minio base path on the bucket only available when `LFS_STORE_TYPE` is `minio`
+- `LFS_MINIO_USE_SSL`: **false**: Minio enabled ssl only available when `LFS_STORE_TYPE` is `minio`
 - `LFS_JWT_SECRET`: **\<empty\>**: LFS authentication secret, change this a unique string.
 - `LFS_HTTP_AUTH_EXPIRY`: **20m**: LFS authentication validity period in time.Duration, pushes taking longer than this may fail.
 - `LFS_MAX_FILE_SIZE`: **0**: Maximum allowed LFS file size in bytes (Set to 0 for no limit).
 - `LFS_LOCK_PAGING_NUM`: **50**: Maximum number of LFS Locks returned per page.
+
 - `REDIRECT_OTHER_PORT`: **false**: If true and `PROTOCOL` is https, allows redirecting http requests on `PORT_TO_REDIRECT` to the https port Gitea listens on.
 - `PORT_TO_REDIRECT`: **80**: Port for the http redirection service to listen on. Used when `REDIRECT_OTHER_PORT` is true.
 - `ENABLE_LETSENCRYPT`: **false**: If enabled you must set `DOMAIN` to valid internet facing domain (ensure DNS is set and port 80 is accessible by letsencrypt validation server).

docs/content/doc/advanced/config-cheat-sheet.zh-cn.md

@@ -69,8 +69,18 @@ menu:
 - `STATIC_CACHE_TIME`: **6h**: 静态资源文件，包括 `custom/`, `public/` 和所有上传的头像的浏览器缓存时间。
 - `ENABLE_GZIP`: 启用应用级别的 GZIP 压缩。
 - `LANDING_PAGE`: 未登录用户的默认页面，可选 `home` 或 `explore`。
+
 - `LFS_START_SERVER`: 是否启用 git-lfs 支持. 可以为 `true` 或 `false`， 默认是 `false`。
+- `LFS_STORE_TYPE`: **local**: LFS 的存储类型，`local` 将存储到磁盘，`minio` 将存储到 s3 兼容的对象服务。
+- `LFS_SERVE_DIRECT`: **false**: 允许直接重定向到存储系统。当前，仅 Minio/S3 是支持的。
 - `LFS_CONTENT_PATH`: 存放 lfs 命令上传的文件的地方，默认是 `data/lfs`。
+- `LFS_MINIO_ENDPOINT`: **localhost:9000**: Minio 地址，仅当 `LFS_STORE_TYPE` 为 `minio` 时有效。
+- `LFS_MINIO_ACCESS_KEY_ID`: Minio accessKeyID，仅当 `LFS_STORE_TYPE` 为 `minio` 时有效。
+- `LFS_MINIO_SECRET_ACCESS_KEY`: Minio secretAccessKey，仅当 `LFS_STORE_TYPE` 为 `minio` 时有效。
+- `LFS_MINIO_BUCKET`: **gitea**: Minio bucket，仅当 `LFS_STORE_TYPE` 为 `minio` 时有效。
+- `LFS_MINIO_LOCATION`: **us-east-1**: Minio location ，仅当 `LFS_STORE_TYPE` 为 `minio` 时有效。
+- `LFS_MINIO_BASE_PATH`: **lfs/**: Minio base path ，仅当 `LFS_STORE_TYPE` 为 `minio` 时有效。
+- `LFS_MINIO_USE_SSL`: **false**: Minio 是否启用 ssl ，仅当 `LFS_STORE_TYPE` 为 `minio` 时有效。
 - `LFS_JWT_SECRET`: LFS 认证密钥，改成自己的。
 
 ## Database (`database`)

integrations/lfs_getobject_test.go

@@ -18,6 +18,7 @@ import (
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/lfs"
 	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/storage"
 
 	"gitea.com/macaron/gzip"
 	gzipp "github.com/klauspost/compress/gzip"
@@ -49,8 +50,10 @@ func storeObjectInRepo(t *testing.T, repositoryID int64, content *[]byte) string
 	lfsID++
 	lfsMetaObject, err = models.NewLFSMetaObject(lfsMetaObject)
 	assert.NoError(t, err)
-	contentStore := &lfs.ContentStore{BasePath: setting.LFS.ContentPath}
-	if !contentStore.Exists(lfsMetaObject) {
+	contentStore := &lfs.ContentStore{ObjectStorage: storage.LFS}
+	exist, err := contentStore.Exists(lfsMetaObject)
+	assert.NoError(t, err)
+	if !exist {
 		err := contentStore.Put(lfsMetaObject, bytes.NewReader(*content))
 		assert.NoError(t, err)
 	}

integrations/mysql.ini.tmpl

@@ -36,13 +36,23 @@ ROOT_URL         = http://localhost:3001/
 DISABLE_SSH      = false
 SSH_LISTEN_HOST  = localhost
 SSH_PORT         = 2201
+APP_DATA_PATH    = integrations/gitea-integration-mysql/data
+BUILTIN_SSH_SERVER_USER = git
 START_SSH_SERVER = true
+OFFLINE_MODE     = false
+
 LFS_START_SERVER = true
 LFS_CONTENT_PATH = integrations/gitea-integration-mysql/datalfs-mysql
-OFFLINE_MODE     = false
 LFS_JWT_SECRET   = Tv_MjmZuHqpIY6GFl12ebgkRAMt4RlWt0v4EHKSXO0w
-APP_DATA_PATH    = integrations/gitea-integration-mysql/data
-BUILTIN_SSH_SERVER_USER = git
+LFS_STORE_TYPE = minio
+LFS_SERVE_DIRECT = false
+LFS_MINIO_ENDPOINT = minio:9000
+LFS_MINIO_ACCESS_KEY_ID = 123456
+LFS_MINIO_SECRET_ACCESS_KEY = 12345678
+LFS_MINIO_BUCKET = gitea
+LFS_MINIO_LOCATION = us-east-1
+LFS_MINIO_BASE_PATH = lfs/
+LFS_MINIO_USE_SSL = false
 
 [attachment]
 STORE_TYPE = minio

models/lfs.go

@@ -10,6 +10,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"path"
 
 	"code.gitea.io/gitea/modules/timeutil"
 
@@ -26,6 +27,15 @@ type LFSMetaObject struct {
 	CreatedUnix  timeutil.TimeStamp `xorm:"created"`
 }
 
+// RelativePath returns the relative path of the lfs object
+func (m *LFSMetaObject) RelativePath() string {
+	if len(m.Oid) < 5 {
+		return m.Oid
+	}
+
+	return path.Join(m.Oid[0:2], m.Oid[2:4], m.Oid[4:])
+}
+
 // Pointer returns the string representation of an LFS pointer file
 func (m *LFSMetaObject) Pointer() string {
 	return fmt.Sprintf("%s\n%s%s\nsize %d\n", LFSMetaFileIdentifier, LFSMetaFileOidPrefix, m.Oid, m.Size)
@@ -202,3 +212,25 @@ func LFSAutoAssociate(metas []*LFSMetaObject, user *User, repoID int64) error {
 
 	return sess.Commit()
 }
+
+// IterateLFS iterates lfs object
+func IterateLFS(f func(mo *LFSMetaObject) error) error {
+	var start int
+	const batchSize = 100
+	for {
+		var mos = make([]*LFSMetaObject, 0, batchSize)
+		if err := x.Limit(batchSize, start).Find(&mos); err != nil {
+			return err
+		}
+		if len(mos) == 0 {
+			return nil
+		}
+		start += len(mos)
+
+		for _, mo := range mos {
+			if err := f(mo); err != nil {
+				return err
+			}
+		}
+	}
+}

models/unit_tests.go

@@ -69,6 +69,7 @@ func MainTest(m *testing.M, pathToGiteaRoot string) {
 	}
 
 	setting.Attachment.Path = filepath.Join(setting.AppDataPath, "attachments")
+	setting.LFS.ContentPath = filepath.Join(setting.AppDataPath, "lfs")
 	if err = storage.Init(); err != nil {
 		fatalTestError("storage.Init: %v\n", err)
 	}

modules/lfs/content_store.go

@@ -10,11 +10,10 @@ import (
 	"errors"
 	"io"
 	"os"
-	"path/filepath"
 
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/log"
-	"code.gitea.io/gitea/modules/util"
+	"code.gitea.io/gitea/modules/storage"
 )
 
 var (
@@ -24,17 +23,15 @@ var (
 
 // ContentStore provides a simple file system based storage.
 type ContentStore struct {
-	BasePath string
+	storage.ObjectStorage
 }
 
 // Get takes a Meta object and retrieves the content from the store, returning
 // it as an io.Reader. If fromByte > 0, the reader starts from that byte
 func (s *ContentStore) Get(meta *models.LFSMetaObject, fromByte int64) (io.ReadCloser, error) {
-	path := filepath.Join(s.BasePath, transformKey(meta.Oid))
-
-	f, err := os.Open(path)
+	f, err := s.Open(meta.RelativePath())
 	if err != nil {
-		log.Error("Whilst trying to read LFS OID[%s]: Unable to open %s Error: %v", meta.Oid, path, err)
+		log.Error("Whilst trying to read LFS OID[%s]: Unable to open Error: %v", meta.Oid, err)
 		return nil, err
 	}
 	if fromByte > 0 {
@@ -48,82 +45,55 @@ func (s *ContentStore) Get(meta *models.LFSMetaObject, fromByte int64) (io.ReadC
 
 // Put takes a Meta object and an io.Reader and writes the content to the store.
 func (s *ContentStore) Put(meta *models.LFSMetaObject, r io.Reader) error {
-	path := filepath.Join(s.BasePath, transformKey(meta.Oid))
-	tmpPath := path + ".tmp"
-
-	dir := filepath.Dir(path)
-	if err := os.MkdirAll(dir, 0750); err != nil {
-		log.Error("Whilst putting LFS OID[%s]: Unable to create the LFS directory: %s Error: %v", meta.Oid, dir, err)
-		return err
-	}
-
-	file, err := os.OpenFile(tmpPath, os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0640)
-	if err != nil {
-		log.Error("Whilst putting LFS OID[%s]: Unable to open temporary file for writing: %s Error: %v", tmpPath, err)
-		return err
-	}
-	defer func() {
-		if err := util.Remove(tmpPath); err != nil {
-			log.Warn("Unable to remove temporary path: %s: Error: %v", tmpPath, err)
-		}
-	}()
-
 	hash := sha256.New()
-	hw := io.MultiWriter(hash, file)
-
-	written, err := io.Copy(hw, r)
+	rd := io.TeeReader(r, hash)
+	p := meta.RelativePath()
+	written, err := s.Save(p, rd)
 	if err != nil {
-		log.Error("Whilst putting LFS OID[%s]: Failed to copy to tmpPath: %s Error: %v", meta.Oid, tmpPath, err)
-		file.Close()
+		log.Error("Whilst putting LFS OID[%s]: Failed to copy to tmpPath: %s Error: %v", meta.Oid, p, err)
 		return err
 	}
-	file.Close()
 
 	if written != meta.Size {
+		if err := s.Delete(p); err != nil {
+			log.Error("Cleaning the LFS OID[%s] failed: %v", meta.Oid, err)
+		}
 		return errSizeMismatch
 	}
 
 	shaStr := hex.EncodeToString(hash.Sum(nil))
 	if shaStr != meta.Oid {
+		if err := s.Delete(p); err != nil {
+			log.Error("Cleaning the LFS OID[%s] failed: %v", meta.Oid, err)
+		}
 		return errHashMismatch
 	}
 
-	if err := os.Rename(tmpPath, path); err != nil {
-		log.Error("Whilst putting LFS OID[%s]: Unable to move tmp file to final destination: %s Error: %v", meta.Oid, path, err)
-		return err
-	}
-
 	return nil
 }
 
 // Exists returns true if the object exists in the content store.
-func (s *ContentStore) Exists(meta *models.LFSMetaObject) bool {
-	path := filepath.Join(s.BasePath, transformKey(meta.Oid))
-	if _, err := os.Stat(path); os.IsNotExist(err) {
-		return false
+func (s *ContentStore) Exists(meta *models.LFSMetaObject) (bool, error) {
+	_, err := s.ObjectStorage.Stat(meta.RelativePath())
+	if err != nil {
+		if os.IsNotExist(err) {
+			return false, nil
+		}
+		return false, err
 	}
-	return true
+	return true, nil
 }
 
 // Verify returns true if the object exists in the content store and size is correct.
 func (s *ContentStore) Verify(meta *models.LFSMetaObject) (bool, error) {
-	path := filepath.Join(s.BasePath, transformKey(meta.Oid))
-
-	fi, err := os.Stat(path)
-	if os.IsNotExist(err) || err == nil && fi.Size() != meta.Size {
+	p := meta.RelativePath()
+	fi, err := s.ObjectStorage.Stat(p)
+	if os.IsNotExist(err) || (err == nil && fi.Size() != meta.Size) {
 		return false, nil
 	} else if err != nil {
-		log.Error("Unable stat file: %s for LFS OID[%s] Error: %v", path, meta.Oid, err)
+		log.Error("Unable stat file: %s for LFS OID[%s] Error: %v", p, meta.Oid, err)
 		return false, err
 	}
 
 	return true, nil
 }
-
-func transformKey(key string) string {
-	if len(key) < 5 {
-		return key
-	}
-
-	return filepath.Join(key[0:2], key[2:4], key[4:])
-}