diff --git a/components/content-service-api/go/config/config.go b/components/content-service-api/go/config/config.go index 6ddd5c24e65313..e9f9208050504f 100644 --- a/components/content-service-api/go/config/config.go +++ b/components/content-service-api/go/config/config.go @@ -105,6 +105,8 @@ type MinIOConfig struct { Region string `json:"region"` ParallelUpload uint `json:"parallelUpload,omitempty"` + + BucketName string `json:"bucket"` } type PProf struct { diff --git a/components/content-service/go.mod b/components/content-service/go.mod index 60809f65ad10c0..b928d86ffa817b 100644 --- a/components/content-service/go.mod +++ b/components/content-service/go.mod @@ -11,7 +11,7 @@ require ( github.com/golang/mock v1.6.0 github.com/google/go-cmp v0.5.8 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect - github.com/minio/minio-go/v7 v7.0.11 + github.com/minio/minio-go/v7 v7.0.26 github.com/opencontainers/go-digest v1.0.0 github.com/opencontainers/image-spec v1.0.2 github.com/opentracing/opentracing-go v1.2.0 @@ -65,6 +65,7 @@ require ( github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.13.5 // indirect github.com/klauspost/cpuid v1.3.1 // indirect github.com/magiconair/properties v1.8.5 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect diff --git a/components/content-service/go.sum b/components/content-service/go.sum index 0a5e9dfd196f3a..0f62bc8b00657e 100644 --- a/components/content-service/go.sum +++ b/components/content-service/go.sum @@ -554,6 +554,8 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.13.5 h1:9O69jUPDcsT9fEm74W92rZL9FQY7rCdaXVneq+yyzl4= +github.com/klauspost/compress v1.13.5/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s= github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4= @@ -589,8 +591,8 @@ github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3N github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4= github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= -github.com/minio/minio-go/v7 v7.0.11 h1:7utSkCtMQPYYB1UB8FR3d0QSiOWE6F/JYXon29imYek= -github.com/minio/minio-go/v7 v7.0.11/go.mod h1:WoyW+ySKAKjY98B9+7ZbI8z8S3jaxaisdcvj9TGlazA= +github.com/minio/minio-go/v7 v7.0.26 h1:D0HK+8793etZfRY/vHhDmFaP+vmT41K3K4JV9vmZCBQ= +github.com/minio/minio-go/v7 v7.0.26/go.mod h1:x81+AX5gHSfCSqw7jxRKHvxUXMlE5uKX0Vb75Xk5yYg= github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= @@ -891,7 +893,6 @@ golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= @@ -1405,7 +1406,6 @@ gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKW gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= diff --git a/components/content-service/pkg/layer/provider_test.go b/components/content-service/pkg/layer/provider_test.go index c886b925d84578..b499cbd591d973 100644 --- a/components/content-service/pkg/layer/provider_test.go +++ b/components/content-service/pkg/layer/provider_test.go @@ -244,11 +244,11 @@ func (s *testStorage) DeleteBucket(ctx context.Context, bucket string) error { return nil } -func (*testStorage) BackupObject(workspaceID string, name string) string { +func (*testStorage) BackupObject(ownerID string, workspaceID string, name string) string { return "" } -func (*testStorage) InstanceObject(workspaceID string, instanceID string, name string) string { +func (*testStorage) InstanceObject(ownerID string, workspaceID string, instanceID string, name string) string { return "" } diff --git a/components/content-service/pkg/service/headless-log-service.go b/components/content-service/pkg/service/headless-log-service.go index 35044bd3bbdb39..974a54e28a8400 100644 --- a/components/content-service/pkg/service/headless-log-service.go +++ b/components/content-service/pkg/service/headless-log-service.go @@ -54,7 +54,7 @@ func (ls *HeadlessLogService) LogDownloadURL(ctx context.Context, req *api.LogDo span.SetTag("instanceId", req.InstanceId) defer tracing.FinishSpan(span, &err) - blobName := ls.s.InstanceObject(req.WorkspaceId, req.InstanceId, logs.UploadedHeadlessLogPath(req.TaskId)) + blobName := ls.s.InstanceObject(req.OwnerId, req.WorkspaceId, req.InstanceId, logs.UploadedHeadlessLogPath(req.TaskId)) info, err := ls.s.SignDownload(ctx, ls.s.Bucket(req.OwnerId), blobName, &storage.SignedURLOptions{}) if err != nil { log.WithFields(log.OWI(req.OwnerId, req.WorkspaceId, "")). @@ -87,7 +87,7 @@ func (ls *HeadlessLogService) ListLogs(ctx context.Context, req *api.ListLogsReq // we do not need to check whether the bucket exists because ListObjects() does that for us // all files under this prefix are headless log files, named after their respective taskId - prefix := ls.s.InstanceObject(req.WorkspaceId, req.InstanceId, logs.UploadedHeadlessLogPathPrefix) + prefix := ls.s.InstanceObject(req.OwnerId, req.WorkspaceId, req.InstanceId, logs.UploadedHeadlessLogPathPrefix) objects, err := da.ListObjects(ctx, prefix) if err != nil { return nil, err diff --git a/components/content-service/pkg/service/headless-log-service_test.go b/components/content-service/pkg/service/headless-log-service_test.go index e4771d0db4fe19..e4a4b2a365f424 100644 --- a/components/content-service/pkg/service/headless-log-service_test.go +++ b/components/content-service/pkg/service/headless-log-service_test.go @@ -70,7 +70,7 @@ func TestListLogs(t *testing.T) { daFactory: daFactory, } - s.EXPECT().InstanceObject(gomock.Any(), gomock.Any(), gomock.Any()). + s.EXPECT().InstanceObject(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). Return("") da.EXPECT().Init(gomock.Any(), gomock.Eq(OwnerId), gomock.Eq(WorkspaceId), gomock.Not(gomock.Eq(""))). Times(1) diff --git a/components/content-service/pkg/service/workspace-service.go b/components/content-service/pkg/service/workspace-service.go index 0c17ba613eea08..44c4d33667ae6e 100644 --- a/components/content-service/pkg/service/workspace-service.go +++ b/components/content-service/pkg/service/workspace-service.go @@ -44,7 +44,7 @@ func (cs *WorkspaceService) WorkspaceDownloadURL(ctx context.Context, req *api.W span.SetTag("workspaceId", req.WorkspaceId) defer tracing.FinishSpan(span, &err) - blobName := cs.s.BackupObject(req.WorkspaceId, storage.DefaultBackup) + blobName := cs.s.BackupObject(req.OwnerId, req.WorkspaceId, storage.DefaultBackup) info, err := cs.s.SignDownload(ctx, cs.s.Bucket(req.OwnerId), blobName, &storage.SignedURLOptions{}) if err != nil { @@ -73,7 +73,7 @@ func (cs *WorkspaceService) DeleteWorkspace(ctx context.Context, req *api.Delete defer tracing.FinishSpan(span, &err) if req.IncludeSnapshots { - prefix := cs.s.BackupObject(req.WorkspaceId, "") + prefix := cs.s.BackupObject(req.OwnerId, req.WorkspaceId, "") if !strings.HasSuffix(prefix, "/") { prefix = prefix + "/" } @@ -89,7 +89,7 @@ func (cs *WorkspaceService) DeleteWorkspace(ctx context.Context, req *api.Delete return &api.DeleteWorkspaceResponse{}, nil } - blobName := cs.s.BackupObject(req.WorkspaceId, storage.DefaultBackup) + blobName := cs.s.BackupObject(req.OwnerId, req.WorkspaceId, storage.DefaultBackup) err = cs.s.DeleteObject(ctx, cs.s.Bucket(req.OwnerId), &storage.DeleteObjectQuery{Name: blobName}) if err != nil { if errors.Is(err, storage.ErrNotFound) { @@ -100,7 +100,7 @@ func (cs *WorkspaceService) DeleteWorkspace(ctx context.Context, req *api.Delete return nil, status.Error(codes.Unknown, err.Error()) } - trailPrefix := cs.s.BackupObject(req.WorkspaceId, "trail-") + trailPrefix := cs.s.BackupObject(req.OwnerId, req.WorkspaceId, "trail-") err = cs.s.DeleteObject(ctx, cs.s.Bucket(req.OwnerId), &storage.DeleteObjectQuery{Prefix: trailPrefix}) if err != nil { if errors.Is(err, storage.ErrNotFound) { @@ -121,7 +121,7 @@ func (cs *WorkspaceService) WorkspaceSnapshotExists(ctx context.Context, req *ap span.SetTag("filename", req.Filename) defer tracing.FinishSpan(span, &err) - exists, err := cs.s.ObjectExists(ctx, cs.s.Bucket(req.OwnerId), cs.s.BackupObject(req.WorkspaceId, req.Filename)) + exists, err := cs.s.ObjectExists(ctx, cs.s.Bucket(req.OwnerId), cs.s.BackupObject(req.OwnerId, req.WorkspaceId, req.Filename)) if err != nil { return nil, status.Error(codes.Unknown, err.Error()) } diff --git a/components/content-service/pkg/storage/gcloud.go b/components/content-service/pkg/storage/gcloud.go index 1b2fb289038d39..9d5e1958b6dc9b 100644 --- a/components/content-service/pkg/storage/gcloud.go +++ b/components/content-service/pkg/storage/gcloud.go @@ -870,11 +870,11 @@ func (p *PresignedGCPStorage) ObjectExists(ctx context.Context, bucket, obj stri } // BackupObject returns a backup's object name that a direct downloader would download -func (p *PresignedGCPStorage) BackupObject(workspaceID string, name string) string { +func (p *PresignedGCPStorage) BackupObject(ownerID string, workspaceID string, name string) string { return fmt.Sprintf("workspaces/%s", gcpWorkspaceBackupObjectName(workspaceID, name)) } // InstanceObject returns a instance's object name that a direct downloader would download -func (p *PresignedGCPStorage) InstanceObject(workspaceID string, instanceID string, name string) string { - return p.BackupObject(workspaceID, InstanceObjectName(instanceID, name)) +func (p *PresignedGCPStorage) InstanceObject(ownerID string, workspaceID string, instanceID string, name string) string { + return p.BackupObject(ownerID, workspaceID, InstanceObjectName(instanceID, name)) } diff --git a/components/content-service/pkg/storage/minio.go b/components/content-service/pkg/storage/minio.go index 9d339c4f1a4535..6149855810ef91 100644 --- a/components/content-service/pkg/storage/minio.go +++ b/components/content-service/pkg/storage/minio.go @@ -10,6 +10,7 @@ import ( "io" "net/http" "os" + "path/filepath" "strings" "time" @@ -129,6 +130,7 @@ func (rs *DirectMinIOStorage) Init(ctx context.Context, owner, workspace, instan rs.Username = owner rs.WorkspaceName = workspace rs.InstanceID = instance + err = rs.Validate() if err != nil { return err @@ -310,17 +312,21 @@ func (rs *DirectMinIOStorage) Upload(ctx context.Context, source string, name st return } -func minioBucketName(ownerID string) string { +func minioBucketName(ownerID, bucketName string) string { + if bucketName != "" { + return bucketName + } + return fmt.Sprintf("gitpod-user-%s", ownerID) } -func minioWorkspaceBackupObjectName(workspaceID string, name string) string { - return fmt.Sprintf("workspaces/%s/%s", workspaceID, name) +func minioWorkspaceBackupObjectName(ownerID, workspaceID, name string) string { + return filepath.Join(ownerID, "workspaces", workspaceID, name) } // Bucket provides the bucket name for a particular user func (rs *DirectMinIOStorage) Bucket(ownerID string) string { - return minioBucketName(ownerID) + return minioBucketName(ownerID, rs.MinIOConfig.BucketName) } // BackupObject returns a backup's object name that a direct downloader would download @@ -329,11 +335,15 @@ func (rs *DirectMinIOStorage) BackupObject(name string) string { } func (rs *DirectMinIOStorage) bucketName() string { - return minioBucketName(rs.Username) + return minioBucketName(rs.Username, rs.MinIOConfig.BucketName) } func (rs *DirectMinIOStorage) objectName(name string) string { - return minioWorkspaceBackupObjectName(rs.WorkspaceName, name) + var username string + if rs.MinIOConfig.BucketName != "" { + username = rs.Username + } + return minioWorkspaceBackupObjectName(username, rs.WorkspaceName, name) } func newPresignedMinIOAccess(cfg config.MinIOConfig) (*presignedMinIOStorage, error) { @@ -517,7 +527,7 @@ func annotationToAmzMetaHeader(annotation string) string { // Bucket provides the bucket name for a particular user func (s *presignedMinIOStorage) Bucket(ownerID string) string { - return minioBucketName(ownerID) + return minioBucketName(ownerID, s.MinIOConfig.BucketName) } // BlobObject returns a blob's object name @@ -526,13 +536,17 @@ func (s *presignedMinIOStorage) BlobObject(name string) (string, error) { } // BackupObject returns a backup's object name that a direct downloader would download -func (s *presignedMinIOStorage) BackupObject(workspaceID string, name string) string { - return minioWorkspaceBackupObjectName(workspaceID, name) +func (s *presignedMinIOStorage) BackupObject(ownerID string, workspaceID, name string) string { + var username string + if s.MinIOConfig.BucketName != "" { + username = ownerID + } + return minioWorkspaceBackupObjectName(username, workspaceID, name) } // InstanceObject returns a instance's object name that a direct downloader would download -func (s *presignedMinIOStorage) InstanceObject(workspaceID string, instanceID string, name string) string { - return s.BackupObject(workspaceID, InstanceObjectName(instanceID, name)) +func (s *presignedMinIOStorage) InstanceObject(ownerID string, workspaceID string, instanceID string, name string) string { + return s.BackupObject(ownerID, workspaceID, InstanceObjectName(instanceID, name)) } func translateMinioError(err error) error { diff --git a/components/content-service/pkg/storage/minio_test.go b/components/content-service/pkg/storage/minio_test.go new file mode 100644 index 00000000000000..dcc13c51854d66 --- /dev/null +++ b/components/content-service/pkg/storage/minio_test.go @@ -0,0 +1,132 @@ +// Copyright (c) 2022 Gitpod GmbH. All rights reserved. +// Licensed under the GNU Affero General Public License (AGPL). +// See License-AGPL.txt in the project root for license information. + +package storage + +import ( + "context" + "testing" + + config "github.com/gitpod-io/gitpod/content-service/api/config" +) + +func TestMinioBucketName(t *testing.T) { + tests := []struct { + Name string + BucketNameConfig string + OwnerID string + ExpectedBucket string + }{ + { + Name: "no dedicated bucket", + BucketNameConfig: "", + OwnerID: "fake-owner-id", + ExpectedBucket: "gitpod-user-fake-owner-id", + }, + { + Name: "with dedicated bucket", + BucketNameConfig: "root-bucket", + OwnerID: "fake-owner-id", + ExpectedBucket: "root-bucket", + }, + } + + for _, test := range tests { + t.Run(test.Name, func(t *testing.T) { + cfg := config.MinIOConfig{ + Endpoint: "fake", + AccessKeyID: "fake_access_key", + SecretAccessKey: "fake_secret", + Region: "none", + BucketName: test.BucketNameConfig, + } + minio, err := newDirectMinIOAccess(cfg) + if err != nil { + t.Fatalf("failed to create minio access: '%v'", err) + } + + actualBucketName := minio.Bucket(test.OwnerID) + if actualBucketName != test.ExpectedBucket { + t.Fatalf("[minio] unexpected bucket name: is '%s' but expected '%s'", actualBucketName, test.ExpectedBucket) + } + + presignedMinio, err := newPresignedMinIOAccess(cfg) + if err != nil { + t.Fatalf("failed to create presigned minio access: '%v'", err) + } + + actualBucketName = presignedMinio.Bucket(test.OwnerID) + if actualBucketName != test.ExpectedBucket { + t.Fatalf("[presigned minio] unexpected bucket name: is '%s' but expected '%s'", actualBucketName, test.ExpectedBucket) + } + }) + } +} + +func TestMinioBackupObject(t *testing.T) { + tests := []struct { + Name string + BucketNameConfig string + Username string + Workspace string + InstanceID string + ObjectName string + ExpectedBackupObject string + }{ + { + Name: "no dedicated bucket", + BucketNameConfig: "", + Username: "test-user", + Workspace: "gitpodio-gitpod-2cx8z8e643x", + InstanceID: "fa9aa2af-b6de-45fc-8b48-534bb440429f", + ObjectName: "backup.tar", + ExpectedBackupObject: "workspaces/gitpodio-gitpod-2cx8z8e643x/backup.tar", + }, + { + Name: "with dedicated bucket", + BucketNameConfig: "root-bucket", + Username: "test-user", + Workspace: "gitpodio-gitpod-2cx8z8e643x", + InstanceID: "fa9aa2af-b6de-45fc-8b48-534bb440429f", + ObjectName: "backup.tar", + ExpectedBackupObject: "test-user/workspaces/gitpodio-gitpod-2cx8z8e643x/backup.tar", + }, + } + + for _, test := range tests { + t.Run(test.Name, func(t *testing.T) { + cfg := config.MinIOConfig{ + Endpoint: "fake", + AccessKeyID: "fake_access_key", + SecretAccessKey: "fake_secret", + Region: "none", + BucketName: test.BucketNameConfig, + } + minio, err := newDirectMinIOAccess(cfg) + if err != nil { + t.Fatalf("failed to create minio access: '%v'", err) + } + err = minio.Init(context.Background(), test.Username, test.Workspace, test.InstanceID) + if err != nil { + t.Fatalf("failed to init minio access: '%v'", err) + } + + actualBackupObject := minio.BackupObject(test.ObjectName) + if actualBackupObject != test.ExpectedBackupObject { + t.Fatalf("[minio] unexpected backup object name: is '%s' but expected '%s'", actualBackupObject, test.ExpectedBackupObject) + } + + presignedMinio, err := newPresignedMinIOAccess(cfg) + if err != nil { + t.Fatalf("failed to create presigned minio access: '%v'", err) + } + + actualBackupObject = presignedMinio.BackupObject(test.Username, test.Workspace, test.ObjectName) + if actualBackupObject != test.ExpectedBackupObject { + t.Fatalf("[presigned minio] unexpected backup object name: is '%s' but expected '%s'", actualBackupObject, test.ExpectedBackupObject) + } + + }) + } +} diff --git a/components/content-service/pkg/storage/mock/mock.go b/components/content-service/pkg/storage/mock/mock.go index d826ff5d3572a8..62c8b4bc141e24 100644 --- a/components/content-service/pkg/storage/mock/mock.go +++ b/components/content-service/pkg/storage/mock/mock.go @@ -41,17 +41,17 @@ func (m *MockPresignedAccess) EXPECT() *MockPresignedAccessMockRecorder { } // BackupObject mocks base method. -func (m *MockPresignedAccess) BackupObject(arg0, arg1 string) string { +func (m *MockPresignedAccess) BackupObject(arg0, arg1, arg2 string) string { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "BackupObject", arg0, arg1) + ret := m.ctrl.Call(m, "BackupObject", arg0, arg1, arg2) ret0, _ := ret[0].(string) return ret0 } // BackupObject indicates an expected call of BackupObject. -func (mr *MockPresignedAccessMockRecorder) BackupObject(arg0, arg1 interface{}) *gomock.Call { +func (mr *MockPresignedAccessMockRecorder) BackupObject(arg0, arg1, arg2 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BackupObject", reflect.TypeOf((*MockPresignedAccess)(nil).BackupObject), arg0, arg1) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BackupObject", reflect.TypeOf((*MockPresignedAccess)(nil).BackupObject), arg0, arg1, arg2) } // BlobObject mocks base method. @@ -141,17 +141,17 @@ func (mr *MockPresignedAccessMockRecorder) EnsureExists(arg0, arg1 interface{}) } // InstanceObject mocks base method. -func (m *MockPresignedAccess) InstanceObject(arg0, arg1, arg2 string) string { +func (m *MockPresignedAccess) InstanceObject(arg0, arg1, arg2, arg3 string) string { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "InstanceObject", arg0, arg1, arg2) + ret := m.ctrl.Call(m, "InstanceObject", arg0, arg1, arg2, arg3) ret0, _ := ret[0].(string) return ret0 } // InstanceObject indicates an expected call of InstanceObject. -func (mr *MockPresignedAccessMockRecorder) InstanceObject(arg0, arg1, arg2 interface{}) *gomock.Call { +func (mr *MockPresignedAccessMockRecorder) InstanceObject(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InstanceObject", reflect.TypeOf((*MockPresignedAccess)(nil).InstanceObject), arg0, arg1, arg2) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InstanceObject", reflect.TypeOf((*MockPresignedAccess)(nil).InstanceObject), arg0, arg1, arg2, arg3) } // ObjectExists mocks base method. diff --git a/components/content-service/pkg/storage/noop.go b/components/content-service/pkg/storage/noop.go index 6dd4581ba94191..b62957a1963e3d 100644 --- a/components/content-service/pkg/storage/noop.go +++ b/components/content-service/pkg/storage/noop.go @@ -121,11 +121,11 @@ func (p *PresignedNoopStorage) ObjectExists(ctx context.Context, bucket, obj str } // BackupObject returns a backup's object name that a direct downloader would download -func (*PresignedNoopStorage) BackupObject(workspaceID string, name string) string { +func (*PresignedNoopStorage) BackupObject(ownerID string, workspaceID string, name string) string { return "" } // InstanceObject returns a instance's object name that a direct downloader would download -func (*PresignedNoopStorage) InstanceObject(workspaceID string, instanceID string, name string) string { +func (*PresignedNoopStorage) InstanceObject(ownerID string, workspaceID string, instanceID string, name string) string { return "" } diff --git a/components/content-service/pkg/storage/storage.go b/components/content-service/pkg/storage/storage.go index f38dac80faff2c..88ffbceabeda94 100644 --- a/components/content-service/pkg/storage/storage.go +++ b/components/content-service/pkg/storage/storage.go @@ -89,10 +89,10 @@ type PresignedAccess interface { ObjectExists(ctx context.Context, bucket string, path string) (bool, error) // BackupObject returns a backup's object name that a direct downloader would download - BackupObject(workspaceID string, name string) string + BackupObject(ownerID string, workspaceID string, name string) string // InstanceObject returns a instance's object name that a direct downloader would download - InstanceObject(workspaceID string, instanceID string, name string) string + InstanceObject(ownerID string, workspaceID string, instanceID string, name string) string } // ObjectMeta describtes the metadata of a remote object diff --git a/components/supervisor/go.mod b/components/supervisor/go.mod index 83ed595c0b7071..099a7ee290a6e0 100644 --- a/components/supervisor/go.mod +++ b/components/supervisor/go.mod @@ -37,7 +37,6 @@ require ( ) require ( - github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/prometheus/client_golang v1.12.1 github.com/prometheus/client_model v0.2.0 github.com/prometheus/common v0.32.1 @@ -64,6 +63,7 @@ require ( github.com/golang/protobuf v1.5.2 // indirect github.com/googleapis/gax-go/v2 v2.3.0 // indirect github.com/googleapis/go-type-adapters v1.0.0 // indirect + github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect github.com/hashicorp/golang-lru v0.5.1 // indirect github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect @@ -75,7 +75,7 @@ require ( github.com/kr/text v0.2.0 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect github.com/minio/md5-simd v1.1.0 // indirect - github.com/minio/minio-go/v7 v7.0.11 // indirect + github.com/minio/minio-go/v7 v7.0.26 // indirect github.com/minio/sha256-simd v0.1.1 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect diff --git a/components/supervisor/go.sum b/components/supervisor/go.sum index efdfa1b7d961a4..a2fcc37216a9c4 100644 --- a/components/supervisor/go.sum +++ b/components/supervisor/go.sum @@ -259,7 +259,6 @@ github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -364,8 +363,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182aff github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4= github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= -github.com/minio/minio-go/v7 v7.0.11 h1:7utSkCtMQPYYB1UB8FR3d0QSiOWE6F/JYXon29imYek= -github.com/minio/minio-go/v7 v7.0.11/go.mod h1:WoyW+ySKAKjY98B9+7ZbI8z8S3jaxaisdcvj9TGlazA= +github.com/minio/minio-go/v7 v7.0.26 h1:D0HK+8793etZfRY/vHhDmFaP+vmT41K3K4JV9vmZCBQ= +github.com/minio/minio-go/v7 v7.0.26/go.mod h1:x81+AX5gHSfCSqw7jxRKHvxUXMlE5uKX0Vb75Xk5yYg= github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= @@ -534,7 +533,6 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 h1:HWj/xjIHfjYU5nVXpTM0s39J9CbLn7Cc5a7IC5rwsMQ= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= @@ -977,7 +975,6 @@ gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8X gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= -gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/segmentio/analytics-go.v3 v3.1.0 h1:UzxH1uaGZRpMKDhJyBz0pexz6yUoBU3x8bJsRk/HV6U= diff --git a/components/ws-daemon/go.mod b/components/ws-daemon/go.mod index 0ccd7b6f360ae6..8d65b839b199b0 100644 --- a/components/ws-daemon/go.mod +++ b/components/ws-daemon/go.mod @@ -84,7 +84,7 @@ require ( github.com/klauspost/pgzip v1.2.5 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect github.com/minio/md5-simd v1.1.0 // indirect - github.com/minio/minio-go/v7 v7.0.11 // indirect + github.com/minio/minio-go/v7 v7.0.26 // indirect github.com/minio/sha256-simd v0.1.1 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/moby/locker v1.0.1 // indirect diff --git a/components/ws-daemon/go.sum b/components/ws-daemon/go.sum index 9049c08e56d938..7e95a07a172455 100644 --- a/components/ws-daemon/go.sum +++ b/components/ws-daemon/go.sum @@ -629,8 +629,8 @@ github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3N github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4= github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= -github.com/minio/minio-go/v7 v7.0.11 h1:7utSkCtMQPYYB1UB8FR3d0QSiOWE6F/JYXon29imYek= -github.com/minio/minio-go/v7 v7.0.11/go.mod h1:WoyW+ySKAKjY98B9+7ZbI8z8S3jaxaisdcvj9TGlazA= +github.com/minio/minio-go/v7 v7.0.26 h1:D0HK+8793etZfRY/vHhDmFaP+vmT41K3K4JV9vmZCBQ= +github.com/minio/minio-go/v7 v7.0.26/go.mod h1:x81+AX5gHSfCSqw7jxRKHvxUXMlE5uKX0Vb75Xk5yYg= github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= @@ -954,7 +954,6 @@ golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= @@ -1477,7 +1476,6 @@ gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKW gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= diff --git a/components/ws-manager/go.mod b/components/ws-manager/go.mod index 63115fdcd8495c..5821499afb106b 100644 --- a/components/ws-manager/go.mod +++ b/components/ws-manager/go.mod @@ -73,10 +73,11 @@ require ( github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.13.5 // indirect github.com/klauspost/cpuid v1.3.1 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect github.com/minio/md5-simd v1.1.0 // indirect - github.com/minio/minio-go/v7 v7.0.11 // indirect + github.com/minio/minio-go/v7 v7.0.26 // indirect github.com/minio/sha256-simd v0.1.1 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect diff --git a/components/ws-manager/go.sum b/components/ws-manager/go.sum index 9f62189f5776dd..17dcc3d9a98945 100644 --- a/components/ws-manager/go.sum +++ b/components/ws-manager/go.sum @@ -311,7 +311,6 @@ github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -393,6 +392,8 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.13.5 h1:9O69jUPDcsT9fEm74W92rZL9FQY7rCdaXVneq+yyzl4= +github.com/klauspost/compress v1.13.5/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s= github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4= @@ -422,8 +423,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182aff github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4= github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= -github.com/minio/minio-go/v7 v7.0.11 h1:7utSkCtMQPYYB1UB8FR3d0QSiOWE6F/JYXon29imYek= -github.com/minio/minio-go/v7 v7.0.11/go.mod h1:WoyW+ySKAKjY98B9+7ZbI8z8S3jaxaisdcvj9TGlazA= +github.com/minio/minio-go/v7 v7.0.26 h1:D0HK+8793etZfRY/vHhDmFaP+vmT41K3K4JV9vmZCBQ= +github.com/minio/minio-go/v7 v7.0.26/go.mod h1:x81+AX5gHSfCSqw7jxRKHvxUXMlE5uKX0Vb75Xk5yYg= github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= @@ -632,7 +633,6 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 h1:HWj/xjIHfjYU5nVXpTM0s39J9CbLn7Cc5a7IC5rwsMQ= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= @@ -1109,7 +1109,6 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= diff --git a/install/installer/go.mod b/install/installer/go.mod index 024129b08a3f79..adbea620baedc9 100644 --- a/install/installer/go.mod +++ b/install/installer/go.mod @@ -184,7 +184,7 @@ require ( github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 // indirect github.com/minio/md5-simd v1.1.0 // indirect - github.com/minio/minio-go/v7 v7.0.11 // indirect + github.com/minio/minio-go/v7 v7.0.26 // indirect github.com/minio/sha256-simd v1.0.0 // indirect github.com/mitchellh/copystructure v1.1.1 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect diff --git a/install/installer/go.sum b/install/installer/go.sum index 3153bc62d9508b..59af531641feb3 100644 --- a/install/installer/go.sum +++ b/install/installer/go.sum @@ -1402,8 +1402,8 @@ github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0 github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4= github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= -github.com/minio/minio-go/v7 v7.0.11 h1:7utSkCtMQPYYB1UB8FR3d0QSiOWE6F/JYXon29imYek= -github.com/minio/minio-go/v7 v7.0.11/go.mod h1:WoyW+ySKAKjY98B9+7ZbI8z8S3jaxaisdcvj9TGlazA= +github.com/minio/minio-go/v7 v7.0.26 h1:D0HK+8793etZfRY/vHhDmFaP+vmT41K3K4JV9vmZCBQ= +github.com/minio/minio-go/v7 v7.0.26/go.mod h1:x81+AX5gHSfCSqw7jxRKHvxUXMlE5uKX0Vb75Xk5yYg= github.com/minio/sha256-simd v0.0.0-20190131020904-2d45a736cd16/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= github.com/minio/sha256-simd v0.0.0-20190328051042-05b4dd3047e5/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= github.com/minio/sha256-simd v0.1.0/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= @@ -2042,7 +2042,6 @@ golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200602180216-279210d13fed/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -2664,7 +2663,6 @@ gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.51.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.63.2 h1:tGK/CyBg7SMzb60vP1M03vNZ3VDu3wGQJwn7Sxi9r3c= gopkg.in/ini.v1 v1.63.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= diff --git a/install/installer/pkg/common/storage.go b/install/installer/pkg/common/storage.go index 63502b82e08e92..930eb3ec0d3c4c 100644 --- a/install/installer/pkg/common/storage.go +++ b/install/installer/pkg/common/storage.go @@ -60,6 +60,8 @@ func StorageConfig(context *RenderContext) storageconfig.StorageConfig { Secure: true, Region: context.Config.Metadata.Region, ParallelUpload: 100, + + BucketName: context.Config.ObjectStorage.S3.BucketName, }, } } diff --git a/install/installer/pkg/components/docker-registry/helm.go b/install/installer/pkg/components/docker-registry/helm.go index b6fc543437558f..b6b5186918319e 100644 --- a/install/installer/pkg/components/docker-registry/helm.go +++ b/install/installer/pkg/components/docker-registry/helm.go @@ -46,7 +46,7 @@ var Helm = common.CompositeHelmFunc( if inCluster && s3Storage != nil { registryValues = append(registryValues, - helm.KeyValue("docker-registry.s3.region", cfg.Config.Metadata.Region), + helm.KeyValue("docker-registry.s3.region", s3Storage.Region), helm.KeyValue("docker-registry.s3.bucket", s3Storage.Bucket), helm.KeyValue("docker-registry.s3.encrypt", "true"), helm.KeyValue("docker-registry.s3.secure", "true"), diff --git a/install/installer/pkg/config/v1/config.go b/install/installer/pkg/config/v1/config.go index c1d76ecbdead7a..ce685fddf1f39c 100644 --- a/install/installer/pkg/config/v1/config.go +++ b/install/installer/pkg/config/v1/config.go @@ -157,6 +157,10 @@ type ObjectStorage struct { type ObjectStorageS3 struct { Endpoint string `json:"endpoint" validate:"required"` Credentials ObjectRef `json:"credentials" validate:"required"` + + // BucketName sets the name of an existing bucket to enable the "single bucket mode" + // If no name is configured, the old "one bucket per user" behaviour kicks in. + BucketName string `json:"bucket"` } type ObjectStorageCloudStorage struct { @@ -201,6 +205,7 @@ type ContainerRegistryExternal struct { type S3Storage struct { Bucket string `json:"bucket" validate:"required"` Certificate ObjectRef `json:"certificate" validate:"required"` + Region string `json:"region"` } type LogLevel string diff --git a/install/kots/manifests/gitpod-installer-job.yaml b/install/kots/manifests/gitpod-installer-job.yaml index 1f0a5c20f6d515..4dd89e45f14fbe 100644 --- a/install/kots/manifests/gitpod-installer-job.yaml +++ b/install/kots/manifests/gitpod-installer-job.yaml @@ -156,6 +156,7 @@ spec: echo "Gitpod: configuring container registry S3 backend" yq e -i ".containerRegistry.s3storage.bucket = \"{{repl ConfigOption "reg_incluster_storage_s3_bucketname" }}\"" "${CONFIG_FILE}" + yq e -i ".containerRegistry.s3storage.region = \"{{repl ConfigOption "reg_incluster_storage_s3_region" }}\"" "${CONFIG_FILE}" yq e -i ".containerRegistry.s3storage.certificate.kind = \"secret\"" "${CONFIG_FILE}" yq e -i ".containerRegistry.s3storage.certificate.name = \"container-registry-s3-backend\"" "${CONFIG_FILE}" fi diff --git a/install/kots/manifests/gitpod-registry-s3-backend.yaml b/install/kots/manifests/gitpod-registry-s3-backend.yaml index 2eef65090d2189..33df230f5b72d7 100644 --- a/install/kots/manifests/gitpod-registry-s3-backend.yaml +++ b/install/kots/manifests/gitpod-registry-s3-backend.yaml @@ -9,7 +9,7 @@ metadata: app: gitpod component: gitpod-installer annotations: - kots.io/when: '{{repl and (ConfigOptionEquals "reg_incluster" "0") (ConfigOptionEquals "reg_incluster_storage" "s3") }}' + kots.io/when: '{{repl ConfigOptionEquals "reg_incluster_storage" "s3" }}' data: s3AccessKey: '{{repl ConfigOption "reg_incluster_storage_s3_accesskey" | Base64Encode }}' s3SecretKey: '{{repl ConfigOption "reg_incluster_storage_s3_secretkey" | Base64Encode }}' diff --git a/install/kots/manifests/kots-config.yaml b/install/kots/manifests/kots-config.yaml index 70228c1dea6376..d53a8e15ac4402 100644 --- a/install/kots/manifests/kots-config.yaml +++ b/install/kots/manifests/kots-config.yaml @@ -52,6 +52,13 @@ spec: when: '{{repl (ConfigOptionEquals "reg_incluster_storage" "s3") }}' help_text: The name of the bucket to act as your S3 storage backend. + - name: reg_incluster_storage_s3_region + title: S3 bucket region + type: text + required: true + when: '{{repl (ConfigOptionEquals "reg_incluster_storage" "s3") }}' + help_text: The AWS region of the S3 bucket. Eg. `eu-central-1`. + - name: reg_incluster_storage_s3_accesskey title: S3 access key type: text diff --git a/test/go.mod b/test/go.mod index ad185fe1f8ceaa..c6e32183f2461e 100644 --- a/test/go.mod +++ b/test/go.mod @@ -75,12 +75,13 @@ require ( github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.13.5 // indirect github.com/klauspost/cpuid v1.3.1 // indirect github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect github.com/mailru/easyjson v0.7.6 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect github.com/minio/md5-simd v1.1.0 // indirect - github.com/minio/minio-go/v7 v7.0.11 // indirect + github.com/minio/minio-go/v7 v7.0.26 // indirect github.com/minio/sha256-simd v0.1.1 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/go-wordwrap v1.0.0 // indirect diff --git a/test/go.sum b/test/go.sum index cadee42b9917af..22837342bde4bb 100644 --- a/test/go.sum +++ b/test/go.sum @@ -394,6 +394,8 @@ github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.13.5 h1:9O69jUPDcsT9fEm74W92rZL9FQY7rCdaXVneq+yyzl4= +github.com/klauspost/compress v1.13.5/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s= github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4= @@ -427,8 +429,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182aff github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4= github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= -github.com/minio/minio-go/v7 v7.0.11 h1:7utSkCtMQPYYB1UB8FR3d0QSiOWE6F/JYXon29imYek= -github.com/minio/minio-go/v7 v7.0.11/go.mod h1:WoyW+ySKAKjY98B9+7ZbI8z8S3jaxaisdcvj9TGlazA= +github.com/minio/minio-go/v7 v7.0.26 h1:D0HK+8793etZfRY/vHhDmFaP+vmT41K3K4JV9vmZCBQ= +github.com/minio/minio-go/v7 v7.0.26/go.mod h1:x81+AX5gHSfCSqw7jxRKHvxUXMlE5uKX0Vb75Xk5yYg= github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= @@ -648,7 +650,6 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 h1:HWj/xjIHfjYU5nVXpTM0s39J9CbLn7Cc5a7IC5rwsMQ= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= @@ -1106,6 +1107,7 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/DATA-DOG/go-sqlmock.v1 v1.3.0 h1:FVCohIoYO7IJoDDVpV2pdq7SgrMH6wHnuTyrdrxJNoY= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -1116,7 +1118,6 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=