Skip to content

PBM-1504 Add support for Workload Identity Authentication #1021

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 10 commits into
base: dev
Choose a base branch
from
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
@@ -22,6 +22,7 @@ require (
)

require (
cloud.google.com/go/compute/metadata v0.3.0 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.2 // indirect
github.com/Microsoft/go-winio v0.4.14 // indirect
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
@@ -56,6 +57,7 @@ require (
golang.org/x/crypto v0.25.0 // indirect
golang.org/x/exp v0.0.0-20240529005216-23cca8864a10 // indirect
golang.org/x/net v0.25.0 // indirect
golang.org/x/oauth2 v0.23.0 // indirect
golang.org/x/sys v0.22.0 // indirect
golang.org/x/term v0.22.0 // indirect
golang.org/x/text v0.16.0 // indirect
4 changes: 4 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2QxYC4trgAKZc=
cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.10.0 h1:n1DH8TPV4qqPTje2RcUBYwtrTWlabVp4n46+74X2pn4=
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.10.0/go.mod h1:HDcZnuGbiyppErN6lB+idp4CKhjbc8gwjto6OPpyggM=
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 h1:sO0/P7g68FrryJzljemN+6GTssUXdANk6aJ7T1ZxnsQ=
@@ -177,6 +179,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac=
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs=
golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
21 changes: 12 additions & 9 deletions packaging/conf/pbm-conf-reference.yml
Original file line number Diff line number Diff line change
@@ -11,28 +11,31 @@
#storage:

## Remote backup storage type. Supported types: S3, filesystem, azure


#---------------------S3 Storage Configuration--------------------------
# type:
# s3:

## Specify the location and name of the bucket that you have configured on the S3
# region:
# bucket:
## Specify the location and name of the bucket that you have configured on the S3
# region:
# bucket:

## The data directory to store backups in.
## The data directory to store backups in.
## When undefined, backups are saved at the root of the bucket.
# prefix:
# prefix:

## The URL to access the bucket for GCS and MinIO
# endpointURL:
# endpointURL:

## The service account used to access the GCS bucket.
## IF undefined, the "default" service account is used.
# serviceAccount:

## S3 access credentials.
# credentials:
# access-key-id:
# access-key-id:
# secret-access-key:
# session-token:
# session-token:

## The size of data chinks (in MB) to upload to the bucket.
# uploadPartSize: 10
43 changes: 43 additions & 0 deletions pbm/storage/s3/s3.go
Original file line number Diff line number Diff line change
@@ -25,6 +25,7 @@ import (
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
"github.com/aws/aws-sdk-go/service/sts"
"golang.org/x/oauth2/google"

"github.com/percona/percona-backup-mongodb/pbm/errors"
"github.com/percona/percona-backup-mongodb/pbm/log"
@@ -43,6 +44,7 @@ type Config struct {
Provider string `bson:"provider,omitempty" json:"provider,omitempty" yaml:"provider,omitempty"`
Region string `bson:"region" json:"region" yaml:"region"`
EndpointURL string `bson:"endpointUrl,omitempty" json:"endpointUrl" yaml:"endpointUrl,omitempty"`
ServiceAccount string `bson:"serviceAccount,omitempty" json:"serviceAccount" yaml:"serviceAccount,omitempty"`
ForcePathStyle *bool `bson:"forcePathStyle,omitempty" json:"forcePathStyle,omitempty" yaml:"forcePathStyle,omitempty"`
Bucket string `bson:"bucket" json:"bucket" yaml:"bucket"`
Prefix string `bson:"prefix,omitempty" json:"prefix,omitempty" yaml:"prefix,omitempty"`
@@ -159,6 +161,9 @@ func (cfg *Config) Equal(other *Config) bool {
if cfg.EndpointURL != other.EndpointURL {
return false
}
if cfg.ServiceAccount != other.ServiceAccount {
return false
}
if cfg.Bucket != other.Bucket {
return false
}
@@ -196,6 +201,9 @@ func (cfg *Config) Cast() error {
if cfg.Region == "" {
cfg.Region = defaultS3Region
}
if cfg.ServiceAccount == "" {
cfg.ServiceAccount = "default"
}
if cfg.ForcePathStyle == nil {
cfg.ForcePathStyle = aws.Bool(true)
}
@@ -550,6 +558,20 @@ func (s *S3) session() (*session.Session, error) {
}})
}

// If using GCE, attempt to retrieve access token from metadata server
if onGCE() {
tokenSource := google.ComputeTokenSource(s.opts.ServiceAccount, "")
token, err := tokenSource.Token()
if err != nil {
return nil, errors.Wrap(err, "get GCP token")
}
providers = append(providers, &credentials.StaticProvider{Value: credentials.Value{
AccessKeyID: "GCP_OAUTH_TOKEN",
SecretAccessKey: "GCP_OATH_TOKEN",
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

is GCP_OATH_TOKEN a typo?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sorry for the delay, no not a typo, the fields are mandatory but they're just set to placeholder values here since they're not used for GCS.

SessionToken: token.AccessToken,
}})
}

awsSession, err := session.NewSession()
if err != nil {
return nil, errors.Wrap(err, "new session")
@@ -617,3 +639,24 @@ func awsLogger(l log.LogEvent) aws.Logger {
l.Debug(msg, xs...)
})
}

func onGCE() bool {
client := http.Client{
Timeout: 100 * time.Millisecond,
}

req, err := http.NewRequest("GET", "http://169.254.169.254/computeMetadata/v1/", nil)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

shouldn't "http://metadata.google.internal" be used instead of the hardcoded IP?

if err != nil {
return false
}

req.Header.Add("Metadata-Flavor", "Google")

resp, err := client.Do(req)
if err != nil {
return false
}
defer resp.Body.Close()

return resp.StatusCode == http.StatusOK
}
26 changes: 26 additions & 0 deletions vendor/cloud.google.com/go/compute/metadata/CHANGES.md

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

202 changes: 202 additions & 0 deletions vendor/cloud.google.com/go/compute/metadata/LICENSE

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

27 changes: 27 additions & 0 deletions vendor/cloud.google.com/go/compute/metadata/README.md
579 changes: 579 additions & 0 deletions vendor/cloud.google.com/go/compute/metadata/metadata.go

Large diffs are not rendered by default.

114 changes: 114 additions & 0 deletions vendor/cloud.google.com/go/compute/metadata/retry.go
26 changes: 26 additions & 0 deletions vendor/cloud.google.com/go/compute/metadata/retry_linux.go
13 changes: 13 additions & 0 deletions vendor/golang.org/x/oauth2/.travis.yml
26 changes: 26 additions & 0 deletions vendor/golang.org/x/oauth2/CONTRIBUTING.md
27 changes: 27 additions & 0 deletions vendor/golang.org/x/oauth2/LICENSE
40 changes: 40 additions & 0 deletions vendor/golang.org/x/oauth2/README.md
94 changes: 94 additions & 0 deletions vendor/golang.org/x/oauth2/authhandler/authhandler.go
198 changes: 198 additions & 0 deletions vendor/golang.org/x/oauth2/deviceauth.go
40 changes: 40 additions & 0 deletions vendor/golang.org/x/oauth2/google/appengine.go
317 changes: 317 additions & 0 deletions vendor/golang.org/x/oauth2/google/default.go
53 changes: 53 additions & 0 deletions vendor/golang.org/x/oauth2/google/doc.go
64 changes: 64 additions & 0 deletions vendor/golang.org/x/oauth2/google/error.go
577 changes: 577 additions & 0 deletions vendor/golang.org/x/oauth2/google/externalaccount/aws.go

Large diffs are not rendered by default.

485 changes: 485 additions & 0 deletions vendor/golang.org/x/oauth2/google/externalaccount/basecredentials.go

Large diffs are not rendered by default.

64 changes: 64 additions & 0 deletions vendor/golang.org/x/oauth2/google/externalaccount/header.go
312 changes: 312 additions & 0 deletions vendor/golang.org/x/oauth2/google/google.go
102 changes: 102 additions & 0 deletions vendor/golang.org/x/oauth2/google/jwt.go
201 changes: 201 additions & 0 deletions vendor/golang.org/x/oauth2/google/sdk.go
6 changes: 6 additions & 0 deletions vendor/golang.org/x/oauth2/internal/doc.go
37 changes: 37 additions & 0 deletions vendor/golang.org/x/oauth2/internal/oauth2.go
Loading