gh2gcs: add yaml config option

cmd/gh2gcs/cmd/BUILD.bazel

@@ -13,6 +13,8 @@ go_library(
         "@com_github_pkg_errors//:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
         "@com_github_spf13_cobra//:go_default_library",
+        "@com_github_spf13_pflag//:go_default_library",
+        "@in_gopkg_yaml_v2//:go_default_library",
     ],
 )
 

cmd/gh2gcs/cmd/root.go

@@ -19,10 +19,13 @@ package cmd
 import (
 	"io/ioutil"
 	"os"
+	"strings"
 
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+	"gopkg.in/yaml.v2"
 
 	"k8s.io/release/pkg/gcp"
 	"k8s.io/release/pkg/gh2gcs"
@@ -32,12 +35,15 @@ import (
 
 // rootCmd represents the base command when called without any subcommands
 var rootCmd = &cobra.Command{
-	Use:               "gh2gcs --org kubernetes --repo release --bucket <bucket> --release-dir <release-dir> [--tags v0.0.0] [--include-prereleases] [--output-dir <temp-dir>] [--download-only]",
+	Use:               "gh2gcs --org kubernetes --repo release --bucket <bucket> --release-dir <release-dir> [--tags v0.0.0] [--include-prereleases] [--output-dir <temp-dir>] [--download-only] [--config <config-file>]",
 	Short:             "gh2gcs uploads GitHub releases to Google Cloud Storage",
 	Example:           "gh2gcs --org kubernetes --repo release --bucket k8s-staging-release-test --release-dir release --tags v0.0.0,v0.0.1",
 	SilenceUsage:      true,
 	SilenceErrors:     true,
 	PersistentPreRunE: initLogging,
+	PreRunE: func(cmd *cobra.Command, args []string) error {
+		return checkRequiredFlags(cmd.Flags())
+	},
 	RunE: func(*cobra.Command, []string) error {
 		return run(opts)
 	},
@@ -53,6 +59,7 @@ type options struct {
 	outputDir          string
 	logLevel           string
 	tags               []string
+	configFilePath     string
 }
 
 var opts = &options{}
@@ -61,12 +68,14 @@ var (
 	orgFlag                = "org"
 	repoFlag               = "repo"
 	tagsFlag               = "tags"
+	configFlag             = "config"
 	includePrereleasesFlag = "include-prereleases"
 	bucketFlag             = "bucket"
 	releaseDirFlag         = "release-dir"
 	outputDirFlag          = "output-dir"
 	downloadOnlyFlag       = "download-only"
 
+	// requiredFlags only if the config flag is not set
 	requiredFlags = []string{
 		orgFlag,
 		repoFlag,
@@ -152,17 +161,39 @@ func init() {
 		"the logging verbosity, either 'panic', 'fatal', 'error', 'warn', 'warning', 'info', 'debug' or 'trace'",
 	)
 
-	for _, flag := range requiredFlags {
-		if err := rootCmd.MarkPersistentFlagRequired(flag); err != nil {
-			logrus.Fatal(err)
-		}
-	}
+	rootCmd.PersistentFlags().StringVar(
+		&opts.configFilePath,
+		configFlag,
+		"",
+		"config file to set all the branch/repositories the user wants to",
+	)
 }
 
 func initLogging(*cobra.Command, []string) error {
 	return log.SetupGlobalLogger(opts.logLevel)
 }
 
+func checkRequiredFlags(flags *pflag.FlagSet) error {
+	if flags.Lookup(configFlag).Changed {
+		return nil
+	}
+
+	checkRequiredFlags := []string{}
+	flags.VisitAll(func(flag *pflag.Flag) {
+		for _, requiredflag := range requiredFlags {
+			if requiredflag == flag.Name && !flag.Changed {
+				checkRequiredFlags = append(checkRequiredFlags, requiredflag)
+			}
+		}
+	})
+
+	if len(checkRequiredFlags) != 0 {
+		return errors.New("Required flag(s) `" + strings.Join(checkRequiredFlags, ", ") + "` not set")
+	}
+
+	return nil
+}
+
 func run(opts *options) error {
 	if err := opts.SetAndValidate(); err != nil {
 		return errors.Wrap(err, "validating gh2gcs options")
@@ -172,44 +203,56 @@ func run(opts *options) error {
 		return errors.Wrap(err, "pre-checking for GCP package usage")
 	}
 
-	// Create a real GitHub API client
-	gh := github.New()
-
-	// TODO: Support downloading releases via yaml config
-	uploadConfig := &gh2gcs.Config{}
-	releaseConfig := &gh2gcs.ReleaseConfig{
-		Org:                opts.org,
-		Repo:               opts.repo,
-		Tags:               []string{},
-		IncludePrereleases: opts.includePrereleases,
-		GCSBucket:          opts.bucket,
-		ReleaseDir:         opts.releaseDir,
-		GCSCopyOptions:     gh2gcs.DefaultGCSCopyOptions,
-	}
-
-	// TODO: Expose certain GCSCopyOptions for user configuration
+	releaseConfigs := &gh2gcs.Config{}
+	if opts.configFilePath != "" {
+		logrus.Infof("Reading the config file %s...", opts.configFilePath)
+		data, err := ioutil.ReadFile(opts.configFilePath)
+		if err != nil {
+			return errors.Wrap(err, "failed to read the file")
+		}
 
-	if len(opts.tags) > 0 {
-		releaseConfig.Tags = opts.tags
-	} else {
-		releaseTags, err := gh.GetReleaseTags(opts.org, opts.repo, opts.includePrereleases)
+		logrus.Info("Parsing the config...")
+		err = yaml.UnmarshalStrict(data, &releaseConfigs)
 		if err != nil {
-			return errors.Wrap(err, "getting release tags")
+			return errors.Wrap(err, "failed to decode the file")
 		}
 
-		releaseConfig.Tags = releaseTags
+		for i, releaseConfig := range releaseConfigs.ReleaseConfigs {
+			releaseConfigs.ReleaseConfigs[i].GCSCopyOptions = gh2gcs.CheckGCSCopyOptions(releaseConfig.GCSCopyOptions)
+		}
+	} else {
+		// TODO: Expose certain GCSCopyOptions for user configuration
+		releaseConfigs.ReleaseConfigs = append(releaseConfigs.ReleaseConfigs, gh2gcs.ReleaseConfig{
+			Org:                opts.org,
+			Repo:               opts.repo,
+			Tags:               opts.tags,
+			IncludePrereleases: opts.includePrereleases,
+			GCSBucket:          opts.bucket,
+			ReleaseDir:         opts.releaseDir,
+			GCSCopyOptions:     gh2gcs.DefaultGCSCopyOptions,
+		})
 	}
 
-	uploadConfig.ReleaseConfigs = append(uploadConfig.ReleaseConfigs, *releaseConfig)
+	// Create a real GitHub API client
+	gh := github.New()
+
+	for _, releaseConfig := range releaseConfigs.ReleaseConfigs {
+		if len(releaseConfig.Tags) == 0 {
+			releaseTags, err := gh.GetReleaseTags(releaseConfig.Org, releaseConfig.Repo, releaseConfig.IncludePrereleases)
+			if err != nil {
+				return errors.Wrap(err, "getting release tags")
+			}
+
+			releaseConfig.Tags = releaseTags
+		}
 
-	for _, rc := range uploadConfig.ReleaseConfigs {
-		if err := gh2gcs.DownloadReleases(&rc, gh, opts.outputDir); err != nil {
+		if err := gh2gcs.DownloadReleases(&releaseConfig, gh, opts.outputDir); err != nil {
 			return errors.Wrap(err, "downloading release assets")
 		}
 		logrus.Infof("Files downloaded to %s directory", opts.outputDir)
 
 		if !opts.downloadOnly {
-			if err := gh2gcs.Upload(&rc, gh, opts.outputDir); err != nil {
+			if err := gh2gcs.Upload(&releaseConfig, gh, opts.outputDir); err != nil {
 				return errors.Wrap(err, "uploading release assets to GCS")
 			}
 		}

go.mod

@@ -22,12 +22,15 @@ require (
 	github.com/sendgrid/sendgrid-go v3.6.0+incompatible
 	github.com/sirupsen/logrus v1.6.0
 	github.com/spf13/cobra v1.0.0
+	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.6.1
 	github.com/yuin/goldmark v1.1.32
 	golang.org/x/net v0.0.0-20200602114024-627f9648deb9
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
 	google.golang.org/api v0.21.0
+	gopkg.in/yaml.v2 v2.2.8
 	k8s.io/apimachinery v0.18.3
+	k8s.io/utils v0.0.0-20200229041039-0a110f9eb7ab
 	sigs.k8s.io/yaml v1.2.0
 )
 

pkg/gcp/gcs/gcs.go

@@ -36,16 +36,16 @@ var (
 
 type Options struct {
 	// gsutil options
-	Concurrent bool
-	Recursive  bool
-	NoClobber  bool
+	Concurrent *bool
+	Recursive  *bool
+	NoClobber  *bool
 
 	// local options
 	// AllowMissing allows a copy operation to be skipped if the source or
 	// destination does not exist. This is useful for scenarios where copy
 	// operations happen in a loop/channel, so a single "failure" does not block
 	// the entire operation.
-	AllowMissing bool
+	AllowMissing *bool
 }
 
 // CopyToGCS copies a local directory to the specified GCS path
@@ -57,7 +57,7 @@ func CopyToGCS(src, gcsPath string, opts *Options) error {
 	if err != nil {
 		logrus.Info("unable to get local source directory info")
 
-		if opts.AllowMissing {
+		if *opts.AllowMissing {
 			logrus.Infof("Source directory (%s) does not exist. Skipping GCS upload.", src)
 			return nil
 		}
@@ -79,17 +79,17 @@ func CopyToLocal(gcsPath, dst string, opts *Options) error {
 func bucketCopy(src, dst string, opts *Options) error {
 	args := []string{}
 
-	if opts.Concurrent {
+	if *opts.Concurrent {
 		logrus.Debug("Setting GCS copy to run concurrently")
 		args = append(args, concurrentFlag)
 	}
 
 	args = append(args, "cp")
-	if opts.Recursive {
+	if *opts.Recursive {
 		logrus.Debug("Setting GCS copy to run recursively")
 		args = append(args, recursiveFlag)
 	}
-	if opts.NoClobber {
+	if *opts.NoClobber {
 		logrus.Debug("Setting GCS copy to not clobber existing files")
 		args = append(args, noClobberFlag)
 	}

pkg/gh2gcs/BUILD.bazel

@@ -9,6 +9,7 @@ go_library(
         "//pkg/gcp/gcs:go_default_library",
         "//pkg/github:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
+        "@io_k8s_utils//pointer:go_default_library",
     ],
 )
 

pkg/gh2gcs/gh2gcs.go

@@ -24,31 +24,33 @@ import (
 
 	"k8s.io/release/pkg/gcp/gcs"
 	"k8s.io/release/pkg/github"
+	"k8s.io/utils/pointer"
 )
 
 // Config contains a slice of `ReleaseConfig` to be used when unmarshalling a
 // yaml config containing multiple repository configs.
 type Config struct {
-	ReleaseConfigs []ReleaseConfig
+	ReleaseConfigs []ReleaseConfig `yaml:"releaseConfigs"`
 }
 
 // ReleaseConfig contains source (GitHub) and destination (GCS) information
 // to perform a copy/upload operation using gh2gcs.
 type ReleaseConfig struct {
-	Org                string
-	Repo               string
-	Tags               []string
-	IncludePrereleases bool
-	GCSBucket          string
-	ReleaseDir         string
-	GCSCopyOptions     *gcs.Options
+	Org                string       `yaml:"org"`
+	Repo               string       `yaml:"repo"`
+	Tags               []string     `yaml:"tags"`
+	IncludePrereleases bool         `yaml:"includePrereleases"`
+	GCSBucket          string       `yaml:"gcsBucket"`
+	ReleaseDir         string       `yaml:"releaseDir"`
+	GCSCopyOptions     *gcs.Options `yaml:"gcsCopyOptions"`
 }
 
+// DefaultGCSCopyOptions have the default options for the GCS copy action
 var DefaultGCSCopyOptions = &gcs.Options{
-	Concurrent:   true,
-	Recursive:    true,
-	NoClobber:    true,
-	AllowMissing: true,
+	Concurrent:   pointer.BoolPtr(true),
+	Recursive:    pointer.BoolPtr(true),
+	NoClobber:    pointer.BoolPtr(true),
+	AllowMissing: pointer.BoolPtr(true),
 }
 
 // DownloadReleases downloads release assets to a local directory
@@ -86,3 +88,29 @@ func Upload(releaseCfg *ReleaseConfig, ghClient *github.GitHub, outputDir string
 
 	return nil
 }
+
+// CheckGCSCopyOptions checks if the user set any config or we need to set the default config
+func CheckGCSCopyOptions(copyOptions *gcs.Options) *gcs.Options {
+	// set the GCS Copy options to default values
+	if copyOptions == nil {
+		return DefaultGCSCopyOptions
+	}
+
+	if copyOptions.AllowMissing == nil {
+		copyOptions.AllowMissing = pointer.BoolPtr(true)
+	}
+
+	if copyOptions.Concurrent == nil {
+		copyOptions.Concurrent = pointer.BoolPtr(true)
+	}
+
+	if copyOptions.NoClobber == nil {
+		copyOptions.NoClobber = pointer.BoolPtr(true)
+	}
+
+	if copyOptions.Recursive == nil {
+		copyOptions.Recursive = pointer.BoolPtr(true)
+	}
+
+	return copyOptions
+}