diff --git a/install/installer/cmd/README.md b/install/installer/cmd/README.md new file mode 100644 index 00000000000000..432b9bcd0dd555 --- /dev/null +++ b/install/installer/cmd/README.md @@ -0,0 +1,35 @@ +# Cmd + +Publicly available commands are part of our user contract. They use [Cobra](https://cobra.dev/) to build the Golang CLI commands. + +## Commands + +### config + +These are designed to manipulate configuration files and generate a configuration file that can be used to install Gitpod. + +#### init + +> This replaces `init`, which is now deprecated + +This should be run first. This will generate a new `gitpod.config.yaml` file with the default values configured. + +#### build-from-envvars + +This builds the config from environment variables. + +#### cluster + +Cluster commands are designed to deploy a Kubernetes resource to the cluster and generate the config value based upon the result. Typically (although not exclusively), these will be Jobs. + +##### shiftfs + +Detects whether ShiftFS is supported on the cluster for building images. If not, this will default to Fuse. + +#### files + +Files commands are designed to be run against the file structure. They may be run directly on the node, or by mounting the file system as a volume in a pod. + +##### containerd + +Detects the containerd settings for a cluster. This will return the location of the containerd socket and the path to the directory. diff --git a/install/installer/cmd/config_build-from-envvars.go b/install/installer/cmd/config_build-from-envvars.go new file mode 100644 index 00000000000000..afa7bd110e1104 --- /dev/null +++ b/install/installer/cmd/config_build-from-envvars.go @@ -0,0 +1,40 @@ +// Copyright (c) 2022 Gitpod GmbH. All rights reserved. +// Licensed under the MIT License. See License-MIT.txt in the project root for license information. + +package cmd + +import ( + "github.com/gitpod-io/gitpod/installer/pkg/config" + "github.com/spf13/cobra" +) + +// configBuildFromEnvvarsCmd represents the validate command +var configBuildFromEnvvarsCmd = &cobra.Command{ + Use: "build-from-envvars", + Short: "Build configuration from environment variables", + RunE: func(cmd *cobra.Command, args []string) error { + if _, err := configFileExistsAndInit(); err != nil { + return err + } + + _, version, cfg, err := loadConfig(configOpts.ConfigFile) + if err != nil { + return err + } + + apiVersion, err := config.LoadConfigVersion(version) + if err != nil { + return err + } + + if err := apiVersion.BuildFromEnvvars(cfg); err != nil { + return err + } + + return saveConfigFile(cfg) + }, +} + +func init() { + configCmd.AddCommand(configBuildFromEnvvarsCmd) +} diff --git a/install/installer/cmd/render.go b/install/installer/cmd/render.go index 8c12f7d3f82771..f4cc8294ac0eba 100644 --- a/install/installer/cmd/render.go +++ b/install/installer/cmd/render.go @@ -245,7 +245,7 @@ func init() { } renderCmd.PersistentFlags().StringVarP(&renderOpts.ConfigFN, "config", "c", getEnvvar("GITPOD_INSTALLER_CONFIG", filepath.Join(dir, "gitpod.config.yaml")), "path to the config file, use - for stdin") - renderCmd.PersistentFlags().StringVarP(&renderOpts.Namespace, "namespace", "n", "default", "namespace to deploy to") + renderCmd.PersistentFlags().StringVarP(&renderOpts.Namespace, "namespace", "n", getEnvvar("NAMESPACE", "default"), "namespace to deploy to") renderCmd.Flags().BoolVar(&renderOpts.ValidateConfigDisabled, "no-validation", false, "if set, the config will not be validated before running") renderCmd.Flags().BoolVar(&renderOpts.UseExperimentalConfig, "use-experimental-config", false, "enable the use of experimental config that is prone to be changed") renderCmd.Flags().StringVar(&renderOpts.FilesDir, "output-split-files", "", "path to output individual Kubernetes manifests to") diff --git a/install/installer/cmd/validate_cluster.go b/install/installer/cmd/validate_cluster.go index 4813554466c654..23d68e8bf502a9 100644 --- a/install/installer/cmd/validate_cluster.go +++ b/install/installer/cmd/validate_cluster.go @@ -111,5 +111,5 @@ func init() { validateClusterCmd.PersistentFlags().StringVar(&validateClusterOpts.Kube.Config, "kubeconfig", "", "path to the kubeconfig file") validateClusterCmd.PersistentFlags().StringVarP(&validateClusterOpts.Config, "config", "c", getEnvvar("GITPOD_INSTALLER_CONFIG", filepath.Join(dir, "gitpod.config.yaml")), "path to the config file") - validateClusterCmd.PersistentFlags().StringVarP(&validateClusterOpts.Namespace, "namespace", "n", "default", "namespace to deploy to") + validateClusterCmd.PersistentFlags().StringVarP(&validateClusterOpts.Namespace, "namespace", "n", getEnvvar("NAMESPACE", "default"), "namespace to deploy to") } diff --git a/install/installer/go.mod b/install/installer/go.mod index 6091ae2affec33..7036590ad15554 100644 --- a/install/installer/go.mod +++ b/install/installer/go.mod @@ -122,6 +122,7 @@ require ( github.com/go-redis/redis/v7 v7.4.1 // indirect github.com/go-redis/redis/v8 v8.11.5 // indirect github.com/go-sql-driver/mysql v1.6.0 // indirect + github.com/go-test/deep v1.0.8 github.com/gobwas/glob v0.2.3 // indirect github.com/goccy/go-yaml v1.9.5 // indirect github.com/godbus/dbus/v5 v5.0.6 // indirect diff --git a/install/installer/go.sum b/install/installer/go.sum index c57e19d6947931..eb40fea0683265 100644 --- a/install/installer/go.sum +++ b/install/installer/go.sum @@ -658,6 +658,8 @@ github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg78 github.com/go-test/deep v1.0.2-0.20181118220953-042da051cf31/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/go-test/deep v1.0.2/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/go-test/deep v1.0.7 h1:/VSMRlnY/JSyqxQUzQLKVMAskpY/NZKFA5j2P+0pP2M= +github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= +github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= github.com/gobuffalo/envy v1.7.1/go.mod h1:FurDp9+EDPE4aIUS3ZLyD+7/9fpx7YRt/ukY6jIHf0w= github.com/gobuffalo/flect v0.2.2/go.mod h1:vmkQwuZYhN5Pc4ljYQZzP+1sq+NEkK+lh20jmEmX3jc= diff --git a/install/installer/pkg/config/loader.go b/install/installer/pkg/config/loader.go index 976b44bfdfa1c6..e3ceee74bf072e 100644 --- a/install/installer/pkg/config/loader.go +++ b/install/installer/pkg/config/loader.go @@ -51,6 +51,9 @@ type ConfigVersion interface { // CheckDeprecated checks for deprecated config params. // Returns key/value pair of deprecated params/values and any error messages (used for conflicting params) CheckDeprecated(cfg interface{}) (map[string]interface{}, []string) + + // BuildFromEnvvars builds the configuration file from assigned envvars + BuildFromEnvvars(cfg interface{}) error } // AddVersion adds a new version. diff --git a/install/installer/pkg/config/v1/config.go b/install/installer/pkg/config/v1/config.go index 51d99af1770322..665fb58f0dbe17 100644 --- a/install/installer/pkg/config/v1/config.go +++ b/install/installer/pkg/config/v1/config.go @@ -34,6 +34,13 @@ func (v version) Factory() interface{} { }, } } + +const ( + defaultRepositoryUrl = "eu.gcr.io/gitpod-core-dev/build" + defaultOpenVSXURL = "https://open-vsx.org" + defaultMetadataRegion = "local" +) + func (v version) Defaults(in interface{}) error { cfg, ok := in.(*Config) if !ok { @@ -41,14 +48,14 @@ func (v version) Defaults(in interface{}) error { } cfg.Kind = InstallationFull - cfg.Repository = "eu.gcr.io/gitpod-core-dev/build" + cfg.Repository = defaultRepositoryUrl cfg.Observability = Observability{ LogLevel: LogLevelInfo, } cfg.Certificate.Kind = ObjectRefSecret cfg.Certificate.Name = "https-certificates" cfg.Database.InCluster = pointer.Bool(true) - cfg.Metadata.Region = "local" + cfg.Metadata.Region = defaultMetadataRegion cfg.Metadata.InstallationShortname = InstallationShortNameOldDefault // TODO(gpl): we're tied to "default" here because that's what we put into static bridges in the past cfg.ObjectStorage.InCluster = pointer.Bool(true) cfg.ObjectStorage.Resources = &Resources{ @@ -69,7 +76,7 @@ func (v version) Defaults(in interface{}) error { cfg.Workspace.PVC.Size = resource.MustParse("30Gi") cfg.Workspace.PVC.StorageClass = "" cfg.Workspace.PVC.SnapshotClass = "" - cfg.OpenVSX.URL = "https://open-vsx.org" + cfg.OpenVSX.URL = defaultOpenVSXURL cfg.DisableDefinitelyGP = true return nil diff --git a/install/installer/pkg/config/v1/envvars.go b/install/installer/pkg/config/v1/envvars.go new file mode 100644 index 00000000000000..0273393c1cace7 --- /dev/null +++ b/install/installer/pkg/config/v1/envvars.go @@ -0,0 +1,431 @@ +// Copyright (c) 2022 Gitpod GmbH. All rights reserved. +// Licensed under the MIT License. See License-MIT.txt in the project root for license information. + +package config + +import ( + "fmt" + "os" + "reflect" + "sort" + "strings" + + "github.com/gitpod-io/gitpod/common-go/log" + "github.com/gitpod-io/gitpod/installer/pkg/config" + "github.com/gitpod-io/gitpod/installer/pkg/config/v1/experimental" + v1 "k8s.io/api/core/v1" + "k8s.io/utils/pointer" + "sigs.k8s.io/yaml" +) + +// isTruthy performs a case-insensitive match for values parsed as true +func isTruthy(input string) bool { + trueValues := []string{ + "1", + "y", + "yes", + "on", + "true", + "t", + } + + for _, v := range trueValues { + if strings.ToUpper(input) == strings.ToUpper(v) { + return true + } + } + + return false +} + +// ConfigEnvvars this maps the external environment variables to Golang values and forms part of our public contract +type ConfigEnvvars struct { + AdvancedModeEnabled bool `env:"ADVANCED_MODE_ENABLED"` + ComponentProxyServiceType string `env:"COMPONENT_PROXY_SERVICE_SERVICETYPE"` + ConfigPatch string `env:"CONFIG_PATCH"` + CustomizationPatch string `env:"CUSTOMIZATION_PATCH"` + DBCloudSQLEnabled bool `env:"DB_CLOUDSQL_ENABLED"` + DBCloudSQLInstance string `env:"DB_CLOUDSQL_INSTANCE"` + DBCloudSQLServiceAccountName string `env:"DB_CLOUDSQL_SERVICE_ACCOUNT_NAME"` + DBExternalCertificateName string `env:"DB_EXTERNAL_CERTIFICATE_NAME"` + DBInClusterEnabled bool `env:"DB_INCLUSTER_ENABLED" envDefault:"1"` + Domain string `env:"DOMAIN"` + Distribution string `env:"DISTRIBUTION"` + ExternalDockerConfig string `env:"EXTERNAL_DOCKER_CONFIG_JSON"` + HTTPProxyName string `env:"HTTP_PROXY_NAME"` + ImagePullSecretName string `env:"IMAGE_PULL_SECRET_NAME"` + LicenseName string `env:"LICENSE_NAME"` + LocalRegistryAddress string `env:"LOCAL_REGISTRY_ADDRESS"` + LocalRegistryEnabled bool `env:"HAS_LOCAL_REGISTRY"` + LocalRegistryHost string `env:"LOCAL_REGISTRY_HOST"` + LocalRegistryImagePullConfig string `env:"LOCAL_REGISTRY_IMAGE_PULL_DOCKER_CONFIG_JSON"` + OpenVSXUrl string `env:"OPEN_VSX_URL"` + RegistryDockerConfigEnabled bool `env:"REG_DOCKER_CONFIG_ENABLED"` + RegistryDockerConfig string `env:"REG_DOCKER_CONFIG_JSON"` + RegistryInClusterEnabled bool `env:"REG_INCLUSTER_ENABLED" envDefault:"1"` + RegistryInClusterStorageType string `env:"REG_INCLUSTER_STORAGE"` + RegistryInClusterStorageS3BucketName string `env:"REG_INCLUSTER_STORAGE_S3_BUCKETNAME"` + RegistryInClusterStorageS3CertName string `env:"REG_INCLUSTER_STORAGE_S3_CERTIFICATE_NAME"` + RegistryInClusterStorageS3Endpoint string `env:"REG_INCLUSTER_STORAGE_S3_ENDPOINT"` + RegistryInClusterStorageS3Region string `env:"REG_INCLUSTER_STORAGE_S3_REGION"` + RegistryExternalCertName string `env:"REG_EXTERNAL_CERTIFICATE_NAME"` + RegistryExternalURL string `env:"REG_URL"` + SSHGatewayEnabled bool `env:"SSH_GATEWAY"` + SSHGatewayHostKeyName string `env:"SSH_GATEWAY_HOST_KEY_NAME"` + StorageProvider string `env:"STORE_PROVIDER" envDefault:"incluster"` + StorageRegion string `env:"STORE_REGION"` + StorageAzureCredsName string `env:"STORE_AZURE_CREDENTIALS_NAME"` + StorageGCPProjectName string `env:"STORE_GCP_PROJECT"` + StorageGCPServiceAccountName string `env:"STORE_GCP_SERVICE_ACCOUNT_NAME"` + StorageS3Bucket string `env:"STORE_S3_BUCKET"` + StorageS3CredsName string `env:"STORE_S3_CREDENTIALS_NAME"` + StorageS3Endpoint string `env:"STORE_S3_ENDPOINT"` + TLSSelfSignedEnabled bool `env:"TLS_SELF_SIGNED_ENABLED"` + TLSCertManagerEnabled bool `env:"CERT_MANAGER_ENABLED"` + TLSCustomCACertEnabled bool `env:"TLS_CUSTOM_CA_CRT_ENABLED"` + TLSCustomCACertCredsName string `env:"TLS_CUSTOM_CA_CRT_CREDENTIALS_NAME"` + UserManagementBlockEnabled bool `env:"USER_MANAGEMENT_BLOCK_ENABLED"` + UserManagementBlockPassList []string `env:"USER_MANAGEMENT_BLOCK_PASSLIST"` +} + +func (c *ConfigEnvvars) load() { + t := reflect.TypeOf(*c) + v := reflect.ValueOf(c).Elem() + + for i := 0; i < t.NumField(); i++ { + field := t.Field(i) + tag := field.Tag.Get("env") + defaultTag := field.Tag.Get("envDefault") + + envvar, ok := os.LookupEnv(tag) + if !ok { + envvar = defaultTag + } + + valueField := v.Field(i) + if valueField.IsValid() && valueField.CanSet() { + switch valueField.Kind() { + case reflect.Bool: + valueField.SetBool(isTruthy(envvar)) + case reflect.Slice: + // This only currently works for slices of strings + sliceValue := strings.Split(envvar, " ") + slice := reflect.MakeSlice(reflect.SliceOf(field.Type.Elem()), 0, len(sliceValue)) + + for _, i := range sliceValue { + // Clean up any whitespace and append to slice + slice = reflect.Append(slice, reflect.ValueOf(strings.TrimSpace(i))) + } + + valueField.Set(slice) + default: + valueField.SetString(envvar) + } + } + } +} + +type dockerConfigJson struct { + Auths map[string]interface{} `json:"auths"` +} + +// BuildFromEnvvars complete the configuration based on envvars - this may remove existing values +// **IMPORTANT** this function does not guarantee validity of the generated configuration +// +// This will build the same config file given the same set of envvars, regardless of the initial +// config.yaml. This means that this function could be run multiple times and always give a consistent +// output. This is to ensure that conflicting config values are never rendered - for example, if the +// existing config had an external database and then an in-cluster database was set, we could end up +// with both in-cluster and external databases configured. This would then fail validation checks and +// would not be installable. +// +// The only exceptions to this are parameters which are **NEVER** set by an envvar, such as containerd +// location or ShiftFS/Fuse status. +func (v version) BuildFromEnvvars(in interface{}) error { + cfg, ok := in.(*Config) + if !ok { + return config.ErrInvalidType + } + + envvars := ConfigEnvvars{} + envvars.load() + + log.Infof("Detected envvars: %+v", envvars) + + cfg.Domain = envvars.Domain + + if licenseName := envvars.LicenseName; licenseName != "" { + log.Infof("Setting license name: %s", licenseName) + cfg.License = &ObjectRef{ + Kind: ObjectRefSecret, + Name: licenseName, + } + } else { + cfg.License = nil + } + + if httpProxy := envvars.HTTPProxyName; httpProxy != "" { + log.Infof("Setting HTTP proxy name: %s", httpProxy) + cfg.HTTPProxy = &ObjectRef{ + Kind: ObjectRefSecret, + Name: httpProxy, + } + } else { + cfg.HTTPProxy = nil + } + + log.Info("Setting Open VSX URL") + cfg.OpenVSX = OpenVSX{ + URL: func() string { + if v := envvars.OpenVSXUrl; v != "" { + log.Infof("Setting Open VSX URL to %s", v) + return v + } + + log.Info("Setting Open VSX URL to default") + return defaultOpenVSXURL + }(), + } + + log.Infof("DB incluster: %v", envvars.DBInClusterEnabled) + + cfg.Database.InCluster = pointer.Bool(true) + cfg.Database.CloudSQL = nil + cfg.Database.External = nil + + if !envvars.DBInClusterEnabled { + cfg.Database.InCluster = pointer.Bool(false) + + if envvars.DBCloudSQLEnabled { + log.Info("Configuring CloudSQLProxy for database") + cfg.Database.CloudSQL = &DatabaseCloudSQL{ + Instance: envvars.DBCloudSQLInstance, + ServiceAccount: ObjectRef{ + Kind: ObjectRefSecret, + Name: envvars.DBCloudSQLServiceAccountName, + }, + } + } else { + log.Info("Configuring external database") + cfg.Database.External = &DatabaseExternal{ + Certificate: ObjectRef{ + Kind: ObjectRefSecret, + Name: envvars.DBExternalCertificateName, + }, + } + } + } + + cfg.Repository = defaultRepositoryUrl + cfg.ImagePullSecrets = nil + cfg.DropImageRepo = nil + cfg.ContainerRegistry.PrivateBaseImageAllowList = []string{} + + if envvars.LocalRegistryEnabled { + log.Info("Configuring mirrored container registry for airgapped installation") + + cfg.Repository = envvars.LocalRegistryAddress + cfg.ImagePullSecrets = append(cfg.ImagePullSecrets, ObjectRef{ + Kind: ObjectRefSecret, + Name: envvars.ImagePullSecretName, + }) + cfg.DropImageRepo = pointer.Bool(true) + + cfg.ContainerRegistry.PrivateBaseImageAllowList = append(cfg.ContainerRegistry.PrivateBaseImageAllowList, envvars.LocalRegistryHost) + } + + if envvars.RegistryDockerConfigEnabled { + log.Info("Extracting servers from the custom registry authentication") + var registryDockerConfig dockerConfigJson + if err := yaml.Unmarshal([]byte(envvars.RegistryDockerConfig), ®istryDockerConfig); err != nil { + return err + } + + // Sort the registry auths to make the tests consistent + registryHosts := make([]string, 0, len(registryDockerConfig.Auths)) + for k := range registryDockerConfig.Auths { + registryHosts = append(registryHosts, k) + } + sort.Strings(registryHosts) + cfg.ContainerRegistry.PrivateBaseImageAllowList = append(cfg.ContainerRegistry.PrivateBaseImageAllowList, registryHosts...) + } + + if len(cfg.ContainerRegistry.PrivateBaseImageAllowList) > 0 { + // Docker should always be in allow list if other values set + cfg.ContainerRegistry.PrivateBaseImageAllowList = append(cfg.ContainerRegistry.PrivateBaseImageAllowList, "docker.io") + } + + log.Infof("Registry incluster: %v", envvars.RegistryInClusterEnabled) + + cfg.ContainerRegistry.InCluster = pointer.Bool(true) + cfg.ContainerRegistry.External = nil + cfg.ContainerRegistry.S3Storage = nil + if !envvars.RegistryInClusterEnabled { + log.Info("Configuring external container registry") + + cfg.ContainerRegistry.InCluster = pointer.Bool(false) + cfg.ContainerRegistry.External = &ContainerRegistryExternal{ + URL: envvars.RegistryExternalURL, + Certificate: ObjectRef{ + Kind: ObjectRefSecret, + Name: envvars.RegistryExternalCertName, + }, + } + } else { + if envvars.RegistryInClusterStorageType == "s3" { + log.Info("Configuring in-cluster container registry S3 storage") + + cfg.ContainerRegistry.S3Storage = &S3Storage{ + Region: envvars.RegistryInClusterStorageS3Region, + Endpoint: envvars.RegistryInClusterStorageS3Endpoint, + Bucket: envvars.RegistryInClusterStorageS3BucketName, + Certificate: ObjectRef{ + Kind: ObjectRefSecret, + Name: envvars.RegistryInClusterStorageS3CertName, + }, + } + } + } + + log.Infof("Storage provider: %s", envvars.StorageProvider) + + cfg.ObjectStorage.InCluster = pointer.Bool(true) + cfg.Metadata.Region = defaultMetadataRegion + cfg.ObjectStorage.Azure = nil + cfg.ObjectStorage.CloudStorage = nil + cfg.ObjectStorage.S3 = nil + if storageProvider := envvars.StorageProvider; storageProvider != "incluster" { + log.Info("Configuring the storage provider") + + cfg.Metadata.Region = envvars.StorageRegion + cfg.ObjectStorage.InCluster = pointer.Bool(false) + + switch storageProvider { + case "azure": + log.Infof("Configuring storage for Azure") + cfg.ObjectStorage.Azure = &ObjectStorageAzure{ + Credentials: ObjectRef{ + Kind: ObjectRefSecret, + Name: envvars.StorageAzureCredsName, + }, + } + case "gcp": + log.Infof("Configuring storage for GCP") + cfg.ObjectStorage.CloudStorage = &ObjectStorageCloudStorage{ + Project: envvars.StorageGCPProjectName, + ServiceAccount: ObjectRef{ + Kind: ObjectRefSecret, + Name: envvars.StorageGCPServiceAccountName, + }, + } + case "s3": + log.Infof("Configuring storage for S3") + cfg.ObjectStorage.S3 = &ObjectStorageS3{ + Endpoint: envvars.StorageS3Endpoint, + BucketName: envvars.StorageS3Bucket, + Credentials: ObjectRef{ + Kind: ObjectRefSecret, + Name: envvars.StorageS3CredsName, + }, + } + default: + return fmt.Errorf("unknown storage provider: %s", storageProvider) + } + } + + cfg.SSHGatewayHostKey = nil + if envvars.SSHGatewayEnabled { + log.Info("Configuring SSH gateway host key") + cfg.SSHGatewayHostKey = &ObjectRef{ + Kind: ObjectRefSecret, + Name: envvars.SSHGatewayHostKeyName, + } + } + + cfg.CustomCACert = nil + cfg.CustomCACert = nil + if envvars.TLSSelfSignedEnabled { + log.Info("Generating a self-signed certificate with the internal CA") + cfg.CustomCACert = &ObjectRef{ + Kind: ObjectRefSecret, + Name: "ca-issuer-ca", // This comes from common/constants.go + } + } else if !envvars.TLSCertManagerEnabled && envvars.TLSCustomCACertEnabled { + log.Info("Setting the CA to be used for the certificate") + cfg.CustomCACert = &ObjectRef{ + Kind: ObjectRefSecret, + Name: envvars.TLSCustomCACertCredsName, + } + } + + cfg.BlockNewUsers.Enabled = false + cfg.BlockNewUsers.Passlist = []string{} + if envvars.UserManagementBlockEnabled { + log.Info("Enabling user management") + cfg.BlockNewUsers.Enabled = true + + for _, domain := range envvars.UserManagementBlockPassList { + log.Infof("Adding domain %s to block new users pass list", domain) + cfg.BlockNewUsers.Passlist = append(cfg.BlockNewUsers.Passlist, domain) + } + } + + cfg.Components = nil + cfg.Customization = nil + if envvars.AdvancedModeEnabled { + log.Info("Applying advanced configuration") + + if serviceType := envvars.ComponentProxyServiceType; serviceType != "" { + log.Infof("Applying proxy service type: %s", serviceType) + cfg.Components = &Components{ + Proxy: &ProxyComponent{ + Service: &ComponentTypeService{ + ServiceType: (*v1.ServiceType)(&serviceType), + }, + }, + } + } + + if customizationPatch := envvars.CustomizationPatch; customizationPatch != "" { + log.Info("Applying customization") + + var customization Config + err := yaml.Unmarshal([]byte(customizationPatch), &customization) + if err != nil { + return err + } + + if customization.Customization == nil { + log.Info("No customization components added") + } else { + log.Infof("Adding %+v customization param(s)", *customization.Customization) + } + + cfg.Customization = customization.Customization + } + } else { + log.Info("No advanced configuration applied") + } + + if telemetryValue := envvars.Distribution; telemetryValue != "" { + if cfg.Experimental == nil { + cfg.Experimental = &experimental.Config{} + } + + if cfg.Experimental.Telemetry == nil { + cfg.Experimental.Telemetry = &experimental.TelemetryConfig{} + } + cfg.Experimental.Telemetry.Data.Platform = telemetryValue + } + + if cfgPatch := envvars.ConfigPatch; cfgPatch != "" { + log.Warnf("Applying patch customization - this may overwrite all settings: %+v", cfgPatch) + + err := yaml.Unmarshal([]byte(cfgPatch), &cfg) + if err != nil { + return err + } + } + + return nil +} diff --git a/install/installer/pkg/config/v1/envvars_test.go b/install/installer/pkg/config/v1/envvars_test.go new file mode 100644 index 00000000000000..36b938bd5f1e3f --- /dev/null +++ b/install/installer/pkg/config/v1/envvars_test.go @@ -0,0 +1,96 @@ +// Copyright (c) 2022 Gitpod GmbH. All rights reserved. +// Licensed under the MIT License. See License-MIT.txt in the project root for license information. + +package config_test + +import ( + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/gitpod-io/gitpod/common-go/log" + "github.com/gitpod-io/gitpod/installer/pkg/config" + "github.com/go-test/deep" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" + "sigs.k8s.io/yaml" +) + +func TestMain(m *testing.M) { + // Set to highest logging level to keep console clean + log.Log.Logger.SetLevel(logrus.PanicLevel) + + os.Exit(m.Run()) +} + +type envvarTestData struct { + Envvars map[string]string `json:"envvars"` +} + +func TestBuildFromEnvvars(t *testing.T) { + baseDir := "testdata/envvars" + + dir, err := ioutil.ReadDir(baseDir) + require.NoError(t, err) + + var testCases []struct { + Name string + } + + for _, d := range dir { + if d.IsDir() { + testCases = append(testCases, struct{ Name string }{ + Name: d.Name(), + }) + } + } + + for _, testCase := range testCases { + t.Run(testCase.Name, func(t *testing.T) { + cfg, version, err := config.Load("", true) + require.NoError(t, err) + + apiVersion, err := config.LoadConfigVersion(version) + require.NoError(t, err) + + testPath := filepath.Join(baseDir, testCase.Name) + envvarsFile := filepath.Join(testPath, "envvars.yaml") + expectFile := filepath.Join(testPath, "expect.yaml") + + // Load the envvars + envF, err := os.OpenFile(envvarsFile, os.O_RDONLY, 0644) + defer envF.Close() + require.NoError(t, err) + + envContent, err := ioutil.ReadAll(envF) + require.NoError(t, err) + + var env envvarTestData + err = yaml.Unmarshal(envContent, &env) + require.NoError(t, err) + + // Load the expected output + expectF, err := os.OpenFile(expectFile, os.O_RDONLY, 0644) + defer expectF.Close() + require.NoError(t, err) + + expectContent, err := ioutil.ReadAll(expectF) + require.NoError(t, err) + + for k, v := range env.Envvars { + t.Setenv(k, v) + } + + err = apiVersion.BuildFromEnvvars(cfg) + require.NoError(t, err) + + expect, _, err := config.Load(string(expectContent), true) + require.NoError(t, err) + + if diff := deep.Equal(cfg, expect); diff != nil { + t.Error(diff) + } + }) + } +} diff --git a/install/installer/pkg/config/v1/testdata/envvars/README.md b/install/installer/pkg/config/v1/testdata/envvars/README.md new file mode 100644 index 00000000000000..15684430261f84 --- /dev/null +++ b/install/installer/pkg/config/v1/testdata/envvars/README.md @@ -0,0 +1,23 @@ +# Envvars + +This test is designed to ensure that the environment variables create the expected +configuration file. + +- create a new directory with a descriptive test name +- create an `envvars.yaml` file - this contains a map of the environment variables that will be loaded for your test +- create an `expect.yaml` file - this contains the config YAML that you expect to be generated by the test + +## Example envvars.yaml + +```yaml +envvars: + DOMAIN: gitpod.io +``` + +## Example expect.yaml + +```yaml +domain: gitpod.io +``` + +This does not require the factory/default values that will be generated automatically diff --git a/install/installer/pkg/config/v1/testdata/envvars/additional-registry/envvars.yaml b/install/installer/pkg/config/v1/testdata/envvars/additional-registry/envvars.yaml new file mode 100644 index 00000000000000..99933c5c342216 --- /dev/null +++ b/install/installer/pkg/config/v1/testdata/envvars/additional-registry/envvars.yaml @@ -0,0 +1,7 @@ +# Copyright (c) 2022 Gitpod GmbH. All rights reserved. +# Licensed under the MIT License. See License-MIT.txt in the project root for license information. + +envvars: + DOMAIN: gitpod.io + REG_DOCKER_CONFIG_ENABLED: "1" + REG_DOCKER_CONFIG_JSON: '{ "auths": { "host1": "", "host2": "", "host3": "" } }' diff --git a/install/installer/pkg/config/v1/testdata/envvars/additional-registry/expect.yaml b/install/installer/pkg/config/v1/testdata/envvars/additional-registry/expect.yaml new file mode 100644 index 00000000000000..be5a8e1156c36d --- /dev/null +++ b/install/installer/pkg/config/v1/testdata/envvars/additional-registry/expect.yaml @@ -0,0 +1,11 @@ +# Copyright (c) 2022 Gitpod GmbH. All rights reserved. +# Licensed under the MIT License. See License-MIT.txt in the project root for license information. + +domain: gitpod.io +containerRegistry: + inCluster: true + privateBaseImageAllowList: + - host1 + - host2 + - host3 + - docker.io diff --git a/install/installer/pkg/config/v1/testdata/envvars/airgapped-registry/envvars.yaml b/install/installer/pkg/config/v1/testdata/envvars/airgapped-registry/envvars.yaml new file mode 100644 index 00000000000000..8576ac54c5330e --- /dev/null +++ b/install/installer/pkg/config/v1/testdata/envvars/airgapped-registry/envvars.yaml @@ -0,0 +1,9 @@ +# Copyright (c) 2022 Gitpod GmbH. All rights reserved. +# Licensed under the MIT License. See License-MIT.txt in the project root for license information. + +envvars: + DOMAIN: gitpod.io + HAS_LOCAL_REGISTRY: "1" + IMAGE_PULL_SECRET_NAME: local-registry-pull-secret + LOCAL_REGISTRY_ADDRESS: local-registry-address.com + LOCAL_REGISTRY_HOST: local-registry-host.com diff --git a/install/installer/pkg/config/v1/testdata/envvars/airgapped-registry/expect.yaml b/install/installer/pkg/config/v1/testdata/envvars/airgapped-registry/expect.yaml new file mode 100644 index 00000000000000..3392116a98b932 --- /dev/null +++ b/install/installer/pkg/config/v1/testdata/envvars/airgapped-registry/expect.yaml @@ -0,0 +1,14 @@ +# Copyright (c) 2022 Gitpod GmbH. All rights reserved. +# Licensed under the MIT License. See License-MIT.txt in the project root for license information. + +containerRegistry: + inCluster: true + privateBaseImageAllowList: + - local-registry-host.com + - docker.io +domain: gitpod.io +dropImageRepo: true +imagePullSecrets: + - kind: secret + name: local-registry-pull-secret +repository: local-registry-address.com diff --git a/install/installer/pkg/config/v1/testdata/envvars/aws/envvars.yaml b/install/installer/pkg/config/v1/testdata/envvars/aws/envvars.yaml new file mode 100644 index 00000000000000..cc03a15dcf7eb3 --- /dev/null +++ b/install/installer/pkg/config/v1/testdata/envvars/aws/envvars.yaml @@ -0,0 +1,18 @@ +# Copyright (c) 2022 Gitpod GmbH. All rights reserved. +# Licensed under the MIT License. See License-MIT.txt in the project root for license information. + +envvars: + DB_INCLUSTER_ENABLED: "0" + DB_EXTERNAL_CERTIFICATE_NAME: database-secret + DOMAIN: gitpod.io + REG_INCLUSTER_ENABLED: "1" + REG_INCLUSTER_STORAGE_S3_BUCKETNAME: container-s3-bucket + REG_INCLUSTER_STORAGE_S3_CERTIFICATE_NAME: container-s3-secret + REG_INCLUSTER_STORAGE_S3_ENDPOINT: container-s3-bucket.com + REG_INCLUSTER_STORAGE_S3_REGION: container-s3-region + REG_INCLUSTER_STORAGE: s3 + STORE_PROVIDER: s3 + STORE_REGION: s3-region + STORE_S3_BUCKET: s3-bucket + STORE_S3_CREDENTIALS_NAME: s3-secret + STORE_S3_ENDPOINT: s3-endpoint.com diff --git a/install/installer/pkg/config/v1/testdata/envvars/aws/expect.yaml b/install/installer/pkg/config/v1/testdata/envvars/aws/expect.yaml new file mode 100644 index 00000000000000..7be3f76000d5f9 --- /dev/null +++ b/install/installer/pkg/config/v1/testdata/envvars/aws/expect.yaml @@ -0,0 +1,29 @@ +# Copyright (c) 2022 Gitpod GmbH. All rights reserved. +# Licensed under the MIT License. See License-MIT.txt in the project root for license information. + +containerRegistry: + inCluster: true + s3storage: + bucket: container-s3-bucket + endpoint: container-s3-bucket.com + region: container-s3-region + certificate: + kind: secret + name: container-s3-secret +database: + inCluster: false + external: + certificate: + kind: secret + name: database-secret +domain: gitpod.io +metadata: + region: s3-region +objectStorage: + inCluster: false + s3: + endpoint: s3-endpoint.com + credentials: + kind: secret + name: s3-secret + bucket: s3-bucket diff --git a/install/installer/pkg/config/v1/testdata/envvars/azure/envvars.yaml b/install/installer/pkg/config/v1/testdata/envvars/azure/envvars.yaml new file mode 100644 index 00000000000000..6c031222c83f0b --- /dev/null +++ b/install/installer/pkg/config/v1/testdata/envvars/azure/envvars.yaml @@ -0,0 +1,13 @@ +# Copyright (c) 2022 Gitpod GmbH. All rights reserved. +# Licensed under the MIT License. See License-MIT.txt in the project root for license information. + +envvars: + DB_INCLUSTER_ENABLED: "0" + DB_EXTERNAL_CERTIFICATE_NAME: azure-db-secret + DOMAIN: gitpod.io + REG_INCLUSTER_ENABLED: "0" + REG_EXTERNAL_CERTIFICATE_NAME: azure-reg-secret + REG_URL: azure-reg-url + STORE_PROVIDER: azure + STORE_REGION: azure-region + STORE_AZURE_CREDENTIALS_NAME: azure-store-secret diff --git a/install/installer/pkg/config/v1/testdata/envvars/azure/expect.yaml b/install/installer/pkg/config/v1/testdata/envvars/azure/expect.yaml new file mode 100644 index 00000000000000..93dffbd40b36ac --- /dev/null +++ b/install/installer/pkg/config/v1/testdata/envvars/azure/expect.yaml @@ -0,0 +1,25 @@ +# Copyright (c) 2022 Gitpod GmbH. All rights reserved. +# Licensed under the MIT License. See License-MIT.txt in the project root for license information. + +containerRegistry: + inCluster: false + external: + url: azure-reg-url + certificate: + kind: secret + name: azure-reg-secret +database: + inCluster: false + external: + certificate: + kind: secret + name: azure-db-secret +domain: gitpod.io +metadata: + region: azure-region +objectStorage: + inCluster: false + azure: + credentials: + kind: secret + name: azure-store-secret diff --git a/install/installer/pkg/config/v1/testdata/envvars/config-options/envvars.yaml b/install/installer/pkg/config/v1/testdata/envvars/config-options/envvars.yaml new file mode 100644 index 00000000000000..dbd3492dafbaa0 --- /dev/null +++ b/install/installer/pkg/config/v1/testdata/envvars/config-options/envvars.yaml @@ -0,0 +1,15 @@ +# Copyright (c) 2022 Gitpod GmbH. All rights reserved. +# Licensed under the MIT License. See License-MIT.txt in the project root for license information. + +envvars: + DOMAIN: test.gitpod.io + DISTRIBUTION: distribution-name + HTTP_PROXY_NAME: http-proxy-settings + LICENSE_NAME: gitpod-license + LOCAL_REGISTRY_ADDRESS: mylocalregistry.com + IMAGE_PULL_SECRET_NAME: image-pull-secret + OPEN_VSX_URL: https://my-openvsx.com + SSH_GATEWAY: "1" + SSH_GATEWAY_HOST_KEY_NAME: ssh-gateway-secret + USER_MANAGEMENT_BLOCK_ENABLED: "1" + USER_MANAGEMENT_BLOCK_PASSLIST: gitpod.io domain.com domain2.com diff --git a/install/installer/pkg/config/v1/testdata/envvars/config-options/expect.yaml b/install/installer/pkg/config/v1/testdata/envvars/config-options/expect.yaml new file mode 100644 index 00000000000000..feaa2d6d9a0a78 --- /dev/null +++ b/install/installer/pkg/config/v1/testdata/envvars/config-options/expect.yaml @@ -0,0 +1,26 @@ +# Copyright (c) 2022 Gitpod GmbH. All rights reserved. +# Licensed under the MIT License. See License-MIT.txt in the project root for license information. + +blockNewUsers: + enabled: true + passlist: + - gitpod.io + - domain.com + - domain2.com +domain: test.gitpod.io +httpProxy: + kind: secret + name: http-proxy-settings +license: + kind: secret + name: gitpod-license +openVSX: + url: https://my-openvsx.com +sshGatewayHostKey: + kind: secret + name: ssh-gateway-secret + +experimental: + telemetry: + data: + platform: distribution-name diff --git a/install/installer/pkg/config/v1/testdata/envvars/config-patch/envvars.yaml b/install/installer/pkg/config/v1/testdata/envvars/config-patch/envvars.yaml new file mode 100644 index 00000000000000..829a0fd33ae7ad --- /dev/null +++ b/install/installer/pkg/config/v1/testdata/envvars/config-patch/envvars.yaml @@ -0,0 +1,6 @@ +# Copyright (c) 2022 Gitpod GmbH. All rights reserved. +# Licensed under the MIT License. See License-MIT.txt in the project root for license information. + +envvars: + CONFIG_PATCH: "domain: override.gitpod.io" + DOMAIN: gitpod.io diff --git a/install/installer/pkg/config/v1/testdata/envvars/config-patch/expect.yaml b/install/installer/pkg/config/v1/testdata/envvars/config-patch/expect.yaml new file mode 100644 index 00000000000000..eea42fbf027412 --- /dev/null +++ b/install/installer/pkg/config/v1/testdata/envvars/config-patch/expect.yaml @@ -0,0 +1,4 @@ +# Copyright (c) 2022 Gitpod GmbH. All rights reserved. +# Licensed under the MIT License. See License-MIT.txt in the project root for license information. + +domain: override.gitpod.io diff --git a/install/installer/pkg/config/v1/testdata/envvars/customization-patch/envvars.yaml b/install/installer/pkg/config/v1/testdata/envvars/customization-patch/envvars.yaml new file mode 100644 index 00000000000000..1833ff6ab1115d --- /dev/null +++ b/install/installer/pkg/config/v1/testdata/envvars/customization-patch/envvars.yaml @@ -0,0 +1,7 @@ +# Copyright (c) 2022 Gitpod GmbH. All rights reserved. +# Licensed under the MIT License. See License-MIT.txt in the project root for license information. + +envvars: + ADVANCED_MODE_ENABLED: "1" + CUSTOMIZATION_PATCH: "customization:\n - apiVersion: \"*\"\n kind: \"*\"\n metadata:\n name: \"*\"\n annotations:\n appliedToAll: value\n hello: world\n labels:\n appliedToAll: value\n hello: world" + DOMAIN: gitpod.io diff --git a/install/installer/pkg/config/v1/testdata/envvars/customization-patch/expect.yaml b/install/installer/pkg/config/v1/testdata/envvars/customization-patch/expect.yaml new file mode 100644 index 00000000000000..d4e653338f2eeb --- /dev/null +++ b/install/installer/pkg/config/v1/testdata/envvars/customization-patch/expect.yaml @@ -0,0 +1,15 @@ +# Copyright (c) 2022 Gitpod GmbH. All rights reserved. +# Licensed under the MIT License. See License-MIT.txt in the project root for license information. + +customization: + - apiVersion: "*" + kind: "*" + metadata: + name: "*" + annotations: + appliedToAll: value + hello: world + labels: + appliedToAll: value + hello: world +domain: gitpod.io diff --git a/install/installer/pkg/config/v1/testdata/envvars/gcp/envvars.yaml b/install/installer/pkg/config/v1/testdata/envvars/gcp/envvars.yaml new file mode 100644 index 00000000000000..2b7db57d5c8bc1 --- /dev/null +++ b/install/installer/pkg/config/v1/testdata/envvars/gcp/envvars.yaml @@ -0,0 +1,16 @@ +# Copyright (c) 2022 Gitpod GmbH. All rights reserved. +# Licensed under the MIT License. See License-MIT.txt in the project root for license information. + +envvars: + DB_INCLUSTER_ENABLED: "0" + DB_CLOUDSQL_SERVICE_ACCOUNT_NAME: gcp-db-service-account + DB_CLOUDSQL_INSTANCE: gcp-db-instance + DB_CLOUDSQL_ENABLED: "1" + DOMAIN: gitpod.io + REG_INCLUSTER_ENABLED: "0" + REG_EXTERNAL_CERTIFICATE_NAME: gcp-reg-secret + REG_URL: gcp-reg-url + STORE_PROVIDER: gcp + STORE_REGION: gcp-region + STORE_GCP_PROJECT: gcp-project-name + STORE_GCP_SERVICE_ACCOUNT_NAME: gcp-store-secret diff --git a/install/installer/pkg/config/v1/testdata/envvars/gcp/expect.yaml b/install/installer/pkg/config/v1/testdata/envvars/gcp/expect.yaml new file mode 100644 index 00000000000000..b9ca310939bbbd --- /dev/null +++ b/install/installer/pkg/config/v1/testdata/envvars/gcp/expect.yaml @@ -0,0 +1,27 @@ +# Copyright (c) 2022 Gitpod GmbH. All rights reserved. +# Licensed under the MIT License. See License-MIT.txt in the project root for license information. + +containerRegistry: + inCluster: false + external: + url: gcp-reg-url + certificate: + kind: secret + name: gcp-reg-secret +database: + inCluster: false + cloudSQL: + instance: gcp-db-instance + serviceAccount: + kind: secret + name: gcp-db-service-account +domain: gitpod.io +metadata: + region: gcp-region +objectStorage: + inCluster: false + cloudStorage: + project: gcp-project-name + serviceAccount: + kind: secret + name: gcp-store-secret diff --git a/install/installer/pkg/config/v1/testdata/envvars/minimal/envvars.yaml b/install/installer/pkg/config/v1/testdata/envvars/minimal/envvars.yaml new file mode 100644 index 00000000000000..eb43ad2d555deb --- /dev/null +++ b/install/installer/pkg/config/v1/testdata/envvars/minimal/envvars.yaml @@ -0,0 +1,5 @@ +# Copyright (c) 2022 Gitpod GmbH. All rights reserved. +# Licensed under the MIT License. See License-MIT.txt in the project root for license information. + +envvars: + DOMAIN: gitpod.io diff --git a/install/installer/pkg/config/v1/testdata/envvars/minimal/expect.yaml b/install/installer/pkg/config/v1/testdata/envvars/minimal/expect.yaml new file mode 100644 index 00000000000000..1b6bed9ae0630c --- /dev/null +++ b/install/installer/pkg/config/v1/testdata/envvars/minimal/expect.yaml @@ -0,0 +1,4 @@ +# Copyright (c) 2022 Gitpod GmbH. All rights reserved. +# Licensed under the MIT License. See License-MIT.txt in the project root for license information. + +domain: gitpod.io diff --git a/install/installer/pkg/config/v1/testdata/envvars/service-type/envvars.yaml b/install/installer/pkg/config/v1/testdata/envvars/service-type/envvars.yaml new file mode 100644 index 00000000000000..ca2c9c370e3126 --- /dev/null +++ b/install/installer/pkg/config/v1/testdata/envvars/service-type/envvars.yaml @@ -0,0 +1,7 @@ +# Copyright (c) 2022 Gitpod GmbH. All rights reserved. +# Licensed under the MIT License. See License-MIT.txt in the project root for license information. + +envvars: + ADVANCED_MODE_ENABLED: "1" + COMPONENT_PROXY_SERVICE_SERVICETYPE: ClusterIP + DOMAIN: gitpod.io diff --git a/install/installer/pkg/config/v1/testdata/envvars/service-type/expect.yaml b/install/installer/pkg/config/v1/testdata/envvars/service-type/expect.yaml new file mode 100644 index 00000000000000..15ed13486acc9c --- /dev/null +++ b/install/installer/pkg/config/v1/testdata/envvars/service-type/expect.yaml @@ -0,0 +1,8 @@ +# Copyright (c) 2022 Gitpod GmbH. All rights reserved. +# Licensed under the MIT License. See License-MIT.txt in the project root for license information. + +domain: gitpod.io +components: + proxy: + service: + serviceType: ClusterIP