Use structured configuration

Author: astefanutti

.github/workflows/e2e_tests.yaml

@@ -73,9 +73,6 @@ jobs:
           echo Deploying CodeFlare operator
           IMG="${REGISTRY_ADDRESS}"/codeflare-operator
           make image-push -e IMG="${IMG}"
-          # FIXME
-          kubectl create ns openshift-operators
-          kubectl apply -n openshift-operators -f config/default/instascale-configmap.yaml
           make deploy -e IMG="${IMG}"
           kubectl wait --timeout=120s --for=condition=Available=true deployment -n openshift-operators codeflare-operator-manager
 

.github/workflows/olm_tests.yaml

@@ -117,9 +117,6 @@ jobs:
 
       - name: Deploy CodeFlare stack
         run: |
-          # FIXME
-          kubectl apply -n openshift-operators -f config/default/instascale-configmap.yaml
-
           make setup-e2e
 
       - name: Build operator and catalog image

Dockerfile

@@ -7,8 +7,9 @@ COPY go.mod go.mod
 COPY go.sum go.sum
 RUN go mod download
 
-# Copy the go source
+# Copy the Go sources
 COPY main.go main.go
+COPY pkg/ pkg/
 
 # Build
 USER root

config/default/instascale-configmap.yaml

@@ -28,8 +28,6 @@ bases:
 resources:
 # Add metrics service
 - metrics_service.yaml
-# TODO: refactor configuration
-- instascale-configmap.yaml
 
 # Mount the controller config file for loading manager configurations
 # through a ComponentConfig type

config/default/kustomization.yaml

@@ -37,8 +37,6 @@ spec:
         args:
         - "--health-probe-bind-address=:8081"
         - "--metrics-bind-address=0.0.0.0:8082"
-        - "--leader-elect"
-        - "--configs-namespace=openshift-operators"
         image: controller:latest
         imagePullPolicy: Always
         name: manager

config/manager/manager.yaml

@@ -7,20 +7,19 @@ require (
 	github.com/openshift-eng/openshift-goimports v0.0.0-20230304234052-c70783e636f2
 	github.com/openshift/api v0.0.0-20230213134911-7ba313770556
 	github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c
-	github.com/project-codeflare/instascale v0.0.0-00010101000000-000000000000
-	github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.1-0.20230904071404-ce734452edc6
+	github.com/project-codeflare/instascale v0.0.9-0.20230913044449-3db3f4b4e9b1
+	github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.2-0.20230913035103-6e7bbdc8fb1f
 	github.com/ray-project/kuberay/ray-operator v0.0.0-20230908233208-a8f730e5a2b6
 	go.uber.org/zap v1.24.0
 	k8s.io/api v0.26.3
 	k8s.io/apimachinery v0.26.3
 	k8s.io/client-go v0.26.3
+	k8s.io/component-base v0.26.2
+	k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5
 	sigs.k8s.io/controller-runtime v0.14.6
 )
 
-replace (
-	github.com/project-codeflare/instascale => github.com/Bobbins228/instascale v0.0.0-20230724130835-2a8ebf58690f
-	sigs.k8s.io/custom-metrics-apiserver => sigs.k8s.io/custom-metrics-apiserver v1.25.1-0.20230306170449-63d8c93851f3
-)
+replace sigs.k8s.io/custom-metrics-apiserver => sigs.k8s.io/custom-metrics-apiserver v1.25.1-0.20230306170449-63d8c93851f3
 
 require (
 	github.com/NYTimes/gziphandler v1.1.1 // indirect
@@ -125,13 +124,11 @@ require (
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/apiextensions-apiserver v0.26.1 // indirect
 	k8s.io/apiserver v0.26.2 // indirect
-	k8s.io/component-base v0.26.2 // indirect
 	k8s.io/klog v1.0.0 // indirect
 	k8s.io/klog/v2 v2.90.1 // indirect
 	k8s.io/kms v0.26.2 // indirect
 	k8s.io/kube-openapi v0.0.0-20230303024457-afdc3dddf62d // indirect
 	k8s.io/metrics v0.26.2 // indirect
-	k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 // indirect
 	sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.35 // indirect
 	sigs.k8s.io/custom-metrics-apiserver v0.0.0 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect

go.mod

@@ -33,8 +33,6 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl
 cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
-github.com/Bobbins228/instascale v0.0.0-20230724130835-2a8ebf58690f h1:aEtM6OZ2dCY97VnVsVYZl2GJOe+6WegPGLgLXQt5BGk=
-github.com/Bobbins228/instascale v0.0.0-20230724130835-2a8ebf58690f/go.mod h1:kwBkC+o4W+a1dpftXbVQjvGCVyu2206HlZtcp1+ZjyY=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
@@ -460,8 +458,10 @@ github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZ
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
-github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.1-0.20230904071404-ce734452edc6 h1:YT6C0Z84TgftYcoNUd/dwvXxGJvXy67EeWAMQjpKvdU=
-github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.1-0.20230904071404-ce734452edc6/go.mod h1:Yge6GRNpO9YIDfeL+XOcCE9xbmfCTD5C1h5dlW87mxQ=
+github.com/project-codeflare/instascale v0.0.9-0.20230913044449-3db3f4b4e9b1 h1:SNc05sEO/KBuRwep+u7J/ZrT06IhncjTHBvopc4LDlU=
+github.com/project-codeflare/instascale v0.0.9-0.20230913044449-3db3f4b4e9b1/go.mod h1:n2fIzs06LmGZFB3vRfQcXOkPebN3Hwd5Y+SD5v0hnrs=
+github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.2-0.20230913035103-6e7bbdc8fb1f h1:xrhSm18A/Gvmm9cia05eAGkj0i9fTMqeTExz8rQTHJU=
+github.com/project-codeflare/multi-cluster-app-dispatcher v1.34.2-0.20230913035103-6e7bbdc8fb1f/go.mod h1:Yge6GRNpO9YIDfeL+XOcCE9xbmfCTD5C1h5dlW87mxQ=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=

go.sum

@@ -22,18 +22,22 @@ import (
 	"time"
 
 	instascale "github.com/project-codeflare/instascale/controllers"
-	mcadoptions "github.com/project-codeflare/multi-cluster-app-dispatcher/cmd/kar-controllers/app/options"
 	mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1"
+	mcadconfig "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/config"
 	mcad "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/controller/queuejob"
 	"go.uber.org/zap/zapcore"
 
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
+	configv1alpha1 "k8s.io/component-base/config/v1alpha1"
+	"k8s.io/utils/pointer"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
+
+	"github.com/project-codeflare/codeflare-operator/pkg/config"
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
 	// to ensure that exec-entrypoint and run can make use of them.
 	// +kubebuilder:scaffold:imports
@@ -51,73 +55,60 @@ func init() {
 
 func main() {
 	var metricsAddr string
-	var enableLeaderElection bool
 	var probeAddr string
-	var configsNamespace string
-	var ocmSecretNamespace string
 
-	// Operator
 	flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
 	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
-	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
-		"Enable leader election for controller manager. "+
-			"Enabling this will ensure there is only one active controller manager.")
-
-	// InstScale
-	flag.StringVar(&configsNamespace, "configs-namespace", "kube-system", "The namespace containing the Instacale configmap")
-	flag.StringVar(&ocmSecretNamespace, "ocm-secret-namespace", "default", "The namespace containing the OCM secret")
-
-	mcadOptions := mcadoptions.NewServerOption()
 
 	zapOptions := zap.Options{
 		Development: true,
 		TimeEncoder: zapcore.TimeEncoderOfLayout(time.RFC3339),
 	}
-
-	flagSet := flag.NewFlagSet(os.Args[0], flag.ExitOnError)
-	flag.CommandLine.VisitAll(func(f *flag.Flag) {
-		if f.Name != "kubeconfig" {
-			flagSet.Var(f.Value, f.Name, f.Usage)
-		}
-	})
-
-	zapOptions.BindFlags(flagSet)
-	mcadOptions.AddFlags(flagSet)
-	_ = flagSet.Parse(os.Args[1:])
+	zapOptions.BindFlags(flag.CommandLine)
 
 	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&zapOptions)))
 
 	ctx := ctrl.SetupSignalHandler()
 
+	cfg := config.CodeFlareOperatorConfiguration{
+		LeaderElection: &configv1alpha1.LeaderElectionConfiguration{},
+		MCAD:           &mcadconfig.MCADConfiguration{},
+		InstaScale:     &config.InstaScaleConfiguration{},
+	}
+
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
-		Scheme:                 scheme,
-		MetricsBindAddress:     metricsAddr,
-		Port:                   9443,
-		HealthProbeBindAddress: probeAddr,
-		LeaderElection:         enableLeaderElection,
-		LeaderElectionID:       "5a3ca514.codeflare.dev",
+		Scheme:                     scheme,
+		MetricsBindAddress:         metricsAddr,
+		HealthProbeBindAddress:     probeAddr,
+		LeaderElection:             pointer.BoolDeref(cfg.LeaderElection.LeaderElect, false),
+		LeaderElectionID:           cfg.LeaderElection.ResourceName,
+		LeaderElectionNamespace:    cfg.LeaderElection.ResourceNamespace,
+		LeaderElectionResourceLock: cfg.LeaderElection.ResourceLock,
+		LeaseDuration:              &cfg.LeaderElection.LeaseDuration.Duration,
+		RetryPeriod:                &cfg.LeaderElection.RetryPeriod.Duration,
+		RenewDeadline:              &cfg.LeaderElection.RenewDeadline.Duration,
 	})
 	if err != nil {
 		setupLog.Error(err, "unable to start manager")
 		os.Exit(1)
 	}
 
-	mcadQueueController := mcad.NewJobController(mgr.GetConfig(), mcadOptions)
+	mcadQueueController := mcad.NewJobController(mgr.GetConfig(), cfg.MCAD, &mcadconfig.MCADConfigurationExtended{})
 	if mcadQueueController == nil {
 		// FIXME
 		os.Exit(1)
 	}
 	mcadQueueController.Run(ctx.Done())
 
-	instascaleController := &instascale.AppWrapperReconciler{
-		Client:             mgr.GetClient(),
-		Scheme:             mgr.GetScheme(),
-		ConfigsNamespace:   configsNamespace,
-		OcmSecretNamespace: ocmSecretNamespace,
+	if pointer.BoolDeref(cfg.InstaScale.Enabled, false) {
+		instaScaleController := &instascale.AppWrapperReconciler{
+			Client: mgr.GetClient(),
+			Scheme: mgr.GetScheme(),
+			Config: cfg.InstaScale.InstaScaleConfiguration,
+		}
+		exitOnError(instaScaleController.SetupWithManager(mgr), "Error setting up InstaScale controller")
 	}
 
-	exitOnError(instascaleController.SetupWithManager(mgr), "Error setting up InstaScale controller")
-
 	if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
 		setupLog.Error(err, "unable to set up health check")
 		os.Exit(1)

main.go

@@ -0,0 +1,46 @@
+/*
+Copyright 2023.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package config
+
+import (
+	instascale "github.com/project-codeflare/instascale/pkg/config"
+	mcad "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/config"
+
+	configv1alpha1 "k8s.io/component-base/config/v1alpha1"
+)
+
+type CodeFlareOperatorConfiguration struct {
+	// LeaderElection is the LeaderElection config to be used when configuring
+	// the manager.Manager leader election
+	LeaderElection *configv1alpha1.LeaderElectionConfiguration `json:"leaderElection,omitempty"`
+
+	// The MCAD controller configuration
+	MCAD *mcad.MCADConfiguration `json:"mcad,omitempty"`
+
+	// The InstaScale controller configuration
+	InstaScale *InstaScaleConfiguration `json:"instascale,omitempty"`
+}
+
+type InstaScaleConfiguration struct {
+	// enabled controls whether the InstaScale controller is started.
+	// It may default to true on platforms that InstaScale supports.
+	// Otherwise, it defaults to false.
+	Enabled *bool `json:"enabled,omitempty"`
+
+	// The InstaScale controller configuration
+	instascale.InstaScaleConfiguration `json:",inline,omitempty"`
+}