Add RayCluster mutating webhook

PROJECT

@@ -12,8 +12,12 @@ projectName: codeflare-operator
 repo: github.com/project-codeflare/codeflare-operator
 resources:
 - controller: true
-  domain: codeflare.dev
+  domain: ray.io
   group: ray
   kind: RayCluster
+  path: github.com/project-codeflare/codeflare-operator/pkg/controllers
   version: v1
+  webhooks:
+    defaulting: true
+    webhookVersion: v1
 version: "3"

config/default/kustomization.yaml

@@ -14,10 +14,10 @@ commonLabels:
   app.kubernetes.io/part-of: codeflare
 
 bases:
-- ../rbac
-- ../manager
+  - ../rbac
+  - ../manager
 # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'.
 # - ../prometheus
 
 resources:
-- metrics_service.yaml
+  - metrics_service.yaml

config/openshift/kustomization.yaml

@@ -0,0 +1,22 @@
+# Adds namespace to all resources.
+namespace: openshift-operators
+
+# Value of this field is prepended to the
+# names of all resources, e.g. a deployment named
+# "wordpress" becomes "alices-wordpress".
+# Note that it should also match with the prefix (text before '-') of the namespace
+# field above.
+namePrefix: codeflare-operator-
+
+# Labels to add to all resources and selectors.
+commonLabels:
+  app.kubernetes.io/name: codeflare-operator
+  app.kubernetes.io/part-of: codeflare
+
+bases:
+  - ../default
+  - ../webhook
+
+patches:
+- path: manager_webhook_patch.yaml
+- path: webhookcainjection_patch.yaml

config/openshift/manager_webhook_patch.yaml

@@ -0,0 +1,23 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: manager
+  namespace: system
+spec:
+  template:
+    spec:
+      containers:
+      - name: manager
+        ports:
+        - containerPort: 9443
+          name: webhook-server
+          protocol: TCP
+        volumeMounts:
+        - mountPath: /tmp/k8s-webhook-server/serving-certs
+          name: cert
+          readOnly: true
+      volumes:
+      - name: cert
+        secret:
+          defaultMode: 420
+          secretName: codeflare-operator-raycluster-webhook-cert

config/openshift/webhookcainjection_patch.yaml

@@ -0,0 +1,7 @@
+# This patch add annotation to admission webhook config
+apiVersion: admissionregistration.k8s.io/v1
+kind: MutatingWebhookConfiguration
+metadata:
+  name: mutating-webhook-configuration
+  annotations:
+    service.beta.openshift.io/inject-cabundle: "true"

config/webhook/kustomization.yaml

@@ -0,0 +1,6 @@
+resources:
+- manifests.yaml
+- service.yaml
+
+configurations:
+- kustomizeconfig.yaml

config/webhook/kustomizeconfig.yaml

@@ -0,0 +1,25 @@
+# the following config is for teaching kustomize where to look at when substituting vars.
+# It requires kustomize v2.1.0 or newer to work properly.
+nameReference:
+- kind: Service
+  version: v1
+  fieldSpecs:
+  - kind: MutatingWebhookConfiguration
+    group: admissionregistration.k8s.io
+    path: webhooks/clientConfig/service/name
+  - kind: ValidatingWebhookConfiguration
+    group: admissionregistration.k8s.io
+    path: webhooks/clientConfig/service/name
+
+namespace:
+- kind: MutatingWebhookConfiguration
+  group: admissionregistration.k8s.io
+  path: webhooks/clientConfig/service/namespace
+  create: true
+- kind: ValidatingWebhookConfiguration
+  group: admissionregistration.k8s.io
+  path: webhooks/clientConfig/service/namespace
+  create: true
+
+varReference:
+- path: metadata/annotations

config/webhook/manifests.yaml

@@ -0,0 +1,27 @@
+---
+apiVersion: admissionregistration.k8s.io/v1
+kind: MutatingWebhookConfiguration
+metadata:
+  creationTimestamp: null
+  name: mutating-webhook-configuration
+webhooks:
+- admissionReviewVersions:
+  - v1
+  clientConfig:
+    service:
+      name: webhook-service
+      namespace: system
+      path: /mutate-ray-io-v1-raycluster
+  failurePolicy: Fail
+  name: mraycluster.kb.io
+  rules:
+  - apiGroups:
+    - ray.io
+    apiVersions:
+    - v1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - rayclusters
+  sideEffects: None

config/webhook/service.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: webhook-service
+  namespace: openshift-operators
+  annotations:
+    service.beta.openshift.io/serving-cert-secret-name: codeflare-operator-raycluster-webhook-cert
+spec:
+  ports:
+    - port: 443
+      protocol: TCP
+      targetPort: 9443
+  selector:
+    app.kubernetes.io/part-of: codeflare
+    app.kubernetes.io/name: codeflare-operator

main.go

@@ -147,9 +147,21 @@ func main() {
 	})
 	exitOnError(err, "unable to start manager")
 
-	ok, err := HasAPIResourceForGVK(kubeClient.DiscoveryClient, rayv1.GroupVersion.WithKind("RayCluster"))
+	OpenShift := isOpenShift(ctx, kubeClient.DiscoveryClient)
+
+	if OpenShift {
+		// TODO: setup the RayCluster webhook on vanilla Kubernetes
+		exitOnError(controllers.SetupRayClusterWebhookWithManager(mgr, cfg.KubeRay), "error setting up RayCluster webhook")
+	}
+
+	ok, err := hasAPIResourceForGVK(kubeClient.DiscoveryClient, rayv1.GroupVersion.WithKind("RayCluster"))
 	if ok {
-		rayClusterController := controllers.RayClusterReconciler{Client: mgr.GetClient(), Scheme: mgr.GetScheme(), Config: cfg.KubeRay}
+		rayClusterController := controllers.RayClusterReconciler{
+			Client:      mgr.GetClient(),
+			Scheme:      mgr.GetScheme(),
+			Config:      cfg.KubeRay,
+			IsOpenShift: OpenShift,
+		}
 		exitOnError(rayClusterController.SetupWithManager(mgr), "Error setting up RayCluster controller")
 	} else if err != nil {
 		exitOnError(err, "Could not determine if RayCluster CR present on cluster.")
@@ -205,7 +217,7 @@ func createConfigMap(ctx context.Context, client kubernetes.Interface, ns, name
 	return err
 }
 
-func HasAPIResourceForGVK(dc discovery.DiscoveryInterface, gvk schema.GroupVersionKind) (bool, error) {
+func hasAPIResourceForGVK(dc discovery.DiscoveryInterface, gvk schema.GroupVersionKind) (bool, error) {
 	gv, kind := gvk.ToAPIVersionAndKind()
 	if resources, err := dc.ServerResourcesForGroupVersion(gv); err != nil {
 		if apierrors.IsNotFound(err) {
@@ -245,3 +257,20 @@ func exitOnError(err error, msg string) {
 		os.Exit(1)
 	}
 }
+
+func isOpenShift(ctx context.Context, dc discovery.DiscoveryInterface) bool {
+	logger := ctrl.LoggerFrom(ctx)
+	apiGroupList, err := dc.ServerGroups()
+	if err != nil {
+		logger.Info("Error while querying ServerGroups, assuming we're on Vanilla Kubernetes")
+		return false
+	}
+	for i := 0; i < len(apiGroupList.Groups); i++ {
+		if strings.HasSuffix(apiGroupList.Groups[i].Name, ".openshift.io") {
+			logger.Info("We detected being on OpenShift!")
+			return true
+		}
+	}
+	logger.Info("We detected being on Vanilla Kubernetes!")
+	return false
+}