Skip to content

Commit a62b141

Browse files
ChristianZaccariaChristianZaccaria
committed
Add OAuth config to reconciler struct
1 parent bd3f2f7 commit a62b141

File tree

4 files changed

+69
-89
lines changed

4 files changed

+69
-89
lines changed

‎go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ module github.com/project-codeflare/codeflare-operator
33
go 1.20
44

55
require (
6-
github.com/go-logr/logr v1.2.4
76
github.com/onsi/ginkgo/v2 v2.11.0
87
github.com/onsi/gomega v1.27.10
98
github.com/openshift/api v0.0.0-20230213134911-7ba313770556
@@ -43,6 +42,7 @@ require (
4342
github.com/evanphx/json-patch/v5 v5.6.0 // indirect
4443
github.com/felixge/httpsnoop v1.0.3 // indirect
4544
github.com/fsnotify/fsnotify v1.6.0 // indirect
45+
github.com/go-logr/logr v1.2.4 // indirect
4646
github.com/go-logr/stdr v1.2.2 // indirect
4747
github.com/go-logr/zapr v1.2.4 // indirect
4848
github.com/go-openapi/jsonpointer v0.19.6 // indirect

‎main.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -189,7 +189,7 @@ func main() {
189189

190190
v, err := HasAPIResourceForGVK(kubeClient.DiscoveryClient, rayv1.GroupVersion.WithKind("RayCluster"))
191191
if v {
192-
rayClusterController := controllers.RayClusterReconciler{Client: mgr.GetClient(), Scheme: mgr.GetScheme()}
192+
rayClusterController := controllers.RayClusterReconciler{Client: mgr.GetClient(), Scheme: mgr.GetScheme(), Config: cfg}
193193
exitOnError(rayClusterController.SetupWithManager(mgr), "Error setting up RayCluster controller")
194194
} else if err != nil {
195195
exitOnError(err, "Could not determine if RayCluster CR present on cluster.")

‎pkg/controllers/raycluster_controller.go

Lines changed: 21 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ type RayClusterReconciler struct {
5252
routeClient *routev1client.RouteV1Client
5353
Scheme *runtime.Scheme
5454
CookieSalt string
55+
Config *config.CodeFlareOperatorConfiguration
5556
}
5657

5758
const (
@@ -60,14 +61,13 @@ const (
6061
oAuthFinalizer = "ray.openshift.ai/oauth-finalizer"
6162
oAuthServicePort = 443
6263
oAuthServicePortName = "oauth-proxy"
63-
regularServicePortName = "dashboard"
64+
ingressServicePortName = "dashboard"
6465
logRequeueing = "requeueing"
6566
)
6667

6768
var (
68-
deletePolicy = metav1.DeletePropagationForeground
69-
deleteOptions = client.DeleteOptions{PropagationPolicy: &deletePolicy}
70-
configInstance *config.CodeFlareOperatorConfiguration
69+
deletePolicy = metav1.DeletePropagationForeground
70+
deleteOptions = client.DeleteOptions{PropagationPolicy: &deletePolicy}
7171
)
7272

7373
// +kubebuilder:rbac:groups=ray.io,resources=rayclusters,verbs=get;list;watch;create;update;patch;delete
@@ -101,9 +101,9 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request)
101101
return ctrl.Result{}, client.IgnoreNotFound(err)
102102
}
103103

104-
isLocalInteractive := annotationBoolVal(logger, &cluster, "sdk.codeflare.dev/local_interactive")
105-
isOpenShift, ingressHost := getClusterType(logger, r.kubeClient, &cluster)
106-
ingressDomain := cluster.ObjectMeta.Annotations["sdk.codeflare.dev/ingress_domain"]
104+
isLocalInteractive := annotationBoolVal(ctx, &cluster, "sdk.codeflare.dev/local_interactive", false)
105+
ingressDomain := "" // FIX - CFO will retrieve it.
106+
isOpenShift, ingressHost := getClusterType(ctx, r.kubeClient, &cluster, ingressDomain)
107107

108108
if cluster.ObjectMeta.DeletionTimestamp.IsZero() {
109109
if !controllerutil.ContainsFinalizer(&cluster, oAuthFinalizer) {
@@ -138,63 +138,63 @@ func (r *RayClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request)
138138
return ctrl.Result{}, nil
139139
}
140140

141-
if cluster.Status.State != "suspended" && isRayDashboardOAuthEnabled() && isOpenShift {
141+
if cluster.Status.State != "suspended" && r.isRayDashboardOAuthEnabled() && isOpenShift {
142142
logger.Info("Creating OAuth Objects")
143143
_, err := r.routeClient.Routes(cluster.Namespace).Apply(ctx, desiredClusterRoute(&cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true})
144144
if err != nil {
145145
logger.Error(err, "Failed to update OAuth Route")
146+
return ctrl.Result{RequeueAfter: requeueTime}, err
146147
}
147148

148149
_, err = r.kubeClient.CoreV1().Secrets(cluster.Namespace).Apply(ctx, desiredOAuthSecret(&cluster, r), metav1.ApplyOptions{FieldManager: controllerName, Force: true})
149150
if err != nil {
150151
logger.Error(err, "Failed to create OAuth Secret")
152+
return ctrl.Result{RequeueAfter: requeueTime}, err
151153
}
152154

153155
_, err = r.kubeClient.CoreV1().Services(cluster.Namespace).Apply(ctx, desiredOAuthService(&cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true})
154156
if err != nil {
155157
logger.Error(err, "Failed to update OAuth Service")
158+
return ctrl.Result{RequeueAfter: requeueTime}, err
156159
}
157160

158161
_, err = r.kubeClient.CoreV1().ServiceAccounts(cluster.Namespace).Apply(ctx, desiredServiceAccount(&cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true})
159162
if err != nil {
160163
logger.Error(err, "Failed to update OAuth ServiceAccount")
164+
return ctrl.Result{RequeueAfter: requeueTime}, err
161165
}
162166

163167
_, err = r.kubeClient.RbacV1().ClusterRoleBindings().Apply(ctx, desiredOAuthClusterRoleBinding(&cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true})
164168
if err != nil {
165169
logger.Error(err, "Failed to update OAuth ClusterRoleBinding")
170+
return ctrl.Result{RequeueAfter: requeueTime}, err
166171
}
167172

168-
} else if cluster.Status.State != "suspended" && !isRayDashboardOAuthEnabled() && isOpenShift {
169-
logger.Info("Creating Dashboard Route")
170-
_, err := r.routeClient.Routes(cluster.Namespace).Apply(ctx, createRoute(&cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true})
171-
if err != nil {
172-
logger.Error(err, "Failed to update Dashboard Route")
173-
}
174-
if isLocalInteractive && ingressDomain != "" {
173+
if isLocalInteractive {
175174
logger.Info("Creating RayClient Route")
176-
_, err := r.routeClient.Routes(cluster.Namespace).Apply(ctx, createRayClientRoute(&cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true})
175+
_, err := r.routeClient.Routes(cluster.Namespace).Apply(ctx, desiredRayClientRoute(&cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true})
177176
if err != nil {
178177
logger.Error(err, "Failed to update RayClient Route")
178+
return ctrl.Result{RequeueAfter: requeueTime}, err
179179
}
180180
}
181-
return ctrl.Result{}, nil
182181

183-
} else if cluster.Status.State != "suspended" && !isRayDashboardOAuthEnabled() && !isOpenShift {
182+
} else if cluster.Status.State != "suspended" && !r.isRayDashboardOAuthEnabled() && !isOpenShift {
184183
logger.Info("Creating Dashboard Ingress")
185-
_, err := r.kubeClient.NetworkingV1().Ingresses(cluster.Namespace).Apply(ctx, createIngressApplyConfiguration(&cluster, ingressHost), metav1.ApplyOptions{FieldManager: controllerName, Force: true})
184+
_, err := r.kubeClient.NetworkingV1().Ingresses(cluster.Namespace).Apply(ctx, desiredClusterIngress(&cluster, ingressHost), metav1.ApplyOptions{FieldManager: controllerName, Force: true})
186185
if err != nil {
187186
// This log is info level since errors are not fatal and are expected
188187
logger.Info("WARN: Failed to update Dashboard Ingress", "error", err.Error(), logRequeueing, true)
188+
return ctrl.Result{RequeueAfter: requeueTime}, err
189189
}
190190
if isLocalInteractive && ingressDomain != "" {
191191
logger.Info("Creating RayClient Ingress")
192-
_, err := r.kubeClient.NetworkingV1().Ingresses(cluster.Namespace).Apply(ctx, createRayClientIngress(&cluster), metav1.ApplyOptions{FieldManager: controllerName, Force: true})
192+
_, err := r.kubeClient.NetworkingV1().Ingresses(cluster.Namespace).Apply(ctx, desiredRayClientIngress(&cluster, ingressDomain), metav1.ApplyOptions{FieldManager: controllerName, Force: true})
193193
if err != nil {
194194
logger.Error(err, "Failed to update RayClient Ingress")
195+
return ctrl.Result{RequeueAfter: requeueTime}, err
195196
}
196197
}
197-
return ctrl.Result{}, nil
198198
}
199199

200200
return ctrl.Result{}, nil

‎pkg/controllers/support.go

Lines changed: 46 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@ import (
66
"strconv"
77
"strings"
88

9-
"github.com/go-logr/logr"
109
rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1"
1110

1211
networkingv1 "k8s.io/api/networking/v1"
@@ -27,26 +26,11 @@ func serviceNameFromCluster(cluster *rayv1.RayCluster) string {
2726
return cluster.Name + "-head-svc"
2827
}
2928

30-
func createRoute(cluster *rayv1.RayCluster) *routeapply.RouteApplyConfiguration {
31-
return routeapply.Route(dashboardNameFromCluster(cluster), cluster.Namespace).
32-
WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}).
33-
WithSpec(routeapply.RouteSpec().
34-
WithTo(routeapply.RouteTargetReference().WithKind("Service").WithName(serviceNameFromCluster(cluster))).
35-
WithPort(routeapply.RoutePort().WithTargetPort(intstr.FromString(regularServicePortName))).
36-
WithTLS(routeapply.TLSConfig().
37-
WithTermination("edge")),
38-
).
39-
WithOwnerReferences(
40-
v1.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion),
41-
)
42-
}
43-
44-
func createRayClientRoute(cluster *rayv1.RayCluster) *routeapply.RouteApplyConfiguration {
45-
ingress_domain := cluster.ObjectMeta.Annotations["sdk.codeflare.dev/ingress_domain"]
29+
func desiredRayClientRoute(cluster *rayv1.RayCluster) *routeapply.RouteApplyConfiguration {
4630
return routeapply.Route(rayClientNameFromCluster(cluster), cluster.Namespace).
4731
WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}).
4832
WithSpec(routeapply.RouteSpec().
49-
WithHost(rayClientNameFromCluster(cluster) + "-" + cluster.Namespace + "." + ingress_domain).
33+
WithHost(rayClientNameFromCluster(cluster) + "-" + cluster.Namespace).
5034
WithTo(routeapply.RouteTargetReference().WithKind("Service").WithName(serviceNameFromCluster(cluster)).WithWeight(100)).
5135
WithPort(routeapply.RoutePort().WithTargetPort(intstr.FromString("client"))).
5236
WithTLS(routeapply.TLSConfig().WithTermination("passthrough")),
@@ -57,8 +41,7 @@ func createRayClientRoute(cluster *rayv1.RayCluster) *routeapply.RouteApplyConfi
5741
}
5842

5943
// Create an Ingress object for the RayCluster
60-
func createRayClientIngress(cluster *rayv1.RayCluster) *networkingv1ac.IngressApplyConfiguration {
61-
ingress_domain := cluster.ObjectMeta.Annotations["sdk.codeflare.dev/ingress_domain"]
44+
func desiredRayClientIngress(cluster *rayv1.RayCluster, ingressDomain string) *networkingv1ac.IngressApplyConfiguration {
6245
return networkingv1ac.Ingress(rayClientNameFromCluster(cluster), cluster.Namespace).
6346
WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}).
6447
WithAnnotations(map[string]string{
@@ -74,7 +57,7 @@ func createRayClientIngress(cluster *rayv1.RayCluster) *networkingv1ac.IngressAp
7457
WithSpec(networkingv1ac.IngressSpec().
7558
WithIngressClassName("nginx").
7659
WithRules(networkingv1ac.IngressRule().
77-
WithHost(rayClientNameFromCluster(cluster) + "-" + cluster.Namespace + "." + ingress_domain).
60+
WithHost(rayClientNameFromCluster(cluster) + "-" + cluster.Namespace + "." + ingressDomain).
7861
WithHTTP(networkingv1ac.HTTPIngressRuleValue().
7962
WithPaths(networkingv1ac.HTTPIngressPath().
8063
WithPath("/").
@@ -94,7 +77,7 @@ func createRayClientIngress(cluster *rayv1.RayCluster) *networkingv1ac.IngressAp
9477
}
9578

9679
// Create an Ingress object for the RayCluster
97-
func createIngressApplyConfiguration(cluster *rayv1.RayCluster, ingressHost string) *networkingv1ac.IngressApplyConfiguration {
80+
func desiredClusterIngress(cluster *rayv1.RayCluster, ingressHost string) *networkingv1ac.IngressApplyConfiguration {
9881
return networkingv1ac.Ingress(dashboardNameFromCluster(cluster), cluster.Namespace).
9982
WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}).
10083
WithOwnerReferences(v1.OwnerReference().
@@ -104,7 +87,7 @@ func createIngressApplyConfiguration(cluster *rayv1.RayCluster, ingressHost stri
10487
WithUID(types.UID(cluster.UID))).
10588
WithSpec(networkingv1ac.IngressSpec().
10689
WithRules(networkingv1ac.IngressRule().
107-
WithHost(ingressHost). // kind host name or ingress_domain
90+
WithHost(ingressHost). // KinD hostname or ingressDomain
10891
WithHTTP(networkingv1ac.HTTPIngressRuleValue().
10992
WithPaths(networkingv1ac.HTTPIngressPath().
11093
WithPath("/").
@@ -113,7 +96,7 @@ func createIngressApplyConfiguration(cluster *rayv1.RayCluster, ingressHost stri
11396
WithService(networkingv1ac.IngressServiceBackend().
11497
WithName(serviceNameFromCluster(cluster)).
11598
WithPort(networkingv1ac.ServiceBackendPort().
116-
WithName(regularServicePortName),
99+
WithName(ingressServicePortName),
117100
),
118101
),
119102
),
@@ -143,59 +126,56 @@ func getDiscoveryClient(config *rest.Config) (*discovery.DiscoveryClient, error)
143126

144127
// Check where we are running. We are trying to distinguish here whether
145128
// this is vanilla kubernetes cluster or Openshift
146-
func getClusterType(logger logr.Logger, clientset *kubernetes.Clientset, cluster *rayv1.RayCluster) (bool, string) {
129+
func getClusterType(ctx context.Context, clientset *kubernetes.Clientset, cluster *rayv1.RayCluster, ingressDomain string) (bool, string) {
147130
// The discovery package is used to discover APIs supported by a Kubernetes API server.
148-
ingress_domain := cluster.ObjectMeta.Annotations["sdk.codeflare.dev/ingress_domain"]
131+
logger := ctrl.LoggerFrom(ctx)
149132
config, err := ctrl.GetConfig()
150-
if err == nil && config != nil {
151-
dclient, err := getDiscoveryClient(config)
152-
if err == nil && dclient != nil {
153-
apiGroupList, err := dclient.ServerGroups()
154-
if err != nil {
155-
logger.Info("Error while querying ServerGroups, assuming we're on Vanilla Kubernetes")
156-
return false, ""
157-
} else {
158-
for i := 0; i < len(apiGroupList.Groups); i++ {
159-
if strings.HasSuffix(apiGroupList.Groups[i].Name, ".openshift.io") {
160-
logger.Info("We detected being on OpenShift!")
161-
return true, ""
162-
}
163-
}
164-
onKind, _ := isOnKindCluster(clientset)
165-
if onKind && ingress_domain == "" {
166-
logger.Info("We detected being on a KinD cluster!")
167-
return false, "kind"
168-
} else {
169-
logger.Info("We detected being on Vanilla Kubernetes!")
170-
return false, fmt.Sprintf("ray-dashboard-%s-%s.%s", cluster.Name, cluster.Namespace, ingress_domain)
171-
}
172-
}
173-
} else {
174-
logger.Info("Cannot retrieve a DiscoveryClient, assuming we're on Vanilla Kubernetes")
175-
return false, fmt.Sprintf("ray-dashboard-%s-%s.%s", cluster.Name, cluster.Namespace, ingress_domain)
176-
}
177-
} else {
133+
if err != nil && config == nil {
178134
logger.Info("Cannot retrieve config, assuming we're on Vanilla Kubernetes")
179-
return false, fmt.Sprintf("ray-dashboard-%s-%s.%s", cluster.Name, cluster.Namespace, ingress_domain)
135+
return false, fmt.Sprintf("ray-dashboard-%s-%s.%s", cluster.Name, cluster.Namespace, ingressDomain)
180136
}
137+
dclient, err := getDiscoveryClient(config)
138+
if err != nil && dclient == nil {
139+
logger.Info("Cannot retrieve a DiscoveryClient, assuming we're on Vanilla Kubernetes")
140+
return false, fmt.Sprintf("ray-dashboard-%s-%s.%s", cluster.Name, cluster.Namespace, ingressDomain)
141+
}
142+
apiGroupList, err := dclient.ServerGroups()
143+
if err != nil {
144+
logger.Info("Error while querying ServerGroups, assuming we're on Vanilla Kubernetes")
145+
return false, ""
146+
}
147+
for i := 0; i < len(apiGroupList.Groups); i++ {
148+
if strings.HasSuffix(apiGroupList.Groups[i].Name, ".openshift.io") {
149+
logger.Info("We detected being on OpenShift!")
150+
return true, ""
151+
}
152+
}
153+
onKind, _ := isOnKindCluster(clientset)
154+
if onKind && ingressDomain == "" {
155+
logger.Info("We detected being on a KinD cluster!")
156+
return false, "kind"
157+
}
158+
logger.Info("We detected being on Vanilla Kubernetes!")
159+
return false, fmt.Sprintf("ray-dashboard-%s-%s.%s", cluster.Name, cluster.Namespace, ingressDomain)
181160
}
182161

183-
func isRayDashboardOAuthEnabled() bool {
184-
if configInstance.KubeRay != nil && configInstance.KubeRay.RayDashboardOAuthEnabled != nil {
185-
return *configInstance.KubeRay.RayDashboardOAuthEnabled
162+
func (r *RayClusterReconciler) isRayDashboardOAuthEnabled() bool {
163+
if r.Config != nil && r.Config.KubeRay != nil && r.Config.KubeRay.RayDashboardOAuthEnabled != nil {
164+
return *r.Config.KubeRay.RayDashboardOAuthEnabled
186165
}
187166
return true
188167
}
189168

190-
func annotationBoolVal(logger logr.Logger, cluster *rayv1.RayCluster, annotation string) bool {
191-
val := cluster.ObjectMeta.Annotations[annotation]
169+
func annotationBoolVal(ctx context.Context, cluster *rayv1.RayCluster, annotation string, defaultValue bool) bool {
170+
logger := ctrl.LoggerFrom(ctx)
171+
val, exists := cluster.ObjectMeta.Annotations[annotation]
172+
if !exists || val == "" {
173+
return defaultValue
174+
}
192175
boolVal, err := strconv.ParseBool(val)
193176
if err != nil {
194-
logger.Error(err, "Could not convert", annotation, "value to bool", val)
195-
}
196-
if boolVal {
197-
return true
198-
} else {
199-
return false
177+
logger.Error(err, "Could not convert annotation value to bool", "annotation", annotation, "value", val)
178+
return defaultValue
200179
}
180+
return boolVal
201181
}

0 commit comments

Comments
 (0)