Skip to content

Commit 02e9c95

Browse files
ChristianZaccariaChristianZaccaria
committed
Add support file for RC Controller
1 parent f36c3ec commit 02e9c95

File tree

2 files changed

+186
-172
lines changed

2 files changed

+186
-172
lines changed

‎pkg/controllers/raycluster_controller.go

Lines changed: 0 additions & 172 deletions
Original file line numberDiff line numberDiff line change
@@ -22,29 +22,22 @@ import (
2222
"crypto/sha1"
2323
"encoding/base64"
2424

25-
"fmt"
2625
"strconv"
27-
"strings"
2826

2927
"github.com/go-logr/logr"
3028
"github.com/project-codeflare/codeflare-operator/pkg/config"
3129
rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1"
3230

3331
corev1 "k8s.io/api/core/v1"
34-
networkingv1 "k8s.io/api/networking/v1"
3532
rbacv1 "k8s.io/api/rbac/v1"
3633
"k8s.io/apimachinery/pkg/api/errors"
3734
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
3835
"k8s.io/apimachinery/pkg/runtime"
39-
"k8s.io/apimachinery/pkg/types"
4036
"k8s.io/apimachinery/pkg/util/intstr"
4137
coreapply "k8s.io/client-go/applyconfigurations/core/v1"
4238
v1 "k8s.io/client-go/applyconfigurations/meta/v1"
43-
networkingv1ac "k8s.io/client-go/applyconfigurations/networking/v1"
4439
rbacapply "k8s.io/client-go/applyconfigurations/rbac/v1"
45-
"k8s.io/client-go/discovery"
4640
"k8s.io/client-go/kubernetes"
47-
"k8s.io/client-go/rest"
4841
ctrl "sigs.k8s.io/controller-runtime"
4942
"sigs.k8s.io/controller-runtime/pkg/client"
5043
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
@@ -352,168 +345,3 @@ func (r *RayClusterReconciler) SetupWithManager(mgr ctrl.Manager) error {
352345
Complete(r)
353346
}
354347

355-
func serviceNameFromCluster(cluster *rayv1.RayCluster) string {
356-
return cluster.Name + "-head-svc"
357-
}
358-
359-
func createRoute(cluster *rayv1.RayCluster) *routeapply.RouteApplyConfiguration {
360-
return routeapply.Route(dashboardNameFromCluster(cluster), cluster.Namespace).
361-
WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}).
362-
WithSpec(routeapply.RouteSpec().
363-
WithTo(routeapply.RouteTargetReference().WithKind("Service").WithName(serviceNameFromCluster(cluster))).
364-
WithPort(routeapply.RoutePort().WithTargetPort(intstr.FromString(regularServicePortName))).
365-
WithTLS(routeapply.TLSConfig().
366-
WithTermination("edge")),
367-
).
368-
WithOwnerReferences(
369-
v1.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion),
370-
)
371-
}
372-
373-
func createRayClientRoute(cluster *rayv1.RayCluster) *routeapply.RouteApplyConfiguration {
374-
ingress_domain := cluster.ObjectMeta.Annotations["sdk.codeflare.dev/ingress_domain"]
375-
return routeapply.Route(rayClientNameFromCluster(cluster), cluster.Namespace).
376-
WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}).
377-
WithSpec(routeapply.RouteSpec().
378-
WithHost(rayClientNameFromCluster(cluster) + "-" + cluster.Namespace + "." + ingress_domain).
379-
WithTo(routeapply.RouteTargetReference().WithKind("Service").WithName(serviceNameFromCluster(cluster)).WithWeight(100)).
380-
WithPort(routeapply.RoutePort().WithTargetPort(intstr.FromString("client"))).
381-
WithTLS(routeapply.TLSConfig().WithTermination("passthrough")),
382-
).
383-
WithOwnerReferences(
384-
v1.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion),
385-
)
386-
}
387-
388-
// Create an Ingress object for the RayCluster
389-
func createRayClientIngress(cluster *rayv1.RayCluster) *networkingv1ac.IngressApplyConfiguration {
390-
ingress_domain := cluster.ObjectMeta.Annotations["sdk.codeflare.dev/ingress_domain"]
391-
return networkingv1ac.Ingress(rayClientNameFromCluster(cluster), cluster.Namespace).
392-
WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}).
393-
WithAnnotations(map[string]string{
394-
"nginx.ingress.kubernetes.io/rewrite-target": "/",
395-
"nginx.ingress.kubernetes.io/ssl-redirect": "true",
396-
"nginx.ingress.kubernetes.io/ssl-passthrough": "true",
397-
}).
398-
WithOwnerReferences(v1.OwnerReference().
399-
WithAPIVersion(cluster.APIVersion).
400-
WithKind(cluster.Kind).
401-
WithName(cluster.Name).
402-
WithUID(types.UID(cluster.UID))).
403-
WithSpec(networkingv1ac.IngressSpec().
404-
WithIngressClassName("nginx").
405-
WithRules(networkingv1ac.IngressRule().
406-
WithHost(rayClientNameFromCluster(cluster) + "-" + cluster.Namespace + "." + ingress_domain).
407-
WithHTTP(networkingv1ac.HTTPIngressRuleValue().
408-
WithPaths(networkingv1ac.HTTPIngressPath().
409-
WithPath("/").
410-
WithPathType(networkingv1.PathTypeImplementationSpecific).
411-
WithBackend(networkingv1ac.IngressBackend().
412-
WithService(networkingv1ac.IngressServiceBackend().
413-
WithName(serviceNameFromCluster(cluster)).
414-
WithPort(networkingv1ac.ServiceBackendPort().
415-
WithNumber(10001),
416-
),
417-
),
418-
),
419-
),
420-
),
421-
),
422-
)
423-
// Optionally, add TLS configuration here if needed
424-
}
425-
426-
// Create an Ingress object for the RayCluster
427-
func createIngressApplyConfiguration(cluster *rayv1.RayCluster, ingressHost string) *networkingv1ac.IngressApplyConfiguration {
428-
return networkingv1ac.Ingress(dashboardNameFromCluster(cluster), cluster.Namespace).
429-
WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}).
430-
WithOwnerReferences(v1.OwnerReference().
431-
WithAPIVersion(cluster.APIVersion).
432-
WithKind(cluster.Kind).
433-
WithName(cluster.Name).
434-
WithUID(types.UID(cluster.UID))).
435-
WithSpec(networkingv1ac.IngressSpec().
436-
WithRules(networkingv1ac.IngressRule().
437-
WithHost(ingressHost). // kind host name or ingress_domain
438-
WithHTTP(networkingv1ac.HTTPIngressRuleValue().
439-
WithPaths(networkingv1ac.HTTPIngressPath().
440-
WithPath("/").
441-
WithPathType(networkingv1.PathTypePrefix).
442-
WithBackend(networkingv1ac.IngressBackend().
443-
WithService(networkingv1ac.IngressServiceBackend().
444-
WithName(serviceNameFromCluster(cluster)).
445-
WithPort(networkingv1ac.ServiceBackendPort().
446-
WithName(regularServicePortName),
447-
),
448-
),
449-
),
450-
),
451-
),
452-
),
453-
)
454-
// Optionally, add TLS configuration here if needed
455-
}
456-
457-
// isOnKindCluster checks if the current cluster is a KinD cluster.
458-
// It searches for a node with a label commonly used by KinD clusters.
459-
func isOnKindCluster(clientset *kubernetes.Clientset) (bool, error) {
460-
nodes, err := clientset.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{
461-
LabelSelector: "kubernetes.io/hostname=kind-control-plane",
462-
})
463-
if err != nil {
464-
return false, err
465-
}
466-
// If we find one or more nodes with the label, assume it's a KinD cluster.
467-
return len(nodes.Items) > 0, nil
468-
}
469-
470-
// getDiscoveryClient returns a discovery client for the current reconciler
471-
func getDiscoveryClient(config *rest.Config) (*discovery.DiscoveryClient, error) {
472-
return discovery.NewDiscoveryClientForConfig(config)
473-
}
474-
475-
// Check where we are running. We are trying to distinguish here whether
476-
// this is vanilla kubernetes cluster or Openshift
477-
func getClusterType(logger logr.Logger, clientset *kubernetes.Clientset, cluster *rayv1.RayCluster) (bool, string) {
478-
// The discovery package is used to discover APIs supported by a Kubernetes API server.
479-
ingress_domain := cluster.ObjectMeta.Annotations["sdk.codeflare.dev/ingress_domain"]
480-
config, err := ctrl.GetConfig()
481-
if err == nil && config != nil {
482-
dclient, err := getDiscoveryClient(config)
483-
if err == nil && dclient != nil {
484-
apiGroupList, err := dclient.ServerGroups()
485-
if err != nil {
486-
logger.Info("Error while querying ServerGroups, assuming we're on Vanilla Kubernetes")
487-
return false, ""
488-
} else {
489-
for i := 0; i < len(apiGroupList.Groups); i++ {
490-
if strings.HasSuffix(apiGroupList.Groups[i].Name, ".openshift.io") {
491-
logger.Info("We detected being on OpenShift!")
492-
return true, ""
493-
}
494-
}
495-
onKind, _ := isOnKindCluster(clientset)
496-
if onKind && ingress_domain == "" {
497-
logger.Info("We detected being on a KinD cluster!")
498-
return false, "kind"
499-
} else {
500-
logger.Info("We detected being on Vanilla Kubernetes!")
501-
return false, fmt.Sprintf("ray-dashboard-%s-%s.%s", cluster.Name, cluster.Namespace, ingress_domain)
502-
}
503-
}
504-
} else {
505-
logger.Info("Cannot retrieve a DiscoveryClient, assuming we're on Vanilla Kubernetes")
506-
return false, fmt.Sprintf("ray-dashboard-%s-%s.%s", cluster.Name, cluster.Namespace, ingress_domain)
507-
}
508-
} else {
509-
logger.Info("Cannot retrieve config, assuming we're on Vanilla Kubernetes")
510-
return false, fmt.Sprintf("ray-dashboard-%s-%s.%s", cluster.Name, cluster.Namespace, ingress_domain)
511-
}
512-
}
513-
514-
func isRayDashboardOAuthEnabled() bool {
515-
if configInstance.KubeRay != nil && configInstance.KubeRay.RayDashboardOAuthEnabled != nil {
516-
return *configInstance.KubeRay.RayDashboardOAuthEnabled
517-
}
518-
return true
519-
}

‎pkg/controllers/support.go

Lines changed: 186 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,186 @@
1+
package controllers
2+
3+
import (
4+
"fmt"
5+
"strings"
6+
"context"
7+
ctrl "sigs.k8s.io/controller-runtime"
8+
9+
"github.com/go-logr/logr"
10+
routeapply "github.com/openshift/client-go/route/applyconfigurations/route/v1"
11+
rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1"
12+
13+
networkingv1 "k8s.io/api/networking/v1"
14+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
15+
"k8s.io/apimachinery/pkg/types"
16+
"k8s.io/apimachinery/pkg/util/intstr"
17+
v1 "k8s.io/client-go/applyconfigurations/meta/v1"
18+
networkingv1ac "k8s.io/client-go/applyconfigurations/networking/v1"
19+
"k8s.io/client-go/discovery"
20+
"k8s.io/client-go/kubernetes"
21+
"k8s.io/client-go/rest"
22+
)
23+
24+
func serviceNameFromCluster(cluster *rayv1.RayCluster) string {
25+
return cluster.Name + "-head-svc"
26+
}
27+
28+
func createRoute(cluster *rayv1.RayCluster) *routeapply.RouteApplyConfiguration {
29+
return routeapply.Route(dashboardNameFromCluster(cluster), cluster.Namespace).
30+
WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}).
31+
WithSpec(routeapply.RouteSpec().
32+
WithTo(routeapply.RouteTargetReference().WithKind("Service").WithName(serviceNameFromCluster(cluster))).
33+
WithPort(routeapply.RoutePort().WithTargetPort(intstr.FromString(regularServicePortName))).
34+
WithTLS(routeapply.TLSConfig().
35+
WithTermination("edge")),
36+
).
37+
WithOwnerReferences(
38+
v1.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion),
39+
)
40+
}
41+
42+
func createRayClientRoute(cluster *rayv1.RayCluster) *routeapply.RouteApplyConfiguration {
43+
ingress_domain := cluster.ObjectMeta.Annotations["sdk.codeflare.dev/ingress_domain"]
44+
return routeapply.Route(rayClientNameFromCluster(cluster), cluster.Namespace).
45+
WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}).
46+
WithSpec(routeapply.RouteSpec().
47+
WithHost(rayClientNameFromCluster(cluster) + "-" + cluster.Namespace + "." + ingress_domain).
48+
WithTo(routeapply.RouteTargetReference().WithKind("Service").WithName(serviceNameFromCluster(cluster)).WithWeight(100)).
49+
WithPort(routeapply.RoutePort().WithTargetPort(intstr.FromString("client"))).
50+
WithTLS(routeapply.TLSConfig().WithTermination("passthrough")),
51+
).
52+
WithOwnerReferences(
53+
v1.OwnerReference().WithUID(cluster.UID).WithName(cluster.Name).WithKind(cluster.Kind).WithAPIVersion(cluster.APIVersion),
54+
)
55+
}
56+
57+
// Create an Ingress object for the RayCluster
58+
func createRayClientIngress(cluster *rayv1.RayCluster) *networkingv1ac.IngressApplyConfiguration {
59+
ingress_domain := cluster.ObjectMeta.Annotations["sdk.codeflare.dev/ingress_domain"]
60+
return networkingv1ac.Ingress(rayClientNameFromCluster(cluster), cluster.Namespace).
61+
WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}).
62+
WithAnnotations(map[string]string{
63+
"nginx.ingress.kubernetes.io/rewrite-target": "/",
64+
"nginx.ingress.kubernetes.io/ssl-redirect": "true",
65+
"nginx.ingress.kubernetes.io/ssl-passthrough": "true",
66+
}).
67+
WithOwnerReferences(v1.OwnerReference().
68+
WithAPIVersion(cluster.APIVersion).
69+
WithKind(cluster.Kind).
70+
WithName(cluster.Name).
71+
WithUID(types.UID(cluster.UID))).
72+
WithSpec(networkingv1ac.IngressSpec().
73+
WithIngressClassName("nginx").
74+
WithRules(networkingv1ac.IngressRule().
75+
WithHost(rayClientNameFromCluster(cluster) + "-" + cluster.Namespace + "." + ingress_domain).
76+
WithHTTP(networkingv1ac.HTTPIngressRuleValue().
77+
WithPaths(networkingv1ac.HTTPIngressPath().
78+
WithPath("/").
79+
WithPathType(networkingv1.PathTypeImplementationSpecific).
80+
WithBackend(networkingv1ac.IngressBackend().
81+
WithService(networkingv1ac.IngressServiceBackend().
82+
WithName(serviceNameFromCluster(cluster)).
83+
WithPort(networkingv1ac.ServiceBackendPort().
84+
WithNumber(10001),
85+
),
86+
),
87+
),
88+
),
89+
),
90+
),
91+
)
92+
}
93+
94+
// Create an Ingress object for the RayCluster
95+
func createIngressApplyConfiguration(cluster *rayv1.RayCluster, ingressHost string) *networkingv1ac.IngressApplyConfiguration {
96+
return networkingv1ac.Ingress(dashboardNameFromCluster(cluster), cluster.Namespace).
97+
WithLabels(map[string]string{"ray.io/cluster-name": cluster.Name}).
98+
WithOwnerReferences(v1.OwnerReference().
99+
WithAPIVersion(cluster.APIVersion).
100+
WithKind(cluster.Kind).
101+
WithName(cluster.Name).
102+
WithUID(types.UID(cluster.UID))).
103+
WithSpec(networkingv1ac.IngressSpec().
104+
WithRules(networkingv1ac.IngressRule().
105+
WithHost(ingressHost). // kind host name or ingress_domain
106+
WithHTTP(networkingv1ac.HTTPIngressRuleValue().
107+
WithPaths(networkingv1ac.HTTPIngressPath().
108+
WithPath("/").
109+
WithPathType(networkingv1.PathTypePrefix).
110+
WithBackend(networkingv1ac.IngressBackend().
111+
WithService(networkingv1ac.IngressServiceBackend().
112+
WithName(serviceNameFromCluster(cluster)).
113+
WithPort(networkingv1ac.ServiceBackendPort().
114+
WithName(regularServicePortName),
115+
),
116+
),
117+
),
118+
),
119+
),
120+
),
121+
)
122+
}
123+
124+
// isOnKindCluster checks if the current cluster is a KinD cluster.
125+
// It searches for a node with a label commonly used by KinD clusters.
126+
func isOnKindCluster(clientset *kubernetes.Clientset) (bool, error) {
127+
nodes, err := clientset.CoreV1().Nodes().List(context.TODO(), metav1.ListOptions{
128+
LabelSelector: "kubernetes.io/hostname=kind-control-plane",
129+
})
130+
if err != nil {
131+
return false, err
132+
}
133+
// If we find one or more nodes with the label, assume it's a KinD cluster.
134+
return len(nodes.Items) > 0, nil
135+
}
136+
137+
// getDiscoveryClient returns a discovery client for the current reconciler
138+
func getDiscoveryClient(config *rest.Config) (*discovery.DiscoveryClient, error) {
139+
return discovery.NewDiscoveryClientForConfig(config)
140+
}
141+
142+
// Check where we are running. We are trying to distinguish here whether
143+
// this is vanilla kubernetes cluster or Openshift
144+
func getClusterType(logger logr.Logger, clientset *kubernetes.Clientset, cluster *rayv1.RayCluster) (bool, string) {
145+
// The discovery package is used to discover APIs supported by a Kubernetes API server.
146+
ingress_domain := cluster.ObjectMeta.Annotations["sdk.codeflare.dev/ingress_domain"]
147+
config, err := ctrl.GetConfig()
148+
if err == nil && config != nil {
149+
dclient, err := getDiscoveryClient(config)
150+
if err == nil && dclient != nil {
151+
apiGroupList, err := dclient.ServerGroups()
152+
if err != nil {
153+
logger.Info("Error while querying ServerGroups, assuming we're on Vanilla Kubernetes")
154+
return false, ""
155+
} else {
156+
for i := 0; i < len(apiGroupList.Groups); i++ {
157+
if strings.HasSuffix(apiGroupList.Groups[i].Name, ".openshift.io") {
158+
logger.Info("We detected being on OpenShift!")
159+
return true, ""
160+
}
161+
}
162+
onKind, _ := isOnKindCluster(clientset)
163+
if onKind && ingress_domain == "" {
164+
logger.Info("We detected being on a KinD cluster!")
165+
return false, "kind"
166+
} else {
167+
logger.Info("We detected being on Vanilla Kubernetes!")
168+
return false, fmt.Sprintf("ray-dashboard-%s-%s.%s", cluster.Name, cluster.Namespace, ingress_domain)
169+
}
170+
}
171+
} else {
172+
logger.Info("Cannot retrieve a DiscoveryClient, assuming we're on Vanilla Kubernetes")
173+
return false, fmt.Sprintf("ray-dashboard-%s-%s.%s", cluster.Name, cluster.Namespace, ingress_domain)
174+
}
175+
} else {
176+
logger.Info("Cannot retrieve config, assuming we're on Vanilla Kubernetes")
177+
return false, fmt.Sprintf("ray-dashboard-%s-%s.%s", cluster.Name, cluster.Namespace, ingress_domain)
178+
}
179+
}
180+
181+
func isRayDashboardOAuthEnabled() bool {
182+
if configInstance.KubeRay != nil && configInstance.KubeRay.RayDashboardOAuthEnabled != nil {
183+
return *configInstance.KubeRay.RayDashboardOAuthEnabled
184+
}
185+
return true
186+
}

0 commit comments

Comments
 (0)