From 32a0e3d742254557db8ae9a537b097a18828d4e3 Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Thu, 27 Jul 2023 13:46:17 +0200 Subject: [PATCH] Add Ray cluster REST API support in test support package --- go.mod | 6 +- go.sum | 10 ++- test/support/client.go | 14 ++++ test/support/conditions.go | 15 ++++ test/support/mcad.go | 3 +- test/support/ray.go | 3 +- test/support/ray_api.go | 45 +++++++++++ test/support/ray_cluster_client.go | 115 +++++++++++++++++++++++++++++ test/support/route.go | 33 +++++++++ 9 files changed, 236 insertions(+), 8 deletions(-) create mode 100644 test/support/ray_api.go create mode 100644 test/support/ray_cluster_client.go create mode 100644 test/support/route.go diff --git a/go.mod b/go.mod index 7d8c01c8f..8cbc5ec9b 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,8 @@ require ( github.com/manifestival/manifestival v0.7.2 github.com/onsi/ginkgo/v2 v2.9.2 github.com/onsi/gomega v1.27.6 + github.com/openshift/api v0.0.0-20230213134911-7ba313770556 + github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c github.com/project-codeflare/multi-cluster-app-dispatcher v1.32.0 github.com/ray-project/kuberay/ray-operator v0.0.0-20230614221720-085c29d40fa9 go.uber.org/zap v1.24.0 @@ -36,7 +38,7 @@ require ( github.com/golang/protobuf v1.5.3 // indirect github.com/google/gnostic v0.5.7-v3refs // indirect github.com/google/go-cmp v0.5.9 // indirect - github.com/google/gofuzz v1.1.0 // indirect + github.com/google/gofuzz v1.2.0 // indirect github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 // indirect github.com/google/uuid v1.1.2 // indirect github.com/imdario/mergo v0.3.12 // indirect @@ -56,7 +58,7 @@ require ( go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect golang.org/x/net v0.8.0 // indirect - golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b // indirect + golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 // indirect golang.org/x/sys v0.6.0 // indirect golang.org/x/term v0.6.0 // indirect golang.org/x/text v0.8.0 // indirect diff --git a/go.sum b/go.sum index c143ff7a9..815e362bb 100644 --- a/go.sum +++ b/go.sum @@ -280,8 +280,9 @@ github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -418,6 +419,10 @@ github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y github.com/onsi/gomega v1.10.2/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= +github.com/openshift/api v0.0.0-20230213134911-7ba313770556 h1:7W2fOhJicyEff24VaF7ASNzPtYvr+iSCVft4SIBAzaE= +github.com/openshift/api v0.0.0-20230213134911-7ba313770556/go.mod h1:aQ6LDasvHMvHZXqLHnX2GRmnfTWCF/iIwz8EMTTIE9A= +github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c h1:CV76yFOTXmq9VciBR3Bve5ZWzSxdft7gaMVB3kS0rwg= +github.com/openshift/client-go v0.0.0-20221019143426-16aed247da5c/go.mod h1:lFMO8mLHXWFzSdYvGNo8ivF9SfF6zInA8ZGw4phRnUE= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= @@ -635,8 +640,9 @@ golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b h1:clP8eMhB30EHdc0bd2Twtq6kgU7yl5ub2cQLSdrv1Dg= golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5 h1:OSnWWcOd/CtWQC2cYSBgbTSJv3ciqd8r54ySIW2y3RE= +golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/test/support/client.go b/test/support/client.go index a8efa76d2..73931d645 100644 --- a/test/support/client.go +++ b/test/support/client.go @@ -24,6 +24,8 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/clientcmd" + routev1 "github.com/openshift/client-go/route/clientset/versioned" + codeflareclient "github.com/project-codeflare/codeflare-operator/client/clientset/versioned" mcadclient "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/client/clientset/controller-versioned" rayclient "github.com/ray-project/kuberay/ray-operator/pkg/client/clientset/versioned" @@ -31,6 +33,7 @@ import ( type Client interface { Core() kubernetes.Interface + Route() routev1.Interface CodeFlare() codeflareclient.Interface MCAD() mcadclient.Interface Ray() rayclient.Interface @@ -38,6 +41,7 @@ type Client interface { type testClient struct { core kubernetes.Interface + route routev1.Interface codeflare codeflareclient.Interface mcad mcadclient.Interface ray rayclient.Interface @@ -49,6 +53,10 @@ func (t *testClient) Core() kubernetes.Interface { return t.core } +func (t *testClient) Route() routev1.Interface { + return t.route +} + func (t *testClient) CodeFlare() codeflareclient.Interface { return t.codeflare } @@ -75,6 +83,11 @@ func newTestClient() (Client, error) { return nil, err } + routeClient, err := routev1.NewForConfig(cfg) + if err != nil { + return nil, err + } + codeFlareClient, err := codeflareclient.NewForConfig(cfg) if err != nil { return nil, err @@ -92,6 +105,7 @@ func newTestClient() (Client, error) { return &testClient{ core: kubeClient, + route: routeClient, codeflare: codeFlareClient, mcad: mcadClient, ray: rayClient, diff --git a/test/support/conditions.go b/test/support/conditions.go index 16b26583e..70d0ebe9c 100644 --- a/test/support/conditions.go +++ b/test/support/conditions.go @@ -21,6 +21,8 @@ import ( appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" + + routev1 "github.com/openshift/api/route/v1" ) type conditionType interface { @@ -39,6 +41,10 @@ func ConditionStatus[T conditionType](conditionType T) func(any) corev1.Conditio if c := getDeploymentCondition(o.Status.Conditions, appsv1.DeploymentConditionType(conditionType)); c != nil { return c.Status } + case *routev1.Route: + if c := getRouteCondition(o.Status.Ingress[0].Conditions, routev1.RouteIngressConditionType(conditionType)); c != nil { + return c.Status + } } return corev1.ConditionUnknown @@ -64,3 +70,12 @@ func getDeploymentCondition(conditions []appsv1.DeploymentCondition, conditionTy } return nil } + +func getRouteCondition(conditions []routev1.RouteIngressCondition, conditionType routev1.RouteIngressConditionType) *routev1.RouteIngressCondition { + for _, c := range conditions { + if c.Type == conditionType { + return &c + } + } + return nil +} diff --git a/test/support/mcad.go b/test/support/mcad.go index 4f268985f..607a94f74 100644 --- a/test/support/mcad.go +++ b/test/support/mcad.go @@ -18,11 +18,10 @@ package support import ( "github.com/onsi/gomega" + mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" ) func AppWrapper(t Test, namespace *corev1.Namespace, name string) func(g gomega.Gomega) *mcadv1beta1.AppWrapper { diff --git a/test/support/ray.go b/test/support/ray.go index 52bf2d651..4dc6ebc1b 100644 --- a/test/support/ray.go +++ b/test/support/ray.go @@ -20,10 +20,9 @@ import ( "encoding/json" "github.com/onsi/gomega" + rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1" ) const RayJobDefaultClusterSelectorKey = "ray.io/cluster" diff --git a/test/support/ray_api.go b/test/support/ray_api.go new file mode 100644 index 000000000..9004d92ed --- /dev/null +++ b/test/support/ray_api.go @@ -0,0 +1,45 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package support + +import ( + "github.com/onsi/gomega" +) + +func GetRayJobAPIDetails(t Test, rayClient RayClusterClient, jobID string) *RayJobDetailsResponse { + t.T().Helper() + return RayJobAPIDetails(t, rayClient, jobID)(t) +} + +func WriteRayJobAPILogs(t Test, rayClient RayClusterClient, jobID string) { + t.T().Helper() + logs, err := rayClient.GetJobLogs(jobID) + t.Expect(err).NotTo(gomega.HaveOccurred()) + WriteToOutputDir(t, jobID, Log, []byte(logs)) +} + +func RayJobAPIDetails(t Test, rayClient RayClusterClient, jobID string) func(g gomega.Gomega) *RayJobDetailsResponse { + return func(g gomega.Gomega) *RayJobDetailsResponse { + jobDetails, err := rayClient.GetJobDetails(jobID) + t.Expect(err).NotTo(gomega.HaveOccurred()) + return jobDetails + } +} + +func GetRayJobAPIDetailsStatus(jobDetails *RayJobDetailsResponse) string { + return jobDetails.Status +} diff --git a/test/support/ray_cluster_client.go b/test/support/ray_cluster_client.go new file mode 100644 index 000000000..18dc0c290 --- /dev/null +++ b/test/support/ray_cluster_client.go @@ -0,0 +1,115 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package support + +import ( + "bytes" + "encoding/json" + "io/ioutil" + "net/http" + "net/url" +) + +type RayJobSetup struct { + EntryPoint string `json:"entrypoint"` + RuntimeEnv map[string]any `json:"runtime_env"` +} + +type RayJobResponse struct { + JobID string `json:"job_id"` + SubmissionID string `json:"submission_id"` +} + +type RayJobDetailsResponse struct { + JobID string `json:"job_id"` + SubmissionID string `json:"submission_id"` + Status string `json:"status"` +} + +type RayJobLogsResponse struct { + Logs string `json:"logs"` +} + +var _ RayClusterClient = (*rayClusterClient)(nil) + +type rayClusterClient struct { + endpoint url.URL +} + +type RayClusterClient interface { + CreateJob(job *RayJobSetup) (*RayJobResponse, error) + GetJobDetails(jobID string) (*RayJobDetailsResponse, error) + GetJobLogs(jobID string) (string, error) +} + +func NewRayClusterClient(dashboardEndpoint url.URL) RayClusterClient { + return &rayClusterClient{endpoint: dashboardEndpoint} +} + +func (client *rayClusterClient) CreateJob(job *RayJobSetup) (response *RayJobResponse, err error) { + marshalled, err := json.Marshal(job) + if err != nil { + return + } + + createJobURL := client.endpoint.String() + "/api/jobs/" + resp, err := http.Post(createJobURL, "application/json", bytes.NewReader(marshalled)) + if err != nil { + return + } + + respData, err := ioutil.ReadAll(resp.Body) + if err != nil { + return + } + + err = json.Unmarshal(respData, &response) + return +} + +func (client *rayClusterClient) GetJobDetails(jobID string) (response *RayJobDetailsResponse, err error) { + createJobURL := client.endpoint.String() + "/api/jobs/" + jobID + resp, err := http.Get(createJobURL) + if err != nil { + return + } + + respData, err := ioutil.ReadAll(resp.Body) + if err != nil { + return + } + + err = json.Unmarshal(respData, &response) + return +} + +func (client *rayClusterClient) GetJobLogs(jobID string) (logs string, err error) { + createJobURL := client.endpoint.String() + "/api/jobs/" + jobID + "/logs" + resp, err := http.Get(createJobURL) + if err != nil { + return + } + + respData, err := ioutil.ReadAll(resp.Body) + if err != nil { + return + } + + jobLogs := RayJobLogsResponse{} + err = json.Unmarshal(respData, &jobLogs) + return jobLogs.Logs, err +} diff --git a/test/support/route.go b/test/support/route.go new file mode 100644 index 000000000..b105643fc --- /dev/null +++ b/test/support/route.go @@ -0,0 +1,33 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package support + +import ( + "github.com/onsi/gomega" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + routev1 "github.com/openshift/api/route/v1" +) + +func Route(t Test, namespace, name string) func(g gomega.Gomega) *routev1.Route { + return func(g gomega.Gomega) *routev1.Route { + route, err := t.Client().Route().RouteV1().Routes(namespace).Get(t.Ctx(), name, metav1.GetOptions{}) + g.Expect(err).NotTo(gomega.HaveOccurred()) + return route + } +}