diff --git a/tests/e2e/local_interactive_sdk_kind_test.py b/tests/e2e/local_interactive_sdk_kind_test.py index 1cbc5f624..647ac4727 100644 --- a/tests/e2e/local_interactive_sdk_kind_test.py +++ b/tests/e2e/local_interactive_sdk_kind_test.py @@ -19,13 +19,13 @@ def setup_method(self): def teardown_method(self): delete_namespace(self) + delete_kueue_resources(self) def test_local_interactives(self): self.setup_method() create_namespace(self) create_kueue_resources(self) self.run_local_interactives() - self.teardown_method() def run_local_interactives(self): ray_image = get_ray_image() diff --git a/tests/e2e/local_interactive_sdk_oauth_test.py b/tests/e2e/local_interactive_sdk_oauth_test.py index 5210fe250..c3fd1d851 100644 --- a/tests/e2e/local_interactive_sdk_oauth_test.py +++ b/tests/e2e/local_interactive_sdk_oauth_test.py @@ -19,13 +19,13 @@ def setup_method(self): def teardown_method(self): delete_namespace(self) + delete_kueue_resources(self) def test_local_interactives(self): self.setup_method() create_namespace(self) create_kueue_resources(self) self.run_local_interactives() - self.teardown_method() def run_local_interactives(self): ray_image = get_ray_image() diff --git a/tests/e2e/mnist_raycluster_sdk_kind_test.py b/tests/e2e/mnist_raycluster_sdk_kind_test.py index ece7069f1..630bc5df4 100644 --- a/tests/e2e/mnist_raycluster_sdk_kind_test.py +++ b/tests/e2e/mnist_raycluster_sdk_kind_test.py @@ -19,13 +19,13 @@ def setup_method(self): def teardown_method(self): delete_namespace(self) + delete_kueue_resources(self) def test_mnist_ray_cluster_sdk_kind(self): self.setup_method() create_namespace(self) create_kueue_resources(self) self.run_mnist_raycluster_sdk_kind() - self.teardown_method() def run_mnist_raycluster_sdk_kind(self): ray_image = get_ray_image() diff --git a/tests/e2e/mnist_raycluster_sdk_oauth_test.py b/tests/e2e/mnist_raycluster_sdk_oauth_test.py index 548c80ad0..e489c39f8 100644 --- a/tests/e2e/mnist_raycluster_sdk_oauth_test.py +++ b/tests/e2e/mnist_raycluster_sdk_oauth_test.py @@ -19,13 +19,13 @@ def setup_method(self): def teardown_method(self): delete_namespace(self) + delete_kueue_resources(self) def test_mnist_ray_cluster_sdk_auth(self): self.setup_method() create_namespace(self) create_kueue_resources(self) self.run_mnist_raycluster_sdk_oauth() - self.teardown_method() def run_mnist_raycluster_sdk_oauth(self): ray_image = get_ray_image() diff --git a/tests/e2e/support.py b/tests/e2e/support.py index ab9f5c696..a3dfd89bd 100644 --- a/tests/e2e/support.py +++ b/tests/e2e/support.py @@ -25,6 +25,21 @@ def create_namespace(self): self.api_instance.create_namespace(namespace_body) +def create_new_resource_flavor(self): + self.resource_flavor = f"test-resource-flavor-{random_choice()}" + create_resource_flavor(self, self.resource_flavor) + + +def create_new_cluster_queue(self): + self.cluster_queue = f"test-cluster-queue-{random_choice()}" + create_cluster_queue(self, self.cluster_queue, self.resource_flavor) + + +def create_new_local_queue(self): + self.local_queue = f"test-local-queue-{random_choice()}" + create_local_queue(self, self.cluster_queue, self.local_queue) + + def create_namespace_with_name(self, namespace_name): self.namespace = namespace_name try: @@ -59,18 +74,7 @@ def run_oc_command(args): return None -def create_kueue_resources( - self, - cluster_queue="cluster-queue-mnist", - flavor="default-flavor-mnist", - local_queue="local-queue-mnist", -): - print("creating Kueue resources ...") - resource_flavor_json = { - "apiVersion": "kueue.x-k8s.io/v1beta1", - "kind": "ResourceFlavor", - "metadata": {"name": flavor}, - } +def create_cluster_queue(self, cluster_queue, flavor): cluster_queue_json = { "apiVersion": "kueue.x-k8s.io/v1beta1", "kind": "ClusterQueue", @@ -94,54 +98,69 @@ def create_kueue_resources( ], }, } - local_queue_json = { - "apiVersion": "kueue.x-k8s.io/v1beta1", - "kind": "LocalQueue", - "metadata": { - "namespace": self.namespace, - "name": local_queue, - "annotations": {"kueue.x-k8s.io/default-queue": "true"}, - }, - "spec": {"clusterQueue": cluster_queue}, - } try: - # Check if resource flavor exists + # Check if cluster-queue exists self.custom_api.get_cluster_custom_object( group="kueue.x-k8s.io", - plural="resourceflavors", + plural="clusterqueues", version="v1beta1", - name=flavor, + name=cluster_queue, ) - print(f"'{flavor}' already exists") + print(f"'{cluster_queue}' already exists") except: - # create kueue resource flavor + # create cluster-queue self.custom_api.create_cluster_custom_object( group="kueue.x-k8s.io", - plural="resourceflavors", + plural="clusterqueues", version="v1beta1", - body=resource_flavor_json, + body=cluster_queue_json, ) - print(f"'{flavor}' created!") + print(f"'{cluster_queue}' created") + + self.cluster_queue = cluster_queue + + +def create_resource_flavor(self, flavor): + resource_flavor_json = { + "apiVersion": "kueue.x-k8s.io/v1beta1", + "kind": "ResourceFlavor", + "metadata": {"name": flavor}, + } try: - # Check if cluster-queue exists + # Check if resource flavor exists self.custom_api.get_cluster_custom_object( group="kueue.x-k8s.io", - plural="clusterqueues", + plural="resourceflavors", version="v1beta1", - name=cluster_queue, + name=flavor, ) - print(f"'{cluster_queue}' already exists") + print(f"'{flavor}' already exists") except: - # create cluster-queue + # create kueue resource flavor self.custom_api.create_cluster_custom_object( group="kueue.x-k8s.io", - plural="clusterqueues", + plural="resourceflavors", version="v1beta1", - body=cluster_queue_json, + body=resource_flavor_json, ) - print(f"'{cluster_queue}' created") + print(f"'{flavor}' created!") + + self.resource_flavor = flavor + + +def create_local_queue(self, cluster_queue, local_queue): + local_queue_json = { + "apiVersion": "kueue.x-k8s.io/v1beta1", + "kind": "LocalQueue", + "metadata": { + "namespace": self.namespace, + "name": local_queue, + "annotations": {"kueue.x-k8s.io/default-queue": "true"}, + }, + "spec": {"clusterQueue": cluster_queue}, + } try: # Check if local-queue exists in given namespace @@ -163,3 +182,38 @@ def create_kueue_resources( body=local_queue_json, ) print(f"'{local_queue}' created in namespace '{self.namespace}'") + + self.local_queue = local_queue + + +def create_kueue_resources(self): + print("creating Kueue resources ...") + create_new_resource_flavor(self) + create_new_cluster_queue(self) + create_new_local_queue(self) + + +def delete_kueue_resources(self): + # Delete if given cluster-queue exists + try: + self.custom_api.delete_cluster_custom_object( + group="kueue.x-k8s.io", + plural="clusterqueues", + version="v1beta1", + name=self.cluster_queue, + ) + print(f"\n'{self.cluster_queue}' cluster-queue deleted") + except Exception as e: + print(f"\nError deleting cluster-queue '{self.cluster_queue}' : {e}") + + # Delete if given resource-flavor exists + try: + self.custom_api.delete_cluster_custom_object( + group="kueue.x-k8s.io", + plural="resourceflavors", + version="v1beta1", + name=self.resource_flavor, + ) + print(f"'{self.resource_flavor}' resource-flavor deleted") + except Exception as e: + print(f"\nError deleting resource-flavor '{self.resource_flavor}' : {e}") diff --git a/tests/upgrade/raycluster_sdk_upgrade_test.py b/tests/upgrade/raycluster_sdk_upgrade_test.py index 8a17654a5..d83e62898 100644 --- a/tests/upgrade/raycluster_sdk_upgrade_test.py +++ b/tests/upgrade/raycluster_sdk_upgrade_test.py @@ -10,6 +10,10 @@ from codeflare_sdk.utils.kube_api_helpers import _kube_api_error_handling namespace = "test-ns-rayupgrade" +# Global variables for kueue resources +cluster_queue = "cluster-queue-mnist" +flavor = "default-flavor-mnist" +local_queue = "local-queue-mnist" # Creates a Ray cluster @@ -18,9 +22,12 @@ def setup_method(self): initialize_kubernetes_client(self) create_namespace_with_name(self, namespace) try: - create_kueue_resources(self) + create_cluster_queue(self, cluster_queue, flavor) + create_resource_flavor(self, flavor) + create_local_queue(self, cluster_queue, local_queue) except Exception as e: delete_namespace(self) + delete_kueue_resources(self) return _kube_api_error_handling(e) def test_mnist_ray_cluster_sdk_auth(self):