From 1487f81f91b97e0f6806ddd1fde1e6897455547c Mon Sep 17 00:00:00 2001 From: abhijeet-dhumal Date: Wed, 5 Jun 2024 12:48:30 +0530 Subject: [PATCH 1/3] add provision to delete kueue resources created during test execution --- tests/e2e/local_interactive_sdk_kind_test.py | 2 +- tests/e2e/local_interactive_sdk_oauth_test.py | 2 +- .../e2e/mnist_raycluster_sdk_aw_kind_test.py | 1 + tests/e2e/mnist_raycluster_sdk_kind_test.py | 1 + tests/e2e/mnist_raycluster_sdk_oauth_test.py | 2 +- tests/e2e/support.py | 38 +++++++++++++++++-- tests/upgrade/raycluster_sdk_upgrade_test.py | 1 + 7 files changed, 41 insertions(+), 6 deletions(-) diff --git a/tests/e2e/local_interactive_sdk_kind_test.py b/tests/e2e/local_interactive_sdk_kind_test.py index 1cbc5f624..647ac4727 100644 --- a/tests/e2e/local_interactive_sdk_kind_test.py +++ b/tests/e2e/local_interactive_sdk_kind_test.py @@ -19,13 +19,13 @@ def setup_method(self): def teardown_method(self): delete_namespace(self) + delete_kueue_resources(self) def test_local_interactives(self): self.setup_method() create_namespace(self) create_kueue_resources(self) self.run_local_interactives() - self.teardown_method() def run_local_interactives(self): ray_image = get_ray_image() diff --git a/tests/e2e/local_interactive_sdk_oauth_test.py b/tests/e2e/local_interactive_sdk_oauth_test.py index 5210fe250..c3fd1d851 100644 --- a/tests/e2e/local_interactive_sdk_oauth_test.py +++ b/tests/e2e/local_interactive_sdk_oauth_test.py @@ -19,13 +19,13 @@ def setup_method(self): def teardown_method(self): delete_namespace(self) + delete_kueue_resources(self) def test_local_interactives(self): self.setup_method() create_namespace(self) create_kueue_resources(self) self.run_local_interactives() - self.teardown_method() def run_local_interactives(self): ray_image = get_ray_image() diff --git a/tests/e2e/mnist_raycluster_sdk_aw_kind_test.py b/tests/e2e/mnist_raycluster_sdk_aw_kind_test.py index 2aa5da16d..3ca3498d0 100644 --- a/tests/e2e/mnist_raycluster_sdk_aw_kind_test.py +++ b/tests/e2e/mnist_raycluster_sdk_aw_kind_test.py @@ -25,6 +25,7 @@ def test_mnist_ray_cluster_sdk_kind(self): create_namespace(self) create_kueue_resources(self) self.run_mnist_raycluster_sdk_kind() + self.teardown_method() def run_mnist_raycluster_sdk_kind(self): ray_image = get_ray_image() diff --git a/tests/e2e/mnist_raycluster_sdk_kind_test.py b/tests/e2e/mnist_raycluster_sdk_kind_test.py index ece7069f1..b486efc8f 100644 --- a/tests/e2e/mnist_raycluster_sdk_kind_test.py +++ b/tests/e2e/mnist_raycluster_sdk_kind_test.py @@ -19,6 +19,7 @@ def setup_method(self): def teardown_method(self): delete_namespace(self) + delete_kueue_resources(self) def test_mnist_ray_cluster_sdk_kind(self): self.setup_method() diff --git a/tests/e2e/mnist_raycluster_sdk_oauth_test.py b/tests/e2e/mnist_raycluster_sdk_oauth_test.py index 548c80ad0..e489c39f8 100644 --- a/tests/e2e/mnist_raycluster_sdk_oauth_test.py +++ b/tests/e2e/mnist_raycluster_sdk_oauth_test.py @@ -19,13 +19,13 @@ def setup_method(self): def teardown_method(self): delete_namespace(self) + delete_kueue_resources(self) def test_mnist_ray_cluster_sdk_auth(self): self.setup_method() create_namespace(self) create_kueue_resources(self) self.run_mnist_raycluster_sdk_oauth() - self.teardown_method() def run_mnist_raycluster_sdk_oauth(self): ray_image = get_ray_image() diff --git a/tests/e2e/support.py b/tests/e2e/support.py index ab9f5c696..8503b95e1 100644 --- a/tests/e2e/support.py +++ b/tests/e2e/support.py @@ -59,11 +59,17 @@ def run_oc_command(args): return None +# Global variables for kueue resources +cluster_queue = "cluster-queue-mnist" +flavor = "default-flavor-mnist" +local_queue = "local-queue-mnist" + + def create_kueue_resources( self, - cluster_queue="cluster-queue-mnist", - flavor="default-flavor-mnist", - local_queue="local-queue-mnist", + cluster_queue=cluster_queue, + flavor=flavor, + local_queue=local_queue, ): print("creating Kueue resources ...") resource_flavor_json = { @@ -163,3 +169,29 @@ def create_kueue_resources( body=local_queue_json, ) print(f"'{local_queue}' created in namespace '{self.namespace}'") + + +def delete_kueue_resources(self, cluster_queue=cluster_queue, flavor=flavor): + # Delete if given cluster-queue exists + try: + self.custom_api.delete_cluster_custom_object( + group="kueue.x-k8s.io", + plural="clusterqueues", + version="v1beta1", + name=cluster_queue, + ) + print(f"\n'{cluster_queue}' cluster-queue deleted") + except Exception as e: + print(f"\nError deleting cluster-queue '{cluster_queue}' : {e}") + + # Delete if given resource-flavor exists + try: + self.custom_api.delete_cluster_custom_object( + group="kueue.x-k8s.io", + plural="resourceflavors", + version="v1beta1", + name=flavor, + ) + print(f"'{flavor}' resource-flavor deleted") + except Exception as e: + print(f"\nError deleting resource-flavor '{flavor}' : {e}") diff --git a/tests/upgrade/raycluster_sdk_upgrade_test.py b/tests/upgrade/raycluster_sdk_upgrade_test.py index 8a17654a5..ef09d2e7c 100644 --- a/tests/upgrade/raycluster_sdk_upgrade_test.py +++ b/tests/upgrade/raycluster_sdk_upgrade_test.py @@ -21,6 +21,7 @@ def setup_method(self): create_kueue_resources(self) except Exception as e: delete_namespace(self) + delete_kueue_resources(self) return _kube_api_error_handling(e) def test_mnist_ray_cluster_sdk_auth(self): From 21c39aaaa98041e6daf73bb1e81b1572b2721cfc Mon Sep 17 00:00:00 2001 From: abhijeet-dhumal Date: Thu, 6 Jun 2024 20:04:47 +0530 Subject: [PATCH 2/3] update kueue resource creation flow to create resources using unique name for avoiding naming conflicts and to execute tests parallely --- .../e2e/mnist_raycluster_sdk_aw_kind_test.py | 1 - tests/e2e/mnist_raycluster_sdk_kind_test.py | 1 - tests/e2e/support.py | 124 +++++++++++------- 3 files changed, 73 insertions(+), 53 deletions(-) diff --git a/tests/e2e/mnist_raycluster_sdk_aw_kind_test.py b/tests/e2e/mnist_raycluster_sdk_aw_kind_test.py index 3ca3498d0..2aa5da16d 100644 --- a/tests/e2e/mnist_raycluster_sdk_aw_kind_test.py +++ b/tests/e2e/mnist_raycluster_sdk_aw_kind_test.py @@ -25,7 +25,6 @@ def test_mnist_ray_cluster_sdk_kind(self): create_namespace(self) create_kueue_resources(self) self.run_mnist_raycluster_sdk_kind() - self.teardown_method() def run_mnist_raycluster_sdk_kind(self): ray_image = get_ray_image() diff --git a/tests/e2e/mnist_raycluster_sdk_kind_test.py b/tests/e2e/mnist_raycluster_sdk_kind_test.py index b486efc8f..630bc5df4 100644 --- a/tests/e2e/mnist_raycluster_sdk_kind_test.py +++ b/tests/e2e/mnist_raycluster_sdk_kind_test.py @@ -26,7 +26,6 @@ def test_mnist_ray_cluster_sdk_kind(self): create_namespace(self) create_kueue_resources(self) self.run_mnist_raycluster_sdk_kind() - self.teardown_method() def run_mnist_raycluster_sdk_kind(self): ray_image = get_ray_image() diff --git a/tests/e2e/support.py b/tests/e2e/support.py index 8503b95e1..a3dfd89bd 100644 --- a/tests/e2e/support.py +++ b/tests/e2e/support.py @@ -25,6 +25,21 @@ def create_namespace(self): self.api_instance.create_namespace(namespace_body) +def create_new_resource_flavor(self): + self.resource_flavor = f"test-resource-flavor-{random_choice()}" + create_resource_flavor(self, self.resource_flavor) + + +def create_new_cluster_queue(self): + self.cluster_queue = f"test-cluster-queue-{random_choice()}" + create_cluster_queue(self, self.cluster_queue, self.resource_flavor) + + +def create_new_local_queue(self): + self.local_queue = f"test-local-queue-{random_choice()}" + create_local_queue(self, self.cluster_queue, self.local_queue) + + def create_namespace_with_name(self, namespace_name): self.namespace = namespace_name try: @@ -59,24 +74,7 @@ def run_oc_command(args): return None -# Global variables for kueue resources -cluster_queue = "cluster-queue-mnist" -flavor = "default-flavor-mnist" -local_queue = "local-queue-mnist" - - -def create_kueue_resources( - self, - cluster_queue=cluster_queue, - flavor=flavor, - local_queue=local_queue, -): - print("creating Kueue resources ...") - resource_flavor_json = { - "apiVersion": "kueue.x-k8s.io/v1beta1", - "kind": "ResourceFlavor", - "metadata": {"name": flavor}, - } +def create_cluster_queue(self, cluster_queue, flavor): cluster_queue_json = { "apiVersion": "kueue.x-k8s.io/v1beta1", "kind": "ClusterQueue", @@ -100,54 +98,69 @@ def create_kueue_resources( ], }, } - local_queue_json = { - "apiVersion": "kueue.x-k8s.io/v1beta1", - "kind": "LocalQueue", - "metadata": { - "namespace": self.namespace, - "name": local_queue, - "annotations": {"kueue.x-k8s.io/default-queue": "true"}, - }, - "spec": {"clusterQueue": cluster_queue}, - } try: - # Check if resource flavor exists + # Check if cluster-queue exists self.custom_api.get_cluster_custom_object( group="kueue.x-k8s.io", - plural="resourceflavors", + plural="clusterqueues", version="v1beta1", - name=flavor, + name=cluster_queue, ) - print(f"'{flavor}' already exists") + print(f"'{cluster_queue}' already exists") except: - # create kueue resource flavor + # create cluster-queue self.custom_api.create_cluster_custom_object( group="kueue.x-k8s.io", - plural="resourceflavors", + plural="clusterqueues", version="v1beta1", - body=resource_flavor_json, + body=cluster_queue_json, ) - print(f"'{flavor}' created!") + print(f"'{cluster_queue}' created") + + self.cluster_queue = cluster_queue + + +def create_resource_flavor(self, flavor): + resource_flavor_json = { + "apiVersion": "kueue.x-k8s.io/v1beta1", + "kind": "ResourceFlavor", + "metadata": {"name": flavor}, + } try: - # Check if cluster-queue exists + # Check if resource flavor exists self.custom_api.get_cluster_custom_object( group="kueue.x-k8s.io", - plural="clusterqueues", + plural="resourceflavors", version="v1beta1", - name=cluster_queue, + name=flavor, ) - print(f"'{cluster_queue}' already exists") + print(f"'{flavor}' already exists") except: - # create cluster-queue + # create kueue resource flavor self.custom_api.create_cluster_custom_object( group="kueue.x-k8s.io", - plural="clusterqueues", + plural="resourceflavors", version="v1beta1", - body=cluster_queue_json, + body=resource_flavor_json, ) - print(f"'{cluster_queue}' created") + print(f"'{flavor}' created!") + + self.resource_flavor = flavor + + +def create_local_queue(self, cluster_queue, local_queue): + local_queue_json = { + "apiVersion": "kueue.x-k8s.io/v1beta1", + "kind": "LocalQueue", + "metadata": { + "namespace": self.namespace, + "name": local_queue, + "annotations": {"kueue.x-k8s.io/default-queue": "true"}, + }, + "spec": {"clusterQueue": cluster_queue}, + } try: # Check if local-queue exists in given namespace @@ -170,19 +183,28 @@ def create_kueue_resources( ) print(f"'{local_queue}' created in namespace '{self.namespace}'") + self.local_queue = local_queue + -def delete_kueue_resources(self, cluster_queue=cluster_queue, flavor=flavor): +def create_kueue_resources(self): + print("creating Kueue resources ...") + create_new_resource_flavor(self) + create_new_cluster_queue(self) + create_new_local_queue(self) + + +def delete_kueue_resources(self): # Delete if given cluster-queue exists try: self.custom_api.delete_cluster_custom_object( group="kueue.x-k8s.io", plural="clusterqueues", version="v1beta1", - name=cluster_queue, + name=self.cluster_queue, ) - print(f"\n'{cluster_queue}' cluster-queue deleted") + print(f"\n'{self.cluster_queue}' cluster-queue deleted") except Exception as e: - print(f"\nError deleting cluster-queue '{cluster_queue}' : {e}") + print(f"\nError deleting cluster-queue '{self.cluster_queue}' : {e}") # Delete if given resource-flavor exists try: @@ -190,8 +212,8 @@ def delete_kueue_resources(self, cluster_queue=cluster_queue, flavor=flavor): group="kueue.x-k8s.io", plural="resourceflavors", version="v1beta1", - name=flavor, + name=self.resource_flavor, ) - print(f"'{flavor}' resource-flavor deleted") + print(f"'{self.resource_flavor}' resource-flavor deleted") except Exception as e: - print(f"\nError deleting resource-flavor '{flavor}' : {e}") + print(f"\nError deleting resource-flavor '{self.resource_flavor}' : {e}") From de1ee93f5dac881b403837b1cb7f6ed080d42a7d Mon Sep 17 00:00:00 2001 From: abhijeet-dhumal Date: Fri, 7 Jun 2024 11:58:00 +0530 Subject: [PATCH 3/3] Update upgrade test to create kueue resources with specfic pre-defined names --- tests/upgrade/raycluster_sdk_upgrade_test.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/upgrade/raycluster_sdk_upgrade_test.py b/tests/upgrade/raycluster_sdk_upgrade_test.py index ef09d2e7c..d83e62898 100644 --- a/tests/upgrade/raycluster_sdk_upgrade_test.py +++ b/tests/upgrade/raycluster_sdk_upgrade_test.py @@ -10,6 +10,10 @@ from codeflare_sdk.utils.kube_api_helpers import _kube_api_error_handling namespace = "test-ns-rayupgrade" +# Global variables for kueue resources +cluster_queue = "cluster-queue-mnist" +flavor = "default-flavor-mnist" +local_queue = "local-queue-mnist" # Creates a Ray cluster @@ -18,7 +22,9 @@ def setup_method(self): initialize_kubernetes_client(self) create_namespace_with_name(self, namespace) try: - create_kueue_resources(self) + create_cluster_queue(self, cluster_queue, flavor) + create_resource_flavor(self, flavor) + create_local_queue(self, cluster_queue, local_queue) except Exception as e: delete_namespace(self) delete_kueue_resources(self)