From 8f732635aa76074bf09ca5b45aca872670eeb6f3 Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Mon, 23 Sep 2024 09:31:57 +0100 Subject: [PATCH 01/14] Add UI table with RayCluster specs and action buttons --- demo-notebooks/guided-demos/ipywidgets.ipynb | 568 +++++++++++++++++++ src/codeflare_sdk/__init__.py | 1 + src/codeflare_sdk/cluster/__init__.py | 1 + src/codeflare_sdk/cluster/cluster.py | 178 +++++- src/codeflare_sdk/cluster/model.py | 4 +- src/codeflare_sdk/utils/pretty_print.py | 2 +- tests/unit_test.py | 9 +- 7 files changed, 756 insertions(+), 7 deletions(-) create mode 100644 demo-notebooks/guided-demos/ipywidgets.ipynb diff --git a/demo-notebooks/guided-demos/ipywidgets.ipynb b/demo-notebooks/guided-demos/ipywidgets.ipynb new file mode 100644 index 000000000..5d79cca5b --- /dev/null +++ b/demo-notebooks/guided-demos/ipywidgets.ipynb @@ -0,0 +1,568 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "8d4a42f6", + "metadata": {}, + "source": [ + "In this notebook, we will go through the basics of using the SDK to:\n", + " - Spin up a Ray cluster with our desired resources\n", + " - View the status and specs of our Ray cluster\n", + " - Take down the Ray cluster when finished" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "301094f1", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found existing installation: codeflare-sdk 0.0.0.dev0\n", + "Uninstalling codeflare-sdk-0.0.0.dev0:\n", + " Successfully uninstalled codeflare-sdk-0.0.0.dev0\n", + "Note: you may need to restart the kernel to use updated packages.\n", + "Defaulting to user installation because normal site-packages is not writeable\n", + "Processing /home/christianzaccaria/Documents/GitHub/codeflare-sdk/dist/codeflare_sdk-0.0.0.dev0-py3-none-any.whl\n", + "Requirement already satisfied: cryptography==40.0.2 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from codeflare-sdk==0.0.0.dev0) (40.0.2)\n", + "Requirement already satisfied: executing==1.2.0 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from codeflare-sdk==0.0.0.dev0) (1.2.0)\n", + "Requirement already satisfied: ipywidgets==8.1.2 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from codeflare-sdk==0.0.0.dev0) (8.1.2)\n", + "Requirement already satisfied: kubernetes<27,>=25.3.0 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from codeflare-sdk==0.0.0.dev0) (26.1.0)\n", + "Requirement already satisfied: openshift-client==1.0.18 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from codeflare-sdk==0.0.0.dev0) (1.0.18)\n", + "Requirement already satisfied: pydantic<2 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from codeflare-sdk==0.0.0.dev0) (1.10.14)\n", + "Requirement already satisfied: ray==2.23.0 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (2.23.0)\n", + "Requirement already satisfied: rich<13.0,>=12.5 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from codeflare-sdk==0.0.0.dev0) (12.6.0)\n", + "Requirement already satisfied: setuptools<=73.0.0 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from codeflare-sdk==0.0.0.dev0) (69.1.1)\n", + "Requirement already satisfied: cffi>=1.12 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from cryptography==40.0.2->codeflare-sdk==0.0.0.dev0) (1.16.0)\n", + "Requirement already satisfied: comm>=0.1.3 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ipywidgets==8.1.2->codeflare-sdk==0.0.0.dev0) (0.2.0)\n", + "Requirement already satisfied: ipython>=6.1.0 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ipywidgets==8.1.2->codeflare-sdk==0.0.0.dev0) (8.18.1)\n", + "Requirement already satisfied: traitlets>=4.3.1 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ipywidgets==8.1.2->codeflare-sdk==0.0.0.dev0) (5.14.0)\n", + "Requirement already satisfied: widgetsnbextension~=4.0.10 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ipywidgets==8.1.2->codeflare-sdk==0.0.0.dev0) (4.0.11)\n", + "Requirement already satisfied: jupyterlab-widgets~=3.0.10 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ipywidgets==8.1.2->codeflare-sdk==0.0.0.dev0) (3.0.11)\n", + "Requirement already satisfied: six in /home/christianzaccaria/.local/lib/python3.9/site-packages (from openshift-client==1.0.18->codeflare-sdk==0.0.0.dev0) (1.16.0)\n", + "Requirement already satisfied: pyyaml in /home/christianzaccaria/.local/lib/python3.9/site-packages (from openshift-client==1.0.18->codeflare-sdk==0.0.0.dev0) (6.0.1)\n", + "Requirement already satisfied: paramiko in /home/christianzaccaria/.local/lib/python3.9/site-packages (from openshift-client==1.0.18->codeflare-sdk==0.0.0.dev0) (3.4.0)\n", + "Requirement already satisfied: click>=7.0 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ray==2.23.0->ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (8.1.7)\n", + "Requirement already satisfied: filelock in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ray==2.23.0->ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (3.13.1)\n", + "Requirement already satisfied: jsonschema in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ray==2.23.0->ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (4.21.1)\n", + "Requirement already satisfied: msgpack<2.0.0,>=1.0.0 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ray==2.23.0->ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (1.0.8)\n", + "Requirement already satisfied: packaging in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ray==2.23.0->ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (23.2)\n", + "Requirement already satisfied: protobuf!=3.19.5,>=3.15.3 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ray==2.23.0->ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (4.25.3)\n", + "Requirement already satisfied: aiosignal in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ray==2.23.0->ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (1.3.1)\n", + "Requirement already satisfied: frozenlist in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ray==2.23.0->ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (1.4.1)\n", + "Requirement already satisfied: requests in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ray==2.23.0->ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (2.31.0)\n", + "Requirement already satisfied: numpy>=1.20 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (1.26.4)\n", + "Requirement already satisfied: pandas>=1.3 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (2.2.1)\n", + "Requirement already satisfied: pyarrow>=6.0.1 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (15.0.0)\n", + "Requirement already satisfied: fsspec in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (2024.2.0)\n", + "Requirement already satisfied: aiohttp>=3.7 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (3.9.3)\n", + "Requirement already satisfied: aiohttp-cors in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (0.7.0)\n", + "Requirement already satisfied: colorful in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (0.5.6)\n", + "Requirement already satisfied: py-spy>=0.2.0 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (0.3.14)\n", + "Requirement already satisfied: opencensus in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (0.11.4)\n", + "Requirement already satisfied: prometheus-client>=0.7.1 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (0.20.0)\n", + "Requirement already satisfied: smart-open in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (7.0.1)\n", + "Requirement already satisfied: virtualenv!=20.21.1,>=20.0.24 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (20.21.0)\n", + "Requirement already satisfied: grpcio>=1.32.0 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (1.62.0)\n", + "Requirement already satisfied: memray in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (1.10.0)\n", + "Requirement already satisfied: certifi>=14.05.14 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from kubernetes<27,>=25.3.0->codeflare-sdk==0.0.0.dev0) (2024.2.2)\n", + "Requirement already satisfied: python-dateutil>=2.5.3 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from kubernetes<27,>=25.3.0->codeflare-sdk==0.0.0.dev0) (2.9.0.post0)\n", + "Requirement already satisfied: google-auth>=1.0.1 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from kubernetes<27,>=25.3.0->codeflare-sdk==0.0.0.dev0) (2.28.1)\n", + "Requirement already satisfied: websocket-client!=0.40.0,!=0.41.*,!=0.42.*,>=0.32.0 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from kubernetes<27,>=25.3.0->codeflare-sdk==0.0.0.dev0) (1.7.0)\n", + "Requirement already satisfied: requests-oauthlib in /home/christianzaccaria/.local/lib/python3.9/site-packages (from kubernetes<27,>=25.3.0->codeflare-sdk==0.0.0.dev0) (1.3.1)\n", + "Requirement already satisfied: urllib3>=1.24.2 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from kubernetes<27,>=25.3.0->codeflare-sdk==0.0.0.dev0) (1.26.18)\n", + "Requirement already satisfied: typing-extensions>=4.2.0 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from pydantic<2->codeflare-sdk==0.0.0.dev0) (4.10.0)\n", + "Requirement already satisfied: commonmark<0.10.0,>=0.9.0 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from rich<13.0,>=12.5->codeflare-sdk==0.0.0.dev0) (0.9.1)\n", + "Requirement already satisfied: pygments<3.0.0,>=2.6.0 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from rich<13.0,>=12.5->codeflare-sdk==0.0.0.dev0) (2.17.2)\n", + "Requirement already satisfied: attrs>=17.3.0 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from aiohttp>=3.7->ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (23.2.0)\n", + "Requirement already satisfied: multidict<7.0,>=4.5 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from aiohttp>=3.7->ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (6.0.5)\n", + "Requirement already satisfied: yarl<2.0,>=1.0 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from aiohttp>=3.7->ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (1.9.4)\n", + "Requirement already satisfied: async-timeout<5.0,>=4.0 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from aiohttp>=3.7->ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (4.0.3)\n", + "Requirement already satisfied: pycparser in /home/christianzaccaria/.local/lib/python3.9/site-packages (from cffi>=1.12->cryptography==40.0.2->codeflare-sdk==0.0.0.dev0) (2.21)\n", + "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from google-auth>=1.0.1->kubernetes<27,>=25.3.0->codeflare-sdk==0.0.0.dev0) (5.3.3)\n", + "Requirement already satisfied: pyasn1-modules>=0.2.1 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from google-auth>=1.0.1->kubernetes<27,>=25.3.0->codeflare-sdk==0.0.0.dev0) (0.3.0)\n", + "Requirement already satisfied: rsa<5,>=3.1.4 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from google-auth>=1.0.1->kubernetes<27,>=25.3.0->codeflare-sdk==0.0.0.dev0) (4.9)\n", + "Requirement already satisfied: decorator in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ipython>=6.1.0->ipywidgets==8.1.2->codeflare-sdk==0.0.0.dev0) (5.1.1)\n", + "Requirement already satisfied: jedi>=0.16 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ipython>=6.1.0->ipywidgets==8.1.2->codeflare-sdk==0.0.0.dev0) (0.19.1)\n", + "Requirement already satisfied: matplotlib-inline in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ipython>=6.1.0->ipywidgets==8.1.2->codeflare-sdk==0.0.0.dev0) (0.1.6)\n", + "Requirement already satisfied: prompt-toolkit<3.1.0,>=3.0.41 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ipython>=6.1.0->ipywidgets==8.1.2->codeflare-sdk==0.0.0.dev0) (3.0.41)\n", + "Requirement already satisfied: stack-data in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ipython>=6.1.0->ipywidgets==8.1.2->codeflare-sdk==0.0.0.dev0) (0.6.3)\n", + "Requirement already satisfied: exceptiongroup in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ipython>=6.1.0->ipywidgets==8.1.2->codeflare-sdk==0.0.0.dev0) (1.1.3)\n", + "Requirement already satisfied: pexpect>4.3 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ipython>=6.1.0->ipywidgets==8.1.2->codeflare-sdk==0.0.0.dev0) (4.8.0)\n", + "Requirement already satisfied: pytz>=2020.1 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from pandas>=1.3->ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (2024.1)\n", + "Requirement already satisfied: tzdata>=2022.7 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from pandas>=1.3->ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (2024.1)\n", + "Requirement already satisfied: distlib<1,>=0.3.6 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from virtualenv!=20.21.1,>=20.0.24->ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (0.3.8)\n", + "Requirement already satisfied: platformdirs<4,>=2.4 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from virtualenv!=20.21.1,>=20.0.24->ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (3.11.0)\n", + "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from jsonschema->ray==2.23.0->ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (2023.12.1)\n", + "Requirement already satisfied: referencing>=0.28.4 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from jsonschema->ray==2.23.0->ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (0.33.0)\n", + "Requirement already satisfied: rpds-py>=0.7.1 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from jsonschema->ray==2.23.0->ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (0.18.0)\n", + "Requirement already satisfied: jinja2>=2.9 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from memray->ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (3.1.3)\n", + "Requirement already satisfied: opencensus-context>=0.1.3 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from opencensus->ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (0.1.3)\n", + "Requirement already satisfied: google-api-core<3.0.0,>=1.0.0 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from opencensus->ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (2.17.1)\n", + "Requirement already satisfied: bcrypt>=3.2 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from paramiko->openshift-client==1.0.18->codeflare-sdk==0.0.0.dev0) (4.1.2)\n", + "Requirement already satisfied: pynacl>=1.5 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from paramiko->openshift-client==1.0.18->codeflare-sdk==0.0.0.dev0) (1.5.0)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from requests->ray==2.23.0->ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (3.3.2)\n", + "Requirement already satisfied: idna<4,>=2.5 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from requests->ray==2.23.0->ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (3.6)\n", + "Requirement already satisfied: oauthlib>=3.0.0 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from requests-oauthlib->kubernetes<27,>=25.3.0->codeflare-sdk==0.0.0.dev0) (3.2.2)\n", + "Requirement already satisfied: wrapt in /home/christianzaccaria/.local/lib/python3.9/site-packages (from smart-open->ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (1.16.0)\n", + "Requirement already satisfied: googleapis-common-protos<2.0.dev0,>=1.56.2 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from google-api-core<3.0.0,>=1.0.0->opencensus->ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (1.62.0)\n", + "Requirement already satisfied: parso<0.9.0,>=0.8.3 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from jedi>=0.16->ipython>=6.1.0->ipywidgets==8.1.2->codeflare-sdk==0.0.0.dev0) (0.8.3)\n", + "Requirement already satisfied: MarkupSafe>=2.0 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from jinja2>=2.9->memray->ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (2.1.5)\n", + "Requirement already satisfied: ptyprocess>=0.5 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from pexpect>4.3->ipython>=6.1.0->ipywidgets==8.1.2->codeflare-sdk==0.0.0.dev0) (0.7.0)\n", + "Requirement already satisfied: wcwidth in /home/christianzaccaria/.local/lib/python3.9/site-packages (from prompt-toolkit<3.1.0,>=3.0.41->ipython>=6.1.0->ipywidgets==8.1.2->codeflare-sdk==0.0.0.dev0) (0.2.13)\n", + "Requirement already satisfied: pyasn1<0.6.0,>=0.4.6 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from pyasn1-modules>=0.2.1->google-auth>=1.0.1->kubernetes<27,>=25.3.0->codeflare-sdk==0.0.0.dev0) (0.5.1)\n", + "Requirement already satisfied: asttokens>=2.1.0 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from stack-data->ipython>=6.1.0->ipywidgets==8.1.2->codeflare-sdk==0.0.0.dev0) (2.4.1)\n", + "Requirement already satisfied: pure-eval in /home/christianzaccaria/.local/lib/python3.9/site-packages (from stack-data->ipython>=6.1.0->ipywidgets==8.1.2->codeflare-sdk==0.0.0.dev0) (0.2.2)\n", + "Installing collected packages: codeflare-sdk\n", + "Successfully installed codeflare-sdk-0.0.0.dev0\n", + "\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.2\u001b[0m\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n", + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], + "source": [ + "%pip uninstall codeflare-sdk -y\n", + "%pip install ../../dist/codeflare_sdk-0.0.0.dev0-py3-none-any.whl" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "b55bc3ea-4ce3-49bf-bb1f-e209de8ca47a", + "metadata": {}, + "outputs": [], + "source": [ + "# Import pieces from codeflare-sdk\\\n", + "from codeflare_sdk import Cluster, ClusterConfiguration, TokenAuthentication, list_cluster_details" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "614daa0c", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'Logged into https://api.chris-aisrhods.xb4x.p3.openshiftapps.com:443'" + ] + }, + "execution_count": 2, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Create authentication object for user permissions\n", + "# IF unused, SDK will automatically check for default kubeconfig, then in-cluster config\n", + "# KubeConfigFileAuthentication can also be used to specify kubeconfig path manually\n", + "auth = TokenAuthentication(\n", + " token = \"sha256~pu4rVc-UrGDzHXIOX22BrV0pFLvjEcSZkZ6ECYKndhY\",\n", + " server = \"https://api.chris-aisrhods.xb4x.p3.openshiftapps.com:443\",\n", + " skip_tls=False\n", + ")\n", + "auth.login()" + ] + }, + { + "cell_type": "markdown", + "id": "bc27f84c", + "metadata": {}, + "source": [ + "Here, we want to define our cluster by specifying the resources we require for our batch workload. Below, we define our cluster object (which generates a corresponding RayCluster).\n", + "\n", + "NOTE: We must specify the `image` which will be used in our RayCluster, we recommend you bring your own image which suits your purposes. \n", + "The example here is a community image." + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "0f4bc870-091f-4e11-9642-cba145710159", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Yaml resources loaded for raytesta122\n" + ] + } + ], + "source": [ + "# Create and configure our cluster object\n", + "# The SDK will try to find the name of your default local queue based on the annotation \"kueue.x-k8s.io/default-queue\": \"true\" unless you specify the local queue manually below\n", + "cluster = Cluster(ClusterConfiguration(\n", + " name='raytesta122',\n", + " namespace=\"default\",\n", + " head_cpus='500m',\n", + " head_memory=2,\n", + " head_extended_resource_requests={'nvidia.com/gpu':2}, # For GPU enabled workloads set the head_extended_resource_requests and worker_extended_resource_requests\n", + " worker_extended_resource_requests={'nvidia.com/gpu':0},\n", + " num_workers=2,\n", + " worker_cpu_requests='250m',\n", + " worker_cpu_limits=1,\n", + " worker_memory_requests=4,\n", + " worker_memory_limits=4,\n", + " # image=\"\", # Optional Field \n", + " write_to_file=False, # When enabled Ray Cluster yaml files are written to /HOME/.codeflare/resources \n", + " # local_queue=\"local-queue-name\" # Specify the local queue manually\n", + "))" + ] + }, + { + "cell_type": "markdown", + "id": "12eef53c", + "metadata": {}, + "source": [ + "Next, we want to bring our cluster up, so we call the `up()` function below to submit our Ray Cluster onto the queue, and begin the process of obtaining our resource cluster." + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "id": "f0884bbc-c224-4ca0-98a0-02dfa09c2200", + "metadata": {}, + "outputs": [], + "source": [ + "# Bring up the cluster\n", + "cluster.up()" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "24136d1f", + "metadata": {}, + "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "bbb9361d9bb14342aff79c436f18ecd4", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "VBox(children=(ToggleButtons(description='Select an existing cluster:', options=('raytesta1', 'raytesta10', 'rโ€ฆ" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "ce0ba412e4a245ec93497b0eebff3ac7", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(Button(description='Delete Cluster', icon='trash', style=ButtonStyle()), Button(description='Viโ€ฆ" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "114adc01c5694abc84ecb134f42917e6", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Output()" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "list_cluster_details(\"default\")" + ] + }, + { + "cell_type": "markdown", + "id": "657ebdfb", + "metadata": {}, + "source": [ + "Now, we want to check on the status of our resource cluster, and wait until it is finally ready for use." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "24e612ff", + "metadata": {}, + "outputs": [], + "source": [ + "from codeflare_sdk import get_cluster\n", + "get_cluster(\"raytest21\", \"default\")" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "995850b2", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
                     ๐Ÿš€ CodeFlare Cluster Details ๐Ÿš€                     \n",
+       "                                                                         \n",
+       " โ•ญโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฎ \n",
+       " โ”‚   Name                                                              โ”‚ \n",
+       " โ”‚   raytestmar2k                                        Inactive โŒ   โ”‚ \n",
+       " โ”‚                                                                     โ”‚ \n",
+       " โ”‚   URI: ray://raytestmar2k-head-svc.default.svc:10001                โ”‚ \n",
+       " โ”‚                                                                     โ”‚ \n",
+       " โ”‚   Dashboard๐Ÿ”—                                                       โ”‚ \n",
+       " โ”‚                                                                     โ”‚ \n",
+       " โ”‚                       Cluster Resources                             โ”‚ \n",
+       " โ”‚   โ•ญโ”€โ”€ Workers โ”€โ”€โ•ฎ  โ•ญโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ Worker specs(each) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฎ         โ”‚ \n",
+       " โ”‚   โ”‚  # Workers  โ”‚  โ”‚  Memory      CPU         GPU         โ”‚         โ”‚ \n",
+       " โ”‚   โ”‚             โ”‚  โ”‚                                      โ”‚         โ”‚ \n",
+       " โ”‚   โ”‚  1          โ”‚  โ”‚  2G~2G       1           0           โ”‚         โ”‚ \n",
+       " โ”‚   โ”‚             โ”‚  โ”‚                                      โ”‚         โ”‚ \n",
+       " โ”‚   โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ  โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ         โ”‚ \n",
+       " โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ \n",
+       "
\n" + ], + "text/plain": [ + "\u001b[3m \u001b[0m\u001b[1;3m ๐Ÿš€ CodeFlare Cluster Details ๐Ÿš€\u001b[0m\u001b[3m \u001b[0m\n", + "\u001b[1m \u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m\n", + " โ•ญโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฎ \n", + " โ”‚ \u001b[1;37;42mName\u001b[0m โ”‚ \n", + " โ”‚ \u001b[1;4mraytestmar2k\u001b[0m Inactive โŒ โ”‚ \n", + " โ”‚ โ”‚ \n", + " โ”‚ \u001b[1mURI:\u001b[0m ray://raytestmar2k-head-svc.default.svc:10001 โ”‚ \n", + " โ”‚ โ”‚ \n", + " โ”‚ \u001b]8;id=601715;https://ray-dashboard-raytestmar2k-default.apps.rosa.chris-aisrhods.xb4x.p3.openshiftapps.com\u001b\\\u001b[4;34mDashboard๐Ÿ”—\u001b[0m\u001b]8;;\u001b\\ โ”‚ \n", + " โ”‚ โ”‚ \n", + " โ”‚ \u001b[3m Cluster Resources \u001b[0m โ”‚ \n", + " โ”‚ โ•ญโ”€โ”€ Workers โ”€โ”€โ•ฎ โ•ญโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ Worker specs(each) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฎ โ”‚ \n", + " โ”‚ โ”‚ \u001b[1m \u001b[0m\u001b[1m# Workers\u001b[0m\u001b[1m \u001b[0m โ”‚ โ”‚ \u001b[1m \u001b[0m\u001b[1mMemory \u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m\u001b[1mCPU \u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m\u001b[1mGPU \u001b[0m\u001b[1m \u001b[0m โ”‚ โ”‚ \n", + " โ”‚ โ”‚ \u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m โ”‚ โ”‚ \u001b[36m \u001b[0m\u001b[36m \u001b[0m\u001b[36m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m โ”‚ โ”‚ \n", + " โ”‚ โ”‚ \u001b[35m \u001b[0m\u001b[35m1 \u001b[0m\u001b[35m \u001b[0m โ”‚ โ”‚ \u001b[36m \u001b[0m\u001b[36m2G~2G \u001b[0m\u001b[36m \u001b[0m\u001b[35m \u001b[0m\u001b[35m1 \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m0 \u001b[0m\u001b[35m \u001b[0m โ”‚ โ”‚ \n", + " โ”‚ โ”‚ \u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m โ”‚ โ”‚ \u001b[36m \u001b[0m\u001b[36m \u001b[0m\u001b[36m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m โ”‚ โ”‚ \n", + " โ”‚ โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ โ”‚ \n", + " โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ \n" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "RayCluster(name='raytestmar2k', status=, head_cpus=2, head_mem='8G', workers=1, worker_mem_min='2G', worker_mem_max='2G', worker_cpu=1, namespace='default', dashboard='https://ray-dashboard-raytestmar2k-default.apps.rosa.chris-aisrhods.xb4x.p3.openshiftapps.com', worker_extended_resources={}, head_extended_resources={})" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cluster.details()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9dda874b", + "metadata": {}, + "outputs": [], + "source": [ + "def format_status(status):\n", + " if status == \"Ready\":\n", + " return 'Ready โœ“'\n", + " elif status == \"Suspended\":\n", + " return 'Suspended ~'\n", + " elif status == \"Starting\":\n", + " return 'Starting โŒ›'\n", + " elif status == \"Failed\":\n", + " return 'Failed โœ—'\n", + " else:\n", + " return status\n", + "\n", + "import ipywidgets as widgets\n", + "import pandas as pd\n", + "from IPython.display import display, HTML\n", + "data = {\n", + " \"name\": [\"RayTest1\", \"RayTest2\", \"RayTest3\", \"RayTest4\"],\n", + " \"namespace\": [\"default\", \"usernamespace\", \"usernamespace\", \"usernamespace\"],\n", + " \"head_gpu\": [0, 1, 2, 0],\n", + " \"worker_gpu\": [2, 0, 1, 0],\n", + " \"min_memory\": [2, 4, 4, 2],\n", + " \"max_memory\": [2, 4, 8, 4],\n", + " \"min_cpu\": [1, 2, 4, 2],\n", + " \"max_cpu\": [1, 4, 8, 2],\n", + " \"status\": [\"Ready\", \"Starting\", \"Suspended\", \"Failed\"],\n", + " \"pods\": [\n", + " [{\"pod\": \"head\", \"name\": \"head-raytest1\", \"status\": \"Ready\"}, {\"pod\": \"worker\", \"name\": \"worker-raytest1-a\", \"status\": \"Ready\"}, {\"pod\": \"worker\", \"name\": \"worker-raytest1-b\", \"status\": \"Ready\"}],\n", + " [{\"pod\": \"head\", \"name\": \"head-raytest2\", \"status\": \"Ready\"}, {\"pod\": \"worker\", \"name\": \"worker-raytest2a\", \"status\": \"Starting\"}],\n", + " [{\"pod\": \"head\", \"name\": \"head-raytest3\", \"status\": \"Suspended\"}, {\"pod\": \"worker\", \"name\": \"worker-raytest3a\", \"status\": \"Suspended\"}],\n", + " [{\"pod\": \"head\", \"name\": \"head-raytest4\", \"status\": \"Failed\"}, {\"pod\": \"worker\", \"name\": \"worker-raytest4a\", \"status\": \"Failed\"}]\n", + " ]\n", + "}\n", + "df = pd.DataFrame(data)\n", + "\n", + "# format to add icons\n", + "df['status'] = df['status'].apply(format_status)\n", + "\n", + "my_output = widgets.Output()\n", + "my_output\n", + "classification_widget = widgets.ToggleButtons(\n", + " options=['RayTest1', \"RayTest2\", \"RayTest3\", \"RayTest4\"],\n", + " description='Select an existing cluster:',\n", + ")\n", + "\n", + "def on_click(change):\n", + " new_value = change[\"new\"]\n", + " my_output.clear_output()\n", + " with my_output:\n", + " selected_data = df[df[\"name\"] == new_value]\n", + " main_table = selected_data[[\"name\", \"namespace\", \"head_gpu\", \"worker_gpu\", \"min_memory\", \"max_memory\", \"min_cpu\", \"max_cpu\", \"status\"]].to_html(escape=False, index=False)\n", + " pod_rows = \"\"\n", + " for pod in selected_data[\"pods\"].values[0]:\n", + " pod_rows += f'{pod[\"pod\"]}{pod[\"name\"]}{format_status(pod[\"status\"])}'\n", + " pods_table = f'
{pod_rows}
PodNameStatus
'\n", + " display(HTML(f'
{main_table}{pods_table}
'))\n", + "\n", + "classification_widget.observe(on_click, names=\"value\")\n", + "display(widgets.VBox([classification_widget, my_output]))\n", + "\n", + "\n", + "list_jobs_button = widgets.Button(\n", + " description='View Jobs',\n", + " icon='suitcase'\n", + " )\n", + "delete_button = widgets.Button(\n", + " description='Delete Cluster',\n", + " icon='trash'\n", + " )\n", + "ray_dashboard_button = widgets.Button(\n", + " description='Open Ray Dashboard',\n", + " icon='dashboard',\n", + " layout=widgets.Layout(width='auto'),\n", + " )\n", + "view_yaml_button = widgets.Button(\n", + " description='View YAML',\n", + " icon='file'\n", + " )\n", + "display(widgets.HBox([delete_button, list_jobs_button, view_yaml_button, ray_dashboard_button]))" + ] + }, + { + "cell_type": "markdown", + "id": "b3a55fe4", + "metadata": {}, + "source": [ + "Let's quickly verify that the specs of the cluster are as expected." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6f1ab7ff", + "metadata": {}, + "outputs": [], + "source": [ + "with my_output:\n", + " display(cluster.details())" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7fd45bc5-03c0-4ae5-9ec5-dd1c30f1a084", + "metadata": {}, + "outputs": [], + "source": [ + "from IPython.display import HTML, display\n", + "import ipywidgets as widgets\n", + "\n", + "def on_click(change):\n", + " new_value = change[\"new\"]\n", + " my_output.clear_output()\n", + " with my_output:\n", + " display(HTML(f'
{df[df[\"name\"]==new_value][[\"name\", \"namespace\", \"head_gpu\", \"worker_gpu\", \"min_memory\", \"max_memory\", \"min_cpu\", \"max_cpu\", \"status\"]].to_html(escape=False, index=False)}
'))\n", + "\n", + "classification_widget.observe(on_click, names=\"value\")\n", + "display(widgets.VBox([classification_widget, my_output], layout=widgets.Layout(border='2px solid black')))\n", + "\n", + "list_jobs_button = widgets.Button(description='View Jobs', icon='suitcase')\n", + "delete_button = widgets.Button(description='Delete Cluster', icon='trash')\n", + "ray_dashboard_button = widgets.Button(description='Open Ray Dashboard', icon='dashboard', layout=widgets.Layout(width='auto'))\n", + "view_yaml_button = widgets.Button(description='View YAML', icon='file')\n", + "buttons_container = widgets.HBox([delete_button, list_jobs_button, view_yaml_button, ray_dashboard_button], layout=widgets.Layout(border='2px solid black'))\n", + "\n", + "display(buttons_container)\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "id": "5af8cd32", + "metadata": {}, + "source": [ + "Finally, we bring our resource cluster down and release/terminate the associated resources, bringing everything back to the way it was before our cluster was brought up." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5f36db0f-31f6-4373-9503-dc3c1c4c3f57", + "metadata": {}, + "outputs": [], + "source": [ + "cluster.down()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0d41b90e", + "metadata": {}, + "outputs": [], + "source": [ + "auth.logout()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.18" + }, + "vscode": { + "interpreter": { + "hash": "f9f85f796d01129d0dd105a088854619f454435301f6ffec2fea96ecbd9be4ac" + } + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/src/codeflare_sdk/__init__.py b/src/codeflare_sdk/__init__.py index 0390a3d2f..fa512fbf2 100644 --- a/src/codeflare_sdk/__init__.py +++ b/src/codeflare_sdk/__init__.py @@ -14,6 +14,7 @@ get_cluster, list_all_queued, list_all_clusters, + list_cluster_details, ) from .job import RayJobClient diff --git a/src/codeflare_sdk/cluster/__init__.py b/src/codeflare_sdk/cluster/__init__.py index 0b1849e51..237cff0a9 100644 --- a/src/codeflare_sdk/cluster/__init__.py +++ b/src/codeflare_sdk/cluster/__init__.py @@ -19,6 +19,7 @@ get_cluster, list_all_queued, list_all_clusters, + list_cluster_details, ) from .awload import AWManager diff --git a/src/codeflare_sdk/cluster/cluster.py b/src/codeflare_sdk/cluster/cluster.py index 7c652a186..09b4b8433 100644 --- a/src/codeflare_sdk/cluster/cluster.py +++ b/src/codeflare_sdk/cluster/cluster.py @@ -19,6 +19,7 @@ """ import re +import subprocess from time import sleep from typing import List, Optional, Tuple, Dict @@ -34,6 +35,10 @@ from ..utils.kube_api_helpers import _kube_api_error_handling from ..utils.generate_yaml import is_openshift_cluster +import ipywidgets as widgets +from IPython.display import display, HTML, Javascript +import pandas as pd + from .config import ClusterConfiguration from .model import ( AppWrapper, @@ -592,6 +597,173 @@ def get_current_namespace(): # pragma: no cover except KeyError: return None +# format_status takes a RayCluster status and applies colors and icons based on the status. +def format_status(status): + if status == RayClusterStatus.READY: + return 'Ready โœ“' + elif status == RayClusterStatus.SUSPENDED: + return 'Suspended โ„๏ธ' + elif status == RayClusterStatus.FAILED: + return 'Failed โœ—' + elif status == RayClusterStatus.UNHEALTHY: + return 'Unhealthy' + elif status == RayClusterStatus.UNKNOWN: + return 'Unknown' + else: + return status + +def _fetch_cluster_data(namespace): + rayclusters = list_all_clusters(namespace, False) + names = [item.name for item in rayclusters] + namespaces = [item.namespace for item in rayclusters] + head_extended_resources = [ + f"{list(item.head_extended_resources.keys())[0]}: {list(item.head_extended_resources.values())[0]}" + if item.head_extended_resources else "nvidia.com/gpu: 0" + for item in rayclusters + ] + worker_extended_resources = [ + f"{list(item.worker_extended_resources.keys())[0]}: {list(item.worker_extended_resources.values())[0]}" + if item.worker_extended_resources else "nvidia.com/gpu: 0" + for item in rayclusters + ] + head_cpus = [item.head_cpus if item.head_cpus else 0 for item in rayclusters] + head_mem = [item.head_mem if item.head_mem else 0 for item in rayclusters] + worker_cpu_min = [item.worker_cpu_min if item.worker_cpu_min else 0 for item in rayclusters] + worker_cpu_max = [item.worker_cpu_max if item.worker_cpu_max else 0 for item in rayclusters] + worker_mem_min = [item.worker_mem_min if item.worker_mem_min else 0 for item in rayclusters] + worker_mem_max = [item.worker_mem_max if item.worker_mem_max else 0 for item in rayclusters] + status = [item.status.name for item in rayclusters] + + status = [format_status(item.status) for item in rayclusters] + + data = { + "name": names, + "namespace": namespaces, + "head gpus": head_extended_resources, + "worker gpus": worker_extended_resources, + "head cpus": head_cpus, + "head memory": head_mem, + "worker cpu requests": worker_cpu_min, + "worker cpu limits": worker_cpu_max, + "worker memory requests": worker_mem_min, + "worker memory limits": worker_mem_max, + "status": status + } + return pd.DataFrame(data) + + +def list_cluster_details(namespace: str): + df = _fetch_cluster_data(namespace) + + my_output = widgets.Output() + if df["name"].empty: + print(f"No clusters found in the {namespace} namespace.") + else: + classification_widget = widgets.ToggleButtons( + options=df["name"].tolist(), value=None, + description='Select an existing cluster:', + ) + + def on_cluster_click(change): + new_value = change["new"] + my_output.clear_output() + with my_output: + display(HTML(df[df["name"]==new_value][["name", "namespace", "head gpus", "worker gpus", "head cpus", "head memory", "worker memory requests", "worker memory limits", "status"]].to_html(escape=False, index=False, border=2))) + + classification_widget.observe(on_cluster_click, names="value") + display(widgets.VBox([classification_widget, my_output])) + + def on_delete_button_clicked(b): + cluster_name = classification_widget.value + namespace = df[df["name"]==classification_widget.value]["namespace"].values[0] + delete_cluster(cluster_name, namespace) + my_output.clear_output() + print(f"Cluster {cluster_name} in the {namespace} namespace was deleted successfully.") + # Refresh the dataframe + new_df = _fetch_cluster_data(namespace) + classification_widget.options = new_df["name"].tolist() + + + # out Output widget is used to execute JavaScript code to open the Ray dashboard URL in a new browser tab + out = widgets.Output() + def on_ray_dashboard_button_clicked(b): + cluster_name = classification_widget.value + namespace = df[df["name"]==classification_widget.value]["namespace"].values[0] + + cluster = Cluster(ClusterConfiguration(cluster_name, namespace)) + dashboard_url = cluster.cluster_dashboard_uri() + + my_output.clear_output() + with out: + display(Javascript(f'window.open("{dashboard_url}", "_blank");')) + print(f"Opening Ray Dashboard for {cluster_name}:\n{dashboard_url}") + + def on_list_jobs_button_clicked(b): + cluster_name = classification_widget.value + namespace = df[df["name"]==classification_widget.value]["namespace"].values[0] + + cluster = Cluster(ClusterConfiguration(cluster_name, namespace)) + dashboard_url = cluster.cluster_dashboard_uri() + + my_output.clear_output() + with out: + display(Javascript(f'window.open("{dashboard_url}/#/jobs", "_blank");')) + + list_jobs_button = widgets.Button( + description='View Jobs', + icon='suitcase', + tooltip="Open the Ray Job Dashboard" + ) + list_jobs_button.on_click(on_list_jobs_button_clicked) + + delete_button = widgets.Button( + description='Delete Cluster', + icon='trash', + ) + delete_button.on_click(on_delete_button_clicked) + + ray_dashboard_button = widgets.Button( + description='Open Ray Dashboard', + icon='dashboard', + layout=widgets.Layout(width='auto'), + ) + ray_dashboard_button.on_click(on_ray_dashboard_button_clicked) + + display(widgets.HBox([delete_button, list_jobs_button, ray_dashboard_button]), out) + + + +def delete_cluster( + cluster_name: str, + namespace: str, # TODO: get current namespace if not provided +): + if _check_aw_exists(cluster_name, namespace): + try: + config_check() + api_instance = client.CustomObjectsApi(api_config_handler()) + api_instance.delete_namespaced_custom_object( + group="workload.codeflare.dev", + version="v1beta2", + namespace=namespace, + plural="appwrappers", + name=cluster_name, + ) + except Exception as e: + return _kube_api_error_handling(e) + else: + try: + config_check() + api_instance = client.CustomObjectsApi(api_config_handler()) + api_instance.delete_namespaced_custom_object( + group="ray.io", + version="v1", + namespace=namespace, + plural="rayclusters", + name=cluster_name, + ) + except Exception as e: + return _kube_api_error_handling(e) + def get_cluster( cluster_name: str, @@ -869,7 +1041,8 @@ def _map_to_ray_cluster(rc) -> Optional[RayCluster]: worker_mem_requests=rc["spec"]["workerGroupSpecs"][0]["template"]["spec"][ "containers" ][0]["resources"]["requests"]["memory"], - worker_cpu=rc["spec"]["workerGroupSpecs"][0]["template"]["spec"]["containers"][ + worker_cpu_min=rc["spec"]["workerGroupSpecs"][0]["template"]["spec"]["containers"][0]["resources"]["requests"]["cpu"], + worker_cpu_max=rc["spec"]["workerGroupSpecs"][0]["template"]["spec"]["containers"][ 0 ]["resources"]["limits"]["cpu"], worker_extended_resources=worker_extended_resources, @@ -910,7 +1083,8 @@ def _copy_to_ray(cluster: Cluster) -> RayCluster: workers=cluster.config.num_workers, worker_mem_requests=cluster.config.worker_memory_requests, worker_mem_limits=cluster.config.worker_memory_limits, - worker_cpu=cluster.config.worker_cpu_requests, + worker_cpu_min=cluster.config.worker_cpu_requests, + worker_cpu_max=cluster.config.worker_cpu_limits, worker_extended_resources=cluster.config.worker_extended_resource_requests, namespace=cluster.config.namespace, dashboard=cluster.cluster_dashboard_uri(), diff --git a/src/codeflare_sdk/cluster/model.py b/src/codeflare_sdk/cluster/model.py index ab7b30ede..ee3b0945f 100644 --- a/src/codeflare_sdk/cluster/model.py +++ b/src/codeflare_sdk/cluster/model.py @@ -21,6 +21,7 @@ from dataclasses import dataclass, field from enum import Enum import typing +from typing import Union class RayClusterStatus(Enum): @@ -80,7 +81,8 @@ class RayCluster: workers: int worker_mem_requests: str worker_mem_limits: str - worker_cpu: int + worker_cpu_min: Union[int, str] + worker_cpu_max: Union[int, str] namespace: str dashboard: str worker_extended_resources: typing.Dict[str, int] = field(default_factory=dict) diff --git a/src/codeflare_sdk/utils/pretty_print.py b/src/codeflare_sdk/utils/pretty_print.py index 4842c9cd2..0befc9616 100644 --- a/src/codeflare_sdk/utils/pretty_print.py +++ b/src/codeflare_sdk/utils/pretty_print.py @@ -137,7 +137,7 @@ def print_clusters(clusters: List[RayCluster]): dashboard = cluster.dashboard workers = str(cluster.workers) memory = f"{cluster.worker_mem_requests}~{cluster.worker_mem_limits}" - cpu = str(cluster.worker_cpu) + cpu = f"{cluster.worker_cpu_min}~{cluster.worker_cpu_max}" gpu = str(cluster.worker_extended_resources.get("nvidia.com/gpu", 0)) #'table0' to display the cluster name, status, url, and dashboard link diff --git a/tests/unit_test.py b/tests/unit_test.py index 388723c50..3a907fe32 100644 --- a/tests/unit_test.py +++ b/tests/unit_test.py @@ -944,7 +944,8 @@ def test_ray_details(mocker, capsys): workers=1, worker_mem_requests="2G", worker_mem_limits="2G", - worker_cpu=1, + worker_cpu_min=1, + worker_cpu_max=1, namespace="ns", dashboard="fake-uri", head_cpu_requests=2, @@ -982,7 +983,8 @@ def test_ray_details(mocker, capsys): assert ray1.workers == ray2.workers assert ray1.worker_mem_requests == ray2.worker_mem_requests assert ray1.worker_mem_limits == ray2.worker_mem_limits - assert ray1.worker_cpu == ray2.worker_cpu + assert ray1.worker_cpu_min == ray2.worker_cpu_min + assert ray1.worker_cpu_max == ray2.worker_cpu_max assert ray1.worker_extended_resources == ray2.worker_extended_resources try: print_clusters([ray1, ray2]) @@ -2360,7 +2362,8 @@ def test_cluster_status(mocker): workers=1, worker_mem_requests=2, worker_mem_limits=2, - worker_cpu=1, + worker_cpu_min=1, + worker_cpu_max=1, namespace="ns", dashboard="fake-uri", head_cpu_requests=2, From c3c62387e61808e4b6b75ae4b5a5da5607591d8c Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Mon, 23 Sep 2024 09:56:53 +0100 Subject: [PATCH 02/14] Update cpu and mem names from UI table --- demo-notebooks/guided-demos/ipywidgets.ipynb | 6 ++-- src/codeflare_sdk/cluster/cluster.py | 38 +++++++++++--------- src/codeflare_sdk/cluster/model.py | 4 +-- src/codeflare_sdk/utils/pretty_print.py | 2 +- tests/unit_test.py | 12 +++---- 5 files changed, 33 insertions(+), 29 deletions(-) diff --git a/demo-notebooks/guided-demos/ipywidgets.ipynb b/demo-notebooks/guided-demos/ipywidgets.ipynb index 5d79cca5b..580ac120e 100644 --- a/demo-notebooks/guided-demos/ipywidgets.ipynb +++ b/demo-notebooks/guided-demos/ipywidgets.ipynb @@ -244,7 +244,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "bbb9361d9bb14342aff79c436f18ecd4", + "model_id": "6cd1ac11ee7248cc9f76b8b5349eabb1", "version_major": 2, "version_minor": 0 }, @@ -258,7 +258,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "ce0ba412e4a245ec93497b0eebff3ac7", + "model_id": "b5f79fe4d91e405aa513cbf599b86eb6", "version_major": 2, "version_minor": 0 }, @@ -272,7 +272,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "114adc01c5694abc84ecb134f42917e6", + "model_id": "4127f8292b064d1c935c9fc0dbe715f4", "version_major": 2, "version_minor": 0 }, diff --git a/src/codeflare_sdk/cluster/cluster.py b/src/codeflare_sdk/cluster/cluster.py index 09b4b8433..7727d3730 100644 --- a/src/codeflare_sdk/cluster/cluster.py +++ b/src/codeflare_sdk/cluster/cluster.py @@ -626,12 +626,14 @@ def _fetch_cluster_data(namespace): if item.worker_extended_resources else "nvidia.com/gpu: 0" for item in rayclusters ] - head_cpus = [item.head_cpus if item.head_cpus else 0 for item in rayclusters] - head_mem = [item.head_mem if item.head_mem else 0 for item in rayclusters] - worker_cpu_min = [item.worker_cpu_min if item.worker_cpu_min else 0 for item in rayclusters] - worker_cpu_max = [item.worker_cpu_max if item.worker_cpu_max else 0 for item in rayclusters] - worker_mem_min = [item.worker_mem_min if item.worker_mem_min else 0 for item in rayclusters] - worker_mem_max = [item.worker_mem_max if item.worker_mem_max else 0 for item in rayclusters] + head_cpu_requests = [item.head_cpu_requests if item.head_cpu_requests else 0 for item in rayclusters] + head_cpu_limits = [item.head_cpu_limits if item.head_cpu_limits else 0 for item in rayclusters] + head_mem_requests = [item.head_mem_requests if item.head_mem_requests else 0 for item in rayclusters] + head_mem_limits = [item.head_mem_limits if item.head_mem_limits else 0 for item in rayclusters] + worker_cpu_requests = [item.worker_cpu_requests if item.worker_cpu_requests else 0 for item in rayclusters] + worker_cpu_limits = [item.worker_cpu_limits if item.worker_cpu_limits else 0 for item in rayclusters] + worker_mem_requests = [item.worker_mem_requests if item.worker_mem_requests else 0 for item in rayclusters] + worker_mem_limits = [item.worker_mem_limits if item.worker_mem_limits else 0 for item in rayclusters] status = [item.status.name for item in rayclusters] status = [format_status(item.status) for item in rayclusters] @@ -641,12 +643,14 @@ def _fetch_cluster_data(namespace): "namespace": namespaces, "head gpus": head_extended_resources, "worker gpus": worker_extended_resources, - "head cpus": head_cpus, - "head memory": head_mem, - "worker cpu requests": worker_cpu_min, - "worker cpu limits": worker_cpu_max, - "worker memory requests": worker_mem_min, - "worker memory limits": worker_mem_max, + "head cpu requests": head_cpu_requests, + "head cpu limits": head_cpu_limits, + "head memory requests": head_mem_requests, + "head memory limits": head_mem_limits, + "worker cpu requests": worker_cpu_requests, + "worker cpu limits": worker_cpu_limits, + "worker memory requests": worker_mem_requests, + "worker memory limits": worker_mem_limits, "status": status } return pd.DataFrame(data) @@ -668,7 +672,7 @@ def on_cluster_click(change): new_value = change["new"] my_output.clear_output() with my_output: - display(HTML(df[df["name"]==new_value][["name", "namespace", "head gpus", "worker gpus", "head cpus", "head memory", "worker memory requests", "worker memory limits", "status"]].to_html(escape=False, index=False, border=2))) + display(HTML(df[df["name"]==new_value][["name", "namespace", "head gpus", "worker gpus", "head cpu requests", "head cpu limits", "head memory requests", "head memory limits", "worker memory requests", "worker memory limits", "status"]].to_html(escape=False, index=False, border=2))) classification_widget.observe(on_cluster_click, names="value") display(widgets.VBox([classification_widget, my_output])) @@ -1041,8 +1045,8 @@ def _map_to_ray_cluster(rc) -> Optional[RayCluster]: worker_mem_requests=rc["spec"]["workerGroupSpecs"][0]["template"]["spec"][ "containers" ][0]["resources"]["requests"]["memory"], - worker_cpu_min=rc["spec"]["workerGroupSpecs"][0]["template"]["spec"]["containers"][0]["resources"]["requests"]["cpu"], - worker_cpu_max=rc["spec"]["workerGroupSpecs"][0]["template"]["spec"]["containers"][ + worker_cpu_requests=rc["spec"]["workerGroupSpecs"][0]["template"]["spec"]["containers"][0]["resources"]["requests"]["cpu"], + worker_cpu_limits=rc["spec"]["workerGroupSpecs"][0]["template"]["spec"]["containers"][ 0 ]["resources"]["limits"]["cpu"], worker_extended_resources=worker_extended_resources, @@ -1083,8 +1087,8 @@ def _copy_to_ray(cluster: Cluster) -> RayCluster: workers=cluster.config.num_workers, worker_mem_requests=cluster.config.worker_memory_requests, worker_mem_limits=cluster.config.worker_memory_limits, - worker_cpu_min=cluster.config.worker_cpu_requests, - worker_cpu_max=cluster.config.worker_cpu_limits, + worker_cpu_requests=cluster.config.worker_cpu_requests, + worker_cpu_limits=cluster.config.worker_cpu_limits, worker_extended_resources=cluster.config.worker_extended_resource_requests, namespace=cluster.config.namespace, dashboard=cluster.cluster_dashboard_uri(), diff --git a/src/codeflare_sdk/cluster/model.py b/src/codeflare_sdk/cluster/model.py index ee3b0945f..7b2d2ee33 100644 --- a/src/codeflare_sdk/cluster/model.py +++ b/src/codeflare_sdk/cluster/model.py @@ -81,8 +81,8 @@ class RayCluster: workers: int worker_mem_requests: str worker_mem_limits: str - worker_cpu_min: Union[int, str] - worker_cpu_max: Union[int, str] + worker_cpu_requests: Union[int, str] + worker_cpu_limits: Union[int, str] namespace: str dashboard: str worker_extended_resources: typing.Dict[str, int] = field(default_factory=dict) diff --git a/src/codeflare_sdk/utils/pretty_print.py b/src/codeflare_sdk/utils/pretty_print.py index 0befc9616..a1410af35 100644 --- a/src/codeflare_sdk/utils/pretty_print.py +++ b/src/codeflare_sdk/utils/pretty_print.py @@ -137,7 +137,7 @@ def print_clusters(clusters: List[RayCluster]): dashboard = cluster.dashboard workers = str(cluster.workers) memory = f"{cluster.worker_mem_requests}~{cluster.worker_mem_limits}" - cpu = f"{cluster.worker_cpu_min}~{cluster.worker_cpu_max}" + cpu = f"{cluster.worker_cpu_requests}~{cluster.worker_cpu_limits}" gpu = str(cluster.worker_extended_resources.get("nvidia.com/gpu", 0)) #'table0' to display the cluster name, status, url, and dashboard link diff --git a/tests/unit_test.py b/tests/unit_test.py index 3a907fe32..40c187de0 100644 --- a/tests/unit_test.py +++ b/tests/unit_test.py @@ -944,8 +944,8 @@ def test_ray_details(mocker, capsys): workers=1, worker_mem_requests="2G", worker_mem_limits="2G", - worker_cpu_min=1, - worker_cpu_max=1, + worker_cpu_requests=1, + worker_cpu_limits=1, namespace="ns", dashboard="fake-uri", head_cpu_requests=2, @@ -983,8 +983,8 @@ def test_ray_details(mocker, capsys): assert ray1.workers == ray2.workers assert ray1.worker_mem_requests == ray2.worker_mem_requests assert ray1.worker_mem_limits == ray2.worker_mem_limits - assert ray1.worker_cpu_min == ray2.worker_cpu_min - assert ray1.worker_cpu_max == ray2.worker_cpu_max + assert ray1.worker_cpu_requests == ray2.worker_cpu_requests + assert ray1.worker_cpu_limits == ray2.worker_cpu_limits assert ray1.worker_extended_resources == ray2.worker_extended_resources try: print_clusters([ray1, ray2]) @@ -2362,8 +2362,8 @@ def test_cluster_status(mocker): workers=1, worker_mem_requests=2, worker_mem_limits=2, - worker_cpu_min=1, - worker_cpu_max=1, + worker_cpu_requests=1, + worker_cpu_limits=1, namespace="ns", dashboard="fake-uri", head_cpu_requests=2, From 496b691f493b559d210c7ccaff5adca001e5d126 Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Mon, 23 Sep 2024 10:17:15 +0100 Subject: [PATCH 03/14] Merge requests-limits into single column in UI table --- demo-notebooks/guided-demos/ipywidgets.ipynb | 13 ++++-- src/codeflare_sdk/cluster/cluster.py | 42 ++++++++++---------- 2 files changed, 31 insertions(+), 24 deletions(-) diff --git a/demo-notebooks/guided-demos/ipywidgets.ipynb b/demo-notebooks/guided-demos/ipywidgets.ipynb index 580ac120e..7e6d364e2 100644 --- a/demo-notebooks/guided-demos/ipywidgets.ipynb +++ b/demo-notebooks/guided-demos/ipywidgets.ipynb @@ -244,7 +244,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "6cd1ac11ee7248cc9f76b8b5349eabb1", + "model_id": "bb3e74d1cf9249f5a8f42fcb861be1c2", "version_major": 2, "version_minor": 0 }, @@ -258,7 +258,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "b5f79fe4d91e405aa513cbf599b86eb6", + "model_id": "d0733b44ffd2470d9ecaafd333a496d7", "version_major": 2, "version_minor": 0 }, @@ -272,7 +272,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "4127f8292b064d1c935c9fc0dbe715f4", + "model_id": "8abc240e519f42208fcfffe963cdb808", "version_major": 2, "version_minor": 0 }, @@ -282,6 +282,13 @@ }, "metadata": {}, "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Cluster raytesta11 in the default namespace was deleted successfully.\n" + ] } ], "source": [ diff --git a/src/codeflare_sdk/cluster/cluster.py b/src/codeflare_sdk/cluster/cluster.py index 7727d3730..cd8d09a87 100644 --- a/src/codeflare_sdk/cluster/cluster.py +++ b/src/codeflare_sdk/cluster/cluster.py @@ -618,39 +618,39 @@ def _fetch_cluster_data(namespace): namespaces = [item.namespace for item in rayclusters] head_extended_resources = [ f"{list(item.head_extended_resources.keys())[0]}: {list(item.head_extended_resources.values())[0]}" - if item.head_extended_resources else "nvidia.com/gpu: 0" + if item.head_extended_resources else "0" for item in rayclusters ] worker_extended_resources = [ f"{list(item.worker_extended_resources.keys())[0]}: {list(item.worker_extended_resources.values())[0]}" - if item.worker_extended_resources else "nvidia.com/gpu: 0" + if item.worker_extended_resources else "0" for item in rayclusters ] head_cpu_requests = [item.head_cpu_requests if item.head_cpu_requests else 0 for item in rayclusters] head_cpu_limits = [item.head_cpu_limits if item.head_cpu_limits else 0 for item in rayclusters] + head_cpu_rl = [f"{requests}~{limits}" for requests, limits in zip(head_cpu_requests, head_cpu_limits)] head_mem_requests = [item.head_mem_requests if item.head_mem_requests else 0 for item in rayclusters] head_mem_limits = [item.head_mem_limits if item.head_mem_limits else 0 for item in rayclusters] + head_mem_rl = [f"{requests}~{limits}" for requests, limits in zip(head_mem_requests, head_mem_limits)] worker_cpu_requests = [item.worker_cpu_requests if item.worker_cpu_requests else 0 for item in rayclusters] worker_cpu_limits = [item.worker_cpu_limits if item.worker_cpu_limits else 0 for item in rayclusters] + worker_cpu_rl = [f"{requests}~{limits}" for requests, limits in zip(worker_cpu_requests, worker_cpu_limits)] worker_mem_requests = [item.worker_mem_requests if item.worker_mem_requests else 0 for item in rayclusters] worker_mem_limits = [item.worker_mem_limits if item.worker_mem_limits else 0 for item in rayclusters] + worker_mem_rl = [f"{requests}~{limits}" for requests, limits in zip(worker_mem_requests, worker_mem_limits)] status = [item.status.name for item in rayclusters] status = [format_status(item.status) for item in rayclusters] data = { - "name": names, - "namespace": namespaces, - "head gpus": head_extended_resources, - "worker gpus": worker_extended_resources, - "head cpu requests": head_cpu_requests, - "head cpu limits": head_cpu_limits, - "head memory requests": head_mem_requests, - "head memory limits": head_mem_limits, - "worker cpu requests": worker_cpu_requests, - "worker cpu limits": worker_cpu_limits, - "worker memory requests": worker_mem_requests, - "worker memory limits": worker_mem_limits, + "Name": names, + "Namespace": namespaces, + "Head GPUs": head_extended_resources, + "Worker GPUs": worker_extended_resources, + "Head CPU Req~Lim": head_cpu_rl, + "Head Memory Req~Lim": head_mem_rl, + "Worker CPU Req~Lim": worker_cpu_rl, + "Worker Memory Req~Lim": worker_mem_rl, "status": status } return pd.DataFrame(data) @@ -660,11 +660,11 @@ def list_cluster_details(namespace: str): df = _fetch_cluster_data(namespace) my_output = widgets.Output() - if df["name"].empty: + if df["Name"].empty: print(f"No clusters found in the {namespace} namespace.") else: classification_widget = widgets.ToggleButtons( - options=df["name"].tolist(), value=None, + options=df["Name"].tolist(), value=None, description='Select an existing cluster:', ) @@ -672,27 +672,27 @@ def on_cluster_click(change): new_value = change["new"] my_output.clear_output() with my_output: - display(HTML(df[df["name"]==new_value][["name", "namespace", "head gpus", "worker gpus", "head cpu requests", "head cpu limits", "head memory requests", "head memory limits", "worker memory requests", "worker memory limits", "status"]].to_html(escape=False, index=False, border=2))) + display(HTML(df[df["Name"]==new_value][["Name", "Namespace", "Head GPUs", "Head CPU Req~Lim", "Head Memory Req~Lim", "Worker GPUs", "Worker CPU Req~Lim", "Worker Memory Req~Lim", "status"]].to_html(escape=False, index=False, border=2))) classification_widget.observe(on_cluster_click, names="value") display(widgets.VBox([classification_widget, my_output])) def on_delete_button_clicked(b): cluster_name = classification_widget.value - namespace = df[df["name"]==classification_widget.value]["namespace"].values[0] + namespace = df[df["Name"]==classification_widget.value]["Namespace"].values[0] delete_cluster(cluster_name, namespace) my_output.clear_output() print(f"Cluster {cluster_name} in the {namespace} namespace was deleted successfully.") # Refresh the dataframe new_df = _fetch_cluster_data(namespace) - classification_widget.options = new_df["name"].tolist() + classification_widget.options = new_df["Name"].tolist() # out Output widget is used to execute JavaScript code to open the Ray dashboard URL in a new browser tab out = widgets.Output() def on_ray_dashboard_button_clicked(b): cluster_name = classification_widget.value - namespace = df[df["name"]==classification_widget.value]["namespace"].values[0] + namespace = df[df["Name"]==classification_widget.value]["Namespace"].values[0] cluster = Cluster(ClusterConfiguration(cluster_name, namespace)) dashboard_url = cluster.cluster_dashboard_uri() @@ -704,7 +704,7 @@ def on_ray_dashboard_button_clicked(b): def on_list_jobs_button_clicked(b): cluster_name = classification_widget.value - namespace = df[df["name"]==classification_widget.value]["namespace"].values[0] + namespace = df[df["Name"]==classification_widget.value]["Namespace"].values[0] cluster = Cluster(ClusterConfiguration(cluster_name, namespace)) dashboard_url = cluster.cluster_dashboard_uri() From 5460d3e1ab5cfdf99d87fe9224c79af6ae91f053 Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Mon, 23 Sep 2024 12:26:25 +0100 Subject: [PATCH 04/14] Enhance notebook outputs/display on button clicks --- src/codeflare_sdk/cluster/cluster.py | 49 ++++++++++++++++++++-------- 1 file changed, 36 insertions(+), 13 deletions(-) diff --git a/src/codeflare_sdk/cluster/cluster.py b/src/codeflare_sdk/cluster/cluster.py index cd8d09a87..fa26dcb85 100644 --- a/src/codeflare_sdk/cluster/cluster.py +++ b/src/codeflare_sdk/cluster/cluster.py @@ -656,10 +656,12 @@ def _fetch_cluster_data(namespace): return pd.DataFrame(data) -def list_cluster_details(namespace: str): +def list_cluster_details(namespace: str): # TODO: or current namespace as default + global df df = _fetch_cluster_data(namespace) - my_output = widgets.Output() + outputs = widgets.Output() + data_output = widgets.Output() if df["Name"].empty: print(f"No clusters found in the {namespace} namespace.") else: @@ -670,22 +672,38 @@ def list_cluster_details(namespace: str): def on_cluster_click(change): new_value = change["new"] - my_output.clear_output() - with my_output: + data_output.clear_output() + df = _fetch_cluster_data(namespace) + classification_widget.options = df["Name"].tolist() + with data_output: display(HTML(df[df["Name"]==new_value][["Name", "Namespace", "Head GPUs", "Head CPU Req~Lim", "Head Memory Req~Lim", "Worker GPUs", "Worker CPU Req~Lim", "Worker Memory Req~Lim", "status"]].to_html(escape=False, index=False, border=2))) classification_widget.observe(on_cluster_click, names="value") - display(widgets.VBox([classification_widget, my_output])) + display(widgets.VBox([classification_widget, data_output])) def on_delete_button_clicked(b): + global df cluster_name = classification_widget.value namespace = df[df["Name"]==classification_widget.value]["Namespace"].values[0] delete_cluster(cluster_name, namespace) - my_output.clear_output() - print(f"Cluster {cluster_name} in the {namespace} namespace was deleted successfully.") + + sleep(3) # wait for the cluster to be deleted + outputs.clear_output() + with outputs: + print(f"Cluster {cluster_name} in the {namespace} namespace was deleted successfully.") # Refresh the dataframe - new_df = _fetch_cluster_data(namespace) - classification_widget.options = new_df["Name"].tolist() + df = _fetch_cluster_data(namespace) + if df["Name"].empty: + classification_widget.close() + delete_button.close() + list_jobs_button.close() + ray_dashboard_button.close() + data_output.clear_output() + with data_output: + print(f"No clusters found in the {namespace} namespace.") + else: + classification_widget.options = df["Name"].tolist() + # out Output widget is used to execute JavaScript code to open the Ray dashboard URL in a new browser tab @@ -697,10 +715,11 @@ def on_ray_dashboard_button_clicked(b): cluster = Cluster(ClusterConfiguration(cluster_name, namespace)) dashboard_url = cluster.cluster_dashboard_uri() - my_output.clear_output() + outputs.clear_output() + with outputs: + print(f"Opening Ray Dashboard for {cluster_name} cluster:\n{dashboard_url}") with out: display(Javascript(f'window.open("{dashboard_url}", "_blank");')) - print(f"Opening Ray Dashboard for {cluster_name}:\n{dashboard_url}") def on_list_jobs_button_clicked(b): cluster_name = classification_widget.value @@ -709,7 +728,9 @@ def on_list_jobs_button_clicked(b): cluster = Cluster(ClusterConfiguration(cluster_name, namespace)) dashboard_url = cluster.cluster_dashboard_uri() - my_output.clear_output() + outputs.clear_output() + with outputs: + print(f"Opening Ray Jobs Dashboard for {cluster_name} cluster:\n{dashboard_url}/#/jobs") with out: display(Javascript(f'window.open("{dashboard_url}/#/jobs", "_blank");')) @@ -723,17 +744,19 @@ def on_list_jobs_button_clicked(b): delete_button = widgets.Button( description='Delete Cluster', icon='trash', + tooltip="Delete the selected cluster" ) delete_button.on_click(on_delete_button_clicked) ray_dashboard_button = widgets.Button( description='Open Ray Dashboard', icon='dashboard', + tooltip="Open the Ray Dashboard in a new tab", layout=widgets.Layout(width='auto'), ) ray_dashboard_button.on_click(on_ray_dashboard_button_clicked) - display(widgets.HBox([delete_button, list_jobs_button, ray_dashboard_button]), out) + display(widgets.HBox([delete_button, list_jobs_button, ray_dashboard_button]), out, outputs) From a9dcae0ba55053cf63713ca5f66fc7c9f695632c Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Mon, 23 Sep 2024 17:57:50 +0100 Subject: [PATCH 05/14] Refactor and move UI table to widgets.py file --- src/codeflare_sdk/__init__.py | 2 +- src/codeflare_sdk/cluster/__init__.py | 5 +- src/codeflare_sdk/cluster/cluster.py | 198 ------------------------ src/codeflare_sdk/cluster/widgets.py | 213 +++++++++++++++++++++++++- 4 files changed, 216 insertions(+), 202 deletions(-) diff --git a/src/codeflare_sdk/__init__.py b/src/codeflare_sdk/__init__.py index fa512fbf2..29205a36e 100644 --- a/src/codeflare_sdk/__init__.py +++ b/src/codeflare_sdk/__init__.py @@ -14,7 +14,7 @@ get_cluster, list_all_queued, list_all_clusters, - list_cluster_details, + view_clusters, ) from .job import RayJobClient diff --git a/src/codeflare_sdk/cluster/__init__.py b/src/codeflare_sdk/cluster/__init__.py index 237cff0a9..6490a2247 100644 --- a/src/codeflare_sdk/cluster/__init__.py +++ b/src/codeflare_sdk/cluster/__init__.py @@ -19,7 +19,10 @@ get_cluster, list_all_queued, list_all_clusters, - list_cluster_details, +) + +from .widgets import ( + view_clusters, ) from .awload import AWManager diff --git a/src/codeflare_sdk/cluster/cluster.py b/src/codeflare_sdk/cluster/cluster.py index fa26dcb85..8516e921b 100644 --- a/src/codeflare_sdk/cluster/cluster.py +++ b/src/codeflare_sdk/cluster/cluster.py @@ -35,10 +35,6 @@ from ..utils.kube_api_helpers import _kube_api_error_handling from ..utils.generate_yaml import is_openshift_cluster -import ipywidgets as widgets -from IPython.display import display, HTML, Javascript -import pandas as pd - from .config import ClusterConfiguration from .model import ( AppWrapper, @@ -597,200 +593,6 @@ def get_current_namespace(): # pragma: no cover except KeyError: return None -# format_status takes a RayCluster status and applies colors and icons based on the status. -def format_status(status): - if status == RayClusterStatus.READY: - return 'Ready โœ“' - elif status == RayClusterStatus.SUSPENDED: - return 'Suspended โ„๏ธ' - elif status == RayClusterStatus.FAILED: - return 'Failed โœ—' - elif status == RayClusterStatus.UNHEALTHY: - return 'Unhealthy' - elif status == RayClusterStatus.UNKNOWN: - return 'Unknown' - else: - return status - -def _fetch_cluster_data(namespace): - rayclusters = list_all_clusters(namespace, False) - names = [item.name for item in rayclusters] - namespaces = [item.namespace for item in rayclusters] - head_extended_resources = [ - f"{list(item.head_extended_resources.keys())[0]}: {list(item.head_extended_resources.values())[0]}" - if item.head_extended_resources else "0" - for item in rayclusters - ] - worker_extended_resources = [ - f"{list(item.worker_extended_resources.keys())[0]}: {list(item.worker_extended_resources.values())[0]}" - if item.worker_extended_resources else "0" - for item in rayclusters - ] - head_cpu_requests = [item.head_cpu_requests if item.head_cpu_requests else 0 for item in rayclusters] - head_cpu_limits = [item.head_cpu_limits if item.head_cpu_limits else 0 for item in rayclusters] - head_cpu_rl = [f"{requests}~{limits}" for requests, limits in zip(head_cpu_requests, head_cpu_limits)] - head_mem_requests = [item.head_mem_requests if item.head_mem_requests else 0 for item in rayclusters] - head_mem_limits = [item.head_mem_limits if item.head_mem_limits else 0 for item in rayclusters] - head_mem_rl = [f"{requests}~{limits}" for requests, limits in zip(head_mem_requests, head_mem_limits)] - worker_cpu_requests = [item.worker_cpu_requests if item.worker_cpu_requests else 0 for item in rayclusters] - worker_cpu_limits = [item.worker_cpu_limits if item.worker_cpu_limits else 0 for item in rayclusters] - worker_cpu_rl = [f"{requests}~{limits}" for requests, limits in zip(worker_cpu_requests, worker_cpu_limits)] - worker_mem_requests = [item.worker_mem_requests if item.worker_mem_requests else 0 for item in rayclusters] - worker_mem_limits = [item.worker_mem_limits if item.worker_mem_limits else 0 for item in rayclusters] - worker_mem_rl = [f"{requests}~{limits}" for requests, limits in zip(worker_mem_requests, worker_mem_limits)] - status = [item.status.name for item in rayclusters] - - status = [format_status(item.status) for item in rayclusters] - - data = { - "Name": names, - "Namespace": namespaces, - "Head GPUs": head_extended_resources, - "Worker GPUs": worker_extended_resources, - "Head CPU Req~Lim": head_cpu_rl, - "Head Memory Req~Lim": head_mem_rl, - "Worker CPU Req~Lim": worker_cpu_rl, - "Worker Memory Req~Lim": worker_mem_rl, - "status": status - } - return pd.DataFrame(data) - - -def list_cluster_details(namespace: str): # TODO: or current namespace as default - global df - df = _fetch_cluster_data(namespace) - - outputs = widgets.Output() - data_output = widgets.Output() - if df["Name"].empty: - print(f"No clusters found in the {namespace} namespace.") - else: - classification_widget = widgets.ToggleButtons( - options=df["Name"].tolist(), value=None, - description='Select an existing cluster:', - ) - - def on_cluster_click(change): - new_value = change["new"] - data_output.clear_output() - df = _fetch_cluster_data(namespace) - classification_widget.options = df["Name"].tolist() - with data_output: - display(HTML(df[df["Name"]==new_value][["Name", "Namespace", "Head GPUs", "Head CPU Req~Lim", "Head Memory Req~Lim", "Worker GPUs", "Worker CPU Req~Lim", "Worker Memory Req~Lim", "status"]].to_html(escape=False, index=False, border=2))) - - classification_widget.observe(on_cluster_click, names="value") - display(widgets.VBox([classification_widget, data_output])) - - def on_delete_button_clicked(b): - global df - cluster_name = classification_widget.value - namespace = df[df["Name"]==classification_widget.value]["Namespace"].values[0] - delete_cluster(cluster_name, namespace) - - sleep(3) # wait for the cluster to be deleted - outputs.clear_output() - with outputs: - print(f"Cluster {cluster_name} in the {namespace} namespace was deleted successfully.") - # Refresh the dataframe - df = _fetch_cluster_data(namespace) - if df["Name"].empty: - classification_widget.close() - delete_button.close() - list_jobs_button.close() - ray_dashboard_button.close() - data_output.clear_output() - with data_output: - print(f"No clusters found in the {namespace} namespace.") - else: - classification_widget.options = df["Name"].tolist() - - - - # out Output widget is used to execute JavaScript code to open the Ray dashboard URL in a new browser tab - out = widgets.Output() - def on_ray_dashboard_button_clicked(b): - cluster_name = classification_widget.value - namespace = df[df["Name"]==classification_widget.value]["Namespace"].values[0] - - cluster = Cluster(ClusterConfiguration(cluster_name, namespace)) - dashboard_url = cluster.cluster_dashboard_uri() - - outputs.clear_output() - with outputs: - print(f"Opening Ray Dashboard for {cluster_name} cluster:\n{dashboard_url}") - with out: - display(Javascript(f'window.open("{dashboard_url}", "_blank");')) - - def on_list_jobs_button_clicked(b): - cluster_name = classification_widget.value - namespace = df[df["Name"]==classification_widget.value]["Namespace"].values[0] - - cluster = Cluster(ClusterConfiguration(cluster_name, namespace)) - dashboard_url = cluster.cluster_dashboard_uri() - - outputs.clear_output() - with outputs: - print(f"Opening Ray Jobs Dashboard for {cluster_name} cluster:\n{dashboard_url}/#/jobs") - with out: - display(Javascript(f'window.open("{dashboard_url}/#/jobs", "_blank");')) - - list_jobs_button = widgets.Button( - description='View Jobs', - icon='suitcase', - tooltip="Open the Ray Job Dashboard" - ) - list_jobs_button.on_click(on_list_jobs_button_clicked) - - delete_button = widgets.Button( - description='Delete Cluster', - icon='trash', - tooltip="Delete the selected cluster" - ) - delete_button.on_click(on_delete_button_clicked) - - ray_dashboard_button = widgets.Button( - description='Open Ray Dashboard', - icon='dashboard', - tooltip="Open the Ray Dashboard in a new tab", - layout=widgets.Layout(width='auto'), - ) - ray_dashboard_button.on_click(on_ray_dashboard_button_clicked) - - display(widgets.HBox([delete_button, list_jobs_button, ray_dashboard_button]), out, outputs) - - - -def delete_cluster( - cluster_name: str, - namespace: str, # TODO: get current namespace if not provided -): - if _check_aw_exists(cluster_name, namespace): - try: - config_check() - api_instance = client.CustomObjectsApi(api_config_handler()) - api_instance.delete_namespaced_custom_object( - group="workload.codeflare.dev", - version="v1beta2", - namespace=namespace, - plural="appwrappers", - name=cluster_name, - ) - except Exception as e: - return _kube_api_error_handling(e) - else: - try: - config_check() - api_instance = client.CustomObjectsApi(api_config_handler()) - api_instance.delete_namespaced_custom_object( - group="ray.io", - version="v1", - namespace=namespace, - plural="rayclusters", - name=cluster_name, - ) - except Exception as e: - return _kube_api_error_handling(e) - def get_cluster( cluster_name: str, diff --git a/src/codeflare_sdk/cluster/widgets.py b/src/codeflare_sdk/cluster/widgets.py index 351640e04..a005d9c20 100644 --- a/src/codeflare_sdk/cluster/widgets.py +++ b/src/codeflare_sdk/cluster/widgets.py @@ -15,10 +15,17 @@ """ The widgets sub-module contains the ui widgets created using the ipywidgets package. """ -import ipywidgets as widgets -from IPython.display import display import os +from time import sleep import codeflare_sdk +from kubernetes import client +import ipywidgets as widgets +from IPython.display import display, HTML, Javascript +import pandas as pd +from .config import ClusterConfiguration +from .model import RayClusterStatus +from ..utils.kube_api_helpers import _kube_api_error_handling +from .auth import config_check, api_config_handler def cluster_up_down_buttons(cluster: "codeflare_sdk.cluster.Cluster") -> widgets.Button: @@ -89,3 +96,205 @@ def is_notebook() -> bool: return True else: return False + +def view_clusters(namespace: str = None): + """view_clusters function will display existing clusters with their specs, and handle user interactions.""" + from .cluster import Cluster, get_current_namespace + if not namespace: + namespace = get_current_namespace() + + user_output = widgets.Output() + raycluster_data_output = widgets.Output() + url_output = widgets.Output() + + df = _fetch_cluster_data(namespace) + if df.empty: + print(f"No clusters found in the {namespace} namespace.") + else: + classification_widget = widgets.ToggleButtons( + options=df["Name"].tolist(), value=None, + description='Select an existing cluster:', + ) + + classification_widget.observe(lambda selection_change: _on_cluster_click(selection_change, raycluster_data_output, namespace, classification_widget), names="value") + + delete_button = widgets.Button( + description='Delete Cluster', + icon='trash', + tooltip="Delete the selected cluster" + ) + delete_button.on_click(lambda b: _on_delete_button_click(b, classification_widget, df, raycluster_data_output, user_output, delete_button, list_jobs_button, ray_dashboard_button)) + + list_jobs_button = widgets.Button( + description='View Jobs', + icon='suitcase', + tooltip="Open the Ray Job Dashboard" + ) + list_jobs_button.on_click(lambda b: _on_list_jobs_button_click(b, classification_widget, df, user_output, url_output)) + + ray_dashboard_button = widgets.Button( + description='Open Ray Dashboard', + icon='dashboard', + tooltip="Open the Ray Dashboard in a new tab", + layout=widgets.Layout(width='auto'), + ) + ray_dashboard_button.on_click(lambda b: _on_ray_dashboard_button_click(b, classification_widget, df, user_output, url_output)) + + display(widgets.VBox([classification_widget, raycluster_data_output])) + display(widgets.HBox([delete_button, list_jobs_button, ray_dashboard_button]), url_output, user_output) + +# Handles the event when a cluster is selected from the toggle buttons, updating the output with cluster details. +def _on_cluster_click(selection_change, raycluster_data_output: widgets.Output, namespace: str, classification_widget: widgets.ToggleButtons): + new_value = selection_change["new"] + raycluster_data_output.clear_output() + df = _fetch_cluster_data(namespace) + classification_widget.options = df["Name"].tolist() + with raycluster_data_output: + display(HTML(df[df["Name"]==new_value][["Name", "Namespace", "Head GPUs", "Head CPU Req~Lim", "Head Memory Req~Lim", "Worker GPUs", "Worker CPU Req~Lim", "Worker Memory Req~Lim", "status"]].to_html(escape=False, index=False, border=2))) + + +def _on_delete_button_click(b, classification_widget: widgets.ToggleButtons, df: pd.DataFrame, raycluster_data_output: widgets.Output, user_output: widgets.Output, delete_button: widgets.Button, list_jobs_button: widgets.Button, ray_dashboard_button: widgets.Button): + cluster_name = classification_widget.value + namespace = df[df["Name"]==classification_widget.value]["Namespace"].values[0] + + _delete_cluster(cluster_name, namespace) + + sleep(2) # TODO: wait for the cluster to be deleted instead + with user_output: + user_output.clear_output() + print(f"Cluster {cluster_name} in the {namespace} namespace was deleted successfully.") + + # Refresh the dataframe + new_df = _fetch_cluster_data(namespace) + if new_df.empty: + classification_widget.close() + delete_button.close() + list_jobs_button.close() + ray_dashboard_button.close() + with raycluster_data_output: + raycluster_data_output.clear_output() + print(f"No clusters found in the {namespace} namespace.") + else: + classification_widget.options = new_df["Name"].tolist() + +def _on_ray_dashboard_button_click(b, classification_widget: widgets.ToggleButtons, df: pd.DataFrame, user_output: widgets.Output, url_output: widgets.Output): + from codeflare_sdk.cluster import Cluster + cluster_name = classification_widget.value + namespace = df[df["Name"]==classification_widget.value]["Namespace"].values[0] + + cluster = Cluster(ClusterConfiguration(cluster_name, namespace)) + dashboard_url = cluster.cluster_dashboard_uri() + + with user_output: + user_output.clear_output() + print(f"Opening Ray Dashboard for {cluster_name} cluster:\n{dashboard_url}") + with url_output: + display(Javascript(f'window.open("{dashboard_url}", "_blank");')) + +def _on_list_jobs_button_click(b, classification_widget: widgets.ToggleButtons, df: pd.DataFrame, user_output: widgets.Output, url_output: widgets.Output): + from codeflare_sdk.cluster import Cluster + cluster_name = classification_widget.value + namespace = df[df["Name"]==classification_widget.value]["Namespace"].values[0] + + cluster = Cluster(ClusterConfiguration(cluster_name, namespace)) + dashboard_url = cluster.cluster_dashboard_uri() + + with user_output: + user_output.clear_output() + print(f"Opening Ray Jobs Dashboard for {cluster_name} cluster:\n{dashboard_url}/#/jobs") + with url_output: + display(Javascript(f'window.open("{dashboard_url}/#/jobs", "_blank");')) + +def _delete_cluster( + cluster_name: str, + namespace: str, +): + from .cluster import _check_aw_exists + if _check_aw_exists(cluster_name, namespace): + try: + config_check() + api_instance = client.CustomObjectsApi(api_config_handler()) + api_instance.delete_namespaced_custom_object( + group="workload.codeflare.dev", + version="v1beta2", + namespace=namespace, + plural="appwrappers", + name=cluster_name, + ) + except Exception as e: + return _kube_api_error_handling(e) + else: + try: + config_check() + api_instance = client.CustomObjectsApi(api_config_handler()) + api_instance.delete_namespaced_custom_object( + group="ray.io", + version="v1", + namespace=namespace, + plural="rayclusters", + name=cluster_name, + ) + except Exception as e: + return _kube_api_error_handling(e) + + +def _fetch_cluster_data(namespace): + from .cluster import list_all_clusters + rayclusters = list_all_clusters(namespace, False) + if not rayclusters: + return pd.DataFrame() + names = [item.name for item in rayclusters] + namespaces = [item.namespace for item in rayclusters] + head_extended_resources = [ + f"{list(item.head_extended_resources.keys())[0]}: {list(item.head_extended_resources.values())[0]}" + if item.head_extended_resources else "0" + for item in rayclusters + ] + worker_extended_resources = [ + f"{list(item.worker_extended_resources.keys())[0]}: {list(item.worker_extended_resources.values())[0]}" + if item.worker_extended_resources else "0" + for item in rayclusters + ] + head_cpu_requests = [item.head_cpu_requests if item.head_cpu_requests else 0 for item in rayclusters] + head_cpu_limits = [item.head_cpu_limits if item.head_cpu_limits else 0 for item in rayclusters] + head_cpu_rl = [f"{requests}~{limits}" for requests, limits in zip(head_cpu_requests, head_cpu_limits)] + head_mem_requests = [item.head_mem_requests if item.head_mem_requests else 0 for item in rayclusters] + head_mem_limits = [item.head_mem_limits if item.head_mem_limits else 0 for item in rayclusters] + head_mem_rl = [f"{requests}~{limits}" for requests, limits in zip(head_mem_requests, head_mem_limits)] + worker_cpu_requests = [item.worker_cpu_requests if item.worker_cpu_requests else 0 for item in rayclusters] + worker_cpu_limits = [item.worker_cpu_limits if item.worker_cpu_limits else 0 for item in rayclusters] + worker_cpu_rl = [f"{requests}~{limits}" for requests, limits in zip(worker_cpu_requests, worker_cpu_limits)] + worker_mem_requests = [item.worker_mem_requests if item.worker_mem_requests else 0 for item in rayclusters] + worker_mem_limits = [item.worker_mem_limits if item.worker_mem_limits else 0 for item in rayclusters] + worker_mem_rl = [f"{requests}~{limits}" for requests, limits in zip(worker_mem_requests, worker_mem_limits)] + status = [item.status.name for item in rayclusters] + + status = [_format_status(item.status) for item in rayclusters] + + data = { + "Name": names, + "Namespace": namespaces, + "Head GPUs": head_extended_resources, + "Worker GPUs": worker_extended_resources, + "Head CPU Req~Lim": head_cpu_rl, + "Head Memory Req~Lim": head_mem_rl, + "Worker CPU Req~Lim": worker_cpu_rl, + "Worker Memory Req~Lim": worker_mem_rl, + "status": status + } + return pd.DataFrame(data) + +# format_status takes a RayCluster status and applies colors and icons based on the status. +def _format_status(status): + if status == RayClusterStatus.READY: + return 'Ready โœ“' + elif status == RayClusterStatus.SUSPENDED: + return 'Suspended โ„๏ธ' + elif status == RayClusterStatus.FAILED: + return 'Failed โœ—' + elif status == RayClusterStatus.UNHEALTHY: + return 'Unhealthy' + elif status == RayClusterStatus.UNKNOWN: + return 'Unknown' + else: + return status From e7bf08d7ba4e4c9273107e6d812469c0d63a06c0 Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Mon, 23 Sep 2024 19:22:37 +0100 Subject: [PATCH 06/14] Add unit tests for UI table functions --- demo-notebooks/guided-demos/ipywidgets.ipynb | 575 ------------------- src/codeflare_sdk/cluster/widgets.py | 3 +- tests/unit_test.py | 181 +++++- 3 files changed, 174 insertions(+), 585 deletions(-) delete mode 100644 demo-notebooks/guided-demos/ipywidgets.ipynb diff --git a/demo-notebooks/guided-demos/ipywidgets.ipynb b/demo-notebooks/guided-demos/ipywidgets.ipynb deleted file mode 100644 index 7e6d364e2..000000000 --- a/demo-notebooks/guided-demos/ipywidgets.ipynb +++ /dev/null @@ -1,575 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "8d4a42f6", - "metadata": {}, - "source": [ - "In this notebook, we will go through the basics of using the SDK to:\n", - " - Spin up a Ray cluster with our desired resources\n", - " - View the status and specs of our Ray cluster\n", - " - Take down the Ray cluster when finished" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "301094f1", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Found existing installation: codeflare-sdk 0.0.0.dev0\n", - "Uninstalling codeflare-sdk-0.0.0.dev0:\n", - " Successfully uninstalled codeflare-sdk-0.0.0.dev0\n", - "Note: you may need to restart the kernel to use updated packages.\n", - "Defaulting to user installation because normal site-packages is not writeable\n", - "Processing /home/christianzaccaria/Documents/GitHub/codeflare-sdk/dist/codeflare_sdk-0.0.0.dev0-py3-none-any.whl\n", - "Requirement already satisfied: cryptography==40.0.2 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from codeflare-sdk==0.0.0.dev0) (40.0.2)\n", - "Requirement already satisfied: executing==1.2.0 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from codeflare-sdk==0.0.0.dev0) (1.2.0)\n", - "Requirement already satisfied: ipywidgets==8.1.2 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from codeflare-sdk==0.0.0.dev0) (8.1.2)\n", - "Requirement already satisfied: kubernetes<27,>=25.3.0 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from codeflare-sdk==0.0.0.dev0) (26.1.0)\n", - "Requirement already satisfied: openshift-client==1.0.18 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from codeflare-sdk==0.0.0.dev0) (1.0.18)\n", - "Requirement already satisfied: pydantic<2 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from codeflare-sdk==0.0.0.dev0) (1.10.14)\n", - "Requirement already satisfied: ray==2.23.0 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (2.23.0)\n", - "Requirement already satisfied: rich<13.0,>=12.5 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from codeflare-sdk==0.0.0.dev0) (12.6.0)\n", - "Requirement already satisfied: setuptools<=73.0.0 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from codeflare-sdk==0.0.0.dev0) (69.1.1)\n", - "Requirement already satisfied: cffi>=1.12 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from cryptography==40.0.2->codeflare-sdk==0.0.0.dev0) (1.16.0)\n", - "Requirement already satisfied: comm>=0.1.3 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ipywidgets==8.1.2->codeflare-sdk==0.0.0.dev0) (0.2.0)\n", - "Requirement already satisfied: ipython>=6.1.0 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ipywidgets==8.1.2->codeflare-sdk==0.0.0.dev0) (8.18.1)\n", - "Requirement already satisfied: traitlets>=4.3.1 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ipywidgets==8.1.2->codeflare-sdk==0.0.0.dev0) (5.14.0)\n", - "Requirement already satisfied: widgetsnbextension~=4.0.10 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ipywidgets==8.1.2->codeflare-sdk==0.0.0.dev0) (4.0.11)\n", - "Requirement already satisfied: jupyterlab-widgets~=3.0.10 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ipywidgets==8.1.2->codeflare-sdk==0.0.0.dev0) (3.0.11)\n", - "Requirement already satisfied: six in /home/christianzaccaria/.local/lib/python3.9/site-packages (from openshift-client==1.0.18->codeflare-sdk==0.0.0.dev0) (1.16.0)\n", - "Requirement already satisfied: pyyaml in /home/christianzaccaria/.local/lib/python3.9/site-packages (from openshift-client==1.0.18->codeflare-sdk==0.0.0.dev0) (6.0.1)\n", - "Requirement already satisfied: paramiko in /home/christianzaccaria/.local/lib/python3.9/site-packages (from openshift-client==1.0.18->codeflare-sdk==0.0.0.dev0) (3.4.0)\n", - "Requirement already satisfied: click>=7.0 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ray==2.23.0->ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (8.1.7)\n", - "Requirement already satisfied: filelock in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ray==2.23.0->ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (3.13.1)\n", - "Requirement already satisfied: jsonschema in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ray==2.23.0->ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (4.21.1)\n", - "Requirement already satisfied: msgpack<2.0.0,>=1.0.0 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ray==2.23.0->ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (1.0.8)\n", - "Requirement already satisfied: packaging in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ray==2.23.0->ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (23.2)\n", - "Requirement already satisfied: protobuf!=3.19.5,>=3.15.3 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ray==2.23.0->ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (4.25.3)\n", - "Requirement already satisfied: aiosignal in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ray==2.23.0->ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (1.3.1)\n", - "Requirement already satisfied: frozenlist in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ray==2.23.0->ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (1.4.1)\n", - "Requirement already satisfied: requests in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ray==2.23.0->ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (2.31.0)\n", - "Requirement already satisfied: numpy>=1.20 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (1.26.4)\n", - "Requirement already satisfied: pandas>=1.3 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (2.2.1)\n", - "Requirement already satisfied: pyarrow>=6.0.1 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (15.0.0)\n", - "Requirement already satisfied: fsspec in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (2024.2.0)\n", - "Requirement already satisfied: aiohttp>=3.7 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (3.9.3)\n", - "Requirement already satisfied: aiohttp-cors in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (0.7.0)\n", - "Requirement already satisfied: colorful in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (0.5.6)\n", - "Requirement already satisfied: py-spy>=0.2.0 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (0.3.14)\n", - "Requirement already satisfied: opencensus in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (0.11.4)\n", - "Requirement already satisfied: prometheus-client>=0.7.1 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (0.20.0)\n", - "Requirement already satisfied: smart-open in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (7.0.1)\n", - "Requirement already satisfied: virtualenv!=20.21.1,>=20.0.24 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (20.21.0)\n", - "Requirement already satisfied: grpcio>=1.32.0 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (1.62.0)\n", - "Requirement already satisfied: memray in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (1.10.0)\n", - "Requirement already satisfied: certifi>=14.05.14 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from kubernetes<27,>=25.3.0->codeflare-sdk==0.0.0.dev0) (2024.2.2)\n", - "Requirement already satisfied: python-dateutil>=2.5.3 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from kubernetes<27,>=25.3.0->codeflare-sdk==0.0.0.dev0) (2.9.0.post0)\n", - "Requirement already satisfied: google-auth>=1.0.1 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from kubernetes<27,>=25.3.0->codeflare-sdk==0.0.0.dev0) (2.28.1)\n", - "Requirement already satisfied: websocket-client!=0.40.0,!=0.41.*,!=0.42.*,>=0.32.0 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from kubernetes<27,>=25.3.0->codeflare-sdk==0.0.0.dev0) (1.7.0)\n", - "Requirement already satisfied: requests-oauthlib in /home/christianzaccaria/.local/lib/python3.9/site-packages (from kubernetes<27,>=25.3.0->codeflare-sdk==0.0.0.dev0) (1.3.1)\n", - "Requirement already satisfied: urllib3>=1.24.2 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from kubernetes<27,>=25.3.0->codeflare-sdk==0.0.0.dev0) (1.26.18)\n", - "Requirement already satisfied: typing-extensions>=4.2.0 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from pydantic<2->codeflare-sdk==0.0.0.dev0) (4.10.0)\n", - "Requirement already satisfied: commonmark<0.10.0,>=0.9.0 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from rich<13.0,>=12.5->codeflare-sdk==0.0.0.dev0) (0.9.1)\n", - "Requirement already satisfied: pygments<3.0.0,>=2.6.0 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from rich<13.0,>=12.5->codeflare-sdk==0.0.0.dev0) (2.17.2)\n", - "Requirement already satisfied: attrs>=17.3.0 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from aiohttp>=3.7->ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (23.2.0)\n", - "Requirement already satisfied: multidict<7.0,>=4.5 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from aiohttp>=3.7->ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (6.0.5)\n", - "Requirement already satisfied: yarl<2.0,>=1.0 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from aiohttp>=3.7->ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (1.9.4)\n", - "Requirement already satisfied: async-timeout<5.0,>=4.0 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from aiohttp>=3.7->ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (4.0.3)\n", - "Requirement already satisfied: pycparser in /home/christianzaccaria/.local/lib/python3.9/site-packages (from cffi>=1.12->cryptography==40.0.2->codeflare-sdk==0.0.0.dev0) (2.21)\n", - "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from google-auth>=1.0.1->kubernetes<27,>=25.3.0->codeflare-sdk==0.0.0.dev0) (5.3.3)\n", - "Requirement already satisfied: pyasn1-modules>=0.2.1 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from google-auth>=1.0.1->kubernetes<27,>=25.3.0->codeflare-sdk==0.0.0.dev0) (0.3.0)\n", - "Requirement already satisfied: rsa<5,>=3.1.4 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from google-auth>=1.0.1->kubernetes<27,>=25.3.0->codeflare-sdk==0.0.0.dev0) (4.9)\n", - "Requirement already satisfied: decorator in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ipython>=6.1.0->ipywidgets==8.1.2->codeflare-sdk==0.0.0.dev0) (5.1.1)\n", - "Requirement already satisfied: jedi>=0.16 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ipython>=6.1.0->ipywidgets==8.1.2->codeflare-sdk==0.0.0.dev0) (0.19.1)\n", - "Requirement already satisfied: matplotlib-inline in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ipython>=6.1.0->ipywidgets==8.1.2->codeflare-sdk==0.0.0.dev0) (0.1.6)\n", - "Requirement already satisfied: prompt-toolkit<3.1.0,>=3.0.41 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ipython>=6.1.0->ipywidgets==8.1.2->codeflare-sdk==0.0.0.dev0) (3.0.41)\n", - "Requirement already satisfied: stack-data in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ipython>=6.1.0->ipywidgets==8.1.2->codeflare-sdk==0.0.0.dev0) (0.6.3)\n", - "Requirement already satisfied: exceptiongroup in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ipython>=6.1.0->ipywidgets==8.1.2->codeflare-sdk==0.0.0.dev0) (1.1.3)\n", - "Requirement already satisfied: pexpect>4.3 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from ipython>=6.1.0->ipywidgets==8.1.2->codeflare-sdk==0.0.0.dev0) (4.8.0)\n", - "Requirement already satisfied: pytz>=2020.1 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from pandas>=1.3->ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (2024.1)\n", - "Requirement already satisfied: tzdata>=2022.7 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from pandas>=1.3->ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (2024.1)\n", - "Requirement already satisfied: distlib<1,>=0.3.6 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from virtualenv!=20.21.1,>=20.0.24->ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (0.3.8)\n", - "Requirement already satisfied: platformdirs<4,>=2.4 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from virtualenv!=20.21.1,>=20.0.24->ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (3.11.0)\n", - "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from jsonschema->ray==2.23.0->ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (2023.12.1)\n", - "Requirement already satisfied: referencing>=0.28.4 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from jsonschema->ray==2.23.0->ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (0.33.0)\n", - "Requirement already satisfied: rpds-py>=0.7.1 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from jsonschema->ray==2.23.0->ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (0.18.0)\n", - "Requirement already satisfied: jinja2>=2.9 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from memray->ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (3.1.3)\n", - "Requirement already satisfied: opencensus-context>=0.1.3 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from opencensus->ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (0.1.3)\n", - "Requirement already satisfied: google-api-core<3.0.0,>=1.0.0 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from opencensus->ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (2.17.1)\n", - "Requirement already satisfied: bcrypt>=3.2 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from paramiko->openshift-client==1.0.18->codeflare-sdk==0.0.0.dev0) (4.1.2)\n", - "Requirement already satisfied: pynacl>=1.5 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from paramiko->openshift-client==1.0.18->codeflare-sdk==0.0.0.dev0) (1.5.0)\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from requests->ray==2.23.0->ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (3.3.2)\n", - "Requirement already satisfied: idna<4,>=2.5 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from requests->ray==2.23.0->ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (3.6)\n", - "Requirement already satisfied: oauthlib>=3.0.0 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from requests-oauthlib->kubernetes<27,>=25.3.0->codeflare-sdk==0.0.0.dev0) (3.2.2)\n", - "Requirement already satisfied: wrapt in /home/christianzaccaria/.local/lib/python3.9/site-packages (from smart-open->ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (1.16.0)\n", - "Requirement already satisfied: googleapis-common-protos<2.0.dev0,>=1.56.2 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from google-api-core<3.0.0,>=1.0.0->opencensus->ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (1.62.0)\n", - "Requirement already satisfied: parso<0.9.0,>=0.8.3 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from jedi>=0.16->ipython>=6.1.0->ipywidgets==8.1.2->codeflare-sdk==0.0.0.dev0) (0.8.3)\n", - "Requirement already satisfied: MarkupSafe>=2.0 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from jinja2>=2.9->memray->ray[data,default]==2.23.0->codeflare-sdk==0.0.0.dev0) (2.1.5)\n", - "Requirement already satisfied: ptyprocess>=0.5 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from pexpect>4.3->ipython>=6.1.0->ipywidgets==8.1.2->codeflare-sdk==0.0.0.dev0) (0.7.0)\n", - "Requirement already satisfied: wcwidth in /home/christianzaccaria/.local/lib/python3.9/site-packages (from prompt-toolkit<3.1.0,>=3.0.41->ipython>=6.1.0->ipywidgets==8.1.2->codeflare-sdk==0.0.0.dev0) (0.2.13)\n", - "Requirement already satisfied: pyasn1<0.6.0,>=0.4.6 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from pyasn1-modules>=0.2.1->google-auth>=1.0.1->kubernetes<27,>=25.3.0->codeflare-sdk==0.0.0.dev0) (0.5.1)\n", - "Requirement already satisfied: asttokens>=2.1.0 in /home/christianzaccaria/.local/lib/python3.9/site-packages (from stack-data->ipython>=6.1.0->ipywidgets==8.1.2->codeflare-sdk==0.0.0.dev0) (2.4.1)\n", - "Requirement already satisfied: pure-eval in /home/christianzaccaria/.local/lib/python3.9/site-packages (from stack-data->ipython>=6.1.0->ipywidgets==8.1.2->codeflare-sdk==0.0.0.dev0) (0.2.2)\n", - "Installing collected packages: codeflare-sdk\n", - "Successfully installed codeflare-sdk-0.0.0.dev0\n", - "\n", - "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.3.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.2\u001b[0m\n", - "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n", - "Note: you may need to restart the kernel to use updated packages.\n" - ] - } - ], - "source": [ - "%pip uninstall codeflare-sdk -y\n", - "%pip install ../../dist/codeflare_sdk-0.0.0.dev0-py3-none-any.whl" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "b55bc3ea-4ce3-49bf-bb1f-e209de8ca47a", - "metadata": {}, - "outputs": [], - "source": [ - "# Import pieces from codeflare-sdk\\\n", - "from codeflare_sdk import Cluster, ClusterConfiguration, TokenAuthentication, list_cluster_details" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "614daa0c", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'Logged into https://api.chris-aisrhods.xb4x.p3.openshiftapps.com:443'" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Create authentication object for user permissions\n", - "# IF unused, SDK will automatically check for default kubeconfig, then in-cluster config\n", - "# KubeConfigFileAuthentication can also be used to specify kubeconfig path manually\n", - "auth = TokenAuthentication(\n", - " token = \"sha256~pu4rVc-UrGDzHXIOX22BrV0pFLvjEcSZkZ6ECYKndhY\",\n", - " server = \"https://api.chris-aisrhods.xb4x.p3.openshiftapps.com:443\",\n", - " skip_tls=False\n", - ")\n", - "auth.login()" - ] - }, - { - "cell_type": "markdown", - "id": "bc27f84c", - "metadata": {}, - "source": [ - "Here, we want to define our cluster by specifying the resources we require for our batch workload. Below, we define our cluster object (which generates a corresponding RayCluster).\n", - "\n", - "NOTE: We must specify the `image` which will be used in our RayCluster, we recommend you bring your own image which suits your purposes. \n", - "The example here is a community image." - ] - }, - { - "cell_type": "code", - "execution_count": 26, - "id": "0f4bc870-091f-4e11-9642-cba145710159", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Yaml resources loaded for raytesta122\n" - ] - } - ], - "source": [ - "# Create and configure our cluster object\n", - "# The SDK will try to find the name of your default local queue based on the annotation \"kueue.x-k8s.io/default-queue\": \"true\" unless you specify the local queue manually below\n", - "cluster = Cluster(ClusterConfiguration(\n", - " name='raytesta122',\n", - " namespace=\"default\",\n", - " head_cpus='500m',\n", - " head_memory=2,\n", - " head_extended_resource_requests={'nvidia.com/gpu':2}, # For GPU enabled workloads set the head_extended_resource_requests and worker_extended_resource_requests\n", - " worker_extended_resource_requests={'nvidia.com/gpu':0},\n", - " num_workers=2,\n", - " worker_cpu_requests='250m',\n", - " worker_cpu_limits=1,\n", - " worker_memory_requests=4,\n", - " worker_memory_limits=4,\n", - " # image=\"\", # Optional Field \n", - " write_to_file=False, # When enabled Ray Cluster yaml files are written to /HOME/.codeflare/resources \n", - " # local_queue=\"local-queue-name\" # Specify the local queue manually\n", - "))" - ] - }, - { - "cell_type": "markdown", - "id": "12eef53c", - "metadata": {}, - "source": [ - "Next, we want to bring our cluster up, so we call the `up()` function below to submit our Ray Cluster onto the queue, and begin the process of obtaining our resource cluster." - ] - }, - { - "cell_type": "code", - "execution_count": 27, - "id": "f0884bbc-c224-4ca0-98a0-02dfa09c2200", - "metadata": {}, - "outputs": [], - "source": [ - "# Bring up the cluster\n", - "cluster.up()" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "24136d1f", - "metadata": {}, - "outputs": [ - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "bb3e74d1cf9249f5a8f42fcb861be1c2", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "VBox(children=(ToggleButtons(description='Select an existing cluster:', options=('raytesta1', 'raytesta10', 'rโ€ฆ" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "d0733b44ffd2470d9ecaafd333a496d7", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "HBox(children=(Button(description='Delete Cluster', icon='trash', style=ButtonStyle()), Button(description='Viโ€ฆ" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "application/vnd.jupyter.widget-view+json": { - "model_id": "8abc240e519f42208fcfffe963cdb808", - "version_major": 2, - "version_minor": 0 - }, - "text/plain": [ - "Output()" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Cluster raytesta11 in the default namespace was deleted successfully.\n" - ] - } - ], - "source": [ - "list_cluster_details(\"default\")" - ] - }, - { - "cell_type": "markdown", - "id": "657ebdfb", - "metadata": {}, - "source": [ - "Now, we want to check on the status of our resource cluster, and wait until it is finally ready for use." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "24e612ff", - "metadata": {}, - "outputs": [], - "source": [ - "from codeflare_sdk import get_cluster\n", - "get_cluster(\"raytest21\", \"default\")" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "id": "995850b2", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
                     ๐Ÿš€ CodeFlare Cluster Details ๐Ÿš€                     \n",
-       "                                                                         \n",
-       " โ•ญโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฎ \n",
-       " โ”‚   Name                                                              โ”‚ \n",
-       " โ”‚   raytestmar2k                                        Inactive โŒ   โ”‚ \n",
-       " โ”‚                                                                     โ”‚ \n",
-       " โ”‚   URI: ray://raytestmar2k-head-svc.default.svc:10001                โ”‚ \n",
-       " โ”‚                                                                     โ”‚ \n",
-       " โ”‚   Dashboard๐Ÿ”—                                                       โ”‚ \n",
-       " โ”‚                                                                     โ”‚ \n",
-       " โ”‚                       Cluster Resources                             โ”‚ \n",
-       " โ”‚   โ•ญโ”€โ”€ Workers โ”€โ”€โ•ฎ  โ•ญโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ Worker specs(each) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฎ         โ”‚ \n",
-       " โ”‚   โ”‚  # Workers  โ”‚  โ”‚  Memory      CPU         GPU         โ”‚         โ”‚ \n",
-       " โ”‚   โ”‚             โ”‚  โ”‚                                      โ”‚         โ”‚ \n",
-       " โ”‚   โ”‚  1          โ”‚  โ”‚  2G~2G       1           0           โ”‚         โ”‚ \n",
-       " โ”‚   โ”‚             โ”‚  โ”‚                                      โ”‚         โ”‚ \n",
-       " โ”‚   โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ  โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ         โ”‚ \n",
-       " โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ \n",
-       "
\n" - ], - "text/plain": [ - "\u001b[3m \u001b[0m\u001b[1;3m ๐Ÿš€ CodeFlare Cluster Details ๐Ÿš€\u001b[0m\u001b[3m \u001b[0m\n", - "\u001b[1m \u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m\n", - " โ•ญโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฎ \n", - " โ”‚ \u001b[1;37;42mName\u001b[0m โ”‚ \n", - " โ”‚ \u001b[1;4mraytestmar2k\u001b[0m Inactive โŒ โ”‚ \n", - " โ”‚ โ”‚ \n", - " โ”‚ \u001b[1mURI:\u001b[0m ray://raytestmar2k-head-svc.default.svc:10001 โ”‚ \n", - " โ”‚ โ”‚ \n", - " โ”‚ \u001b]8;id=601715;https://ray-dashboard-raytestmar2k-default.apps.rosa.chris-aisrhods.xb4x.p3.openshiftapps.com\u001b\\\u001b[4;34mDashboard๐Ÿ”—\u001b[0m\u001b]8;;\u001b\\ โ”‚ \n", - " โ”‚ โ”‚ \n", - " โ”‚ \u001b[3m Cluster Resources \u001b[0m โ”‚ \n", - " โ”‚ โ•ญโ”€โ”€ Workers โ”€โ”€โ•ฎ โ•ญโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ Worker specs(each) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฎ โ”‚ \n", - " โ”‚ โ”‚ \u001b[1m \u001b[0m\u001b[1m# Workers\u001b[0m\u001b[1m \u001b[0m โ”‚ โ”‚ \u001b[1m \u001b[0m\u001b[1mMemory \u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m\u001b[1mCPU \u001b[0m\u001b[1m \u001b[0m\u001b[1m \u001b[0m\u001b[1mGPU \u001b[0m\u001b[1m \u001b[0m โ”‚ โ”‚ \n", - " โ”‚ โ”‚ \u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m โ”‚ โ”‚ \u001b[36m \u001b[0m\u001b[36m \u001b[0m\u001b[36m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m โ”‚ โ”‚ \n", - " โ”‚ โ”‚ \u001b[35m \u001b[0m\u001b[35m1 \u001b[0m\u001b[35m \u001b[0m โ”‚ โ”‚ \u001b[36m \u001b[0m\u001b[36m2G~2G \u001b[0m\u001b[36m \u001b[0m\u001b[35m \u001b[0m\u001b[35m1 \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m0 \u001b[0m\u001b[35m \u001b[0m โ”‚ โ”‚ \n", - " โ”‚ โ”‚ \u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m โ”‚ โ”‚ \u001b[36m \u001b[0m\u001b[36m \u001b[0m\u001b[36m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m\u001b[35m \u001b[0m โ”‚ โ”‚ \n", - " โ”‚ โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ โ”‚ \n", - " โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ \n" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/plain": [ - "RayCluster(name='raytestmar2k', status=, head_cpus=2, head_mem='8G', workers=1, worker_mem_min='2G', worker_mem_max='2G', worker_cpu=1, namespace='default', dashboard='https://ray-dashboard-raytestmar2k-default.apps.rosa.chris-aisrhods.xb4x.p3.openshiftapps.com', worker_extended_resources={}, head_extended_resources={})" - ] - }, - "execution_count": 14, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "cluster.details()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9dda874b", - "metadata": {}, - "outputs": [], - "source": [ - "def format_status(status):\n", - " if status == \"Ready\":\n", - " return 'Ready โœ“'\n", - " elif status == \"Suspended\":\n", - " return 'Suspended ~'\n", - " elif status == \"Starting\":\n", - " return 'Starting โŒ›'\n", - " elif status == \"Failed\":\n", - " return 'Failed โœ—'\n", - " else:\n", - " return status\n", - "\n", - "import ipywidgets as widgets\n", - "import pandas as pd\n", - "from IPython.display import display, HTML\n", - "data = {\n", - " \"name\": [\"RayTest1\", \"RayTest2\", \"RayTest3\", \"RayTest4\"],\n", - " \"namespace\": [\"default\", \"usernamespace\", \"usernamespace\", \"usernamespace\"],\n", - " \"head_gpu\": [0, 1, 2, 0],\n", - " \"worker_gpu\": [2, 0, 1, 0],\n", - " \"min_memory\": [2, 4, 4, 2],\n", - " \"max_memory\": [2, 4, 8, 4],\n", - " \"min_cpu\": [1, 2, 4, 2],\n", - " \"max_cpu\": [1, 4, 8, 2],\n", - " \"status\": [\"Ready\", \"Starting\", \"Suspended\", \"Failed\"],\n", - " \"pods\": [\n", - " [{\"pod\": \"head\", \"name\": \"head-raytest1\", \"status\": \"Ready\"}, {\"pod\": \"worker\", \"name\": \"worker-raytest1-a\", \"status\": \"Ready\"}, {\"pod\": \"worker\", \"name\": \"worker-raytest1-b\", \"status\": \"Ready\"}],\n", - " [{\"pod\": \"head\", \"name\": \"head-raytest2\", \"status\": \"Ready\"}, {\"pod\": \"worker\", \"name\": \"worker-raytest2a\", \"status\": \"Starting\"}],\n", - " [{\"pod\": \"head\", \"name\": \"head-raytest3\", \"status\": \"Suspended\"}, {\"pod\": \"worker\", \"name\": \"worker-raytest3a\", \"status\": \"Suspended\"}],\n", - " [{\"pod\": \"head\", \"name\": \"head-raytest4\", \"status\": \"Failed\"}, {\"pod\": \"worker\", \"name\": \"worker-raytest4a\", \"status\": \"Failed\"}]\n", - " ]\n", - "}\n", - "df = pd.DataFrame(data)\n", - "\n", - "# format to add icons\n", - "df['status'] = df['status'].apply(format_status)\n", - "\n", - "my_output = widgets.Output()\n", - "my_output\n", - "classification_widget = widgets.ToggleButtons(\n", - " options=['RayTest1', \"RayTest2\", \"RayTest3\", \"RayTest4\"],\n", - " description='Select an existing cluster:',\n", - ")\n", - "\n", - "def on_click(change):\n", - " new_value = change[\"new\"]\n", - " my_output.clear_output()\n", - " with my_output:\n", - " selected_data = df[df[\"name\"] == new_value]\n", - " main_table = selected_data[[\"name\", \"namespace\", \"head_gpu\", \"worker_gpu\", \"min_memory\", \"max_memory\", \"min_cpu\", \"max_cpu\", \"status\"]].to_html(escape=False, index=False)\n", - " pod_rows = \"\"\n", - " for pod in selected_data[\"pods\"].values[0]:\n", - " pod_rows += f'{pod[\"pod\"]}{pod[\"name\"]}{format_status(pod[\"status\"])}'\n", - " pods_table = f'
{pod_rows}
PodNameStatus
'\n", - " display(HTML(f'
{main_table}{pods_table}
'))\n", - "\n", - "classification_widget.observe(on_click, names=\"value\")\n", - "display(widgets.VBox([classification_widget, my_output]))\n", - "\n", - "\n", - "list_jobs_button = widgets.Button(\n", - " description='View Jobs',\n", - " icon='suitcase'\n", - " )\n", - "delete_button = widgets.Button(\n", - " description='Delete Cluster',\n", - " icon='trash'\n", - " )\n", - "ray_dashboard_button = widgets.Button(\n", - " description='Open Ray Dashboard',\n", - " icon='dashboard',\n", - " layout=widgets.Layout(width='auto'),\n", - " )\n", - "view_yaml_button = widgets.Button(\n", - " description='View YAML',\n", - " icon='file'\n", - " )\n", - "display(widgets.HBox([delete_button, list_jobs_button, view_yaml_button, ray_dashboard_button]))" - ] - }, - { - "cell_type": "markdown", - "id": "b3a55fe4", - "metadata": {}, - "source": [ - "Let's quickly verify that the specs of the cluster are as expected." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6f1ab7ff", - "metadata": {}, - "outputs": [], - "source": [ - "with my_output:\n", - " display(cluster.details())" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7fd45bc5-03c0-4ae5-9ec5-dd1c30f1a084", - "metadata": {}, - "outputs": [], - "source": [ - "from IPython.display import HTML, display\n", - "import ipywidgets as widgets\n", - "\n", - "def on_click(change):\n", - " new_value = change[\"new\"]\n", - " my_output.clear_output()\n", - " with my_output:\n", - " display(HTML(f'
{df[df[\"name\"]==new_value][[\"name\", \"namespace\", \"head_gpu\", \"worker_gpu\", \"min_memory\", \"max_memory\", \"min_cpu\", \"max_cpu\", \"status\"]].to_html(escape=False, index=False)}
'))\n", - "\n", - "classification_widget.observe(on_click, names=\"value\")\n", - "display(widgets.VBox([classification_widget, my_output], layout=widgets.Layout(border='2px solid black')))\n", - "\n", - "list_jobs_button = widgets.Button(description='View Jobs', icon='suitcase')\n", - "delete_button = widgets.Button(description='Delete Cluster', icon='trash')\n", - "ray_dashboard_button = widgets.Button(description='Open Ray Dashboard', icon='dashboard', layout=widgets.Layout(width='auto'))\n", - "view_yaml_button = widgets.Button(description='View YAML', icon='file')\n", - "buttons_container = widgets.HBox([delete_button, list_jobs_button, view_yaml_button, ray_dashboard_button], layout=widgets.Layout(border='2px solid black'))\n", - "\n", - "display(buttons_container)\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "id": "5af8cd32", - "metadata": {}, - "source": [ - "Finally, we bring our resource cluster down and release/terminate the associated resources, bringing everything back to the way it was before our cluster was brought up." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5f36db0f-31f6-4373-9503-dc3c1c4c3f57", - "metadata": {}, - "outputs": [], - "source": [ - "cluster.down()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0d41b90e", - "metadata": {}, - "outputs": [], - "source": [ - "auth.logout()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.18" - }, - "vscode": { - "interpreter": { - "hash": "f9f85f796d01129d0dd105a088854619f454435301f6ffec2fea96ecbd9be4ac" - } - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/src/codeflare_sdk/cluster/widgets.py b/src/codeflare_sdk/cluster/widgets.py index a005d9c20..a7672adf4 100644 --- a/src/codeflare_sdk/cluster/widgets.py +++ b/src/codeflare_sdk/cluster/widgets.py @@ -97,9 +97,10 @@ def is_notebook() -> bool: else: return False + def view_clusters(namespace: str = None): """view_clusters function will display existing clusters with their specs, and handle user interactions.""" - from .cluster import Cluster, get_current_namespace + from .cluster import get_current_namespace if not namespace: namespace = get_current_namespace() diff --git a/tests/unit_test.py b/tests/unit_test.py index 40c187de0..b88bfb7d4 100644 --- a/tests/unit_test.py +++ b/tests/unit_test.py @@ -76,7 +76,18 @@ gen_names, is_openshift_cluster, ) -from codeflare_sdk.cluster.widgets import cluster_up_down_buttons + +from codeflare_sdk.cluster.widgets import ( + cluster_up_down_buttons, + view_clusters, + _on_cluster_click, + _on_delete_button_click, + _on_list_jobs_button_click, + _on_ray_dashboard_button_click, + _format_status, + _fetch_cluster_data, +) +import pandas as pd import openshift from openshift.selector import Selector @@ -88,9 +99,6 @@ from ray.job_submission import JobSubmissionClient from codeflare_sdk.job.ray_jobs import RayJobClient -import ipywidgets as widgets -from IPython.display import display - # For mocking openshift client results fake_res = openshift.Result("fake") @@ -1008,7 +1016,7 @@ def test_ray_details(mocker, capsys): " โ”‚ โ•ญโ”€โ”€ Workers โ”€โ”€โ•ฎ โ•ญโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ Worker specs(each) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฎ โ”‚ \n" " โ”‚ โ”‚ # Workers โ”‚ โ”‚ Memory CPU GPU โ”‚ โ”‚ \n" " โ”‚ โ”‚ โ”‚ โ”‚ โ”‚ โ”‚ \n" - " โ”‚ โ”‚ 1 โ”‚ โ”‚ 2G~2G 1 0 โ”‚ โ”‚ \n" + " โ”‚ โ”‚ 1 โ”‚ โ”‚ 2G~2G 1~1 0 โ”‚ โ”‚ \n" " โ”‚ โ”‚ โ”‚ โ”‚ โ”‚ โ”‚ \n" " โ”‚ โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ โ”‚ \n" " โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ \n" @@ -1026,7 +1034,7 @@ def test_ray_details(mocker, capsys): " โ”‚ โ•ญโ”€โ”€ Workers โ”€โ”€โ•ฎ โ•ญโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ Worker specs(each) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฎ โ”‚ \n" " โ”‚ โ”‚ # Workers โ”‚ โ”‚ Memory CPU GPU โ”‚ โ”‚ \n" " โ”‚ โ”‚ โ”‚ โ”‚ โ”‚ โ”‚ \n" - " โ”‚ โ”‚ 1 โ”‚ โ”‚ 2G~2G 1 0 โ”‚ โ”‚ \n" + " โ”‚ โ”‚ 1 โ”‚ โ”‚ 2G~2G 1~1 0 โ”‚ โ”‚ \n" " โ”‚ โ”‚ โ”‚ โ”‚ โ”‚ โ”‚ \n" " โ”‚ โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ โ”‚ \n" " โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ \n" @@ -1042,7 +1050,7 @@ def test_ray_details(mocker, capsys): "โ”‚ โ•ญโ”€โ”€ Workers โ”€โ”€โ•ฎ โ•ญโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ Worker specs(each) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฎ โ”‚\n" "โ”‚ โ”‚ # Workers โ”‚ โ”‚ Memory CPU GPU โ”‚ โ”‚\n" "โ”‚ โ”‚ โ”‚ โ”‚ โ”‚ โ”‚\n" - "โ”‚ โ”‚ 1 โ”‚ โ”‚ 2G~2G 1 0 โ”‚ โ”‚\n" + "โ”‚ โ”‚ 1 โ”‚ โ”‚ 2G~2G 1~1 0 โ”‚ โ”‚\n" "โ”‚ โ”‚ โ”‚ โ”‚ โ”‚ โ”‚\n" "โ”‚ โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ โ”‚\n" "โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ\n" @@ -2247,7 +2255,7 @@ def test_list_clusters(mocker, capsys): " โ”‚ โ•ญโ”€โ”€ Workers โ”€โ”€โ•ฎ โ•ญโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ Worker specs(each) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฎ โ”‚ \n" " โ”‚ โ”‚ # Workers โ”‚ โ”‚ Memory CPU GPU โ”‚ โ”‚ \n" " โ”‚ โ”‚ โ”‚ โ”‚ โ”‚ โ”‚ \n" - " โ”‚ โ”‚ 1 โ”‚ โ”‚ 2G~2G 1 0 โ”‚ โ”‚ \n" + " โ”‚ โ”‚ 1 โ”‚ โ”‚ 2G~2G 1~1 0 โ”‚ โ”‚ \n" " โ”‚ โ”‚ โ”‚ โ”‚ โ”‚ โ”‚ \n" " โ”‚ โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ โ”‚ \n" " โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ \n" @@ -2263,7 +2271,7 @@ def test_list_clusters(mocker, capsys): "โ”‚ โ•ญโ”€โ”€ Workers โ”€โ”€โ•ฎ โ•ญโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ Worker specs(each) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฎ โ”‚\n" "โ”‚ โ”‚ # Workers โ”‚ โ”‚ Memory CPU GPU โ”‚ โ”‚\n" "โ”‚ โ”‚ โ”‚ โ”‚ โ”‚ โ”‚\n" - "โ”‚ โ”‚ 1 โ”‚ โ”‚ 2G~2G 1 0 โ”‚ โ”‚\n" + "โ”‚ โ”‚ 1 โ”‚ โ”‚ 2G~2G 1~1 0 โ”‚ โ”‚\n" "โ”‚ โ”‚ โ”‚ โ”‚ โ”‚ โ”‚\n" "โ”‚ โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ โ”‚\n" "โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ\n" @@ -2957,6 +2965,161 @@ def test_is_notebook_true(): assert is_notebook() is True +@patch.dict("os.environ", {"JPY_SESSION_NAME": "example-test"}) # Mock Jupyter environment variable +def test_view_clusters(mocker): + # Mock Kubernetes API responses + mocker.patch("kubernetes.client.ApisApi.get_api_versions") + mocker.patch( + "kubernetes.client.CustomObjectsApi.list_cluster_custom_object", + return_value={"items": []} + ) + + + test_df=pd.DataFrame({ + "Name": ["test-cluster"], + "Namespace": ["default"], + "Head GPUs": ["0"], + "Worker GPUs": ["0"], + "Head CPU Req~Lim": ["1~1"], + "Head Memory Req~Lim": ["1Gi~1Gi"], + "Worker CPU Req~Lim": ["1~1"], + "Worker Memory Req~Lim": ["1Gi~1Gi"], + "status": ['Ready โœ“'] + }) + + # Mock the _fetch_cluster_data function to return a test DataFrame + mocker.patch( + "codeflare_sdk.cluster.widgets._fetch_cluster_data", + return_value=test_df + ) + + # Mock the Cluster class and related methods + mocker.patch("codeflare_sdk.cluster.Cluster") + mocker.patch("codeflare_sdk.cluster.ClusterConfiguration") + + with patch("ipywidgets.ToggleButtons") as MockToggleButtons, \ + patch("ipywidgets.Button") as MockButton, \ + patch("ipywidgets.Output") as MockOutput, \ + patch("ipywidgets.HBox"), \ + patch("ipywidgets.VBox"), \ + patch("IPython.display.display") as mock_display, \ + patch("IPython.display.HTML"), \ + patch("codeflare_sdk.cluster.widgets.Javascript") as mock_javascript: + + # Create mock widget instances + mock_toggle = MagicMock() + mock_delete_button = MagicMock() + mock_list_jobs_button = MagicMock() + mock_ray_dashboard_button = MagicMock() + mock_output = MagicMock() + + # Set the return values for the mocked widgets + MockToggleButtons.return_value = mock_toggle + MockButton.side_effect = [mock_delete_button, mock_list_jobs_button, mock_ray_dashboard_button] + MockOutput.return_value = mock_output + + # Call the function under test + view_clusters(namespace="default") + + # Simulate selecting a cluster + mock_toggle.value = "test-cluster" + selection_change = {"new": "test-cluster"} + _on_cluster_click(selection_change, mock_output, "default", mock_toggle) + + # Assert that the toggle options are set correctly + mock_toggle.observe.assert_called() + + # Simulate clicking the delete button + _on_delete_button_click(None, mock_toggle, test_df, mock_output, mock_output, + mock_delete_button, mock_list_jobs_button, mock_ray_dashboard_button) + + # Simulate clicking the list jobs button + _on_list_jobs_button_click(None, mock_toggle, test_df, mock_output, mock_output) + mock_javascript.assert_called() + + # Simulate clicking the Ray dashboard button + _on_ray_dashboard_button_click(None, mock_toggle, test_df, mock_output, mock_output) + mock_javascript.assert_called() + + + +def test_format_status(): + # Test each possible status + test_cases = [ + (RayClusterStatus.READY, 'Ready โœ“'), + (RayClusterStatus.SUSPENDED, 'Suspended โ„๏ธ'), + (RayClusterStatus.FAILED, 'Failed โœ—'), + (RayClusterStatus.UNHEALTHY, 'Unhealthy'), + (RayClusterStatus.UNKNOWN, 'Unknown'), + ] + + for status, expected_output in test_cases: + assert _format_status(status) == expected_output, f"Failed for status: {status}" + + # Test an unrecognized status + unrecognized_status = 'NotAStatus' + assert _format_status(unrecognized_status) == 'NotAStatus', "Failed for unrecognized status" + +def test_fetch_cluster_data(mocker): + # Create mock RayCluster objects + mock_raycluster1 = MagicMock(spec=RayCluster) + mock_raycluster1.name = 'test-cluster-1' + mock_raycluster1.namespace = 'default' + mock_raycluster1.head_extended_resources = {'nvidia.com/gpu': '1'} + mock_raycluster1.worker_extended_resources = {'nvidia.com/gpu': '2'} + mock_raycluster1.head_cpu_requests = '500m' + mock_raycluster1.head_cpu_limits = '1000m' + mock_raycluster1.head_mem_requests = '1Gi' + mock_raycluster1.head_mem_limits = '2Gi' + mock_raycluster1.worker_cpu_requests = '1000m' + mock_raycluster1.worker_cpu_limits = '2000m' + mock_raycluster1.worker_mem_requests = '2Gi' + mock_raycluster1.worker_mem_limits = '4Gi' + mock_raycluster1.status = MagicMock() + mock_raycluster1.status.name = 'READY' + mock_raycluster1.status = RayClusterStatus.READY + + mock_raycluster2 = MagicMock(spec=RayCluster) + mock_raycluster2.name = 'test-cluster-2' + mock_raycluster2.namespace = 'default' + mock_raycluster2.head_extended_resources = {} + mock_raycluster2.worker_extended_resources = {} + mock_raycluster2.head_cpu_requests = None + mock_raycluster2.head_cpu_limits = None + mock_raycluster2.head_mem_requests = None + mock_raycluster2.head_mem_limits = None + mock_raycluster2.worker_cpu_requests = None + mock_raycluster2.worker_cpu_limits = None + mock_raycluster2.worker_mem_requests = None + mock_raycluster2.worker_mem_limits = None + mock_raycluster2.status = MagicMock() + mock_raycluster2.status.name = 'SUSPENDED' + mock_raycluster2.status = RayClusterStatus.SUSPENDED + + with patch('codeflare_sdk.cluster.cluster.list_all_clusters', return_value=[mock_raycluster1, mock_raycluster2]): + # Call the function under test + df = _fetch_cluster_data(namespace='default') + + # Expected DataFrame + expected_data = { + "Name": ['test-cluster-1', 'test-cluster-2'], + "Namespace": ['default', 'default'], + "Head GPUs": ['nvidia.com/gpu: 1', '0'], + "Worker GPUs": ['nvidia.com/gpu: 2', '0'], + "Head CPU Req~Lim": ['500m~1000m', '0~0'], + "Head Memory Req~Lim": ['1Gi~2Gi', '0~0'], + "Worker CPU Req~Lim": ['1000m~2000m', '0~0'], + "Worker Memory Req~Lim": ['2Gi~4Gi', '0~0'], + "status": [ + 'Ready โœ“', + 'Suspended โ„๏ธ' + ] + } + + expected_df = pd.DataFrame(expected_data) + + # Assert that the DataFrame matches expected + pd.testing.assert_frame_equal(df.reset_index(drop=True), expected_df.reset_index(drop=True)) # Make sure to always keep this function last def test_cleanup(): From d83c67b721b477e1aef034eaf78970a3204bfccf Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Tue, 24 Sep 2024 10:50:15 +0100 Subject: [PATCH 07/14] Add timeout and interval parameters to _delete_cluster function --- src/codeflare_sdk/cluster/widgets.py | 52 +++++++++++++++++++++------- 1 file changed, 39 insertions(+), 13 deletions(-) diff --git a/src/codeflare_sdk/cluster/widgets.py b/src/codeflare_sdk/cluster/widgets.py index a7672adf4..ea8d5e02a 100644 --- a/src/codeflare_sdk/cluster/widgets.py +++ b/src/codeflare_sdk/cluster/widgets.py @@ -17,8 +17,10 @@ """ import os from time import sleep +import time import codeflare_sdk from kubernetes import client +from kubernetes.client.rest import ApiException import ipywidgets as widgets from IPython.display import display, HTML, Javascript import pandas as pd @@ -160,7 +162,6 @@ def _on_delete_button_click(b, classification_widget: widgets.ToggleButtons, df: _delete_cluster(cluster_name, namespace) - sleep(2) # TODO: wait for the cluster to be deleted instead with user_output: user_output.clear_output() print(f"Cluster {cluster_name} in the {namespace} namespace was deleted successfully.") @@ -209,12 +210,16 @@ def _on_list_jobs_button_click(b, classification_widget: widgets.ToggleButtons, def _delete_cluster( cluster_name: str, namespace: str, + timeout: int = 5, + interval: int = 1, ): from .cluster import _check_aw_exists - if _check_aw_exists(cluster_name, namespace): - try: - config_check() - api_instance = client.CustomObjectsApi(api_config_handler()) + + try: + config_check() + api_instance = client.CustomObjectsApi(api_config_handler()) + + if _check_aw_exists(cluster_name, namespace): api_instance.delete_namespaced_custom_object( group="workload.codeflare.dev", version="v1beta2", @@ -222,12 +227,10 @@ def _delete_cluster( plural="appwrappers", name=cluster_name, ) - except Exception as e: - return _kube_api_error_handling(e) - else: - try: - config_check() - api_instance = client.CustomObjectsApi(api_config_handler()) + group = "workload.codeflare.dev" + version = "v1beta2" + plural = "appwrappers" + else: api_instance.delete_namespaced_custom_object( group="ray.io", version="v1", @@ -235,8 +238,31 @@ def _delete_cluster( plural="rayclusters", name=cluster_name, ) - except Exception as e: - return _kube_api_error_handling(e) + group = "ray.io" + version = "v1" + plural = "rayclusters" + + # Wait for the resource to be deleted + while True: + try: + api_instance.get_namespaced_custom_object( + group=group, + version=version, + namespace=namespace, + plural=plural, + name=cluster_name, + ) + # Retry if resource still exists + time.sleep(interval) + timeout -= interval + if timeout <= 0: + raise TimeoutError(f"Timeout waiting for {cluster_name} to be deleted.") + except ApiException as e: + # Resource is deleted + if e.status == 404: + break + except Exception as e: + return _kube_api_error_handling(e) def _fetch_cluster_data(namespace): From 245c9f20e71d0007891252bf71701636f428b149 Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Tue, 24 Sep 2024 15:30:44 +0100 Subject: [PATCH 08/14] Pre-select cluster if exists, and suppress widgets and outputs on creation of Cluster Object, and bug fixes --- src/codeflare_sdk/cluster/widgets.py | 68 ++++++++++++++++------------ tests/unit_test.py | 55 ++++++++++++---------- 2 files changed, 71 insertions(+), 52 deletions(-) diff --git a/src/codeflare_sdk/cluster/widgets.py b/src/codeflare_sdk/cluster/widgets.py index ea8d5e02a..e68c52c42 100644 --- a/src/codeflare_sdk/cluster/widgets.py +++ b/src/codeflare_sdk/cluster/widgets.py @@ -15,6 +15,8 @@ """ The widgets sub-module contains the ui widgets created using the ipywidgets package. """ +import contextlib +import io import os from time import sleep import time @@ -113,38 +115,42 @@ def view_clusters(namespace: str = None): df = _fetch_cluster_data(namespace) if df.empty: print(f"No clusters found in the {namespace} namespace.") - else: - classification_widget = widgets.ToggleButtons( - options=df["Name"].tolist(), value=None, - description='Select an existing cluster:', - ) + return - classification_widget.observe(lambda selection_change: _on_cluster_click(selection_change, raycluster_data_output, namespace, classification_widget), names="value") + classification_widget = widgets.ToggleButtons( + options=df["Name"].tolist(), value=df["Name"].tolist()[0], + description='Select an existing cluster:', + ) + # Setting the initial value to trigger the event handler to display the cluster details. + initial_value = classification_widget.value + _on_cluster_click({"new": initial_value}, raycluster_data_output, namespace, classification_widget) + classification_widget.observe(lambda selection_change: _on_cluster_click(selection_change, raycluster_data_output, namespace, classification_widget), names="value") - delete_button = widgets.Button( - description='Delete Cluster', - icon='trash', - tooltip="Delete the selected cluster" - ) - delete_button.on_click(lambda b: _on_delete_button_click(b, classification_widget, df, raycluster_data_output, user_output, delete_button, list_jobs_button, ray_dashboard_button)) + # UI table buttons + delete_button = widgets.Button( + description='Delete Cluster', + icon='trash', + tooltip="Delete the selected cluster" + ) + delete_button.on_click(lambda b: _on_delete_button_click(b, classification_widget, df, raycluster_data_output, user_output, delete_button, list_jobs_button, ray_dashboard_button)) - list_jobs_button = widgets.Button( - description='View Jobs', - icon='suitcase', - tooltip="Open the Ray Job Dashboard" - ) - list_jobs_button.on_click(lambda b: _on_list_jobs_button_click(b, classification_widget, df, user_output, url_output)) + list_jobs_button = widgets.Button( + description='View Jobs', + icon='suitcase', + tooltip="Open the Ray Job Dashboard" + ) + list_jobs_button.on_click(lambda b: _on_list_jobs_button_click(b, classification_widget, df, user_output, url_output)) - ray_dashboard_button = widgets.Button( - description='Open Ray Dashboard', - icon='dashboard', - tooltip="Open the Ray Dashboard in a new tab", - layout=widgets.Layout(width='auto'), - ) - ray_dashboard_button.on_click(lambda b: _on_ray_dashboard_button_click(b, classification_widget, df, user_output, url_output)) + ray_dashboard_button = widgets.Button( + description='Open Ray Dashboard', + icon='dashboard', + tooltip="Open the Ray Dashboard in a new tab", + layout=widgets.Layout(width='auto'), + ) + ray_dashboard_button.on_click(lambda b: _on_ray_dashboard_button_click(b, classification_widget, df, user_output, url_output)) - display(widgets.VBox([classification_widget, raycluster_data_output])) - display(widgets.HBox([delete_button, list_jobs_button, ray_dashboard_button]), url_output, user_output) + display(widgets.VBox([classification_widget, raycluster_data_output])) + display(widgets.HBox([delete_button, list_jobs_button, ray_dashboard_button]), url_output, user_output) # Handles the event when a cluster is selected from the toggle buttons, updating the output with cluster details. def _on_cluster_click(selection_change, raycluster_data_output: widgets.Output, namespace: str, classification_widget: widgets.ToggleButtons): @@ -184,7 +190,9 @@ def _on_ray_dashboard_button_click(b, classification_widget: widgets.ToggleButto cluster_name = classification_widget.value namespace = df[df["Name"]==classification_widget.value]["Namespace"].values[0] - cluster = Cluster(ClusterConfiguration(cluster_name, namespace)) + # Suppress from Cluster Object initialisation widgets and outputs + with widgets.Output(), contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()): + cluster = Cluster(ClusterConfiguration(cluster_name, namespace)) dashboard_url = cluster.cluster_dashboard_uri() with user_output: @@ -198,7 +206,9 @@ def _on_list_jobs_button_click(b, classification_widget: widgets.ToggleButtons, cluster_name = classification_widget.value namespace = df[df["Name"]==classification_widget.value]["Namespace"].values[0] - cluster = Cluster(ClusterConfiguration(cluster_name, namespace)) + # Suppress from Cluster Object initialisation widgets and outputs + with widgets.Output(), contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()): + cluster = Cluster(ClusterConfiguration(cluster_name, namespace)) dashboard_url = cluster.cluster_dashboard_uri() with user_output: diff --git a/tests/unit_test.py b/tests/unit_test.py index b88bfb7d4..c3f59f63c 100644 --- a/tests/unit_test.py +++ b/tests/unit_test.py @@ -2953,7 +2953,6 @@ def test_cluster_up_down_buttons(mocker): @patch.dict("os.environ", {}, clear=True) # Mock environment with no variables def test_is_notebook_false(): from codeflare_sdk.cluster.widgets import is_notebook - assert is_notebook() is False @@ -2962,9 +2961,9 @@ def test_is_notebook_false(): ) # Mock Jupyter environment variable def test_is_notebook_true(): from codeflare_sdk.cluster.widgets import is_notebook - assert is_notebook() is True + @patch.dict("os.environ", {"JPY_SESSION_NAME": "example-test"}) # Mock Jupyter environment variable def test_view_clusters(mocker): # Mock Kubernetes API responses @@ -2974,6 +2973,10 @@ def test_view_clusters(mocker): return_value={"items": []} ) + # Return empty dataframe when no clusters are found + mocker.patch("codeflare_sdk.cluster.cluster.list_all_clusters", return_value=[]) + df = _fetch_cluster_data(namespace="default") + assert df.empty test_df=pd.DataFrame({ "Name": ["test-cluster"], @@ -3029,10 +3032,6 @@ def test_view_clusters(mocker): # Assert that the toggle options are set correctly mock_toggle.observe.assert_called() - # Simulate clicking the delete button - _on_delete_button_click(None, mock_toggle, test_df, mock_output, mock_output, - mock_delete_button, mock_list_jobs_button, mock_ray_dashboard_button) - # Simulate clicking the list jobs button _on_list_jobs_button_click(None, mock_toggle, test_df, mock_output, mock_output) mock_javascript.assert_called() @@ -3041,26 +3040,17 @@ def test_view_clusters(mocker): _on_ray_dashboard_button_click(None, mock_toggle, test_df, mock_output, mock_output) mock_javascript.assert_called() + # Simulate clicking the delete button + _on_delete_button_click(None, mock_toggle, test_df, mock_output, mock_output, + mock_delete_button, mock_list_jobs_button, mock_ray_dashboard_button) -def test_format_status(): - # Test each possible status - test_cases = [ - (RayClusterStatus.READY, 'Ready โœ“'), - (RayClusterStatus.SUSPENDED, 'Suspended โ„๏ธ'), - (RayClusterStatus.FAILED, 'Failed โœ—'), - (RayClusterStatus.UNHEALTHY, 'Unhealthy'), - (RayClusterStatus.UNKNOWN, 'Unknown'), - ] - - for status, expected_output in test_cases: - assert _format_status(status) == expected_output, f"Failed for status: {status}" - - # Test an unrecognized status - unrecognized_status = 'NotAStatus' - assert _format_status(unrecognized_status) == 'NotAStatus', "Failed for unrecognized status" - def test_fetch_cluster_data(mocker): + # Return empty dataframe when no clusters are found + mocker.patch("codeflare_sdk.cluster.cluster.list_all_clusters", return_value=[]) + df = _fetch_cluster_data(namespace="default") + assert df.empty + # Create mock RayCluster objects mock_raycluster1 = MagicMock(spec=RayCluster) mock_raycluster1.name = 'test-cluster-1' @@ -3121,6 +3111,25 @@ def test_fetch_cluster_data(mocker): # Assert that the DataFrame matches expected pd.testing.assert_frame_equal(df.reset_index(drop=True), expected_df.reset_index(drop=True)) + +def test_format_status(): + # Test each possible status + test_cases = [ + (RayClusterStatus.READY, 'Ready โœ“'), + (RayClusterStatus.SUSPENDED, 'Suspended โ„๏ธ'), + (RayClusterStatus.FAILED, 'Failed โœ—'), + (RayClusterStatus.UNHEALTHY, 'Unhealthy'), + (RayClusterStatus.UNKNOWN, 'Unknown'), + ] + + for status, expected_output in test_cases: + assert _format_status(status) == expected_output, f"Failed for status: {status}" + + # Test an unrecognized status + unrecognized_status = 'NotAStatus' + assert _format_status(unrecognized_status) == 'NotAStatus', "Failed for unrecognized status" + + # Make sure to always keep this function last def test_cleanup(): os.remove(f"{aw_dir}unit-test-no-kueue.yaml") From 1bebb25360c4ecd59ba3cae2173454eee3639a19 Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Wed, 25 Sep 2024 15:52:11 +0100 Subject: [PATCH 09/14] Add UI table to regression and functionality tests --- .github/workflows/ui_notebooks_test.yaml | 3 +- .../guided-demos/3_widget_example.ipynb | 18 ++++-- src/codeflare_sdk/cluster/widgets.py | 24 +++----- tests/unit_test.py | 33 ++++------ .../tests/widget_notebook_example.test.ts | 58 +++++++++++++++--- .../widgets-cell-4-linux.png | Bin 9895 -> 3461 bytes .../widgets-cell-5-linux.png | Bin 0 -> 9832 bytes 7 files changed, 88 insertions(+), 48 deletions(-) create mode 100644 ui-tests/tests/widget_notebook_example.test.ts-snapshots/widgets-cell-5-linux.png diff --git a/.github/workflows/ui_notebooks_test.yaml b/.github/workflows/ui_notebooks_test.yaml index 864330b9c..9dff9fafd 100644 --- a/.github/workflows/ui_notebooks_test.yaml +++ b/.github/workflows/ui_notebooks_test.yaml @@ -86,7 +86,8 @@ jobs: jq -r 'del(.cells[] | select(.source[] | contains("Create authentication object for user permissions")))' 3_widget_example.ipynb > 3_widget_example.ipynb.tmp && mv 3_widget_example.ipynb.tmp 3_widget_example.ipynb jq -r 'del(.cells[] | select(.source[] | contains("auth.logout()")))' 3_widget_example.ipynb > 3_widget_example.ipynb.tmp && mv 3_widget_example.ipynb.tmp 3_widget_example.ipynb # Set explicit namespace as SDK need it (currently) to resolve local queues - sed -i "s/head_memory_limits=2,/head_memory_limits=2, namespace='default',/" 3_widget_example.ipynb + sed -i "s/head_memory_limits=2,/head_memory_limits=2, namespace='default', image='quay.io/modh/ray:2.35.0-py39-cu121',/" 3_widget_example.ipynb + sed -i "s/view_clusters()/view_clusters('default')/" 3_widget_example.ipynb working-directory: demo-notebooks/guided-demos - name: Run UI notebook tests diff --git a/demo-notebooks/guided-demos/3_widget_example.ipynb b/demo-notebooks/guided-demos/3_widget_example.ipynb index 4d3d6ea70..c3bb6b862 100644 --- a/demo-notebooks/guided-demos/3_widget_example.ipynb +++ b/demo-notebooks/guided-demos/3_widget_example.ipynb @@ -19,7 +19,7 @@ "outputs": [], "source": [ "# Import pieces from codeflare-sdk\n", - "from codeflare_sdk import Cluster, ClusterConfiguration, TokenAuthentication" + "from codeflare_sdk import Cluster, ClusterConfiguration, TokenAuthentication, view_clusters" ] }, { @@ -61,7 +61,7 @@ "# Create and configure our cluster object\n", "# The SDK will try to find the name of your default local queue based on the annotation \"kueue.x-k8s.io/default-queue\": \"true\" unless you specify the local queue manually below\n", "cluster = Cluster(ClusterConfiguration(\n", - " name='raytest', \n", + " name='raytest',\n", " head_cpu_requests='500m',\n", " head_cpu_limits='500m',\n", " head_memory_requests=2,\n", @@ -73,12 +73,22 @@ " worker_cpu_limits=1,\n", " worker_memory_requests=2,\n", " worker_memory_limits=2,\n", - " # image=\"\", # Optional Field \n", - " write_to_file=False, # When enabled Ray Cluster yaml files are written to /HOME/.codeflare/resources \n", + " # image=\"\", # Optional Field\n", + " write_to_file=False, # When enabled Ray Cluster yaml files are written to /HOME/.codeflare/resources\n", " # local_queue=\"local-queue-name\" # Specify the local queue manually\n", "))" ] }, + { + "cell_type": "code", + "execution_count": null, + "id": "3de6403c", + "metadata": {}, + "outputs": [], + "source": [ + "view_clusters()" + ] + }, { "cell_type": "code", "execution_count": null, diff --git a/src/codeflare_sdk/cluster/widgets.py b/src/codeflare_sdk/cluster/widgets.py index e68c52c42..ae2390fe1 100644 --- a/src/codeflare_sdk/cluster/widgets.py +++ b/src/codeflare_sdk/cluster/widgets.py @@ -253,7 +253,7 @@ def _delete_cluster( plural = "rayclusters" # Wait for the resource to be deleted - while True: + while timeout > 0: try: api_instance.get_namespaced_custom_object( group=group, @@ -321,17 +321,13 @@ def _fetch_cluster_data(namespace): } return pd.DataFrame(data) -# format_status takes a RayCluster status and applies colors and icons based on the status. + def _format_status(status): - if status == RayClusterStatus.READY: - return 'Ready โœ“' - elif status == RayClusterStatus.SUSPENDED: - return 'Suspended โ„๏ธ' - elif status == RayClusterStatus.FAILED: - return 'Failed โœ—' - elif status == RayClusterStatus.UNHEALTHY: - return 'Unhealthy' - elif status == RayClusterStatus.UNKNOWN: - return 'Unknown' - else: - return status + status_map = { + RayClusterStatus.READY: 'Ready โœ“', + RayClusterStatus.SUSPENDED: 'Suspended โ„๏ธ', + RayClusterStatus.FAILED: 'Failed โœ—', + RayClusterStatus.UNHEALTHY: 'Unhealthy', + RayClusterStatus.UNKNOWN: 'Unknown' + } + return status_map.get(status, status) diff --git a/tests/unit_test.py b/tests/unit_test.py index c3f59f63c..b9682996a 100644 --- a/tests/unit_test.py +++ b/tests/unit_test.py @@ -77,16 +77,7 @@ is_openshift_cluster, ) -from codeflare_sdk.cluster.widgets import ( - cluster_up_down_buttons, - view_clusters, - _on_cluster_click, - _on_delete_button_click, - _on_list_jobs_button_click, - _on_ray_dashboard_button_click, - _format_status, - _fetch_cluster_data, -) +import codeflare_sdk.cluster.widgets as cf_widgets import pandas as pd import openshift @@ -2933,7 +2924,7 @@ def test_cluster_up_down_buttons(mocker): MockButton.side_effect = [mock_up_button, mock_down_button] # Call the method under test - cluster_up_down_buttons(cluster) + cf_widgets.cluster_up_down_buttons(cluster) # Simulate checkbox being checked or unchecked mock_wait_ready_check_box.value = True # Simulate checkbox being checked @@ -2975,7 +2966,7 @@ def test_view_clusters(mocker): # Return empty dataframe when no clusters are found mocker.patch("codeflare_sdk.cluster.cluster.list_all_clusters", return_value=[]) - df = _fetch_cluster_data(namespace="default") + df = cf_widgets._fetch_cluster_data(namespace="default") assert df.empty test_df=pd.DataFrame({ @@ -3022,33 +3013,33 @@ def test_view_clusters(mocker): MockOutput.return_value = mock_output # Call the function under test - view_clusters(namespace="default") + cf_widgets.view_clusters(namespace="default") # Simulate selecting a cluster mock_toggle.value = "test-cluster" selection_change = {"new": "test-cluster"} - _on_cluster_click(selection_change, mock_output, "default", mock_toggle) + cf_widgets._on_cluster_click(selection_change, mock_output, "default", mock_toggle) # Assert that the toggle options are set correctly mock_toggle.observe.assert_called() # Simulate clicking the list jobs button - _on_list_jobs_button_click(None, mock_toggle, test_df, mock_output, mock_output) + cf_widgets._on_list_jobs_button_click(None, mock_toggle, test_df, mock_output, mock_output) mock_javascript.assert_called() # Simulate clicking the Ray dashboard button - _on_ray_dashboard_button_click(None, mock_toggle, test_df, mock_output, mock_output) + cf_widgets._on_ray_dashboard_button_click(None, mock_toggle, test_df, mock_output, mock_output) mock_javascript.assert_called() # Simulate clicking the delete button - _on_delete_button_click(None, mock_toggle, test_df, mock_output, mock_output, + cf_widgets._on_delete_button_click(None, mock_toggle, test_df, mock_output, mock_output, mock_delete_button, mock_list_jobs_button, mock_ray_dashboard_button) def test_fetch_cluster_data(mocker): # Return empty dataframe when no clusters are found mocker.patch("codeflare_sdk.cluster.cluster.list_all_clusters", return_value=[]) - df = _fetch_cluster_data(namespace="default") + df = cf_widgets._fetch_cluster_data(namespace="default") assert df.empty # Create mock RayCluster objects @@ -3088,7 +3079,7 @@ def test_fetch_cluster_data(mocker): with patch('codeflare_sdk.cluster.cluster.list_all_clusters', return_value=[mock_raycluster1, mock_raycluster2]): # Call the function under test - df = _fetch_cluster_data(namespace='default') + df = cf_widgets._fetch_cluster_data(namespace='default') # Expected DataFrame expected_data = { @@ -3123,11 +3114,11 @@ def test_format_status(): ] for status, expected_output in test_cases: - assert _format_status(status) == expected_output, f"Failed for status: {status}" + assert cf_widgets._format_status(status) == expected_output, f"Failed for status: {status}" # Test an unrecognized status unrecognized_status = 'NotAStatus' - assert _format_status(unrecognized_status) == 'NotAStatus', "Failed for unrecognized status" + assert cf_widgets._format_status(unrecognized_status) == 'NotAStatus', "Failed for unrecognized status" # Make sure to always keep this function last diff --git a/ui-tests/tests/widget_notebook_example.test.ts b/ui-tests/tests/widget_notebook_example.test.ts index 798c2eb60..95e84d66f 100644 --- a/ui-tests/tests/widget_notebook_example.test.ts +++ b/ui-tests/tests/widget_notebook_example.test.ts @@ -30,11 +30,13 @@ test.describe("Visual Regression", () => { tmpPath, }) => { const notebook = "3_widget_example.ipynb"; + const namespace = 'default'; await page.notebook.openByPath(`${tmpPath}/${notebook}`); await page.notebook.activate(notebook); const captures: (Buffer | null)[] = []; // Array to store cell screenshots const cellCount = await page.notebook.getCellCount(); + console.log(`Cell count: ${cellCount}`); // Run all cells and capture their screenshots await page.notebook.runCellByCell({ @@ -59,25 +61,27 @@ test.describe("Visual Regression", () => { } } - const widgetCellIndex = 3; + // At this point, all cells have been ran, and their screenshots have been captured. + // We now interact with the widgets in the notebook. + const upDownWidgetCellIndex = 3; // 4 on OpenShift - await waitForWidget(page, widgetCellIndex, 'input[type="checkbox"]'); - await waitForWidget(page, widgetCellIndex, 'button:has-text("Cluster Down")'); - await waitForWidget(page, widgetCellIndex, 'button:has-text("Cluster Up")'); + await waitForWidget(page, upDownWidgetCellIndex, 'input[type="checkbox"]'); + await waitForWidget(page, upDownWidgetCellIndex, 'button:has-text("Cluster Down")'); + await waitForWidget(page, upDownWidgetCellIndex, 'button:has-text("Cluster Up")'); - await interactWithWidget(page, widgetCellIndex, 'input[type="checkbox"]', async (checkbox) => { + await interactWithWidget(page, upDownWidgetCellIndex, 'input[type="checkbox"]', async (checkbox) => { await checkbox.click(); const isChecked = await checkbox.isChecked(); expect(isChecked).toBe(true); }); - await interactWithWidget(page, widgetCellIndex, 'button:has-text("Cluster Down")', async (button) => { + await interactWithWidget(page, upDownWidgetCellIndex, 'button:has-text("Cluster Down")', async (button) => { await button.click(); const clusterDownMessage = await page.waitForSelector('text=No instances found, nothing to be done.', { timeout: 5000 }); expect(clusterDownMessage).not.toBeNull(); }); - await interactWithWidget(page, widgetCellIndex, 'button:has-text("Cluster Up")', async (button) => { + await interactWithWidget(page, upDownWidgetCellIndex, 'button:has-text("Cluster Up")', async (button) => { await button.click(); const successMessage = await page.waitForSelector('text=Ray Cluster: \'raytest\' has successfully been created', { timeout: 10000 }); @@ -95,13 +99,51 @@ test.describe("Visual Regression", () => { await runPreviousCell(page, cellCount, '(, True)'); - await interactWithWidget(page, widgetCellIndex, 'button:has-text("Cluster Down")', async (button) => { + await interactWithWidget(page, upDownWidgetCellIndex, 'button:has-text("Cluster Down")', async (button) => { await button.click(); const clusterDownMessage = await page.waitForSelector('text=Ray Cluster: \'raytest\' has successfully been deleted', { timeout: 5000 }); expect(clusterDownMessage).not.toBeNull(); }); await runPreviousCell(page, cellCount, '(, False)'); + + // view_clusters table with buttons + await interactWithWidget(page, upDownWidgetCellIndex, 'input[type="checkbox"]', async (checkbox) => { + await checkbox.click(); + const isChecked = await checkbox.isChecked(); + expect(isChecked).toBe(false); + }); + + await interactWithWidget(page, upDownWidgetCellIndex, 'button:has-text("Cluster Up")', async (button) => { + await button.click(); + const successMessage = await page.waitForSelector('text=Ray Cluster: \'raytest\' has successfully been created', { timeout: 10000 }); + expect(successMessage).not.toBeNull(); + }); + + const viewClustersCellIndex = 4; // 5 on OpenShift + await page.notebook.runCell(cellCount - 2, true); + await interactWithWidget(page, viewClustersCellIndex, 'button:has-text("Open Ray Dashboard")', async (button) => { + await button.click(); + const successMessage = await page.waitForSelector('text=Opening Ray Dashboard for raytest cluster', { timeout: 5000 }); + expect(successMessage).not.toBeNull(); + }); + + await interactWithWidget(page, viewClustersCellIndex, 'button:has-text("View Jobs")', async (button) => { + await button.click(); + const successMessage = await page.waitForSelector('text=Opening Ray Jobs Dashboard for raytest cluster', { timeout: 5000 }); + expect(successMessage).not.toBeNull(); + }); + + await interactWithWidget(page, viewClustersCellIndex, 'button:has-text("Delete Cluster")', async (button) => { + await button.click(); + + const noClustersMessage = await page.waitForSelector(`text=No clusters found in the ${namespace} namespace.`, { timeout: 5000 }); + expect(noClustersMessage).not.toBeNull(); + const successMessage = await page.waitForSelector(`text=Cluster raytest in the ${namespace} namespace was deleted successfully.`, { timeout: 5000 }); + expect(successMessage).not.toBeNull(); + }); + + await runPreviousCell(page, cellCount, '(, False)'); }); }); diff --git a/ui-tests/tests/widget_notebook_example.test.ts-snapshots/widgets-cell-4-linux.png b/ui-tests/tests/widget_notebook_example.test.ts-snapshots/widgets-cell-4-linux.png index 9d881da2855f221ac585377caeb0773a740a7df8..691e7124f118370dd40eb36b4c0e423830a99a33 100644 GIT binary patch literal 3461 zcmYjUc{tQv8@5bBVoYP~8eXJKLWqg#D6GcYhE zfH4m%6Zj2Q)YkzIMlWMsErzllAsU#ZuCJ|W;+wUc>5RFhE7*>PF`acRLSj(aT6Vhm zmPvZ7C&jfGoWx4S(vVD6pOOqHwBvjQ@jbU?tKhFK;F$CpOi3u_PU&PJ$LDTSSyJBl zDvQTV1mt7YS|R0M(LwWuQC$Sg9V`0EFXqbo7kUpX>s8QOBax47`Iy2qkWki{5W#hW z6QO^GkP78K+iQOY>p~2|Tk_m^?VOw(4iN={0G`S|!KWo9QQ)43icCM6}M zr|+0OaHBRIjJh`m%`p`i(;`ps<`7!&cs7;@g2_4FQlc?YOiXOpMJcn}>bmZ+W5=`z zeIp|(Dk^^F2BOdyf^9ixeN|OTi%piQ59gIv1qBWJEA_p7!GHh#x04g?qMSskL2PVn zE}q!jv$26QHa0djHTCkMhcO>NoSm2#OYXdl8lt`n&6kR3@m-(ktUR^(UP?yhl`)Y$ zGQYOgkH6dJ*Tmghw@yK!vhK7eCHx@C3ceEZb!cmAYe9MWLw|q&!pPqj?X*%FbE0OgD(&wAS1 z+iT9PO}FcaVV#|w{r&6o#9?^tTeoiIy>4m>ZjTpsQM!XdP@94T-{)p#)`KLH&!C@Z zi-|lto>vd9lQWpM%8`5)w8Rn!)<|_}eIN508-sKRW)@nJSC(B7DBT zcjR855)rY&U^Ma`+jM`4H92Rbug^%%S7)L>(Z6|ZE^7lc(!^-Fgfa^pPwIJlo1;3gTvYI0WKo= z`BfAZBlBNt5%>fIEO3FLJ8NqmSBgqYyIWcaH8rM113Yl}L**X`ho^{^Ete{rmUhu3vM}EE=HDaI#2cke-^Fny|2NL_`D(FD@;8f-GBY zu{=AQnv|4JajdJa*Ae4ha1>n`tDRn28vFkJvZ5l7px}+7fq;MjIgWjh_-^1KU~m2a z&C}EK_rU=c-gZY=E%3HX+T!Th^XFaN+=d=hgK9fGJj}?*_^_n=OeE~Gs?W-o-rjGQ zl2TF>%=*U^E1_T`?v0y6t*wwJ`L?#UHV_0XIVAobFf=`vYEanU+Nv!DhndO9$ZV~zC+Ug9+1tj)v)iW+R>@qx#MIQm!-M@W zBLGnXA3r~BVF8(_C&bx4HWtXw{?@vAWo~YcMssp-fb;VMiY_iL;xku$uc)f3LNq6Q zr{e%bqO9_=vUwOOJ3G6ladEYlWoc>O3kU%0yU%#ek5gyiB z$}&Oh0IUFMkZIkc#igaCz5RVQgm{##gZwVgNT4GZ8aH5RU2UqfYrZxfK$mK&s~_QT zP@MnTln!273=tdm^yyHEIrGhxw97`&m=^o0ex^h74hK&Y6O-UzwG}e>;&~M6waGc( zs+tLu#|wo9*8NZM@v97NhWI24!` zhjBwRU0hr=4gp?x<{C(eMK!xP$$@ddi}dj$lenRwVQ3=guYiA-sDYttsao-d{MEH3WH?$*{8KR-WfxPp=rpn!isfU1g0m>3MNu=ucNXvkD7 zv14}KPa@3hjasp3sR2me&~RyadA#yI8jIx~4!^)AEnWQS)AfW;vK-sg6G8cYj7Cw> z(P6!&L=y!8SE9l5<9O}8SVdP?S8Kmno`bw&azKvAWHPeUn7Far;UEuXxec$jXmqDG zJyY9TSbk5HOzu2`&T(;aVypy}!7E^EZ$ItlZq#%4;te}dIWNXWC`#|&Cg@dB5uHw- zo}SKp@q(m1*&Goa9Xw^bxN=o3dZIJGuz(5SVyQ3pLE6b#KFztf%r&tsU1^DT;y>j<;CpNnqp9RxLgn-z#eJ$&p(mWpuaSv;Q3N7va?G~i?MKkX|^K1GC@xq_^wH& z?8Cy!)z#ID_j^KRcUD&)Iys$P@tK!bR4k-80&mdaus%6CIaq9_sfO3KljWN%>L*8Y z-9I*H9dhs}Qj0)wyl0+SfA4v#mL0;5a12;Lx3Ld$ii$b}43HmCZL7IP+rOx9iA2yg z%+1X?IXO94BJu{ZRegZOj(zZ8*1@J4JUZCh)F~o1G+2tmxN|i%kStL@R9zsNTT3Gm z&Fg?XJ-O=EckeF${3*c8>$9Q8grr7MQ3<~Aq?!LXgK1JV6O-=O$Ha4ki*0Xl3 zv-2$IHx8d2DppojLPJA=?uKcEnw1q7gYrcoq?MGm$Xx1>_5j~xz(epkE-o&phWEpV z9iKm64%kAab#1P$cJ%h1ISd7i2-sfv=iP1K3wT%|nwpwjd=kfxM^{x5y9*YH<`nky_I|D>4xAyRJ`lL&f&%djLj*Us2SDi{fCJdeuCA_RgF+CM zQ!}EXqDo3imNMTuJM}~8SADusY)o?WioWoyjEv6SUSOq!IEzY3mNsYuzNggHn}LVp zjDCrclD~LSUmT`c1ds#GMpZSlYt}$K322i}k&}x{NoD1&qZ}28-RkdBk^qD$>UGLxT{bTOTotbmb+56ege)c{$LRnE78}k_^5)u-&tc-*z64ImZ;JpU=Bk<~E zC(aB`$S$hVVo1e~zj+x_~4?}u`Kn3$esdi6@6wy~y$w7pcxn(9$# zAgv)f*&}Km*xlXz&yZl`Wfe+CMMa#2OhZ(w@!Pey zg9Cd#y@kL)Bs;r(2dbxFO-D(QY5aS=y}kGM{(mOH1*z{5u|MqW*ad@j^QUzQ5jVlf37T3^kQNhlS39^k!;Ea4{|kbVu}X?>wR={)x3<|Fo|F4S^)efC+v%t!qL z#TW5#&*P#sV~x9}rb}M?@udn25AmS=_OK-Z$|JTPZ|#KwjpW#M5xRoD9yb^G-Vn?q z;qaMegZ9NVC9%)d!$&nsF^x_6JUj-K+hLFgU%Q?d8Nr^Tn`@<+_ zV98-~VyV-W$APlp;El^qb*)R~{4ZTyLMB~~ekJKeNWS{x$;o@A917Ci$m=FV#AZEz zzsW)2{xVk{hfxzF+J1*=W)cz_-q%5Pb8vsG<*(O=FKs(%{AF@IXugwE`dW|D@a{c`2V$b1pM>%BoEpP=2W zifAjv&|vAgP#0eHB3+1giZ|IC~>IPX&V{2qZ%U&H(gDw9qN-e zzBQ22L{BdyDN)R(J*unQU%TK#@ivrISEkX{`(Y6#QP9)hASIf~~*_;uxmt$`^04Nl?``ueFk+=S{0Y<+O|$m5w7 z&MgA;jK8sre#AsLr^ipe_jHz(c^Zw^yB*N0ekpMfb$#Rik^26Y8*W{klG3`^Wa8#Z z6o4?|BBUZHO?|L$n_MhsxVyc&U(}y%>(web_^nsiU^kbuXFH)$_@e0dv!|Gevs9tN z3l$bUSEmgW6i~!Xg^TlryF2hQNfL*Vxc7BIUS4lyWw)UFvYQLt3w@YygAjd3cAllx zJ0wO3M1OH?k-FM7zhgX~VyThy_*~)k&cmG12CXUzufYFKV0>KN|6m@QoP5&bz>uc- zR5K;kWntd8tKKce#YI68_66d}@m7e^;5;lcGO_$*BbBl{i+dKlm0m*w*D$d! z3>Q74=*bf_4%_|iq!7WVD7Nfuoa5taTpCTdcjwd$YFRwUtY8-qqBpR1Flp?A5d0%q>pF=nQwD?w-upM#Fu1~<&1Wgv9da(4TBC2HNa1@t9Jxt^@l25%g8r~(qdP71IGd=Ck%u<1l8-2B3Q{O1m zbq;X4e06$L-!LWQx04w6M&BU4R{mDqj+nS+@^{kq>Z+=G<@KWQh-~bUvzyKw3R(5Z zB%7lDJqnpSQadl)E2zYHILrwwb;8kcT3fARV}={qYWv5NP`uto?}>|*c3=x%V`GoW z;%q&QliRvIyyjRwP3hd%Z);$S(IVI_KjpHnZko?7^!GMv_!*^L_CKzE8Qxp%L?O%B zUHxTh{Utp?l7`(2qiSYm#>nUy4i24&;}NV)>Z`}$;y@y2P*O7BvU=L_u@NUt1-7?b z<=Macu`^Zi*qJ#crs={2gK+wEv?gP&nyhQ}m~s=f_= zp9W!k#--2JT;+7}zUFf)_y=0+CELqbwdtDc@7ZS>); z!NZGuIlDVqqRV6(oxNhMA>V$mx;6g@*b!z`+JN>RJsI6(E68 z<2&kT5W`=2_a&5?^@R%eXViBnH!*b%9Rh+dP{O?*wYWv}vuKgM{js{~Aus{{`0-=!hn6P=z z@k)GluF?<=HkI}+LW6ffZZG-w@A}3@tmJJZPBSxK(e9b6>Lxz`4`*%*#fnOzbgqz| zNi<8=S7PY0{*(duikeLu-5AC-t&1wVWGlxJ`M*bLvx>$Pa> z4@t@AQBW=U-$lP38Tdb(S*51v*eT+Ocn@71C%FDDE+t&ZOGVP=xVXSL zqEjjdrl5w$D37?fY-s`n!!G{fX7dW_B*?!+4ZFuqBIo{lb~ad0@aq03Xudzmx=;(( z^P3F^svp^cN&rN<%XZ9=NaHUYv}U z*06h@{9P#00f|;nC}5E94hy3lwd2+!?}3%lk(EAqH>EisA0}a(Nj^g5?&|Tu_D%Cn zXw9gm3ck5Huf-BvLq(Y(AW<}UJ?7f z4@-iZEA-#rRq^v{XJo|6%Oe;VA~fju^st8a39!8Jx6`!}0@81XEb=A+ex_d1$H6^S zC`qB-*RG?-Yapf6g#?ow+U1FSTO%Q%7F7>YddkT;GBw6+C@$i39F_C3{-%T70}7e=p?yeCFEO>g&euUsI9h_WOQAcRoZ3v!Gzs`5FyX`AZu(Y&m*Pc zWGV<%S1XiPdbkm-5LGBlOkqDc=V@@=9`d;?g^M(vceRNb$L{pY|8k?o|6z~$#K4}J z&5SEB2H|5#T{@zBO;tzr zf(Z5|!V{?|v&?BZmp-&A0gCG*{pZ7Z%3Jhytc!znVq!`3yR~OH8CEP>B;8#g4{DrW zpTA*YX;fBKD5}ZK8eMS%$uknW0pO7fd<*q&yMy3C!tCg0^=#f)>9uH4k^0V8b zO?-S&biW-PNF4szuMC-je2wf}pk`gCoH5YPaO#^)hXo>FexEZkRz9HsW9jtu6>@Qf zYf#S52j!lUUmBs#?(QUi{c2UgwZ<*q$PCg2I;~2w5{i|=K!*9Xjf-<_NN{+MXV;cH zh6o0RKQl9G_n+2*1<2O3M;6pXS+?d~@?(MPx4DG8_gmVncQ+k!Y>c_MWo68{iaaDl zK!1IJLf@J!)c@GmZXb+_jb#Zk%7XohyNcLU=!ns$kFb|d- zydTos-Irt0BPrhYbo?D*Z&uIFg)%aB@6F{}+eI#&{fX%TN}b{O7`Q0l9j3D`X=4S^Jl=~ty^^^;v2=~6t^)QLqXQ~)Pi0@TyTI?ZW_OMgQj^G){h86 z2SL7X6O$xKeE94{S}IA_R##y~b}Q{+3WrOiBA$3~UoS12-BLTN>A76Q)w!F8QqC21 z(PSug4;=nvWW>$O3sgfyF|jF?j9!=rM_e73mscWm z>&m&V!_Vtn@)+0tk=r6c5D(Tx9$Zvz6XU#r<(if zxQ=GQi;F#K94;$sF5NuVLd4Hkj24#aCp=aqW@{pj$p>pegfw{AR8*wz&GIrlX zqcbNZs^?^5!(WS{$9HtIkWs7!jQ9VXW59a2Y(|KO=^NCKtE;OgBqEMB)0ZG#Vm==W zxVUzfns)d1C>=#(x7Ib1zqgo<1y~$R%8GBk#cI?Gg1lAXE+s(j)*y&F~Vk<)8sw=FT;Ck_hee>_8f!n5{ zo_+Dd+4g0P{lpv`hDg{H!i^qGAYo_D4<}|B*DNxAiDadqkg~Of%gK44oJ{-jWfKhM zSr=D-y|cBoT3ZW-WI5d=751hV_7m`jisu)@jqLUd^ALu+nsgo>#4q8ZZ{p0LuQ7Uh zo{&G&zgs~A!x2Itjk@yMUTI;{!6BWpURU6b=87fdNlA+r`%v?t3pLf%riCk9Bu81^$miWsQ@Yy!=J0Tka;IZK5n-i6jIWR zGLx|6@vLlP`@x>>w*#Xwu#^<0@j19&``|%!)18;Xz{T~ohR4Bj1rj|kZ^c*AC0+O) z19Q^e8u!5H$r9xTy7!1O-__N=vGMfuOi6!wfQr`TT`XfIuF2O|%y-NCcVKZ2#J0NgM+<9@Z|M*fA_l@IZKB{p`rG*&N7qS zT=|H~a?mx1R#ARFSui)-I|TD_+UN&WYqXx8)um=y>b$CEm+x+1$ai>$>BQR~>FLeh z=R0R-XAV^8hytz>-MSj1puU^S%Jh>ZZfk`B41WHb+FGXyi-6q^NOVAnCWHlZ)Y=YU zV#uDK>Of<}bP~xaQ#TF|5ho0%*rHG*`D-PJE2`9|)#(pK*c2aL=v-c2R8LI^&($c& zYmpPyc!>=}TK)Tk;YT66)$_ zI@13dGrstlj7BKnO(q$=EjR@}dDn&t&L5Ee>*KeyU^XVE>5&mmYie&a@Qr0;T8Wzx zS+!y7=WIFX&h9Q!PBf#>U3fGymg!&}vHAFhIf488p-|F`-+Kp7inY z;l#Shn*6`bCPI!R`5YgjellSU3fedr>~6QDIUz32kgTh_drI*TTFyj-iHnPih6Z~e zqkI1RUB;yj5onBo_L!>lNY(W`+j&dyuZ zPq?Kt{<9dV!V&JUe=WIh#{bohLi+!^Q~ygo9%U1_`+XFsxmx}-)KguzPrRPgX7VoM`!2f-5_(>4&>zI=ry=o zSy?H-1SaO^g&>dz?ocB$v+Q>$N(nZTzhz})5WgZ$TZ7QRFV%A7MJKLKJdN&cX))~jMh=6)5N;WtKa1(>Q|nVdd-g?ZaA;_1auV*m{@%;0?(^ri z*(#gklf*0dw5@ zHH3zS`uX`?9xgpxHauDu>^PqIxJrPpR=&_nM=Ne@Qt0W-GG?>+TkS1=#F-?;{=oqb zm=qQkWwp73LrG9j(9fUCV4#l9&e+)4hK2@+14jz4-E6fT1zi5R4K2#&7I7R!B`72$ z^er-y;@L9-Lc(UT%=E1C^3k52o`HdZ@$t&4DxTM`i)(5wb7|6-lw1;Bz@bDn2&CJYrd3kqsc38BEZEbDoFJ!Y-7Zw&ktG%kK%42_4GDA6B z1AH-0)Xc2<^Vu`V+mT@aqL ziwidwSMT7Ui2rTvkAH`Xv^1I+6CGW9Z|{3?adByBi?y!EjyXy>g($< zpFRb11t_1dcau|5DFf!JsyYY5VPaw)E;beB<>3MQkdk(zeJ3O!2sg=H-`=(|2P*vU zr%#`JF7`ONxBwj0)YK>h-C4!{05HzYHDqP|ja7OqtgEXl5oTdxB91O8A;J2wQbteD zZ!ndgnwq*@pAh$)mw|!d4zU&uT=12I?aSAj?yQtgpG2jkr2PEuEchQI!C4G$GAcR- z`UogVHw>=zW?O>Q-g!=6FDRVvYh=^ezh-2VmX)o)t}O%~})73;~#%l};uRGBR~@ps$Dh*x3(;Q-NXHE zz_JhK>j+6nfqC4|wnp%S7skd+3=NOAMl#RNT)DZq(a^#!F5HdntgQ>ae^&;Wa&|uX zCHXbfI_`Zo7(i1=$>A~bFwk|=buPInDRU`r9S)Y7OR}>=piAKUo>)foj6$uFuCA_C zC{BCYr{Q5`2&6eUIJmTw1N4jY)pDz=s=PcsP0h^C&(81wzs?(*m`se1>+0#5{O9oU z@`|__$;!&w97;buIU#W9R&K@WTv*UkQevm0qvPas%DXW!x3mG(l=Djl25hE>(;*08 z13Oz=%L*Pb%k!NHuxn&vV?*~IxF7U4k22d!V!wR(l9iqPe6Q2izAlr(vjO^pgW+SfoAlaZIVU2FvHnI95BGpw(y z(CVL@oT%oj3%Nk!;{i|d^YLNHOnYXotgHkbD*O5RCMHF8m5>=58{027UM;r@I?o1$w!WM$@_J#KCx@6BC_2H*FzLk7I~gH09GpV8D&< z-&-HAcDBR)T$geHhXbx6BqRix0q8a=Dp;;k`lvkvP?V#iT;++#h%~<@ep^bA6QJSK z9mF;Uwz|JgZ#1YelGtZcK`iVqV2R~{04^%R4w#ezCS-!f#Jr&Xo1vkj!@|H2!oGI7 z1k8pIDy*!0qYs=JggdRNuKqkY6$U%@@y2SO20d5xR}cuq|EApk&ie%is*#6>2N0v( zAoLpc4-a$iEI}f_4W#=umseZMhlVDtrZyHnIKKpF`L%8>9ubl2$zNH3WG$_!Cr?SR zG?I?V-=Qlcau~I|1gT0yL=*VjXqqAt(QI0g@}CbHzUv)q}Klmu`ZpKxzNX$jB=5Eu1v06M1Y#8^NOUsP>{KmhmvcpON5Qkub z(l-%nKvq~-cx(<*d3aocUQ9>(#yPmPY@ddeC3IeR+^HM za-6y_J|4V8-_zUsBDk%s4HdU&jV=i>u#N;)34~N7!{#6iOb;Y8fS=uigEU^dXZw0! z2Pe#Acnf4?px450C_VyQ0?|;(02s0Z#Q>$)83xCjH_E>ipvVe4U!C?DqZL4G20}Sy_Y(ii+2f zNSFWoTv0c+z# zHA9R5RH!H^qo{@JUAI-i?P+Mz`}^gM+3wZ510j#=9mb~1OcdnhacD&3Yev8{Ks|uW zOc^+`vWp#|1qm2VjGrBwou5$FahwV3=wW7p+**i^RB@O1yOB9%nsOz}<37 zS|`MUl8nrsK&3KdvNt=ctIj~X&DA*6R8;}-uGesJ>9P4zAz58jRaIMCHttVfpIZKS zTbo!g-hQ|r2x*{Kfg%QUPzS1wtE&iHZ@@_y7#OoxZA!u@z`^(T_u1LmKre#u!=({P z74Z`&(**K?H(-q%I4UG8EH^jTqJ{oo+3?6UF)b|$&)Uweld;Z8MMVYBB0_^mOy=FY zzz-VQ+E%8fM4&|)x3#ll4ho{291u^^l9C{6f{+-i1|nSUvQPpfCLA0bAXpGsoXE|l z9Z-Ix_il=6Qz14Rnt zr{sp4tE;P<8<SsAD+=bL<>?*EciV&XK=x>irO;R*=}2_qvTAn$;ifocor=1M{- zcKya-u*38HJDK)(B;O&pep}b|HKw>y^X&|m+fedFfvVfoQk_Ic=t4n5U0G(eaYv-H zlhX`X?z!F~vs%vWVZZ~(qJT6Qf5?!s>l2ZZ8o+()K+RWD!hSCqr%|X02(sv7mG$Ir zZ6eHdkRuV*jEbqAS0~wZbyvlD4WQDA@T2B;mXMO#=%xu!*V3w~si8Sqr>qA`8VHyk z({iv;;!j#@)D9F1DiCZSux@dpkEuhg$N&8KGd9Kz8VWi(K7g4kTbWo`1O)_=laoO} zbaiw*4+ce;$KLeH^77G`5@f_5Be%I3!pHYPNl8gf?G(sj0QO3ShwI%EfM#!R?~Z{3 zMi{b@(aKbbfzF2yd4+`zAo*^OW>>x)P*hX|xdY_%3iE+B3$*!#1#vO4-@ktYA^H}v z^3dAK$Il-Rg{CAYAIwyMdL}(DDKQb$7F>SUXKXsvwG9p6!5+dwpp1aUEEz>DhSHk+ zZrruBwAk2+faZdU_<6TXU4znL1JLCq5|}qF6H~3nA=tSOc}hw!MB?b;Vo>Wy%f_al zqjS^p38ne&A_eFkf5{M|>2KuRcw!$vegu>o^Z;#1=CQF18^ad^r8f%dW>`5GMO#~2 zT)f@ubVF87ZcY3Qq&x6H5l|+T9$mpe71&G_uaLVw2DP}$(XuEgC@3gW?rtt|JzWDO zUQhv=I$CK5Rt22U{tr$-D}n_GyKG=3$)^G75}11co?ePzZ$RPb#iDe+)qTbtyOaiL zYh}-?bq?b5!ddIhzAV=0l)kL2`i`rI*bDlr{tBqzvz>{;{@X($q!wjR6NAcxl!{70 zFyuimB03u4bHRH0V2g${d6G@Qi)0w54Y=@Pe{ORi<+2C`c^QHUu7XFD9>K>l*uWd4 l;m`l$^IHG?(_DcMD8e_)+|<7f9si9XE2$_^EN1xm{{S-+E9f6O^^&dk|+zx#Qg_j%uaLX;Jyah^VZiiU=UBP%2M5e@CXXmGCn z+c^6Ef|UOeCB`p2(>~6UH1N3KBwPy zhU_Oo21VD?)-TieqAt^pMa`KA1C5@5e$gNTk^5BtPSC`02?fC6_v05v55ds}?f?1f z>sB{3sFsh9kE|@FM6g`m)2B~yQ9o*G;-jLX!osW%e(mkqQBzY-Ad-u%Y;2wp6I1D- zCMSc8rkHDRAH4gG&y9}#AS%h9_1V*o9K)_)xhVny0ui^pmF4AkPY#cd-JG4VQGs%K zM@L7Fj=ZlO-oAay7B5_%78H~Zho>5az1!s-(;c6mk1`_bnCR;2+TPx#HGD$$U!Xtj z0s*CiogF85>?6z08wWWhrKpw`Uk8Ws($c;2$VXsK7wzu{DP2uXSAYLTGqDC2=tuv^ zPR-5DVfV0uZ~g>^D-1T^#1Q=(3CZ=aLnw*@} zR9E-eX8{-d%RYg>%F4mZl04% z>y4&X{P)jYqobN1N2MhhJ{>NVQ9Zmky9>jl&~!LvB3^51MYnQsvG;<<6teDbjQoXc znX{uoHM|rQ6y57x{575IYCN;D} z@U335H8vTx`M!M%(ViA5Ep6_kp>Hdn(udZ*_-p?9v?UVv307vZaLwkf?x!l|>$6Ka z8A?hq;gQ+YcsE$Xxl(3E(yLAX=wWr;*_J`Yri1Cl4rYG-bF3#fnMqSP7(s*G?_v1u z-1j`jykEbC_z*QWD_L1AAewh~dA297CL6Jm7Vcr8ub~iSE-UXC=%b zYO(PFQ6EMBHl9Yu#gTIDHzf0I9~~XoZgd~)*EbqeRNrhoz#V4{+>*LTjVX64%PcM~ zgQ15pH&m1#gy!W96{uV+)+HqJ&1cBc4C3HH#6Q!ac0(dls7Tgi>7;Is_L8HxSjgbklZPQhUcr|w z0hca`sWDPZ>(56QvNMaG$Vd5#w)Qr9_|GKQmbSLH8ZE5Yj*v>$yu1lg(waZ>Aybu) z<^{NTk1j)q!aCajhK7+o#&E)Xvas4rMYV`UD)%9+{%lpg%HKct{u-jeY>iP9iT}&@ za4Dvtp}jxZnwQVk9@Q!CyWDwo+5``XlsW@2l#a>8hd3FN$KGV@o^Q+XV&La5rcMB#P9P zUoh3(vvi>E>a^`!V0(Mg={A8xuw;sbZ8{CH)K043=~ix@e28%|0-;MyR3*%zpsXYO zxBcrSInvSD`}%^1r!eH5V#vJ3$KLDua=Pl;%Tq(tb0sZJbYeKV{=S^OV)S=8U8E*0t#)Fp*!@OJS;y1; z3oWdY)1bQ5&EDFUqcv%2AfDY)GhW<-PNZ{QHV%m_LLjycPoy(9xHkzm-@YBXxmm+_ zTskx`ZqN`@Pgnig$qX-W6W&mml)9X6iVBi9kg!!$1XPv~i!8^Zy!lc7ed%<2QuXHt zBU8jz-8xbHt=%2btgP>(#N!_|*1jlPexeIB5=>7Y8cLC}@sQBJ{Q|$1W`Eh_HTf{B zE{b}D@dYCoJtQ71o*1J-gK!V-z@!`XV>gpjJQA#N%{qvcxyE`9Z z9ITSk(mn_cy*SFdJTBvaD$Y)TwN%0()QpXN=T%xdGlSA=a6B;cP*D-jkzPiuva)8F zIC|+*1sX+2Nl`x;Cw*o}rg48&gAp>OV-RUSF*#lnl?oSS4Ddh$>=%Pa)t(4C&7=|! z=fYrNW!#wy<5Ddu0z(q4vnP5`r>?$O^hv~-s8u6q|bKtce|ih zQBAEva=H?XfG;s#~=8%q>vykA;ZG-w0|DraO2)J zUDflPypT)3udOSfPFzPeBrKC=Ud!`nB~3mta8lSee&B!*&Ljfogg|1=gx`EnCx#9? zxil|sB)J@B-;5Q;7MFA}e=w|Y5sv8UdP3Wa_pWC&mf86Bc3>a1Scm%|{J97Tbb*BM z%1ilqR(CcXhqSb}w&dF{x-u5-Dg?pHt3*WBMFqbT=fypr?JugNriMkKoV@E9iRhk?BH^4zp6^c{zb_UpqLH(6tAD}O$+bS@09Ms+z_tE)jU^&Q34~Ic_W7nZvArI#w7t+^wpT$B^rO6INnrfA4VZ`l$t< z7RDVeNr#ieMwl#m(IM}F5{1LvFziKjWP3~M>n|2NV(RPR6oPBgQq!oW-X@nBN1I{1 zMjo5=5Fv4KWSu>Yp54}V(dvfQ&A}qD3<$O98KOoRzJV|!XP$eatyZkf$N)(vpW`8@ z74!Gk7Vn)Q!j0D|l&}28aqRO-vg!V|f9fvPM`Y(*N=iZJ>j^APrEK_myP3GytopXb zFI7~wXu3NHB+|US;WM)%`OZnb>>8bWDE-vbRG-b5%H(AKT!soOviva2K>e?>Ty(eB zeMgzG_p8|B!Wg8JeVv^^E+lmJZRTJ5;Z!sxZVT{bDupcMqqC!-vMsUvY6pHyFiSEXq8B_hZdyx)=L{M)a_Hbo6RU z%8bm{Z{vd5*e6ZQiMXsQtEAakampNJaU%M{!~LjVQH)im_nN_eSMt|Ez%<*PBLT2CUgw$MnojmenDF#$;_&q zm@^AlBsQRv4o?(x8mk#NOOGy(f01%AO;0TZ#5{lQlb#;GxtTc@9bH@;Q@Dv39tL_3 zOz1!YS5fhMrZ1hMrk4Uv&Ckbv+a6Mq_!5~{6L_X%%@!=+zv|@VKu2LuI$}LN!)a_a z85IusaMwFE+{va$ozaH8?7!wFqH=K=U}UV;PQD~(LdDIbQNp5G z2J*jD*Pbc!-PSZTh_<(Pnur46qPN0e{&CM@jaOC{$;q5?zEol3sMnMm=2`xBy+dMB z_?;9V3{oxSMTQ~Z_4Q*p(iB)&L}z;?h@s3%tMLP?2}g&8rS+)F+EN*LYZjJ2E$%#5 z@6@%|hgwD5N*4!GyR);K{>aGU;1DK<9XSR#wq4eY9&%n!d|MBQKHa6($IRuL=j zg8tEX(8ZqJj9Nn6n1+f95E{tYy6UQjnUe1PWTt#o*RA&k@!wT4VhG{ZiH#*3zDRoc ztZH?Q1SWlpo zO4^RRcjW1=H?Mbfl37(HyCdL>D-8KX?7ViA1}vpN1x9ABxV)~jz0OlmEgX$rf*>*3 zo0y2I&0F0_dQLVqkL1f(^QsoMp2$W2ClSY1*XIr38d?Sh9(#J;xjHh6xPB}3qKvEh zHaqK-YuuhA%GK#^zx&lT8Nsf{(#qk9DM^z9P z#!O6W5)xaVU@oq#66cf8lc;O#?eF>eEWfvxmiAM^GkC2=b5|fjA^13)Li$>O2hx#j zKGf3SQY65%xOUphHk+#kA^2B@3b`lDRL0c!xoi)GA z{`1-gt2e(7Wn&#V^>qlT2bY>g3<0~aQ;RJvECF5XDJRVg=0|t`84FVk;w!2p&735$ zX=hs#8QzI#Ku1rQYtl|kBz@&^lpaBWBVfD&rc*_deb3o2-C6_w0tTN}{9#TBHaQnIvm8=K^U zdk|$ldhyvoD^2M5=&uk|w)dIez3jw9rc+YKhSa-7LVSndh}9RdT>Tk7@1gg*9YNK2 z#$mq-Zg$q)+?L-%$diG@H9-m2MGwP!qK1Yts9yVe+evngZcjckHL)m4+1wkprbfgm)I9mE#&^jTyc*okCH>~>&6l!3SJGeDo z-1EC!exno$0U~y@?OPAihv3SJEDg;l4K;=dtC?B${xlDe>e`F9VC?Nx*Tm=NR>#Fx zpo9(SbIW+NPxYTYd-f9AB1%uc?1k}POJOk_%OQsUx7vd?SFp-bs8*8Ocz(z z$fO|DzdJhg>(=^Yfdk2W=8NqCfq{WfXxZY~SXf@WS4~W4h>MGJzV|yU1huB1wfqMV zok0_IV`KUb>51#BD{k!T+=>5FZ6e|hkT7z8Rt$-rkZ0^4RF>={2>qQ2(o8 z{VQcrw>?;2TjLddsu5oBKg&?zYV_u~|7J(l_Wxd+`~Ugwf;HpL)k!Cem}%&5@iw6M z#?PArVc>je*W7s`&hJoWW_C6rA_CFZ*Wx|^yS+Z&pQ6EUcFX#4X==G{Bv`4%52 zEoZl*qhok@ILouphK5T)>t|?8A3l8esgzk-P%t$!b0u`jz`*cQzwrdo_2_rhDm%*P zI9Q$>9aKgKa@os#9R*Ro?=PEWvRCq@TDNE#Z{pjZ;Nl9rdPT`+)-PRWV`Vi{YnvJw zIoKIO#KFOV_4H|rA(?=HK`*H_%YcXO1{d-?0v|Md0eut+)M5);|k z*h=*p2)UvnBF0BY+Y_P9o;B`=pZNLt>FIOWHA@#X9kEDW{)jL#G<>734lcWFPn3j) zhH^n$IuWECI?tXz7j9Y>cG-&i{{4GO$_yw3I5{~5X=1(u^~uotvxT;YOtU>*UHZ+Q zr^mDW0s?#c`>jsvgX@})e&YonQ`Ai&|0=t#KaENn6SOMLUmwkrr=_K(nX4=-8|v+q zm66%l8ZTN}T4H5o6%!MK3M7I4ZD*=6p-kUpptX*H)S~X?rU())E-qD7Byh0Y++6$N z4d%k)V&~7F4-XEw1qJPGZ6l(h_P4jMQEz>|0;^F7+JU9z)zej0KG-g6ZER?mo}Eoh zOr(c>R8O2?CpkxW&JcMS~qvbeCYu(~J6oW}RuTH`+qdIbTmc4cBW7@|3E@sUS3sI6*gSfdS~_NQ>E+fOjO1L7nf7uhwSX^^#58; zZEfw&j*YLcFSr7Tk(ISDPd*V~_yr7@7`P;GQ(P)xlTZKJ==5|5eqVoobw$O<=qTt~ z7w*b&=!#7o9ua}_xrzcGzufoM(_2VTaI&vYPFFWoBFko~{7bK;(O`S~(#_>Dw&vor zC-SGl+1Z)o;@3x58?{QZvV|ojE^cl#{8n+mgx|l9fHDkNVjbOotxO;Ov!LMO>U0}m zUnS>#NLbkN!#^P*A)OV-qr=0B@^VAQqv+^pDiPPDq@>`G5K9Y-KfS%Q{|0|^b0a4& z54n`&ZXXy>aCXPV#f2%Gn5;B;9KYh{H}6kcTwOgKv0lnT{ajgDf%;xb$;(@#nn2~| zKE;`jmlsU5xUB4+E(q;6-fAk-&P?3Y7Zf$757C6%rsc2w0#C>e3Gg{}HMI>pa~^DY zS=rBSZi25~dEZT{4Twa3xh{L0SR6zs+ z0pMXj*T~Pr6gcHOS*jlp5Flue3Xh7ix3(5>SoqD|zPG!}&&%8BcklcDeIU3U8>=cU zjZq=M!J(j~^_r8DlabLBjQbt`3<`C(HO~I<;Ws1nH$_!dPJlDOHVxp6Ca3kt$VglR z-{tOzAUqm67^guK87XNHRM`6j8(@`{o12@5heuQt?7^5Xb%xm5+JdMyJw4sr+}vpf zp7Htf=jYF#@9ph@6VSZ9y*qC$9378hnUyXsE`sZf3k(C7Ha9H|4WpBjljGuEjotFF zJG;0v)Yr$v#%}h_H+dM_+uNI)OQ)6)NFWlqT79lvb#*CJKB}oP(9?gTq4G^v)6;ti z$eM)($oY))bPi_b#Q1o11B1wU19})bIy$%+f<>SM$k2Yi<f2L~rUJ|1u}pZQ=K zpuz+W-IUZ+0to2tv%e{6@E# ze1p8Hx1UP$@}y*Bf@@bmCrnICEiElNIy%5OZEbC4W@b;aYHI}obG`zzg1}?k6S=y& z3dRb2*2>PVw5SNt->+9^MLkd~f)sO$RZQ9#!7ypM%sfj)Q2%z;Lf zl$2oW4A_&7j_%phfZp!@AvAH@sQGsQasakbR3ZYryp>s5ZPU}*nwpa!4FGyb_Et4t z5g2%NeI3kSACQ=$;@|-;V62>9|5|8JP!EW2?(qv6#af^D{t7j=mzQ&UcwA0RY0VVj z!=jUtri!(zva_>;W6sW80fv?sX=tEcULqL{fZr;ss?yWaHV6e!7Lt;k3nFTOYAGVA`n=#xj%o_nh#~HteB9JCcUILvvtN~XPf4aXkjO( zr-vwUw>`Zk6A+gH2m#LnS4)BT-AT`j2imJjO2R5CDw31QlRr#=Jv=1R>tusq*8q?w zut-N3xj)TxU1|qsBN>Z`HEmh$j-WLG?JKJ>mKiuXOj$}w3PAI&I3hUsi3;#8DbYXg zz-F9=+}1w1+sz%kww5H*JU$_z8)0fQ{g91b4>0=6I`Ir9N*SP60~Sr-PPO=~OU?xGQtkbw9Aea2n^?Q%GKh(Z~x zO{a%^!@zs|_%Ww;B|5E%xp_~nR3IMZ!o#N5SXfw4==}NqJwI->-CQGx=ExTs00cJYvnQ-_gUK&1r{0PEsVOd#SdAWnL^UnT$zQPp1@%Y$Scw{6s zYH?Zc1aK;d3r}c4Sf1Hub#Qb97*R}n&BMi2_TvXyNBN?dBLNjvL?+88CJ}!AQ=kO* z=UYL9{qeDoCOy#nLIAL5ZeAWidU;Nc@o27WwO36|4JH;A9=#!&ufsg2QQfn(2?jBD zQkTbm+Rt2zSq0y9+%9O^s_Evml7TsYABW35csagl;5W?ckW6lX8nC4T{;5^(K#PG9)Y|H&p+QnX z;`i&@9^}7{I$6La{r&y3v#CZo>gwtxB_+XlrXK_N5#Jo+-rigB6TGPH(HnX!& zH(neauKT^lfvY?XkYI}4e2_qT7Y>Ra) z$SL;9`a0M$YWBNt$oE_;ykMknu>rPWW7}F^p9ERg#o=AG&d%Q6#Ms!%!h(f`MZL>5$*zuyiV6=; z1)%#85?=4r;*t`8G~ioTS63k4T3TMVWDd^|_W^YR4q95CrWX$`zTa-NI!_?~PHF4% z?92hJzq`8|bj7(3!z8*pZ@p(^W{&MOAq;U|>n8{4NLLu?_U7gzbo2$Nq;{=MazH>= zr77a4LhAdxk9btVuzjzKgGJE0wvLY7Y^{Q(W@3TU{oO4uFK_G3Q5X>Wz>;%`PGpsp zl`#nkKb2WPB6y*{kiu^bG||hKF99wOL`2v4vBwL5G>wW<`d{y(#4yTRd{a?Z2Zad% zK|!GLL9!?H1V~u>>B_F&-c5kSyj#M--@ktY8{%MN15^RMOM;L8X7+@}?`Cm%*%)Mh ze|LOG$;tSw#sHT9kOJvCSq5!&Sd^GaH3OmX=hhPWbFu1SMn(nzD^MFn8YRzf z4(=BW_4M@m5Qu?+0k@qgBMS=|8z9Y0%*+Hlj#h9e1l&D6K~V(12oN0roh9>Iu=p(= z$7XJBZd6q10F2ZkuHk#uw;&SmnUCJ1(CiQi93X^(S9x+d(nQ=i4{aXc;SFsf^Fh`I zSmu0Z`glFlZ)8cp9|x`?T% zDJT_~52P@1a75sZTwh;fU|@i7IEo@C0wV&#HQapIh)m4$1ULt%paAT+?xPgWH_$>P z|9QsQnTDn&#Qk7j-XqXH_Sdhi5F+MC8i-<+^1sDB9|JESpdXsLp}i-&qV3_z<6&PN zFT73rPeM+9GFa7&d;a2f7om>H(`?HT=acWjNQV8~FA~TWC#Pdz?8>8ARy6-CN>O)i zkQf1Z?)vmTC?-1EVW|^y!p{Z+?d+%qygp?J)2??|TpLIQ(>QH@_+SC@^nbrp3!(adJ7mfV?zkL02j}g{I>_I@bUI literal 0 HcmV?d00001 From 745c99164caa58dff29654865fd648c56c3f1f69 Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Wed, 25 Sep 2024 16:50:30 +0100 Subject: [PATCH 10/14] Update codeflare_sdk.egg-info --- .github/workflows/ui_notebooks_test.yaml | 4 ++-- src/codeflare_sdk.egg-info/PKG-INFO | 2 +- src/codeflare_sdk.egg-info/SOURCES.txt | 2 ++ src/codeflare_sdk.egg-info/dependency_links.txt | 1 + 4 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ui_notebooks_test.yaml b/.github/workflows/ui_notebooks_test.yaml index 9dff9fafd..5e8d506d1 100644 --- a/.github/workflows/ui_notebooks_test.yaml +++ b/.github/workflows/ui_notebooks_test.yaml @@ -86,8 +86,8 @@ jobs: jq -r 'del(.cells[] | select(.source[] | contains("Create authentication object for user permissions")))' 3_widget_example.ipynb > 3_widget_example.ipynb.tmp && mv 3_widget_example.ipynb.tmp 3_widget_example.ipynb jq -r 'del(.cells[] | select(.source[] | contains("auth.logout()")))' 3_widget_example.ipynb > 3_widget_example.ipynb.tmp && mv 3_widget_example.ipynb.tmp 3_widget_example.ipynb # Set explicit namespace as SDK need it (currently) to resolve local queues - sed -i "s/head_memory_limits=2,/head_memory_limits=2, namespace='default', image='quay.io/modh/ray:2.35.0-py39-cu121',/" 3_widget_example.ipynb - sed -i "s/view_clusters()/view_clusters('default')/" 3_widget_example.ipynb + sed -i "s|head_memory_limits=2,|head_memory_limits=2, namespace='default', image='quay.io/modh/ray:2.35.0-py39-cu121',|" 3_widget_example.ipynb + sed -i "s|view_clusters()|view_clusters('default')|" 3_widget_example.ipynb working-directory: demo-notebooks/guided-demos - name: Run UI notebook tests diff --git a/src/codeflare_sdk.egg-info/PKG-INFO b/src/codeflare_sdk.egg-info/PKG-INFO index c4061c623..27ec5cbfa 100644 --- a/src/codeflare_sdk.egg-info/PKG-INFO +++ b/src/codeflare_sdk.egg-info/PKG-INFO @@ -1,4 +1,4 @@ Metadata-Version: 2.1 -Name: codeflare-sdk +Name: codeflare_sdk Version: 0.0.0 License-File: LICENSE diff --git a/src/codeflare_sdk.egg-info/SOURCES.txt b/src/codeflare_sdk.egg-info/SOURCES.txt index 42541f1d2..63614a814 100644 --- a/src/codeflare_sdk.egg-info/SOURCES.txt +++ b/src/codeflare_sdk.egg-info/SOURCES.txt @@ -12,9 +12,11 @@ src/codeflare_sdk/cluster/awload.py src/codeflare_sdk/cluster/cluster.py src/codeflare_sdk/cluster/config.py src/codeflare_sdk/cluster/model.py +src/codeflare_sdk/cluster/widgets.py src/codeflare_sdk/job/__init__.py src/codeflare_sdk/job/ray_jobs.py src/codeflare_sdk/utils/__init__.py +src/codeflare_sdk/utils/demos.py src/codeflare_sdk/utils/generate_cert.py src/codeflare_sdk/utils/generate_yaml.py src/codeflare_sdk/utils/kube_api_helpers.py diff --git a/src/codeflare_sdk.egg-info/dependency_links.txt b/src/codeflare_sdk.egg-info/dependency_links.txt index e69de29bb..8b1378917 100644 --- a/src/codeflare_sdk.egg-info/dependency_links.txt +++ b/src/codeflare_sdk.egg-info/dependency_links.txt @@ -0,0 +1 @@ + From a1b01a01d498fd6c4d913ec63275bf09d74da646 Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Wed, 25 Sep 2024 17:46:53 +0100 Subject: [PATCH 11/14] Fix to hide toolbar before capturing snapshots for UI notebook tests --- ui-tests/tests/widget_notebook_example.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ui-tests/tests/widget_notebook_example.test.ts b/ui-tests/tests/widget_notebook_example.test.ts index 95e84d66f..8884ab8f7 100644 --- a/ui-tests/tests/widget_notebook_example.test.ts +++ b/ui-tests/tests/widget_notebook_example.test.ts @@ -34,6 +34,9 @@ test.describe("Visual Regression", () => { await page.notebook.openByPath(`${tmpPath}/${notebook}`); await page.notebook.activate(notebook); + // Hide the cell toolbar before capturing the screenshots + await page.addStyleTag({ content: '.jp-cell-toolbar { display: none !important; }' }); + const captures: (Buffer | null)[] = []; // Array to store cell screenshots const cellCount = await page.notebook.getCellCount(); console.log(`Cell count: ${cellCount}`); @@ -45,7 +48,6 @@ test.describe("Visual Regression", () => { if (cell && (await cell.isVisible())) { captures[cellIndex] = await cell.screenshot(); // Save the screenshot by cell index } - await page.addStyleTag({ content: '.jp-cell-toolbar { display: none !important; }' }); }, }); From f9d90ad15d4f73fa39a02c6b29e10ab31806d550 Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Thu, 26 Sep 2024 12:28:47 +0100 Subject: [PATCH 12/14] Add head comments to functions and add num_workers to data frame --- .../guided-demos/3_widget_example.ipynb | 2 +- src/codeflare_sdk/cluster/cluster.py | 4 +- src/codeflare_sdk/cluster/model.py | 2 +- src/codeflare_sdk/cluster/widgets.py | 67 ++++++++++++++----- src/codeflare_sdk/utils/pretty_print.py | 2 +- tests/unit_test.py | 6 +- 6 files changed, 57 insertions(+), 26 deletions(-) diff --git a/demo-notebooks/guided-demos/3_widget_example.ipynb b/demo-notebooks/guided-demos/3_widget_example.ipynb index c3bb6b862..11521ec72 100644 --- a/demo-notebooks/guided-demos/3_widget_example.ipynb +++ b/demo-notebooks/guided-demos/3_widget_example.ipynb @@ -116,7 +116,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.19" + "version": "3.9.18" }, "vscode": { "interpreter": { diff --git a/src/codeflare_sdk/cluster/cluster.py b/src/codeflare_sdk/cluster/cluster.py index 8516e921b..e97141b33 100644 --- a/src/codeflare_sdk/cluster/cluster.py +++ b/src/codeflare_sdk/cluster/cluster.py @@ -863,7 +863,7 @@ def _map_to_ray_cluster(rc) -> Optional[RayCluster]: name=rc["metadata"]["name"], status=status, # for now we are not using autoscaling so same replicas is fine - workers=rc["spec"]["workerGroupSpecs"][0]["replicas"], + num_workers=rc["spec"]["workerGroupSpecs"][0]["replicas"], worker_mem_limits=rc["spec"]["workerGroupSpecs"][0]["template"]["spec"][ "containers" ][0]["resources"]["limits"]["memory"], @@ -909,7 +909,7 @@ def _copy_to_ray(cluster: Cluster) -> RayCluster: ray = RayCluster( name=cluster.config.name, status=cluster.status(print_to_console=False)[0], - workers=cluster.config.num_workers, + num_workers=cluster.config.num_workers, worker_mem_requests=cluster.config.worker_memory_requests, worker_mem_limits=cluster.config.worker_memory_limits, worker_cpu_requests=cluster.config.worker_cpu_requests, diff --git a/src/codeflare_sdk/cluster/model.py b/src/codeflare_sdk/cluster/model.py index 7b2d2ee33..44be54567 100644 --- a/src/codeflare_sdk/cluster/model.py +++ b/src/codeflare_sdk/cluster/model.py @@ -78,7 +78,7 @@ class RayCluster: head_cpu_limits: int head_mem_requests: str head_mem_limits: str - workers: int + num_workers: int worker_mem_requests: str worker_mem_limits: str worker_cpu_requests: Union[int, str] diff --git a/src/codeflare_sdk/cluster/widgets.py b/src/codeflare_sdk/cluster/widgets.py index ae2390fe1..a4c01f91c 100644 --- a/src/codeflare_sdk/cluster/widgets.py +++ b/src/codeflare_sdk/cluster/widgets.py @@ -18,7 +18,7 @@ import contextlib import io import os -from time import sleep +import warnings import time import codeflare_sdk from kubernetes import client @@ -103,7 +103,13 @@ def is_notebook() -> bool: def view_clusters(namespace: str = None): - """view_clusters function will display existing clusters with their specs, and handle user interactions.""" + """ + view_clusters function will display existing clusters with their specs, and handle user interactions. + """ + if not is_notebook(): + warnings.warn("view_clusters can only be used in a Jupyter Notebook environment.") + return # Exit function if not in Jupyter Notebook + from .cluster import get_current_namespace if not namespace: namespace = get_current_namespace() @@ -112,13 +118,13 @@ def view_clusters(namespace: str = None): raycluster_data_output = widgets.Output() url_output = widgets.Output() - df = _fetch_cluster_data(namespace) - if df.empty: + ray_clusters_df = _fetch_cluster_data(namespace) + if ray_clusters_df.empty: print(f"No clusters found in the {namespace} namespace.") return classification_widget = widgets.ToggleButtons( - options=df["Name"].tolist(), value=df["Name"].tolist()[0], + options=ray_clusters_df["Name"].tolist(), value=ray_clusters_df["Name"].tolist()[0], description='Select an existing cluster:', ) # Setting the initial value to trigger the event handler to display the cluster details. @@ -132,14 +138,14 @@ def view_clusters(namespace: str = None): icon='trash', tooltip="Delete the selected cluster" ) - delete_button.on_click(lambda b: _on_delete_button_click(b, classification_widget, df, raycluster_data_output, user_output, delete_button, list_jobs_button, ray_dashboard_button)) + delete_button.on_click(lambda b: _on_delete_button_click(b, classification_widget, ray_clusters_df, raycluster_data_output, user_output, delete_button, list_jobs_button, ray_dashboard_button)) list_jobs_button = widgets.Button( description='View Jobs', icon='suitcase', tooltip="Open the Ray Job Dashboard" ) - list_jobs_button.on_click(lambda b: _on_list_jobs_button_click(b, classification_widget, df, user_output, url_output)) + list_jobs_button.on_click(lambda b: _on_list_jobs_button_click(b, classification_widget, ray_clusters_df, user_output, url_output)) ray_dashboard_button = widgets.Button( description='Open Ray Dashboard', @@ -147,24 +153,30 @@ def view_clusters(namespace: str = None): tooltip="Open the Ray Dashboard in a new tab", layout=widgets.Layout(width='auto'), ) - ray_dashboard_button.on_click(lambda b: _on_ray_dashboard_button_click(b, classification_widget, df, user_output, url_output)) + ray_dashboard_button.on_click(lambda b: _on_ray_dashboard_button_click(b, classification_widget, ray_clusters_df, user_output, url_output)) display(widgets.VBox([classification_widget, raycluster_data_output])) display(widgets.HBox([delete_button, list_jobs_button, ray_dashboard_button]), url_output, user_output) -# Handles the event when a cluster is selected from the toggle buttons, updating the output with cluster details. + def _on_cluster_click(selection_change, raycluster_data_output: widgets.Output, namespace: str, classification_widget: widgets.ToggleButtons): + """ + _on_cluster_click handles the event when a cluster is selected from the toggle buttons, updating the output with cluster details. + """ new_value = selection_change["new"] raycluster_data_output.clear_output() - df = _fetch_cluster_data(namespace) - classification_widget.options = df["Name"].tolist() + ray_clusters_df = _fetch_cluster_data(namespace) + classification_widget.options = ray_clusters_df["Name"].tolist() with raycluster_data_output: - display(HTML(df[df["Name"]==new_value][["Name", "Namespace", "Head GPUs", "Head CPU Req~Lim", "Head Memory Req~Lim", "Worker GPUs", "Worker CPU Req~Lim", "Worker Memory Req~Lim", "status"]].to_html(escape=False, index=False, border=2))) + display(HTML(ray_clusters_df[ray_clusters_df["Name"]==new_value][["Name", "Namespace", "Num Workers", "Head GPUs", "Head CPU Req~Lim", "Head Memory Req~Lim", "Worker GPUs", "Worker CPU Req~Lim", "Worker Memory Req~Lim", "status"]].to_html(escape=False, index=False, border=2))) -def _on_delete_button_click(b, classification_widget: widgets.ToggleButtons, df: pd.DataFrame, raycluster_data_output: widgets.Output, user_output: widgets.Output, delete_button: widgets.Button, list_jobs_button: widgets.Button, ray_dashboard_button: widgets.Button): +def _on_delete_button_click(b, classification_widget: widgets.ToggleButtons, ray_clusters_df: pd.DataFrame, raycluster_data_output: widgets.Output, user_output: widgets.Output, delete_button: widgets.Button, list_jobs_button: widgets.Button, ray_dashboard_button: widgets.Button): + """ + _on_delete_button_click handles the event when the Delete Button is clicked, deleting the selected cluster. + """ cluster_name = classification_widget.value - namespace = df[df["Name"]==classification_widget.value]["Namespace"].values[0] + namespace = ray_clusters_df[ray_clusters_df["Name"]==classification_widget.value]["Namespace"].values[0] _delete_cluster(cluster_name, namespace) @@ -185,10 +197,13 @@ def _on_delete_button_click(b, classification_widget: widgets.ToggleButtons, df: else: classification_widget.options = new_df["Name"].tolist() -def _on_ray_dashboard_button_click(b, classification_widget: widgets.ToggleButtons, df: pd.DataFrame, user_output: widgets.Output, url_output: widgets.Output): +def _on_ray_dashboard_button_click(b, classification_widget: widgets.ToggleButtons, ray_clusters_df: pd.DataFrame, user_output: widgets.Output, url_output: widgets.Output): + """ + _on_ray_dashboard_button_click handles the event when the Open Ray Dashboard button is clicked, opening the Ray Dashboard in a new tab + """ from codeflare_sdk.cluster import Cluster cluster_name = classification_widget.value - namespace = df[df["Name"]==classification_widget.value]["Namespace"].values[0] + namespace = ray_clusters_df[ray_clusters_df["Name"]==classification_widget.value]["Namespace"].values[0] # Suppress from Cluster Object initialisation widgets and outputs with widgets.Output(), contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()): @@ -201,10 +216,13 @@ def _on_ray_dashboard_button_click(b, classification_widget: widgets.ToggleButto with url_output: display(Javascript(f'window.open("{dashboard_url}", "_blank");')) -def _on_list_jobs_button_click(b, classification_widget: widgets.ToggleButtons, df: pd.DataFrame, user_output: widgets.Output, url_output: widgets.Output): +def _on_list_jobs_button_click(b, classification_widget: widgets.ToggleButtons, ray_clusters_df: pd.DataFrame, user_output: widgets.Output, url_output: widgets.Output): + """ + _on_list_jobs_button_click handles the event when the View Jobs button is clicked, opening the Ray Jobs Dashboard in a new tab + """ from codeflare_sdk.cluster import Cluster cluster_name = classification_widget.value - namespace = df[df["Name"]==classification_widget.value]["Namespace"].values[0] + namespace = ray_clusters_df[ray_clusters_df["Name"]==classification_widget.value]["Namespace"].values[0] # Suppress from Cluster Object initialisation widgets and outputs with widgets.Output(), contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()): @@ -217,12 +235,17 @@ def _on_list_jobs_button_click(b, classification_widget: widgets.ToggleButtons, with url_output: display(Javascript(f'window.open("{dashboard_url}/#/jobs", "_blank");')) + def _delete_cluster( cluster_name: str, namespace: str, timeout: int = 5, interval: int = 1, ): + """ + _delete_cluster function deletes the cluster with the given name and namespace. + It optionally waits for the cluster to be deleted. + """ from .cluster import _check_aw_exists try: @@ -276,12 +299,16 @@ def _delete_cluster( def _fetch_cluster_data(namespace): + """ + _fetch_cluster_data function fetches all clusters and their spec in a given namespace and returns a DataFrame. + """ from .cluster import list_all_clusters rayclusters = list_all_clusters(namespace, False) if not rayclusters: return pd.DataFrame() names = [item.name for item in rayclusters] namespaces = [item.namespace for item in rayclusters] + num_workers = [item.num_workers for item in rayclusters] head_extended_resources = [ f"{list(item.head_extended_resources.keys())[0]}: {list(item.head_extended_resources.values())[0]}" if item.head_extended_resources else "0" @@ -311,6 +338,7 @@ def _fetch_cluster_data(namespace): data = { "Name": names, "Namespace": namespaces, + "Num Workers": num_workers, "Head GPUs": head_extended_resources, "Worker GPUs": worker_extended_resources, "Head CPU Req~Lim": head_cpu_rl, @@ -323,6 +351,9 @@ def _fetch_cluster_data(namespace): def _format_status(status): + """ + _format_status function formats the status enum. + """ status_map = { RayClusterStatus.READY: 'Ready โœ“', RayClusterStatus.SUSPENDED: 'Suspended โ„๏ธ', diff --git a/src/codeflare_sdk/utils/pretty_print.py b/src/codeflare_sdk/utils/pretty_print.py index a1410af35..303313199 100644 --- a/src/codeflare_sdk/utils/pretty_print.py +++ b/src/codeflare_sdk/utils/pretty_print.py @@ -135,7 +135,7 @@ def print_clusters(clusters: List[RayCluster]): ) name = cluster.name dashboard = cluster.dashboard - workers = str(cluster.workers) + workers = str(cluster.num_workers) memory = f"{cluster.worker_mem_requests}~{cluster.worker_mem_limits}" cpu = f"{cluster.worker_cpu_requests}~{cluster.worker_cpu_limits}" gpu = str(cluster.worker_extended_resources.get("nvidia.com/gpu", 0)) diff --git a/tests/unit_test.py b/tests/unit_test.py index b9682996a..92765ceb8 100644 --- a/tests/unit_test.py +++ b/tests/unit_test.py @@ -940,7 +940,7 @@ def test_ray_details(mocker, capsys): ray1 = RayCluster( name="raytest1", status=RayClusterStatus.READY, - workers=1, + num_workers=1, worker_mem_requests="2G", worker_mem_limits="2G", worker_cpu_requests=1, @@ -979,7 +979,7 @@ def test_ray_details(mocker, capsys): assert details == ray2 assert ray2.name == "raytest2" assert ray1.namespace == ray2.namespace - assert ray1.workers == ray2.workers + assert ray1.num_workers == ray2.num_workers assert ray1.worker_mem_requests == ray2.worker_mem_requests assert ray1.worker_mem_limits == ray2.worker_mem_limits assert ray1.worker_cpu_requests == ray2.worker_cpu_requests @@ -2358,7 +2358,7 @@ def test_cluster_status(mocker): fake_ray = RayCluster( name="test", status=RayClusterStatus.UNKNOWN, - workers=1, + num_workers=1, worker_mem_requests=2, worker_mem_limits=2, worker_cpu_requests=1, From 891038cbe582a7f5cfff6375a523d77a6f2640dd Mon Sep 17 00:00:00 2001 From: ChristianZaccaria Date: Thu, 26 Sep 2024 15:47:00 +0100 Subject: [PATCH 13/14] Reformat for pre-commit checks --- .../dependency_links.txt | 1 - src/codeflare_sdk/cluster/cluster.py | 10 +- src/codeflare_sdk/cluster/widgets.py | 238 ++++++++++++++---- tests/unit_test.py | 213 +++++++++++----- .../tests/widget_notebook_example.test.ts | 2 +- 5 files changed, 335 insertions(+), 129 deletions(-) diff --git a/src/codeflare_sdk.egg-info/dependency_links.txt b/src/codeflare_sdk.egg-info/dependency_links.txt index 8b1378917..e69de29bb 100644 --- a/src/codeflare_sdk.egg-info/dependency_links.txt +++ b/src/codeflare_sdk.egg-info/dependency_links.txt @@ -1 +0,0 @@ - diff --git a/src/codeflare_sdk/cluster/cluster.py b/src/codeflare_sdk/cluster/cluster.py index e97141b33..a32d5a4b7 100644 --- a/src/codeflare_sdk/cluster/cluster.py +++ b/src/codeflare_sdk/cluster/cluster.py @@ -870,10 +870,12 @@ def _map_to_ray_cluster(rc) -> Optional[RayCluster]: worker_mem_requests=rc["spec"]["workerGroupSpecs"][0]["template"]["spec"][ "containers" ][0]["resources"]["requests"]["memory"], - worker_cpu_requests=rc["spec"]["workerGroupSpecs"][0]["template"]["spec"]["containers"][0]["resources"]["requests"]["cpu"], - worker_cpu_limits=rc["spec"]["workerGroupSpecs"][0]["template"]["spec"]["containers"][ - 0 - ]["resources"]["limits"]["cpu"], + worker_cpu_requests=rc["spec"]["workerGroupSpecs"][0]["template"]["spec"][ + "containers" + ][0]["resources"]["requests"]["cpu"], + worker_cpu_limits=rc["spec"]["workerGroupSpecs"][0]["template"]["spec"][ + "containers" + ][0]["resources"]["limits"]["cpu"], worker_extended_resources=worker_extended_resources, namespace=rc["metadata"]["namespace"], head_cpu_requests=rc["spec"]["headGroupSpec"]["template"]["spec"]["containers"][ diff --git a/src/codeflare_sdk/cluster/widgets.py b/src/codeflare_sdk/cluster/widgets.py index a4c01f91c..53afa28be 100644 --- a/src/codeflare_sdk/cluster/widgets.py +++ b/src/codeflare_sdk/cluster/widgets.py @@ -107,10 +107,13 @@ def view_clusters(namespace: str = None): view_clusters function will display existing clusters with their specs, and handle user interactions. """ if not is_notebook(): - warnings.warn("view_clusters can only be used in a Jupyter Notebook environment.") - return # Exit function if not in Jupyter Notebook + warnings.warn( + "view_clusters can only be used in a Jupyter Notebook environment." + ) + return # Exit function if not in Jupyter Notebook from .cluster import get_current_namespace + if not namespace: namespace = get_current_namespace() @@ -124,42 +127,76 @@ def view_clusters(namespace: str = None): return classification_widget = widgets.ToggleButtons( - options=ray_clusters_df["Name"].tolist(), value=ray_clusters_df["Name"].tolist()[0], - description='Select an existing cluster:', + options=ray_clusters_df["Name"].tolist(), + value=ray_clusters_df["Name"].tolist()[0], + description="Select an existing cluster:", ) # Setting the initial value to trigger the event handler to display the cluster details. initial_value = classification_widget.value - _on_cluster_click({"new": initial_value}, raycluster_data_output, namespace, classification_widget) - classification_widget.observe(lambda selection_change: _on_cluster_click(selection_change, raycluster_data_output, namespace, classification_widget), names="value") + _on_cluster_click( + {"new": initial_value}, raycluster_data_output, namespace, classification_widget + ) + classification_widget.observe( + lambda selection_change: _on_cluster_click( + selection_change, raycluster_data_output, namespace, classification_widget + ), + names="value", + ) # UI table buttons delete_button = widgets.Button( - description='Delete Cluster', - icon='trash', - tooltip="Delete the selected cluster" - ) - delete_button.on_click(lambda b: _on_delete_button_click(b, classification_widget, ray_clusters_df, raycluster_data_output, user_output, delete_button, list_jobs_button, ray_dashboard_button)) + description="Delete Cluster", + icon="trash", + tooltip="Delete the selected cluster", + ) + delete_button.on_click( + lambda b: _on_delete_button_click( + b, + classification_widget, + ray_clusters_df, + raycluster_data_output, + user_output, + delete_button, + list_jobs_button, + ray_dashboard_button, + ) + ) list_jobs_button = widgets.Button( - description='View Jobs', - icon='suitcase', - tooltip="Open the Ray Job Dashboard" - ) - list_jobs_button.on_click(lambda b: _on_list_jobs_button_click(b, classification_widget, ray_clusters_df, user_output, url_output)) + description="View Jobs", icon="suitcase", tooltip="Open the Ray Job Dashboard" + ) + list_jobs_button.on_click( + lambda b: _on_list_jobs_button_click( + b, classification_widget, ray_clusters_df, user_output, url_output + ) + ) ray_dashboard_button = widgets.Button( - description='Open Ray Dashboard', - icon='dashboard', - tooltip="Open the Ray Dashboard in a new tab", - layout=widgets.Layout(width='auto'), - ) - ray_dashboard_button.on_click(lambda b: _on_ray_dashboard_button_click(b, classification_widget, ray_clusters_df, user_output, url_output)) + description="Open Ray Dashboard", + icon="dashboard", + tooltip="Open the Ray Dashboard in a new tab", + layout=widgets.Layout(width="auto"), + ) + ray_dashboard_button.on_click( + lambda b: _on_ray_dashboard_button_click( + b, classification_widget, ray_clusters_df, user_output, url_output + ) + ) display(widgets.VBox([classification_widget, raycluster_data_output])) - display(widgets.HBox([delete_button, list_jobs_button, ray_dashboard_button]), url_output, user_output) + display( + widgets.HBox([delete_button, list_jobs_button, ray_dashboard_button]), + url_output, + user_output, + ) -def _on_cluster_click(selection_change, raycluster_data_output: widgets.Output, namespace: str, classification_widget: widgets.ToggleButtons): +def _on_cluster_click( + selection_change, + raycluster_data_output: widgets.Output, + namespace: str, + classification_widget: widgets.ToggleButtons, +): """ _on_cluster_click handles the event when a cluster is selected from the toggle buttons, updating the output with cluster details. """ @@ -168,21 +205,51 @@ def _on_cluster_click(selection_change, raycluster_data_output: widgets.Output, ray_clusters_df = _fetch_cluster_data(namespace) classification_widget.options = ray_clusters_df["Name"].tolist() with raycluster_data_output: - display(HTML(ray_clusters_df[ray_clusters_df["Name"]==new_value][["Name", "Namespace", "Num Workers", "Head GPUs", "Head CPU Req~Lim", "Head Memory Req~Lim", "Worker GPUs", "Worker CPU Req~Lim", "Worker Memory Req~Lim", "status"]].to_html(escape=False, index=False, border=2))) - - -def _on_delete_button_click(b, classification_widget: widgets.ToggleButtons, ray_clusters_df: pd.DataFrame, raycluster_data_output: widgets.Output, user_output: widgets.Output, delete_button: widgets.Button, list_jobs_button: widgets.Button, ray_dashboard_button: widgets.Button): + display( + HTML( + ray_clusters_df[ray_clusters_df["Name"] == new_value][ + [ + "Name", + "Namespace", + "Num Workers", + "Head GPUs", + "Head CPU Req~Lim", + "Head Memory Req~Lim", + "Worker GPUs", + "Worker CPU Req~Lim", + "Worker Memory Req~Lim", + "status", + ] + ].to_html(escape=False, index=False, border=2) + ) + ) + + +def _on_delete_button_click( + b, + classification_widget: widgets.ToggleButtons, + ray_clusters_df: pd.DataFrame, + raycluster_data_output: widgets.Output, + user_output: widgets.Output, + delete_button: widgets.Button, + list_jobs_button: widgets.Button, + ray_dashboard_button: widgets.Button, +): """ _on_delete_button_click handles the event when the Delete Button is clicked, deleting the selected cluster. """ cluster_name = classification_widget.value - namespace = ray_clusters_df[ray_clusters_df["Name"]==classification_widget.value]["Namespace"].values[0] + namespace = ray_clusters_df[ray_clusters_df["Name"] == classification_widget.value][ + "Namespace" + ].values[0] _delete_cluster(cluster_name, namespace) with user_output: user_output.clear_output() - print(f"Cluster {cluster_name} in the {namespace} namespace was deleted successfully.") + print( + f"Cluster {cluster_name} in the {namespace} namespace was deleted successfully." + ) # Refresh the dataframe new_df = _fetch_cluster_data(namespace) @@ -197,16 +264,28 @@ def _on_delete_button_click(b, classification_widget: widgets.ToggleButtons, ray else: classification_widget.options = new_df["Name"].tolist() -def _on_ray_dashboard_button_click(b, classification_widget: widgets.ToggleButtons, ray_clusters_df: pd.DataFrame, user_output: widgets.Output, url_output: widgets.Output): + +def _on_ray_dashboard_button_click( + b, + classification_widget: widgets.ToggleButtons, + ray_clusters_df: pd.DataFrame, + user_output: widgets.Output, + url_output: widgets.Output, +): """ _on_ray_dashboard_button_click handles the event when the Open Ray Dashboard button is clicked, opening the Ray Dashboard in a new tab """ from codeflare_sdk.cluster import Cluster + cluster_name = classification_widget.value - namespace = ray_clusters_df[ray_clusters_df["Name"]==classification_widget.value]["Namespace"].values[0] + namespace = ray_clusters_df[ray_clusters_df["Name"] == classification_widget.value][ + "Namespace" + ].values[0] # Suppress from Cluster Object initialisation widgets and outputs - with widgets.Output(), contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()): + with widgets.Output(), contextlib.redirect_stdout( + io.StringIO() + ), contextlib.redirect_stderr(io.StringIO()): cluster = Cluster(ClusterConfiguration(cluster_name, namespace)) dashboard_url = cluster.cluster_dashboard_uri() @@ -216,22 +295,36 @@ def _on_ray_dashboard_button_click(b, classification_widget: widgets.ToggleButto with url_output: display(Javascript(f'window.open("{dashboard_url}", "_blank");')) -def _on_list_jobs_button_click(b, classification_widget: widgets.ToggleButtons, ray_clusters_df: pd.DataFrame, user_output: widgets.Output, url_output: widgets.Output): + +def _on_list_jobs_button_click( + b, + classification_widget: widgets.ToggleButtons, + ray_clusters_df: pd.DataFrame, + user_output: widgets.Output, + url_output: widgets.Output, +): """ _on_list_jobs_button_click handles the event when the View Jobs button is clicked, opening the Ray Jobs Dashboard in a new tab """ from codeflare_sdk.cluster import Cluster + cluster_name = classification_widget.value - namespace = ray_clusters_df[ray_clusters_df["Name"]==classification_widget.value]["Namespace"].values[0] + namespace = ray_clusters_df[ray_clusters_df["Name"] == classification_widget.value][ + "Namespace" + ].values[0] # Suppress from Cluster Object initialisation widgets and outputs - with widgets.Output(), contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()): + with widgets.Output(), contextlib.redirect_stdout( + io.StringIO() + ), contextlib.redirect_stderr(io.StringIO()): cluster = Cluster(ClusterConfiguration(cluster_name, namespace)) dashboard_url = cluster.cluster_dashboard_uri() with user_output: user_output.clear_output() - print(f"Opening Ray Jobs Dashboard for {cluster_name} cluster:\n{dashboard_url}/#/jobs") + print( + f"Opening Ray Jobs Dashboard for {cluster_name} cluster:\n{dashboard_url}/#/jobs" + ) with url_output: display(Javascript(f'window.open("{dashboard_url}/#/jobs", "_blank");')) @@ -289,12 +382,14 @@ def _delete_cluster( time.sleep(interval) timeout -= interval if timeout <= 0: - raise TimeoutError(f"Timeout waiting for {cluster_name} to be deleted.") + raise TimeoutError( + f"Timeout waiting for {cluster_name} to be deleted." + ) except ApiException as e: # Resource is deleted if e.status == 404: break - except Exception as e: + except Exception as e: # pragma: no cover return _kube_api_error_handling(e) @@ -303,6 +398,7 @@ def _fetch_cluster_data(namespace): _fetch_cluster_data function fetches all clusters and their spec in a given namespace and returns a DataFrame. """ from .cluster import list_all_clusters + rayclusters = list_all_clusters(namespace, False) if not rayclusters: return pd.DataFrame() @@ -311,26 +407,58 @@ def _fetch_cluster_data(namespace): num_workers = [item.num_workers for item in rayclusters] head_extended_resources = [ f"{list(item.head_extended_resources.keys())[0]}: {list(item.head_extended_resources.values())[0]}" - if item.head_extended_resources else "0" + if item.head_extended_resources + else "0" for item in rayclusters ] worker_extended_resources = [ f"{list(item.worker_extended_resources.keys())[0]}: {list(item.worker_extended_resources.values())[0]}" - if item.worker_extended_resources else "0" + if item.worker_extended_resources + else "0" + for item in rayclusters + ] + head_cpu_requests = [ + item.head_cpu_requests if item.head_cpu_requests else 0 for item in rayclusters + ] + head_cpu_limits = [ + item.head_cpu_limits if item.head_cpu_limits else 0 for item in rayclusters + ] + head_cpu_rl = [ + f"{requests}~{limits}" + for requests, limits in zip(head_cpu_requests, head_cpu_limits) + ] + head_mem_requests = [ + item.head_mem_requests if item.head_mem_requests else 0 for item in rayclusters + ] + head_mem_limits = [ + item.head_mem_limits if item.head_mem_limits else 0 for item in rayclusters + ] + head_mem_rl = [ + f"{requests}~{limits}" + for requests, limits in zip(head_mem_requests, head_mem_limits) + ] + worker_cpu_requests = [ + item.worker_cpu_requests if item.worker_cpu_requests else 0 + for item in rayclusters + ] + worker_cpu_limits = [ + item.worker_cpu_limits if item.worker_cpu_limits else 0 for item in rayclusters + ] + worker_cpu_rl = [ + f"{requests}~{limits}" + for requests, limits in zip(worker_cpu_requests, worker_cpu_limits) + ] + worker_mem_requests = [ + item.worker_mem_requests if item.worker_mem_requests else 0 for item in rayclusters ] - head_cpu_requests = [item.head_cpu_requests if item.head_cpu_requests else 0 for item in rayclusters] - head_cpu_limits = [item.head_cpu_limits if item.head_cpu_limits else 0 for item in rayclusters] - head_cpu_rl = [f"{requests}~{limits}" for requests, limits in zip(head_cpu_requests, head_cpu_limits)] - head_mem_requests = [item.head_mem_requests if item.head_mem_requests else 0 for item in rayclusters] - head_mem_limits = [item.head_mem_limits if item.head_mem_limits else 0 for item in rayclusters] - head_mem_rl = [f"{requests}~{limits}" for requests, limits in zip(head_mem_requests, head_mem_limits)] - worker_cpu_requests = [item.worker_cpu_requests if item.worker_cpu_requests else 0 for item in rayclusters] - worker_cpu_limits = [item.worker_cpu_limits if item.worker_cpu_limits else 0 for item in rayclusters] - worker_cpu_rl = [f"{requests}~{limits}" for requests, limits in zip(worker_cpu_requests, worker_cpu_limits)] - worker_mem_requests = [item.worker_mem_requests if item.worker_mem_requests else 0 for item in rayclusters] - worker_mem_limits = [item.worker_mem_limits if item.worker_mem_limits else 0 for item in rayclusters] - worker_mem_rl = [f"{requests}~{limits}" for requests, limits in zip(worker_mem_requests, worker_mem_limits)] + worker_mem_limits = [ + item.worker_mem_limits if item.worker_mem_limits else 0 for item in rayclusters + ] + worker_mem_rl = [ + f"{requests}~{limits}" + for requests, limits in zip(worker_mem_requests, worker_mem_limits) + ] status = [item.status.name for item in rayclusters] status = [_format_status(item.status) for item in rayclusters] @@ -345,7 +473,7 @@ def _fetch_cluster_data(namespace): "Head Memory Req~Lim": head_mem_rl, "Worker CPU Req~Lim": worker_cpu_rl, "Worker Memory Req~Lim": worker_mem_rl, - "status": status + "status": status, } return pd.DataFrame(data) @@ -359,6 +487,6 @@ def _format_status(status): RayClusterStatus.SUSPENDED: 'Suspended โ„๏ธ', RayClusterStatus.FAILED: 'Failed โœ—', RayClusterStatus.UNHEALTHY: 'Unhealthy', - RayClusterStatus.UNKNOWN: 'Unknown' + RayClusterStatus.UNKNOWN: 'Unknown', } return status_map.get(status, status) diff --git a/tests/unit_test.py b/tests/unit_test.py index 92765ceb8..ae2af6591 100644 --- a/tests/unit_test.py +++ b/tests/unit_test.py @@ -2944,6 +2944,7 @@ def test_cluster_up_down_buttons(mocker): @patch.dict("os.environ", {}, clear=True) # Mock environment with no variables def test_is_notebook_false(): from codeflare_sdk.cluster.widgets import is_notebook + assert is_notebook() is False @@ -2952,54 +2953,85 @@ def test_is_notebook_false(): ) # Mock Jupyter environment variable def test_is_notebook_true(): from codeflare_sdk.cluster.widgets import is_notebook + assert is_notebook() is True -@patch.dict("os.environ", {"JPY_SESSION_NAME": "example-test"}) # Mock Jupyter environment variable -def test_view_clusters(mocker): +def test_view_clusters(mocker, capsys): + from kubernetes.client.rest import ApiException + + mocker.patch("codeflare_sdk.cluster.widgets.is_notebook", return_value=False) + with pytest.warns( + UserWarning, + match="view_clusters can only be used in a Jupyter Notebook environment.", + ): + result = cf_widgets.view_clusters(namespace="default") + # Assert the function returns None when not in a notebook environment + assert result is None + + mocker.patch("codeflare_sdk.cluster.widgets.is_notebook", return_value=True) + # Mock Kubernetes API responses mocker.patch("kubernetes.client.ApisApi.get_api_versions") mocker.patch( - "kubernetes.client.CustomObjectsApi.list_cluster_custom_object", - return_value={"items": []} + "kubernetes.client.CustomObjectsApi.list_namespaced_custom_object", + return_value={"items": []}, ) + mocker.patch("codeflare_sdk.cluster.cluster._check_aw_exists", return_value=False) # Return empty dataframe when no clusters are found mocker.patch("codeflare_sdk.cluster.cluster.list_all_clusters", return_value=[]) + mocker.patch( + "codeflare_sdk.cluster.cluster.get_current_namespace", + return_value="default", + ) df = cf_widgets._fetch_cluster_data(namespace="default") assert df.empty - test_df=pd.DataFrame({ - "Name": ["test-cluster"], - "Namespace": ["default"], - "Head GPUs": ["0"], - "Worker GPUs": ["0"], - "Head CPU Req~Lim": ["1~1"], - "Head Memory Req~Lim": ["1Gi~1Gi"], - "Worker CPU Req~Lim": ["1~1"], - "Worker Memory Req~Lim": ["1Gi~1Gi"], - "status": ['Ready โœ“'] - }) + cf_widgets.view_clusters() + captured = capsys.readouterr() + assert f"No clusters found in the default namespace." in captured.out + + # Assert the function returns None + assert result is None + + test_df = pd.DataFrame( + { + "Name": ["test-cluster"], + "Namespace": ["default"], + "Num Workers": ["1"], + "Head GPUs": ["0"], + "Worker GPUs": ["0"], + "Head CPU Req~Lim": ["1~1"], + "Head Memory Req~Lim": ["1Gi~1Gi"], + "Worker CPU Req~Lim": ["1~1"], + "Worker Memory Req~Lim": ["1Gi~1Gi"], + "status": ['Ready โœ“'], + } + ) # Mock the _fetch_cluster_data function to return a test DataFrame mocker.patch( - "codeflare_sdk.cluster.widgets._fetch_cluster_data", - return_value=test_df + "codeflare_sdk.cluster.widgets._fetch_cluster_data", return_value=test_df ) # Mock the Cluster class and related methods mocker.patch("codeflare_sdk.cluster.Cluster") mocker.patch("codeflare_sdk.cluster.ClusterConfiguration") - with patch("ipywidgets.ToggleButtons") as MockToggleButtons, \ - patch("ipywidgets.Button") as MockButton, \ - patch("ipywidgets.Output") as MockOutput, \ - patch("ipywidgets.HBox"), \ - patch("ipywidgets.VBox"), \ - patch("IPython.display.display") as mock_display, \ - patch("IPython.display.HTML"), \ - patch("codeflare_sdk.cluster.widgets.Javascript") as mock_javascript: - + with patch("ipywidgets.ToggleButtons") as MockToggleButtons, patch( + "ipywidgets.Button" + ) as MockButton, patch("ipywidgets.Output") as MockOutput, patch( + "ipywidgets.HBox" + ), patch( + "ipywidgets.VBox" + ), patch( + "IPython.display.display" + ) as mock_display, patch( + "IPython.display.HTML" + ), patch( + "codeflare_sdk.cluster.widgets.Javascript" + ) as mock_javascript: # Create mock widget instances mock_toggle = MagicMock() mock_delete_button = MagicMock() @@ -3009,31 +3041,61 @@ def test_view_clusters(mocker): # Set the return values for the mocked widgets MockToggleButtons.return_value = mock_toggle - MockButton.side_effect = [mock_delete_button, mock_list_jobs_button, mock_ray_dashboard_button] + MockButton.side_effect = [ + mock_delete_button, + mock_list_jobs_button, + mock_ray_dashboard_button, + ] MockOutput.return_value = mock_output # Call the function under test - cf_widgets.view_clusters(namespace="default") + cf_widgets.view_clusters() # Simulate selecting a cluster mock_toggle.value = "test-cluster" selection_change = {"new": "test-cluster"} - cf_widgets._on_cluster_click(selection_change, mock_output, "default", mock_toggle) + cf_widgets._on_cluster_click( + selection_change, mock_output, "default", mock_toggle + ) # Assert that the toggle options are set correctly mock_toggle.observe.assert_called() # Simulate clicking the list jobs button - cf_widgets._on_list_jobs_button_click(None, mock_toggle, test_df, mock_output, mock_output) - mock_javascript.assert_called() + cf_widgets._on_list_jobs_button_click( + None, mock_toggle, test_df, mock_output, mock_output + ) + mock_javascript.assert_called_once() # Simulate clicking the Ray dashboard button - cf_widgets._on_ray_dashboard_button_click(None, mock_toggle, test_df, mock_output, mock_output) - mock_javascript.assert_called() + cf_widgets._on_ray_dashboard_button_click( + None, mock_toggle, test_df, mock_output, mock_output + ) + mock_javascript.call_count = 2 + + mocker.patch( + "kubernetes.client.CustomObjectsApi.delete_namespaced_custom_object", + ) + mock_response = mocker.MagicMock() + mock_response.status = 404 + mock_exception = ApiException(http_resp=mock_response) + mocker.patch( + "kubernetes.client.CustomObjectsApi.get_namespaced_custom_object", + side_effect=mock_exception, + ) # Simulate clicking the delete button - cf_widgets._on_delete_button_click(None, mock_toggle, test_df, mock_output, mock_output, - mock_delete_button, mock_list_jobs_button, mock_ray_dashboard_button) + cf_widgets._on_delete_button_click( + None, + mock_toggle, + test_df, + mock_output, + mock_output, + mock_delete_button, + mock_list_jobs_button, + mock_ray_dashboard_button, + ) + MockButton.call_count = 3 def test_fetch_cluster_data(mocker): @@ -3044,25 +3106,27 @@ def test_fetch_cluster_data(mocker): # Create mock RayCluster objects mock_raycluster1 = MagicMock(spec=RayCluster) - mock_raycluster1.name = 'test-cluster-1' - mock_raycluster1.namespace = 'default' - mock_raycluster1.head_extended_resources = {'nvidia.com/gpu': '1'} - mock_raycluster1.worker_extended_resources = {'nvidia.com/gpu': '2'} - mock_raycluster1.head_cpu_requests = '500m' - mock_raycluster1.head_cpu_limits = '1000m' - mock_raycluster1.head_mem_requests = '1Gi' - mock_raycluster1.head_mem_limits = '2Gi' - mock_raycluster1.worker_cpu_requests = '1000m' - mock_raycluster1.worker_cpu_limits = '2000m' - mock_raycluster1.worker_mem_requests = '2Gi' - mock_raycluster1.worker_mem_limits = '4Gi' + mock_raycluster1.name = "test-cluster-1" + mock_raycluster1.namespace = "default" + mock_raycluster1.num_workers = 1 + mock_raycluster1.head_extended_resources = {"nvidia.com/gpu": "1"} + mock_raycluster1.worker_extended_resources = {"nvidia.com/gpu": "2"} + mock_raycluster1.head_cpu_requests = "500m" + mock_raycluster1.head_cpu_limits = "1000m" + mock_raycluster1.head_mem_requests = "1Gi" + mock_raycluster1.head_mem_limits = "2Gi" + mock_raycluster1.worker_cpu_requests = "1000m" + mock_raycluster1.worker_cpu_limits = "2000m" + mock_raycluster1.worker_mem_requests = "2Gi" + mock_raycluster1.worker_mem_limits = "4Gi" mock_raycluster1.status = MagicMock() - mock_raycluster1.status.name = 'READY' + mock_raycluster1.status.name = "READY" mock_raycluster1.status = RayClusterStatus.READY mock_raycluster2 = MagicMock(spec=RayCluster) - mock_raycluster2.name = 'test-cluster-2' - mock_raycluster2.namespace = 'default' + mock_raycluster2.name = "test-cluster-2" + mock_raycluster2.namespace = "default" + mock_raycluster2.num_workers = 2 mock_raycluster2.head_extended_resources = {} mock_raycluster2.worker_extended_resources = {} mock_raycluster2.head_cpu_requests = None @@ -3074,51 +3138,64 @@ def test_fetch_cluster_data(mocker): mock_raycluster2.worker_mem_requests = None mock_raycluster2.worker_mem_limits = None mock_raycluster2.status = MagicMock() - mock_raycluster2.status.name = 'SUSPENDED' + mock_raycluster2.status.name = "SUSPENDED" mock_raycluster2.status = RayClusterStatus.SUSPENDED - with patch('codeflare_sdk.cluster.cluster.list_all_clusters', return_value=[mock_raycluster1, mock_raycluster2]): + with patch( + "codeflare_sdk.cluster.cluster.list_all_clusters", + return_value=[mock_raycluster1, mock_raycluster2], + ): # Call the function under test - df = cf_widgets._fetch_cluster_data(namespace='default') + df = cf_widgets._fetch_cluster_data(namespace="default") # Expected DataFrame expected_data = { - "Name": ['test-cluster-1', 'test-cluster-2'], - "Namespace": ['default', 'default'], - "Head GPUs": ['nvidia.com/gpu: 1', '0'], - "Worker GPUs": ['nvidia.com/gpu: 2', '0'], - "Head CPU Req~Lim": ['500m~1000m', '0~0'], - "Head Memory Req~Lim": ['1Gi~2Gi', '0~0'], - "Worker CPU Req~Lim": ['1000m~2000m', '0~0'], - "Worker Memory Req~Lim": ['2Gi~4Gi', '0~0'], + "Name": ["test-cluster-1", "test-cluster-2"], + "Namespace": ["default", "default"], + "Num Workers": [1, 2], + "Head GPUs": ["nvidia.com/gpu: 1", "0"], + "Worker GPUs": ["nvidia.com/gpu: 2", "0"], + "Head CPU Req~Lim": ["500m~1000m", "0~0"], + "Head Memory Req~Lim": ["1Gi~2Gi", "0~0"], + "Worker CPU Req~Lim": ["1000m~2000m", "0~0"], + "Worker Memory Req~Lim": ["2Gi~4Gi", "0~0"], "status": [ 'Ready โœ“', - 'Suspended โ„๏ธ' - ] + 'Suspended โ„๏ธ', + ], } expected_df = pd.DataFrame(expected_data) # Assert that the DataFrame matches expected - pd.testing.assert_frame_equal(df.reset_index(drop=True), expected_df.reset_index(drop=True)) + pd.testing.assert_frame_equal( + df.reset_index(drop=True), expected_df.reset_index(drop=True) + ) def test_format_status(): # Test each possible status test_cases = [ (RayClusterStatus.READY, 'Ready โœ“'), - (RayClusterStatus.SUSPENDED, 'Suspended โ„๏ธ'), + ( + RayClusterStatus.SUSPENDED, + 'Suspended โ„๏ธ', + ), (RayClusterStatus.FAILED, 'Failed โœ—'), (RayClusterStatus.UNHEALTHY, 'Unhealthy'), (RayClusterStatus.UNKNOWN, 'Unknown'), ] for status, expected_output in test_cases: - assert cf_widgets._format_status(status) == expected_output, f"Failed for status: {status}" + assert ( + cf_widgets._format_status(status) == expected_output + ), f"Failed for status: {status}" # Test an unrecognized status - unrecognized_status = 'NotAStatus' - assert cf_widgets._format_status(unrecognized_status) == 'NotAStatus', "Failed for unrecognized status" + unrecognized_status = "NotAStatus" + assert ( + cf_widgets._format_status(unrecognized_status) == "NotAStatus" + ), "Failed for unrecognized status" # Make sure to always keep this function last diff --git a/ui-tests/tests/widget_notebook_example.test.ts b/ui-tests/tests/widget_notebook_example.test.ts index 8884ab8f7..823a73f47 100644 --- a/ui-tests/tests/widget_notebook_example.test.ts +++ b/ui-tests/tests/widget_notebook_example.test.ts @@ -121,7 +121,7 @@ test.describe("Visual Regression", () => { const successMessage = await page.waitForSelector('text=Ray Cluster: \'raytest\' has successfully been created', { timeout: 10000 }); expect(successMessage).not.toBeNull(); }); - + const viewClustersCellIndex = 4; // 5 on OpenShift await page.notebook.runCell(cellCount - 2, true); await interactWithWidget(page, viewClustersCellIndex, 'button:has-text("Open Ray Dashboard")', async (button) => { From 19b0ee99c6b95ce4eef8e9d115da43b4fab59bdf Mon Sep 17 00:00:00 2001 From: Christian Zaccaria <73656840+ChristianZaccaria@users.noreply.github.com> Date: Fri, 27 Sep 2024 11:00:19 +0100 Subject: [PATCH 14/14] Revert codeflare_sdk.egg-info name --- src/codeflare_sdk.egg-info/PKG-INFO | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/codeflare_sdk.egg-info/PKG-INFO b/src/codeflare_sdk.egg-info/PKG-INFO index 27ec5cbfa..c4061c623 100644 --- a/src/codeflare_sdk.egg-info/PKG-INFO +++ b/src/codeflare_sdk.egg-info/PKG-INFO @@ -1,4 +1,4 @@ Metadata-Version: 2.1 -Name: codeflare_sdk +Name: codeflare-sdk Version: 0.0.0 License-File: LICENSE