Adjust existing e2e tests to provide custom Pypi index URL

Author: sutaakar

README.md

@@ -86,6 +86,21 @@ The e2e tests can be executed locally by running the following commands:
 
    Alternatively, You can run the e2e test(s) from your IDE / debugger.
 
+#### Testing on disconnected cluster
+
+To properly run e2e tests on disconnected cluster user has to provide additional environment variables to properly configure testing environment:
+
+- `CODEFLARE_TEST_PYTORCH_IMAGE` - image tag for image used to run training job using MCAD
+- `CODEFLARE_TEST_RAY_IMAGE` - image tag for Ray cluster image
+- `MNIST_DATASET_URL` - URL where MNIST dataset is available
+- `PIP_INDEX_URL` - URL where PyPI server with needed dependencies is running
+- `PIP_TRUSTED_HOST` - PyPI server hostname
+
+For ODH tests additional environment variables are needed:
+
+- `NOTEBOOK_IMAGE_STREAM_NAME` - name of the ODH Notebook ImageStream to be used
+- `ODH_NAMESPACE` - namespace where ODH is installed
+
 ## Release
 
 1. Invoke [project-codeflare-release.yaml](https://github.com/project-codeflare/codeflare-operator/actions/workflows/project-codeflare-release.yml)

test/e2e/mnist_pytorch_mcad_job_test.go

@@ -81,6 +81,8 @@ func TestMNISTPyTorchMCAD(t *testing.T) {
 							Env: []corev1.EnvVar{
 								{Name: "PYTHONUSERBASE", Value: "/workdir"},
 								{Name: "MNIST_DATASET_URL", Value: GetMnistDatasetURL()},
+								{Name: "PIP_INDEX_URL", Value: GetPipIndexURL()},
+								{Name: "PIP_TRUSTED_HOST", Value: GetPipTrustedHost()},
 							},
 							Command: []string{"/bin/sh", "-c", "pip install -r /test/requirements.txt && torchrun /test/mnist.py"},
 							VolumeMounts: []corev1.VolumeMount{

test/e2e/mnist_rayjob_mcad_raycluster_test.go

@@ -229,6 +229,8 @@ func TestMNISTRayJobMCADRayCluster(t *testing.T) {
     - torchvision==0.12.0
   env_vars:
     MNIST_DATASET_URL: "` + GetMnistDatasetURL() + `"
+    PIP_INDEX_URL: "` + GetPipIndexURL() + `"
+    PIP_TRUSTED_HOST: "` + GetPipTrustedHost() + `"
 `,
 			ClusterSelector: map[string]string{
 				RayJobDefaultClusterSelectorKey: rayCluster.Name,

test/odh/mcad_ray_test.go

@@ -59,6 +59,11 @@ func TestMCADRay(t *testing.T) {
 			APIGroups: []string{"route.openshift.io"},
 			Resources: []string{"routes"},
 		},
+		{
+			Verbs:     []string{"get", "list"},
+			APIGroups: []string{"networking.k8s.io"},
+			Resources: []string{"ingresses"},
+		},
 	}
 
 	// Create cluster wide RBAC, required for SDK OpenShift check

test/odh/notebook.go

@@ -44,6 +44,7 @@ type NotebookProps struct {
 	OpenDataHubNamespace      string
 	ImageStreamName           string
 	ImageStreamTag            string
+	RayImage                  string
 	NotebookConfigMapName     string
 	NotebookConfigMapFileName string
 	NotebookPVC               string
@@ -66,6 +67,7 @@ func createNotebook(test Test, namespace *corev1.Namespace, notebookToken, jupyt
 		OpenDataHubNamespace:      GetOpenDataHubNamespace(),
 		ImageStreamName:           GetNotebookImageStreamName(test),
 		ImageStreamTag:            recommendedTagName,
+		RayImage:                  GetRayImage(),
 		NotebookConfigMapName:     jupyterNotebookConfigMapName,
 		NotebookConfigMapFileName: jupyterNotebookConfigMapFileName,
 		NotebookPVC:               notebookPVC.Name,

test/odh/resources/custom-nb-small.yaml

@@ -55,7 +55,7 @@ spec:
         - name: OCP_TOKEN
           value: {{.KubernetesBearerToken}}
         image: image-registry.openshift-image-registry.svc:5000/{{.OpenDataHubNamespace}}/{{.ImageStreamName}}:{{.ImageStreamTag}}
-        command: ["/bin/sh", "-c", "pip install papermill && oc login --token=${OCP_TOKEN} --server=${OCP_SERVER} --insecure-skip-tls-verify=true && papermill /opt/app-root/notebooks/{{.NotebookConfigMapFileName}} /opt/app-root/src/mcad-out.ipynb -p namespace {{.Namespace}} && sleep infinity"]
+        command: ["/bin/sh", "-c", "pip install papermill && oc login --token=${OCP_TOKEN} --server=${OCP_SERVER} --insecure-skip-tls-verify=true && papermill /opt/app-root/notebooks/{{.NotebookConfigMapFileName}} /opt/app-root/src/mcad-out.ipynb -p namespace {{.Namespace}} -p ray_image {{.RayImage}} && sleep infinity"]
         # args: ["pip install papermill && oc login --token=${OCP_TOKEN} --server=${OCP_SERVER} --insecure-skip-tls-verify=true && papermill /opt/app-root/notebooks/mcad.ipynb /opt/app-root/src/mcad-out.ipynb" ]
         imagePullPolicy: Always
         # livenessProbe:

test/odh/resources/mnist_ray_mini.ipynb

@@ -27,7 +27,8 @@
    "outputs": [],
    "source": [
     "#parameters\n",
-    "namespace = \"default\""
+    "namespace = \"default\"\n",
+    "ray_image = \"has to be specified\""
    ]
   },
   {
@@ -40,7 +41,7 @@
    "outputs": [],
    "source": [
     "# Create our cluster and submit appwrapper\n",
-    "cluster = Cluster(ClusterConfiguration(namespace=namespace, name='mnisttest', head_cpus=1, head_memory=2, num_workers=1, min_cpus=1, max_cpus=1, min_memory=1, max_memory=2, num_gpus=0, instascale=False))"
+    "cluster = Cluster(ClusterConfiguration(namespace=namespace, name='mnisttest', head_cpus=1, head_memory=2, num_workers=1, min_cpus=1, max_cpus=1, min_memory=1, max_memory=2, num_gpus=0, instascale=False, image=ray_image))"
    ]
   },
   {