Setup existing DW tests to run in CFO

test/odh/environment.go

@@ -0,0 +1,49 @@
+/*
+Copyright 2023.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package odh
+
+import (
+	"os"
+
+	. "github.com/project-codeflare/codeflare-common/support"
+)
+
+const (
+	// The environment variable for namespace where ODH is installed to.
+	odhNamespaceEnvVar = "ODH_NAMESPACE"
+	// The environment variable for ODH Notebook ImageStream name
+	notebookImageStreamName = "NOTEBOOK_IMAGE_STREAM_NAME"
+)
+
+func GetOpenDataHubNamespace() string {
+	return lookupEnvOrDefault(odhNamespaceEnvVar, "opendatahub")
+}
+
+func GetNotebookImageStreamName(t Test) string {
+	isName, ok := os.LookupEnv(notebookImageStreamName)
+	if !ok {
+		t.T().Fatalf("Expected environment variable %s not found, please use this environment variable to specify what ImageStream to use for Notebook.", notebookImageStreamName)
+	}
+	return isName
+}
+
+func lookupEnvOrDefault(key, value string) string {
+	if v, ok := os.LookupEnv(key); ok {
+		return v
+	}
+	return value
+}

test/odh/mcad_ray_test.go

@@ -0,0 +1,98 @@
+/*
+Copyright 2023.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package odh
+
+import (
+	"testing"
+
+	. "github.com/onsi/gomega"
+	. "github.com/project-codeflare/codeflare-common/support"
+	mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1"
+	rayv1alpha1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1alpha1"
+
+	rbacv1 "k8s.io/api/rbac/v1"
+)
+
+func TestMCADRay(t *testing.T) {
+	test := With(t)
+
+	// Create a namespace
+	namespace := test.NewTestNamespace()
+
+	// Test configuration
+	jupyterNotebookConfigMapFileName := "mnist_ray_mini.ipynb"
+	config := CreateConfigMap(test, namespace.Name, map[string][]byte{
+		// MNIST Ray Notebook
+		jupyterNotebookConfigMapFileName: ReadFile(test, "resources/mnist_ray_mini.ipynb"),
+		"mnist.py":                       ReadFile(test, "resources/mnist.py"),
+		"requirements.txt":               ReadFile(test, "resources/requirements.txt"),
+	})
+
+	// Create RBAC, retrieve token for user with limited rights
+	policyRules := []rbacv1.PolicyRule{
+		{
+			Verbs:     []string{"get", "create", "delete", "list", "patch", "update"},
+			APIGroups: []string{mcadv1beta1.GroupName},
+			Resources: []string{"appwrappers"},
+		},
+		{
+			Verbs:     []string{"get", "list"},
+			APIGroups: []string{rayv1alpha1.GroupVersion.Group},
+			Resources: []string{"rayclusters", "rayclusters/status"},
+		},
+		{
+			Verbs:     []string{"get", "list"},
+			APIGroups: []string{"route.openshift.io"},
+			Resources: []string{"routes"},
+		},
+	}
+
+	// Create cluster wide RBAC, required for SDK OpenShift check
+	// TODO reevaluate once SDK change OpenShift detection logic
+	clusterPolicyRules := []rbacv1.PolicyRule{
+		{
+			Verbs:         []string{"get", "list"},
+			APIGroups:     []string{"config.openshift.io"},
+			Resources:     []string{"ingresses"},
+			ResourceNames: []string{"cluster"},
+		},
+	}
+
+	sa := CreateServiceAccount(test, namespace.Name)
+	role := CreateRole(test, namespace.Name, policyRules)
+	CreateRoleBinding(test, namespace.Name, sa, role)
+	clusterRole := CreateClusterRole(test, clusterPolicyRules)
+	CreateClusterRoleBinding(test, sa, clusterRole)
+	token := CreateToken(test, namespace.Name, sa)
+
+	// Create Notebook CR
+	createNotebook(test, namespace, token, config.Name, jupyterNotebookConfigMapFileName)
+
+	// Make sure the AppWrapper is created and running
+	test.Eventually(AppWrappers(test, namespace), TestTimeoutLong).
+		Should(
+			And(
+				HaveLen(1),
+				ContainElement(WithTransform(AppWrapperName, HavePrefix("mnisttest"))),
+				ContainElement(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))),
+			),
+		)
+
+	// Make sure the AppWrapper finishes and is deleted
+	test.Eventually(AppWrappers(test, namespace), TestTimeoutLong).
+		Should(HaveLen(0))
+}

test/odh/notebook.go

@@ -0,0 +1,99 @@
+/*
+Copyright 2023.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package odh
+
+import (
+	"bytes"
+	"html/template"
+
+	gomega "github.com/onsi/gomega"
+	. "github.com/project-codeflare/codeflare-common/support"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/util/yaml"
+
+	imagev1 "github.com/openshift/api/image/v1"
+)
+
+const recommendedTagAnnotation = "opendatahub.io/workbench-image-recommended"
+
+var notebookResource = schema.GroupVersionResource{Group: "kubeflow.org", Version: "v1", Resource: "notebooks"}
+
+type NotebookProps struct {
+	IngressDomain             string
+	OpenShiftApiUrl           string
+	KubernetesBearerToken     string
+	Namespace                 string
+	OpenDataHubNamespace      string
+	ImageStreamName           string
+	ImageStreamTag            string
+	NotebookConfigMapName     string
+	NotebookConfigMapFileName string
+	NotebookPVC               string
+}
+
+func createNotebook(test Test, namespace *corev1.Namespace, notebookToken, jupyterNotebookConfigMapName, jupyterNotebookConfigMapFileName string) {
+	// Create PVC for Notebook
+	notebookPVC := CreatePersistentVolumeClaim(test, namespace.Name, "10Gi", corev1.ReadWriteOnce)
+
+	// Retrieve ImageStream tag for
+	is := GetImageStream(test, GetOpenDataHubNamespace(), GetNotebookImageStreamName(test))
+	recommendedTagName := getRecommendedImageStreamTag(test, is)
+
+	// Read the Notebook CR from resources and perform replacements for custom values using go template
+	notebookProps := NotebookProps{
+		IngressDomain:             GetOpenShiftIngressDomain(test),
+		OpenShiftApiUrl:           GetOpenShiftApiUrl(test),
+		KubernetesBearerToken:     notebookToken,
+		Namespace:                 namespace.Name,
+		OpenDataHubNamespace:      GetOpenDataHubNamespace(),
+		ImageStreamName:           GetNotebookImageStreamName(test),
+		ImageStreamTag:            recommendedTagName,
+		NotebookConfigMapName:     jupyterNotebookConfigMapName,
+		NotebookConfigMapFileName: jupyterNotebookConfigMapFileName,
+		NotebookPVC:               notebookPVC.Name,
+	}
+	notebookTemplate, err := files.ReadFile("resources/custom-nb-small.yaml")
+	test.Expect(err).NotTo(gomega.HaveOccurred())
+	parsedNotebookTemplate, err := template.New("notebook").Parse(string(notebookTemplate))
+	test.Expect(err).NotTo(gomega.HaveOccurred())
+
+	// Filter template and store results to the buffer
+	notebookBuffer := new(bytes.Buffer)
+	err = parsedNotebookTemplate.Execute(notebookBuffer, notebookProps)
+	test.Expect(err).NotTo(gomega.HaveOccurred())
+
+	// Create Notebook CR
+	notebookCR := &unstructured.Unstructured{}
+	err = yaml.NewYAMLOrJSONDecoder(notebookBuffer, 8192).Decode(notebookCR)
+	test.Expect(err).NotTo(gomega.HaveOccurred())
+	_, err = test.Client().Dynamic().Resource(notebookResource).Namespace(namespace.Name).Create(test.Ctx(), notebookCR, metav1.CreateOptions{})
+	test.Expect(err).NotTo(gomega.HaveOccurred())
+}
+
+func getRecommendedImageStreamTag(test Test, is *imagev1.ImageStream) (tagName string) {
+	for _, tag := range is.Spec.Tags {
+		if tag.Annotations[recommendedTagAnnotation] == "true" {
+			return tag.Name
+		}
+	}
+	test.T().Fatalf("tag with annotation '%s' not found in ImageStream %s", recommendedTagAnnotation, is.Name)
+	return
+}

test/odh/pytorch_mcad_test.go

@@ -0,0 +1,77 @@
+/*
+Copyright 2023.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package odh
+
+import (
+	"testing"
+
+	. "github.com/onsi/gomega"
+	. "github.com/project-codeflare/codeflare-common/support"
+	mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1"
+
+	rbacv1 "k8s.io/api/rbac/v1"
+)
+
+func TestMnistPyTorchMCAD(t *testing.T) {
+	test := With(t)
+
+	// Create a namespace
+	namespace := test.NewTestNamespace()
+
+	// Test configuration
+	jupyterNotebookConfigMapFileName := "mnist_mcad_mini.ipynb"
+	config := CreateConfigMap(test, namespace.Name, map[string][]byte{
+		// MNIST MCAD Notebook
+		jupyterNotebookConfigMapFileName: ReadFile(test, "resources/mnist_mcad_mini.ipynb"),
+	})
+
+	// Create RBAC, retrieve token for user with limited rights
+	policyRules := []rbacv1.PolicyRule{
+		{
+			Verbs:     []string{"get", "create", "delete", "list", "patch", "update"},
+			APIGroups: []string{mcadv1beta1.GroupName},
+			Resources: []string{"appwrappers"},
+		},
+		// Needed for job.logs()
+		{
+			Verbs:     []string{"get"},
+			APIGroups: []string{""},
+			Resources: []string{"pods/log"},
+		},
+	}
+	sa := CreateServiceAccount(test, namespace.Name)
+	role := CreateRole(test, namespace.Name, policyRules)
+	CreateRoleBinding(test, namespace.Name, sa, role)
+	token := CreateToken(test, namespace.Name, sa)
+
+	// Create Notebook CR
+	createNotebook(test, namespace, token, config.Name, jupyterNotebookConfigMapFileName)
+
+	// Make sure the AppWrapper is created and running
+	test.Eventually(AppWrappers(test, namespace), TestTimeoutLong).
+		Should(
+			And(
+				HaveLen(1),
+				ContainElement(WithTransform(AppWrapperName, HavePrefix("mnistjob"))),
+				ContainElement(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))),
+			),
+		)
+
+	// Make sure the AppWrapper finishes and is deleted
+	test.Eventually(AppWrappers(test, namespace), TestTimeoutLong).
+		Should(HaveLen(0))
+}