Create e2e scenario covering upgrade during training

Author: Srihari1192

.github/workflows/olm_tests.yaml

@@ -127,7 +127,7 @@ jobs:
           export CODEFLARE_TEST_OUTPUT_DIR=${{ env.TEMP_DIR }}
           echo "CODEFLARE_TEST_OUTPUT_DIR=${CODEFLARE_TEST_OUTPUT_DIR}" >> $GITHUB_ENV
           set -euo pipefail
-          go test -timeout 30m -v ./test/upgrade -run TestMNISTRayClusterUp -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest.log | gotestfmt
+          go test -timeout 30m -v ./test/upgrade -run TestMNISTCreateAppWrapper -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest.log | gotestfmt
 
       - name: Update Operator to the built version
         run: |
@@ -163,7 +163,7 @@ jobs:
           export CODEFLARE_TEST_OUTPUT_DIR=${{ env.TEMP_DIR }}
           echo "CODEFLARE_TEST_OUTPUT_DIR=${CODEFLARE_TEST_OUTPUT_DIR}" >> $GITHUB_ENV
           set -euo pipefail
-          go test -timeout 30m -v ./test/upgrade -run TestMnistJobSubmit -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest.log | gotestfmt
+          go test -timeout 30m -v ./test/upgrade -run TestMNISTCheckAppWrapperStatus -json 2>&1 | tee ${CODEFLARE_TEST_OUTPUT_DIR}/gotest.log | gotestfmt
 
       - name: Run e2e tests against built operator
         run: |

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/onsi/gomega v1.27.10
 	github.com/openshift/api v0.0.0-20230213134911-7ba313770556
 	github.com/project-codeflare/codeflare-common v0.0.0-20231129165224-988ba1da9069
+	github.com/project-codeflare/instascale v0.3.1
 	github.com/project-codeflare/multi-cluster-app-dispatcher v1.38.1
 	github.com/ray-project/kuberay/ray-operator v1.0.0-rc.1
 	go.uber.org/zap v1.26.0