chore(make): add e2e test options (#2393)

- Add more config options to the e2e and e2e-local test options; e.g.
  E2E_TEST_SEED for reproducing test runs
- Refactor e2e-local target to extend the e2e target

Signed-off-by: Nick Hale <[email protected]>

Author: njhale

.github/workflows/e2e-kind.yml

@@ -22,5 +22,5 @@ jobs:
       run: |
         kind create cluster
         kind export kubeconfig
-    - name: Run e2e tests
+    - name: Run OLM
       run: make run-local

.github/workflows/e2e-minikube.yml

@@ -21,5 +21,5 @@ jobs:
         sudo mv minikube /bin/
     - name: Setup minikube
       run: minikube config set vm-driver docker
-    - name: Run e2e tests
+    - name: Run OLM
       run: make run-local

.github/workflows/e2e-tests.yml

@@ -14,7 +14,7 @@ jobs:
       - uses: actions/setup-go@v2
         with:
           go-version: '~1.16'
-      - run: make e2e-local NODES=2 JUNIT_DIRECTORY=./artifacts/
+      - run: make e2e-local E2E_NODES=2 JUNIT_DIRECTORY=./artifacts/
       - name: Archive Test Artifacts # test results, failed or not, are always uploaded.
         if: ${{ always() }}
         uses: actions/upload-artifact@v2

Makefile

@@ -30,10 +30,14 @@ GINKGO := $(GO) run github.com/onsi/ginkgo/ginkgo
 BINDATA := $(GO) run github.com/go-bindata/go-bindata/v3/go-bindata
 GIT_COMMIT := $(shell git rev-parse HEAD)
 
+# Phony prerequisite for targets that rely on the go build cache to determine staleness.
 .PHONY: build test run clean vendor schema-check \
 	vendor-update coverage coverage-html e2e \
 	kubebuilder .FORCE
 
+.PHONY: FORCE
+FORCE:
+
 all: test build
 
 test: clean cover.out
@@ -119,29 +123,23 @@ setup-bare: clean e2e.namespace
 	. ./scripts/package_release.sh 1.0.0 test/e2e/resources test/e2e/e2e-bare-values.yaml
 	. ./scripts/install_bare.sh $(shell cat ./e2e.namespace) test/e2e/resources
 
-GINKGO_OPTS := -flakeAttempts 3 -randomizeAllSpecs -v --timeout 120m
+E2E_NODES ?= 1
+E2E_FLAKE_ATTEMPTS ?= 1
+E2E_TIMEOUT ?= 90m
+E2E_OPTS ?= $(if $(E2E_SEED),-seed '$(E2E_SEED)') $(if $(TEST),-focus '$(TEST)') -flakeAttempts $(E2E_FLAKE_ATTEMPTS) -nodes $(E2E_NODES) -timeout $(E2E_TIMEOUT) -randomizeAllSpecs -v
+E2E_INSTALL_NS ?= operator-lifecycle-manager
+E2E_TEST_NS ?= operators
 
-# TODO(tflannag): Remove this target entirely and move downstream
 e2e:
-	$(GINKGO) \
-        ./test/e2e \
-        $(GINKGO_OPTS) \
-        $< -- \
-        -namespace=openshift-operators \
-        -kubeconfig=${KUBECONFIG} \
-        -olmNamespace=openshift-operator-lifecycle-manager \
-        -dummyImage=bitnami/nginx:latest
+	$(GINKGO) $(E2E_OPTS) $(or $(run), ./test/e2e) $< -- -namespace=$(E2E_TEST_NS) -olmNamespace=$(E2E_INSTALL_NS) -dummyImage=bitnami/nginx:latest $(or $(extra_args), -kubeconfig=${KUBECONFIG})
 
-### Start: End To End Tests ###
 
-# Phony prerequisite for targets that rely on the go build cache to determine staleness.
-.PHONY: FORCE
-FORCE:
-
-# main entry point for running end to end tests. used by .github/workflows/e2e-tests.yml See test/e2e/README.md for details
+# See workflows/e2e-tests.yml See test/e2e/README.md for details.
 .PHONY: e2e-local
+e2e-local: extra_args=-kind.images=../test/e2e-local.image.tar
+e2e-local: run=bin/e2e-local.test
 e2e-local: bin/e2e-local.test test/e2e-local.image.tar
-	$(GINKGO) -nodes $(or $(NODES),1) -flakeAttempts 3 -randomizeAllSpecs $(if $(TEST),-focus '$(TEST)') -v -timeout 90m $< -- -namespace=operators -olmNamespace=operator-lifecycle-manager -dummyImage=bitnami/nginx:latest -kind.images=../test/e2e-local.image.tar
+e2e-local: e2e
 
 # this target updates the zz_chart.go file with files found in deploy/chart
 # this will always fire since it has been marked as phony
@@ -161,8 +159,6 @@ test/e2e-local.image.tar: e2e.Dockerfile bin/wait bin/cpb $(CMDS)
 	docker build -t quay.io/operator-framework/olm:local -f $< bin
 	docker save -o $@ quay.io/operator-framework/olm:local
 
-### Finish: End To End Tests ###
-
 e2e-bare: setup-bare
 	. ./scripts/run_e2e_bare.sh $(TEST)