Add e2e test that installs hive and creates a cluster

Makefile

@@ -30,9 +30,14 @@ all: fmt vet test build
 test: generate fmt vet manifests
 	go test ./pkg/... ./cmd/... ./contrib/... -coverprofile cover.out
 
+.PHONY: test-integration
 test-integration: generate
 	go test ./test/integration/... -coverprofile cover.out
 
+.PHONY: test-e2e
+test-e2e:
+	hack/e2e-test.sh
+
 # Builds all of hive's binaries (including utils).
 .PHONY: build
 build: manager hiveutil hiveadmission

contrib/cmd/waitforjob/main.go

@@ -0,0 +1,162 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+
+	batchv1 "k8s.io/api/batch/v1"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
+)
+
+var (
+	defaultJobExistenceTimeout = 3 * time.Minute
+	defaultJobExecutionTimeout = 45 * time.Minute
+)
+
+const (
+	defaultLogLevel = "info"
+)
+
+func main() {
+	opts := &waitForJobOpts{}
+
+	cmd := &cobra.Command{
+		Use:   "waitforjob JOBNAME [OPTIONS]",
+		Short: "wait for job",
+		Long:  "Contains various utilities for running and testing hive",
+		Run: func(cmd *cobra.Command, args []string) {
+			if len(args) != 1 {
+				cmd.Usage()
+				os.Exit(1)
+			}
+			opts.jobName = args[0]
+			opts.Run()
+		},
+	}
+	cmd.PersistentFlags().StringVar(&opts.logLevel, "log-level", defaultLogLevel, "Log level (debug,info,warn,error,fatal)")
+	cmd.PersistentFlags().DurationVar(&opts.existenceTimeout, "job-existence-timeout", defaultJobExistenceTimeout, "Maximum time to wait for the named job to be created")
+	cmd.PersistentFlags().DurationVar(&opts.executionTimeout, "job-execution-timeout", defaultJobExecutionTimeout, "Maximum time to wait for the job to execute")
+	cmd.Execute()
+}
+
+type waitForJobOpts struct {
+	logLevel         string
+	jobName          string
+	existenceTimeout time.Duration
+	executionTimeout time.Duration
+}
+
+func (w *waitForJobOpts) Run() {
+	// Set log level
+	level, err := log.ParseLevel(w.logLevel)
+	if err != nil {
+		log.WithError(err).Fatal("Cannot parse log level")
+	}
+	log.SetLevel(level)
+	log.Debug("debug logging enabled")
+
+	client, namespace, err := w.localClient()
+	if err != nil {
+		log.WithError(err).Fatal("failed to obtain client")
+	}
+
+	if err := w.waitForJobExistence(client, namespace); err != nil {
+		log.WithError(err).Fatal("job existence failed")
+	}
+
+	if err := w.waitForJobExecution(client, namespace); err != nil {
+		log.WithError(err).Fatal("job execution failed")
+	}
+}
+
+func (w *waitForJobOpts) waitForJobExistence(client clientset.Interface, namespace string) error {
+	logger := log.WithField("waiting-for-existence", fmt.Sprintf("job (%s/%s)", namespace, w.jobName))
+	err := wait.PollImmediate(10*time.Second, w.existenceTimeout, func() (bool, error) {
+		logger.Debug("Retrieving job")
+		_, err := client.BatchV1().Jobs(namespace).Get(w.jobName, metav1.GetOptions{})
+		if err != nil {
+			if errors.IsNotFound(err) {
+				logger.Debug("job does not exist yet")
+			} else {
+				logger.WithError(err).Warning("unexpected error retrieving job")
+			}
+			return false, nil
+		}
+		logger.Info("Job found")
+		return true, nil
+	})
+	return err
+}
+
+func (w *waitForJobOpts) waitForJobExecution(client clientset.Interface, namespace string) error {
+	logger := log.WithField("waiting-for-run", fmt.Sprintf("job (%s/%s)", namespace, w.jobName))
+	err := wait.PollImmediate(30*time.Second, w.executionTimeout, func() (bool, error) {
+		logger.Debug("Retrieving job")
+		job, err := client.BatchV1().Jobs(namespace).Get(w.jobName, metav1.GetOptions{})
+		if err != nil {
+			logger.WithError(err).Error("Could not fetch job")
+			return false, err
+		}
+		if isFailed(job) {
+			logger.Error("Job has failed")
+			return false, fmt.Errorf("job %s/%s has failed", namespace, w.jobName)
+		}
+		if isSuccessful(job) {
+			logger.Info("Job has finished successfully")
+			return true, nil
+		}
+		logger.Debug("Job has not completed yet")
+		return false, nil
+	})
+	return err
+}
+
+func (w *waitForJobOpts) localClient() (clientset.Interface, string, error) {
+	log.Debug("Creating cluster client")
+	rules := clientcmd.NewDefaultClientConfigLoadingRules()
+	kubeconfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(rules, &clientcmd.ConfigOverrides{})
+	cfg, err := kubeconfig.ClientConfig()
+	if err != nil {
+		log.WithError(err).Error("Cannot obtain client config")
+		return nil, "", err
+	}
+	namespace, _, err := kubeconfig.Namespace()
+	if err != nil {
+		log.WithError(err).Error("Cannot obtain default namespace from client config")
+		return nil, "", err
+	}
+
+	kubeClient, err := clientset.NewForConfig(cfg)
+	if err != nil {
+		log.WithError(err).Error("Cannot create kubernetes client from client config")
+		return nil, "", err
+	}
+
+	return kubeClient, namespace, nil
+}
+
+func getJobConditionStatus(job *batchv1.Job, conditionType batchv1.JobConditionType) corev1.ConditionStatus {
+	for _, condition := range job.Status.Conditions {
+		if condition.Type == conditionType {
+			return condition.Status
+		}
+	}
+	return corev1.ConditionFalse
+}
+
+func isSuccessful(job *batchv1.Job) bool {
+	return getJobConditionStatus(job, batchv1.JobComplete) == corev1.ConditionTrue
+}
+
+func isFailed(job *batchv1.Job) bool {
+	return getJobConditionStatus(job, batchv1.JobFailed) == corev1.ConditionTrue
+}

hack/e2e-test.sh

@@ -0,0 +1,70 @@
+#!/bin/bash
+
+set -e
+
+component=hive
+TEST_IMAGE=$(eval "echo $IMAGE_FORMAT")
+component=installer
+INSTALLER_IMAGE=$(eval "echo $IMAGE_FORMAT")
+
+ln -s $(which oc) $(pwd)/kubectl
+export PATH=$PATH:$(pwd)
+
+# download kustomize so we can use it for deploying
+opsys=linux
+curl -s https://github.com/api/repos/kubernetes-sigs/kustomize/releases/latest |\
+  grep browser_download |\
+  grep $opsys |\
+  cut -d '"' -f 4 |\
+  xargs curl -O -L
+mv kustomize_*_${opsys}_amd64 kustomize
+chmod u+x kustomize
+
+oc new-project cluster-test
+
+# Install Hive
+make deploy DEPLOY_IMAGE="${TEST_IMAGE}"
+
+CLOUD_CREDS_DIR="/tmp/cluster"
+
+# Create a new cluster deployment
+# TODO: Determine which domain to use to create Hive clusters
+export BASE_DOMAIN="hive-ci.openshift.com"
+export CLUSTER_NAME="$(oc get cluster.cluster.k8s.io -n openshift-cluster-api -o jsonpath='{ .items[].metadata.name }')-1"
+export SSH_PUB_KEY="$(cat ${CLOUD_CREDS_DIR}/ssh-publickey)"
+export PULL_SECRET="$(cat ${CLOUD_CREDS_DIR}/pull-secret)"
+export AWS_ACCESS_KEY_ID="$(cat ${CLOUD_CREDS_DIR}/.awscred | awk '/aws_access_key_id/ { print $3; exit; }')"
+export AWS_SECRET_ACCESS_KEY="$(cat ${CLOUD_CREDS_DIR}/.awscred | awk '/aws_secret_access_key/ { print $3; exit; }')"
+
+function teardown() {
+	oc logs -c hive job/${CLUSTER_NAME}-install &> "${ARTIFACT_DIR}/hive_install_job.log" || true
+	echo "Deleting ClusterDeployment ${CLUSTER_NAME}"
+	oc delete clusterdeployment ${CLUSTER_NAME}
+}
+trap 'teardown' EXIT
+
+# TODO: Determine how to wait for readiness of the validation webhook
+sleep 120
+
+echo "Creating ClusterDeployment ${CLUSTER_NAME}"
+
+oc process -f config/templates/cluster-deployment.yaml \
+   CLUSTER_NAME="${CLUSTER_NAME}" \
+   SSH_KEY="${SSH_PUB_KEY}" \
+   PULL_SECRET="${PULL_SECRET}" \
+   AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID}" \
+   AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY}" \
+   BASE_DOMAIN="${BASE_DOMAIN}" \
+   INSTALLER_IMAGE="${INSTALLER_IMAGE}" \
+   OPENSHIFT_RELEASE_IMAGE="" \
+   TRY_INSTALL_ONCE="true" \
+   | oc apply -f -
+
+# Wait for the cluster deployment to be installed
+SRC_ROOT=$(git rev-parse --show-toplevel)
+
+echo "Waiting for job ${CLUSTER_NAME}-install to start and complete"
+
+go run "${SRC_ROOT}/contrib/cmd/waitforjob/main.go" --log-level=debug "${CLUSTER_NAME}-install"
+
+echo "ClusterDeployment ${CLUSTER_NAME} was installed successfully"