From 42ab2ea1a59d19e1ca75202fb1e99adffc8e1b3d Mon Sep 17 00:00:00 2001 From: Camila Macedo <7708031+camilamacedo86@users.noreply.github.com> Date: Wed, 5 Feb 2025 23:44:00 +0000 Subject: [PATCH] (cleanup): (cleanup): Refactor metrics endpoint tests by extracting shared helpers Refactored the metrics endpoint tests by introducing reusable helper functions. This reduces duplication and improves maintainability across both tests. --- test/e2e/metrics_test.go | 324 +++++++++++++++++---------------------- test/utils/utils.go | 23 +++ 2 files changed, 161 insertions(+), 186 deletions(-) create mode 100644 test/utils/utils.go diff --git a/test/e2e/metrics_test.go b/test/e2e/metrics_test.go index 174bb87d0..b377cb408 100644 --- a/test/e2e/metrics_test.go +++ b/test/e2e/metrics_test.go @@ -1,234 +1,186 @@ +// Package e2e contains end-to-end tests to verify that the metrics endpoints +// for both components. Metrics are exported and accessible by authorized users through +// RBAC and ServiceAccount tokens. +// +// These tests perform the following steps: +// 1. Create a ClusterRoleBinding to grant necessary permissions for accessing metrics. +// 2. Generate a ServiceAccount token for authentication. +// 3. Deploy a curl pod to interact with the metrics endpoint. +// 4. Wait for the curl pod to become ready. +// 5. Execute a curl command from the pod to validate the metrics endpoint. +// 6. Clean up all resources created during the test, such as the ClusterRoleBinding and curl pod. +// +//nolint:gosec package e2e import ( "bytes" "io" "os/exec" + "strings" "testing" "github.com/stretchr/testify/require" + + "github.com/operator-framework/operator-controller/test/utils" ) -// nolint:gosec // TestOperatorControllerMetricsExportedEndpoint verifies that the metrics endpoint for the operator controller -// is exported correctly and accessible by authorized users through RBAC and a ServiceAccount token. -// The test performs the following steps: -// 1. Creates a ClusterRoleBinding to grant necessary permissions for accessing metrics. -// 2. Generates a ServiceAccount token for authentication. -// 3. Deploys a curl pod to interact with the metrics endpoint. -// 4. Waits for the curl pod to become ready. -// 5. Executes a curl command from the pod to validate the metrics endpoint. -// 6. Cleans up all resources created during the test, such as the ClusterRoleBinding and curl pod. func TestOperatorControllerMetricsExportedEndpoint(t *testing.T) { - var ( - token string - curlPod = "curl-metrics" - client = "" - clients = []string{"kubectl", "oc"} + client := utils.FindK8sClient(t) + config := NewMetricsTestConfig( + t, client, + "control-plane=operator-controller-controller-manager", + "operator-controller-metrics-reader", + "operator-controller-metrics-binding", + "operator-controller-controller-manager", + "oper-curl-metrics", + "https://operator-controller-service.NAMESPACE.svc.cluster.local:8443/metrics", ) - t.Log("Looking for k8s client") - for _, c := range clients { - // Would prefer to use `command -v`, but even that may not be installed! - err := exec.Command(c, "version", "--client").Run() - if err == nil { - client = c - break - } - } - if client == "" { - t.Fatal("k8s client not found") + config.run() +} + +// TestCatalogdMetricsExportedEndpoint verifies that the metrics endpoint for catalogd +func TestCatalogdMetricsExportedEndpoint(t *testing.T) { + client := utils.FindK8sClient(t) + config := NewMetricsTestConfig( + t, client, + "control-plane=catalogd-controller-manager", + "catalogd-metrics-reader", + "catalogd-metrics-binding", + "catalogd-controller-manager", + "catalogd-curl-metrics", + "https://catalogd-service.NAMESPACE.svc.cluster.local:7443/metrics", + ) + + config.run() +} + +// MetricsTestConfig holds the necessary configurations for testing metrics endpoints. +type MetricsTestConfig struct { + t *testing.T + client string + namespace string + clusterRole string + clusterBinding string + serviceAccount string + curlPodName string + metricsURL string +} + +// NewMetricsTestConfig initializes a new MetricsTestConfig. +func NewMetricsTestConfig(t *testing.T, client, selector, clusterRole, clusterBinding, serviceAccount, curlPodName, metricsURL string) *MetricsTestConfig { + namespace := getComponentNamespace(t, client, selector) + metricsURL = strings.ReplaceAll(metricsURL, "NAMESPACE", namespace) + + return &MetricsTestConfig{ + t: t, + client: client, + namespace: namespace, + clusterRole: clusterRole, + clusterBinding: clusterBinding, + serviceAccount: serviceAccount, + curlPodName: curlPodName, + metricsURL: metricsURL, } - t.Logf("Using %q as k8s client", client) +} - t.Log("Determining operator-controller namespace") - cmd := exec.Command(client, "get", "pods", "--all-namespaces", "--selector=control-plane=operator-controller-controller-manager", "--output=jsonpath={.items[0].metadata.namespace}") +// run will execute all steps of those tests +func (c *MetricsTestConfig) run() { + c.createMetricsClusterRoleBinding() + token := c.getServiceAccountToken() + c.createCurlMetricsPod() + c.validate(token) + defer c.cleanup() +} + +// createMetricsClusterRoleBinding to binding and expose the metrics +func (c *MetricsTestConfig) createMetricsClusterRoleBinding() { + c.t.Logf("Creating ClusterRoleBinding %s in namespace %s", c.clusterBinding, c.namespace) + cmd := exec.Command(c.client, "create", "clusterrolebinding", c.clusterBinding, + "--clusterrole="+c.clusterRole, + "--serviceaccount="+c.namespace+":"+c.serviceAccount) output, err := cmd.CombinedOutput() - require.NoError(t, err, "Error creating determining operator-controller namespace: %s", string(output)) - namespace := string(output) - if namespace == "" { - t.Fatal("No operator-controller namespace found") - } - t.Logf("Using %q as operator-controller namespace", namespace) - - t.Log("Creating ClusterRoleBinding for operator controller metrics") - cmd = exec.Command(client, "create", "clusterrolebinding", "operator-controller-metrics-binding", - "--clusterrole=operator-controller-metrics-reader", - "--serviceaccount="+namespace+":operator-controller-controller-manager") - output, err = cmd.CombinedOutput() - require.NoError(t, err, "Error creating ClusterRoleBinding: %s", string(output)) - - defer func() { - t.Log("Cleaning up ClusterRoleBinding") - _ = exec.Command(client, "delete", "clusterrolebinding", "operator-controller-metrics-binding", "--ignore-not-found=true").Run() - }() - - t.Log("Generating ServiceAccount token") - tokenCmd := exec.Command(client, "create", "token", "operator-controller-controller-manager", "-n", namespace) - tokenOutput, tokenCombinedOutput, err := stdoutAndCombined(tokenCmd) - require.NoError(t, err, "Error creating token: %s", string(tokenCombinedOutput)) - token = string(bytes.TrimSpace(tokenOutput)) - - t.Log("Creating curl pod to validate the metrics endpoint") - cmd = exec.Command(client, "run", curlPod, - "--image=curlimages/curl:7.87.0", "-n", namespace, + require.NoError(c.t, err, "Error creating ClusterRoleBinding: %s", string(output)) +} + +// getServiceAccountToken return the token requires to have access to the metrics +func (c *MetricsTestConfig) getServiceAccountToken() string { + c.t.Logf("Generating ServiceAccount token at namespace %s", c.namespace) + cmd := exec.Command(c.client, "create", "token", c.serviceAccount, "-n", c.namespace) + tokenOutput, tokenCombinedOutput, err := stdoutAndCombined(cmd) + require.NoError(c.t, err, "Error creating token: %s", string(tokenCombinedOutput)) + return string(bytes.TrimSpace(tokenOutput)) +} + +// createCurlMetricsPod creates the Pod with curl image to allow check if the metrics are working +func (c *MetricsTestConfig) createCurlMetricsPod() { + c.t.Logf("Creating curl pod (%s/%s) to validate the metrics endpoint", c.namespace, c.curlPodName) + cmd := exec.Command(c.client, "run", c.curlPodName, + "--image=curlimages/curl", "-n", c.namespace, "--restart=Never", "--overrides", `{ "spec": { "containers": [{ "name": "curl", - "image": "curlimages/curl:7.87.0", + "image": "curlimages/curl", "command": ["sh", "-c", "sleep 3600"], "securityContext": { "allowPrivilegeEscalation": false, - "capabilities": { - "drop": ["ALL"] - }, + "capabilities": {"drop": ["ALL"]}, "runAsNonRoot": true, "runAsUser": 1000, - "seccompProfile": { - "type": "RuntimeDefault" - } + "seccompProfile": {"type": "RuntimeDefault"} } }], - "serviceAccountName": "operator-controller-controller-manager" + "serviceAccountName": "`+c.serviceAccount+`" } }`) - output, err = cmd.CombinedOutput() - require.NoError(t, err, "Error creating curl pod: %s", string(output)) - - defer func() { - t.Log("Cleaning up curl pod") - _ = exec.Command(client, "delete", "pod", curlPod, "-n", namespace, "--ignore-not-found=true").Run() - }() + output, err := cmd.CombinedOutput() + require.NoError(c.t, err, "Error creating curl pod: %s", string(output)) +} - t.Log("Waiting for the curl pod to be ready") - waitCmd := exec.Command(client, "wait", "--for=condition=Ready", "pod", curlPod, "-n", namespace, "--timeout=60s") +// validate verifies if is possible to access the metrics +func (c *MetricsTestConfig) validate(token string) { + c.t.Log("Waiting for the curl pod to be ready") + waitCmd := exec.Command(c.client, "wait", "--for=condition=Ready", "pod", c.curlPodName, "-n", c.namespace, "--timeout=60s") waitOutput, waitErr := waitCmd.CombinedOutput() - require.NoError(t, waitErr, "Error waiting for curl pod to be ready: %s", string(waitOutput)) - - t.Log("Validating the metrics endpoint") - metricsURL := "https://operator-controller-service." + namespace + ".svc.cluster.local:8443/metrics" - curlCmd := exec.Command(client, "exec", curlPod, "-n", namespace, "--", - "curl", "-v", "-k", "-H", "Authorization: Bearer "+token, metricsURL) - output, err = curlCmd.CombinedOutput() - require.NoError(t, err, "Error calling metrics endpoint: %s", string(output)) - require.Contains(t, string(output), "200 OK", "Metrics endpoint did not return 200 OK") + require.NoError(c.t, waitErr, "Error waiting for curl pod to be ready: %s", string(waitOutput)) + + c.t.Log("Validating the metrics endpoint") + curlCmd := exec.Command(c.client, "exec", c.curlPodName, "-n", c.namespace, "--", + "curl", "-v", "-k", "-H", "Authorization: Bearer "+token, c.metricsURL) + output, err := curlCmd.CombinedOutput() + require.NoError(c.t, err, "Error calling metrics endpoint: %s", string(output)) + require.Contains(c.t, string(output), "200 OK", "Metrics endpoint did not return 200 OK") } -// nolint:gosec -// TestCatalogdMetricsExportedEndpoint verifies that the metrics endpoint for the catalogd -// is exported correctly and accessible by authorized users through RBAC and a ServiceAccount token. -// The test performs the following steps: -// 1. Creates a ClusterRoleBinding to grant necessary permissions for accessing metrics. -// 2. Generates a ServiceAccount token for authentication. -// 3. Deploys a curl pod to interact with the metrics endpoint. -// 4. Waits for the curl pod to become ready. -// 5. Executes a curl command from the pod to validate the metrics endpoint. -// 6. Cleans up all resources created during the test, such as the ClusterRoleBinding and curl pod. -func TestCatalogdMetricsExportedEndpoint(t *testing.T) { - var ( - token string - curlPod = "curl-metrics" - client = "" - clients = []string{"kubectl", "oc"} - ) - - t.Log("Looking for k8s client") - for _, c := range clients { - // Would prefer to use `command -v`, but even that may not be installed! - err := exec.Command(c, "version", "--client").Run() - if err == nil { - client = c - break - } - } - if client == "" { - t.Fatal("k8s client not found") - } - t.Logf("Using %q as k8s client", client) +// cleanup created resources +func (c *MetricsTestConfig) cleanup() { + c.t.Log("Cleaning up resources") + _ = exec.Command(c.client, "delete", "clusterrolebinding", c.clusterBinding, "--ignore-not-found=true").Run() + _ = exec.Command(c.client, "delete", "pod", c.curlPodName, "-n", c.namespace, "--ignore-not-found=true").Run() +} - t.Log("Determining catalogd namespace") - cmd := exec.Command(client, "get", "pods", "--all-namespaces", "--selector=control-plane=catalogd-controller-manager", "--output=jsonpath={.items[0].metadata.namespace}") +// getComponentNamespace returns the namespace where operator-controller or catalogd is running +func getComponentNamespace(t *testing.T, client, selector string) string { + cmd := exec.Command(client, "get", "pods", "--all-namespaces", "--selector="+selector, "--output=jsonpath={.items[0].metadata.namespace}") output, err := cmd.CombinedOutput() - require.NoError(t, err, "Error creating determining catalogd namespace: %s", string(output)) - namespace := string(output) + require.NoError(t, err, "Error determining namespace: %s", string(output)) + + namespace := string(bytes.TrimSpace(output)) if namespace == "" { - t.Fatal("No catalogd namespace found") + t.Fatal("No namespace found for selector " + selector) } - t.Logf("Using %q as catalogd namespace", namespace) - - t.Log("Creating ClusterRoleBinding for metrics access") - cmd = exec.Command(client, "create", "clusterrolebinding", "catalogd-metrics-binding", - "--clusterrole=catalogd-metrics-reader", - "--serviceaccount="+namespace+":catalogd-controller-manager") - output, err = cmd.CombinedOutput() - require.NoError(t, err, "Error creating ClusterRoleBinding: %s", string(output)) - - defer func() { - t.Log("Cleaning up ClusterRoleBinding") - _ = exec.Command(client, "delete", "clusterrolebinding", "catalogd-metrics-binding", "--ignore-not-found=true").Run() - }() - - t.Log("Creating service account token for authentication") - tokenCmd := exec.Command(client, "create", "token", "catalogd-controller-manager", "-n", namespace) - tokenOutput, tokenCombinedOutput, err := stdoutAndCombined(tokenCmd) - require.NoError(t, err, "Error creating token: %s", string(tokenCombinedOutput)) - token = string(bytes.TrimSpace(tokenOutput)) - - t.Log("Creating a pod to run curl commands") - cmd = exec.Command(client, "run", curlPod, - "--image=curlimages/curl:7.87.0", "-n", namespace, - "--restart=Never", - "--overrides", `{ - "spec": { - "containers": [{ - "name": "curl", - "image": "curlimages/curl:7.87.0", - "command": ["sh", "-c", "sleep 3600"], - "securityContext": { - "allowPrivilegeEscalation": false, - "capabilities": { - "drop": ["ALL"] - }, - "runAsNonRoot": true, - "runAsUser": 1000, - "seccompProfile": { - "type": "RuntimeDefault" - } - } - }], - "serviceAccountName": "catalogd-controller-manager" - } - }`) - output, err = cmd.CombinedOutput() - require.NoError(t, err, "Error creating curl pod: %s", string(output)) - - defer func() { - t.Log("Cleaning up curl pod") - _ = exec.Command(client, "delete", "pod", curlPod, "-n", namespace, "--ignore-not-found=true").Run() - }() - - t.Log("Waiting for the curl pod to become ready") - waitCmd := exec.Command(client, "wait", "--for=condition=Ready", "pod", curlPod, "-n", namespace, "--timeout=60s") - waitOutput, waitErr := waitCmd.CombinedOutput() - require.NoError(t, waitErr, "Error waiting for curl pod to be ready: %s", string(waitOutput)) - - t.Log("Validating the metrics endpoint") - metricsURL := "https://catalogd-service." + namespace + ".svc.cluster.local:7443/metrics" - curlCmd := exec.Command(client, "exec", curlPod, "-n", namespace, "--", - "curl", "-v", "-k", "-H", "Authorization: Bearer "+token, metricsURL) - output, err = curlCmd.CombinedOutput() - require.NoError(t, err, "Error calling metrics endpoint: %s", string(output)) - require.Contains(t, string(output), "200 OK", "Metrics endpoint did not return 200 OK") + return namespace } func stdoutAndCombined(cmd *exec.Cmd) ([]byte, []byte, error) { - var outOnly bytes.Buffer - var outAndErr bytes.Buffer + var outOnly, outAndErr bytes.Buffer allWriter := io.MultiWriter(&outOnly, &outAndErr) - cmd.Stderr = &outAndErr cmd.Stdout = allWriter + cmd.Stderr = &outAndErr err := cmd.Run() return outOnly.Bytes(), outAndErr.Bytes(), err } diff --git a/test/utils/utils.go b/test/utils/utils.go new file mode 100644 index 000000000..db6d25a7f --- /dev/null +++ b/test/utils/utils.go @@ -0,0 +1,23 @@ +package utils + +import ( + "os/exec" + "testing" +) + +// FindK8sClient returns the first available Kubernetes CLI client from the system, +// It checks for the existence of each client by running `version --client`. +// If no suitable client is found, the function terminates the test with a failure. +func FindK8sClient(t *testing.T) string { + t.Logf("Finding kubectl client") + clients := []string{"kubectl", "oc"} + for _, c := range clients { + // Would prefer to use `command -v`, but even that may not be installed! + if err := exec.Command(c, "version", "--client").Run(); err == nil { + t.Logf("Using %q as k8s client", c) + return c + } + } + t.Fatal("k8s client not found") + return "" +}