Skip to content

🌱 (cleanup): Refactor metrics endpoint tests by extracting shared helpers #1719

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Feb 7, 2025
Merged

🌱 (cleanup): Refactor metrics endpoint tests by extracting shared helpers #1719

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
324 changes: 138 additions & 186 deletions test/e2e/metrics_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,234 +1,186 @@
// Package e2e contains end-to-end tests to verify that the metrics endpoints
// for both components. Metrics are exported and accessible by authorized users through
// RBAC and ServiceAccount tokens.
//
// These tests perform the following steps:
// 1. Create a ClusterRoleBinding to grant necessary permissions for accessing metrics.
// 2. Generate a ServiceAccount token for authentication.
// 3. Deploy a curl pod to interact with the metrics endpoint.
// 4. Wait for the curl pod to become ready.
// 5. Execute a curl command from the pod to validate the metrics endpoint.
// 6. Clean up all resources created during the test, such as the ClusterRoleBinding and curl pod.
//
//nolint:gosec
package e2e

import (
"bytes"
"io"
"os/exec"
"strings"
"testing"

"github.com/stretchr/testify/require"

"github.com/operator-framework/operator-controller/test/utils"
)

// nolint:gosec
// TestOperatorControllerMetricsExportedEndpoint verifies that the metrics endpoint for the operator controller
// is exported correctly and accessible by authorized users through RBAC and a ServiceAccount token.
// The test performs the following steps:
// 1. Creates a ClusterRoleBinding to grant necessary permissions for accessing metrics.
// 2. Generates a ServiceAccount token for authentication.
// 3. Deploys a curl pod to interact with the metrics endpoint.
// 4. Waits for the curl pod to become ready.
// 5. Executes a curl command from the pod to validate the metrics endpoint.
// 6. Cleans up all resources created during the test, such as the ClusterRoleBinding and curl pod.
func TestOperatorControllerMetricsExportedEndpoint(t *testing.T) {
var (
token string
curlPod = "curl-metrics"
client = ""
clients = []string{"kubectl", "oc"}
client := utils.FindK8sClient(t)
config := NewMetricsTestConfig(
t, client,
"control-plane=operator-controller-controller-manager",
"operator-controller-metrics-reader",
"operator-controller-metrics-binding",
"operator-controller-controller-manager",
"oper-curl-metrics",
"https://operator-controller-service.NAMESPACE.svc.cluster.local:8443/metrics",
)

t.Log("Looking for k8s client")
for _, c := range clients {
// Would prefer to use `command -v`, but even that may not be installed!
err := exec.Command(c, "version", "--client").Run()
if err == nil {
client = c
break
}
}
if client == "" {
t.Fatal("k8s client not found")
config.run()
}

// TestCatalogdMetricsExportedEndpoint verifies that the metrics endpoint for catalogd
func TestCatalogdMetricsExportedEndpoint(t *testing.T) {
client := utils.FindK8sClient(t)
config := NewMetricsTestConfig(
t, client,
"control-plane=catalogd-controller-manager",
"catalogd-metrics-reader",
"catalogd-metrics-binding",
"catalogd-controller-manager",
"catalogd-curl-metrics",
"https://catalogd-service.NAMESPACE.svc.cluster.local:7443/metrics",
)

config.run()
}

// MetricsTestConfig holds the necessary configurations for testing metrics endpoints.
type MetricsTestConfig struct {
t *testing.T
client string
namespace string
clusterRole string
clusterBinding string
serviceAccount string
curlPodName string
metricsURL string
}

// NewMetricsTestConfig initializes a new MetricsTestConfig.
func NewMetricsTestConfig(t *testing.T, client, selector, clusterRole, clusterBinding, serviceAccount, curlPodName, metricsURL string) *MetricsTestConfig {
namespace := getComponentNamespace(t, client, selector)
metricsURL = strings.ReplaceAll(metricsURL, "NAMESPACE", namespace)

return &MetricsTestConfig{
t: t,
client: client,
namespace: namespace,
clusterRole: clusterRole,
clusterBinding: clusterBinding,
serviceAccount: serviceAccount,
curlPodName: curlPodName,
metricsURL: metricsURL,
}
t.Logf("Using %q as k8s client", client)
}

t.Log("Determining operator-controller namespace")
cmd := exec.Command(client, "get", "pods", "--all-namespaces", "--selector=control-plane=operator-controller-controller-manager", "--output=jsonpath={.items[0].metadata.namespace}")
// run will execute all steps of those tests
func (c *MetricsTestConfig) run() {
c.createMetricsClusterRoleBinding()
token := c.getServiceAccountToken()
c.createCurlMetricsPod()
c.validate(token)
defer c.cleanup()
}

// createMetricsClusterRoleBinding to binding and expose the metrics
func (c *MetricsTestConfig) createMetricsClusterRoleBinding() {
c.t.Logf("Creating ClusterRoleBinding %s in namespace %s", c.clusterBinding, c.namespace)
cmd := exec.Command(c.client, "create", "clusterrolebinding", c.clusterBinding,
"--clusterrole="+c.clusterRole,
"--serviceaccount="+c.namespace+":"+c.serviceAccount)
output, err := cmd.CombinedOutput()
require.NoError(t, err, "Error creating determining operator-controller namespace: %s", string(output))
namespace := string(output)
if namespace == "" {
t.Fatal("No operator-controller namespace found")
}
t.Logf("Using %q as operator-controller namespace", namespace)

t.Log("Creating ClusterRoleBinding for operator controller metrics")
cmd = exec.Command(client, "create", "clusterrolebinding", "operator-controller-metrics-binding",
"--clusterrole=operator-controller-metrics-reader",
"--serviceaccount="+namespace+":operator-controller-controller-manager")
output, err = cmd.CombinedOutput()
require.NoError(t, err, "Error creating ClusterRoleBinding: %s", string(output))

defer func() {
t.Log("Cleaning up ClusterRoleBinding")
_ = exec.Command(client, "delete", "clusterrolebinding", "operator-controller-metrics-binding", "--ignore-not-found=true").Run()
}()

t.Log("Generating ServiceAccount token")
tokenCmd := exec.Command(client, "create", "token", "operator-controller-controller-manager", "-n", namespace)
tokenOutput, tokenCombinedOutput, err := stdoutAndCombined(tokenCmd)
require.NoError(t, err, "Error creating token: %s", string(tokenCombinedOutput))
token = string(bytes.TrimSpace(tokenOutput))

t.Log("Creating curl pod to validate the metrics endpoint")
cmd = exec.Command(client, "run", curlPod,
"--image=curlimages/curl:7.87.0", "-n", namespace,
require.NoError(c.t, err, "Error creating ClusterRoleBinding: %s", string(output))
}

// getServiceAccountToken return the token requires to have access to the metrics
func (c *MetricsTestConfig) getServiceAccountToken() string {
c.t.Logf("Generating ServiceAccount token at namespace %s", c.namespace)
cmd := exec.Command(c.client, "create", "token", c.serviceAccount, "-n", c.namespace)
tokenOutput, tokenCombinedOutput, err := stdoutAndCombined(cmd)
require.NoError(c.t, err, "Error creating token: %s", string(tokenCombinedOutput))
return string(bytes.TrimSpace(tokenOutput))
}

// createCurlMetricsPod creates the Pod with curl image to allow check if the metrics are working
func (c *MetricsTestConfig) createCurlMetricsPod() {
c.t.Logf("Creating curl pod (%s/%s) to validate the metrics endpoint", c.namespace, c.curlPodName)
cmd := exec.Command(c.client, "run", c.curlPodName,
"--image=curlimages/curl", "-n", c.namespace,
"--restart=Never",
"--overrides", `{
"spec": {
"containers": [{
"name": "curl",
"image": "curlimages/curl:7.87.0",
"image": "curlimages/curl",
"command": ["sh", "-c", "sleep 3600"],
"securityContext": {
"allowPrivilegeEscalation": false,
"capabilities": {
"drop": ["ALL"]
},
"capabilities": {"drop": ["ALL"]},
"runAsNonRoot": true,
"runAsUser": 1000,
"seccompProfile": {
"type": "RuntimeDefault"
}
"seccompProfile": {"type": "RuntimeDefault"}
}
}],
"serviceAccountName": "operator-controller-controller-manager"
"serviceAccountName": "`+c.serviceAccount+`"
}
}`)
output, err = cmd.CombinedOutput()
require.NoError(t, err, "Error creating curl pod: %s", string(output))

defer func() {
t.Log("Cleaning up curl pod")
_ = exec.Command(client, "delete", "pod", curlPod, "-n", namespace, "--ignore-not-found=true").Run()
}()
output, err := cmd.CombinedOutput()
require.NoError(c.t, err, "Error creating curl pod: %s", string(output))
}

t.Log("Waiting for the curl pod to be ready")
waitCmd := exec.Command(client, "wait", "--for=condition=Ready", "pod", curlPod, "-n", namespace, "--timeout=60s")
// validate verifies if is possible to access the metrics
func (c *MetricsTestConfig) validate(token string) {
c.t.Log("Waiting for the curl pod to be ready")
waitCmd := exec.Command(c.client, "wait", "--for=condition=Ready", "pod", c.curlPodName, "-n", c.namespace, "--timeout=60s")
waitOutput, waitErr := waitCmd.CombinedOutput()
require.NoError(t, waitErr, "Error waiting for curl pod to be ready: %s", string(waitOutput))

t.Log("Validating the metrics endpoint")
metricsURL := "https://operator-controller-service." + namespace + ".svc.cluster.local:8443/metrics"
curlCmd := exec.Command(client, "exec", curlPod, "-n", namespace, "--",
"curl", "-v", "-k", "-H", "Authorization: Bearer "+token, metricsURL)
output, err = curlCmd.CombinedOutput()
require.NoError(t, err, "Error calling metrics endpoint: %s", string(output))
require.Contains(t, string(output), "200 OK", "Metrics endpoint did not return 200 OK")
require.NoError(c.t, waitErr, "Error waiting for curl pod to be ready: %s", string(waitOutput))

c.t.Log("Validating the metrics endpoint")
curlCmd := exec.Command(c.client, "exec", c.curlPodName, "-n", c.namespace, "--",
"curl", "-v", "-k", "-H", "Authorization: Bearer "+token, c.metricsURL)
output, err := curlCmd.CombinedOutput()
require.NoError(c.t, err, "Error calling metrics endpoint: %s", string(output))
require.Contains(c.t, string(output), "200 OK", "Metrics endpoint did not return 200 OK")
}

// nolint:gosec
// TestCatalogdMetricsExportedEndpoint verifies that the metrics endpoint for the catalogd
// is exported correctly and accessible by authorized users through RBAC and a ServiceAccount token.
// The test performs the following steps:
// 1. Creates a ClusterRoleBinding to grant necessary permissions for accessing metrics.
// 2. Generates a ServiceAccount token for authentication.
// 3. Deploys a curl pod to interact with the metrics endpoint.
// 4. Waits for the curl pod to become ready.
// 5. Executes a curl command from the pod to validate the metrics endpoint.
// 6. Cleans up all resources created during the test, such as the ClusterRoleBinding and curl pod.
func TestCatalogdMetricsExportedEndpoint(t *testing.T) {
var (
token string
curlPod = "curl-metrics"
client = ""
clients = []string{"kubectl", "oc"}
)

t.Log("Looking for k8s client")
for _, c := range clients {
// Would prefer to use `command -v`, but even that may not be installed!
err := exec.Command(c, "version", "--client").Run()
if err == nil {
client = c
break
}
}
if client == "" {
t.Fatal("k8s client not found")
}
t.Logf("Using %q as k8s client", client)
// cleanup created resources
func (c *MetricsTestConfig) cleanup() {
c.t.Log("Cleaning up resources")
_ = exec.Command(c.client, "delete", "clusterrolebinding", c.clusterBinding, "--ignore-not-found=true").Run()
_ = exec.Command(c.client, "delete", "pod", c.curlPodName, "-n", c.namespace, "--ignore-not-found=true").Run()
}

t.Log("Determining catalogd namespace")
cmd := exec.Command(client, "get", "pods", "--all-namespaces", "--selector=control-plane=catalogd-controller-manager", "--output=jsonpath={.items[0].metadata.namespace}")
// getComponentNamespace returns the namespace where operator-controller or catalogd is running
func getComponentNamespace(t *testing.T, client, selector string) string {
cmd := exec.Command(client, "get", "pods", "--all-namespaces", "--selector="+selector, "--output=jsonpath={.items[0].metadata.namespace}")
output, err := cmd.CombinedOutput()
require.NoError(t, err, "Error creating determining catalogd namespace: %s", string(output))
namespace := string(output)
require.NoError(t, err, "Error determining namespace: %s", string(output))

namespace := string(bytes.TrimSpace(output))
if namespace == "" {
t.Fatal("No catalogd namespace found")
t.Fatal("No namespace found for selector " + selector)
}
t.Logf("Using %q as catalogd namespace", namespace)

t.Log("Creating ClusterRoleBinding for metrics access")
cmd = exec.Command(client, "create", "clusterrolebinding", "catalogd-metrics-binding",
"--clusterrole=catalogd-metrics-reader",
"--serviceaccount="+namespace+":catalogd-controller-manager")
output, err = cmd.CombinedOutput()
require.NoError(t, err, "Error creating ClusterRoleBinding: %s", string(output))

defer func() {
t.Log("Cleaning up ClusterRoleBinding")
_ = exec.Command(client, "delete", "clusterrolebinding", "catalogd-metrics-binding", "--ignore-not-found=true").Run()
}()

t.Log("Creating service account token for authentication")
tokenCmd := exec.Command(client, "create", "token", "catalogd-controller-manager", "-n", namespace)
tokenOutput, tokenCombinedOutput, err := stdoutAndCombined(tokenCmd)
require.NoError(t, err, "Error creating token: %s", string(tokenCombinedOutput))
token = string(bytes.TrimSpace(tokenOutput))

t.Log("Creating a pod to run curl commands")
cmd = exec.Command(client, "run", curlPod,
"--image=curlimages/curl:7.87.0", "-n", namespace,
"--restart=Never",
"--overrides", `{
"spec": {
"containers": [{
"name": "curl",
"image": "curlimages/curl:7.87.0",
"command": ["sh", "-c", "sleep 3600"],
"securityContext": {
"allowPrivilegeEscalation": false,
"capabilities": {
"drop": ["ALL"]
},
"runAsNonRoot": true,
"runAsUser": 1000,
"seccompProfile": {
"type": "RuntimeDefault"
}
}
}],
"serviceAccountName": "catalogd-controller-manager"
}
}`)
output, err = cmd.CombinedOutput()
require.NoError(t, err, "Error creating curl pod: %s", string(output))

defer func() {
t.Log("Cleaning up curl pod")
_ = exec.Command(client, "delete", "pod", curlPod, "-n", namespace, "--ignore-not-found=true").Run()
}()

t.Log("Waiting for the curl pod to become ready")
waitCmd := exec.Command(client, "wait", "--for=condition=Ready", "pod", curlPod, "-n", namespace, "--timeout=60s")
waitOutput, waitErr := waitCmd.CombinedOutput()
require.NoError(t, waitErr, "Error waiting for curl pod to be ready: %s", string(waitOutput))

t.Log("Validating the metrics endpoint")
metricsURL := "https://catalogd-service." + namespace + ".svc.cluster.local:7443/metrics"
curlCmd := exec.Command(client, "exec", curlPod, "-n", namespace, "--",
"curl", "-v", "-k", "-H", "Authorization: Bearer "+token, metricsURL)
output, err = curlCmd.CombinedOutput()
require.NoError(t, err, "Error calling metrics endpoint: %s", string(output))
require.Contains(t, string(output), "200 OK", "Metrics endpoint did not return 200 OK")
return namespace
}

func stdoutAndCombined(cmd *exec.Cmd) ([]byte, []byte, error) {
var outOnly bytes.Buffer
var outAndErr bytes.Buffer
var outOnly, outAndErr bytes.Buffer
allWriter := io.MultiWriter(&outOnly, &outAndErr)
cmd.Stderr = &outAndErr
cmd.Stdout = allWriter
cmd.Stderr = &outAndErr
err := cmd.Run()
return outOnly.Bytes(), outAndErr.Bytes(), err
}
23 changes: 23 additions & 0 deletions test/utils/utils.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
package utils

import (
"os/exec"
"testing"
)

// FindK8sClient returns the first available Kubernetes CLI client from the system,
// It checks for the existence of each client by running `version --client`.
// If no suitable client is found, the function terminates the test with a failure.
func FindK8sClient(t *testing.T) string {
t.Logf("Finding kubectl client")
clients := []string{"kubectl", "oc"}
for _, c := range clients {
// Would prefer to use `command -v`, but even that may not be installed!
if err := exec.Command(c, "version", "--client").Run(); err == nil {
t.Logf("Using %q as k8s client", c)
return c
}
}
t.Fatal("k8s client not found")
return ""
}