Add basic KUTTL test

Adds a basic KUTTL test for creating a cluster with tablespaces;
Updates the github test to add the feature gate;

Issue: [sc-17759]

Author: benjaminjb

.github/workflows/test.yaml

@@ -130,6 +130,7 @@ jobs:
             --env 'RELATED_IMAGE_POSTGRES_13_GIS_3.1=registry.developers.crunchydata.com/crunchydata/crunchy-postgres-gis:ubi8-13.9-3.1-2' \
             --env 'RELATED_IMAGE_POSTGRES_14=registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi8-14.6-2' \
             --env 'RELATED_IMAGE_POSTGRES_14_GIS_3.1=registry.developers.crunchydata.com/crunchydata/crunchy-postgres-gis:ubi8-14.6-3.1-2' \
+            --env 'PGO_FEATURE_GATES=TablespaceVolumes=true' \
             --name 'postgres-operator' ubuntu \
             postgres-operator
       - name: Install kuttl

testing/kuttl/e2e/tablespace-enabled/00--cluster.yaml

@@ -0,0 +1,52 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: tablespace-script
+data:
+  script.sql: |
+    CREATE TABLESPACE trial OWNER "tablespace-enabled" LOCATION '/tablespaces/library/data';
+    CREATE TABLESPACE castle OWNER "tablespace-enabled" LOCATION '/tablespaces/user/data';
+---
+apiVersion: postgres-operator.crunchydata.com/v1beta1
+kind: PostgresCluster
+metadata:
+  name: tablespace-enabled
+spec:
+  databaseInitSQL:
+    name: tablespace-script
+    key: script.sql
+  postgresVersion: ${KUTTL_PG_VERSION}
+  instances:
+    - name: instance1
+      dataVolumeClaimSpec:
+        accessModes:
+        - "ReadWriteOnce"
+        resources:
+          requests:
+            storage: 1Gi
+      tablespaceVolumes:
+      - name: user
+        dataVolumeClaimSpec:
+          accessModes:
+          - "ReadWriteOnce"
+          resources:
+            requests:
+              storage: 1Gi
+      - name: library
+        dataVolumeClaimSpec:
+          accessModes:
+          - "ReadWriteOnce"
+          resources:
+            requests:
+              storage: 1Gi
+  backups:
+    pgbackrest:
+      repos:
+      - name: repo1
+        volume:
+          volumeClaimSpec:
+            accessModes:
+            - "ReadWriteOnce"
+            resources:
+              requests:
+                storage: 1Gi

testing/kuttl/e2e/tablespace-enabled/00-assert.yaml

@@ -0,0 +1,24 @@
+apiVersion: postgres-operator.crunchydata.com/v1beta1
+kind: PostgresCluster
+metadata:
+  name: tablespace-enabled
+status:
+  instances:
+    - name: instance1
+      readyReplicas: 1
+      replicas: 1
+      updatedReplicas: 1
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  labels:
+    postgres-operator.crunchydata.com/cluster: tablespace-enabled
+    postgres-operator.crunchydata.com/pgbackrest-backup: replica-create
+status:
+  succeeded: 1
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: tablespace-enabled-primary

testing/kuttl/e2e/tablespace-enabled/01--psql-connect.yaml

@@ -0,0 +1,43 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: psql-connect
+spec:
+  backoffLimit: 6
+  template:
+    spec:
+      restartPolicy: Never
+      containers:
+        - name: psql
+          image: ${KUTTL_PSQL_IMAGE}
+          env:
+            - name: PGURI
+              valueFrom: { secretKeyRef: { name: tablespace-enabled-pguser-tablespace-enabled, key: uri } }
+
+            # Do not wait indefinitely.
+            - { name: PGCONNECT_TIMEOUT, value: '5' }
+
+          # Note: the `$$$$` is reduced to `$$` by Kubernetes.
+          # - https://kubernetes.io/docs/tasks/inject-data-application/
+          command:
+            - psql
+            - $(PGURI)
+            - --quiet
+            - --echo-errors
+            - --set=ON_ERROR_STOP=1
+            - --command
+            - |
+              DO $$$$
+              DECLARE
+                tbsp_count integer;
+              BEGIN
+                SELECT COUNT(*) INTO tbsp_count FROM pg_tablespace WHERE spcname = 'trial';
+                ASSERT tbsp_count = 1, 'tablespace not found';
+                SELECT COUNT(*) INTO tbsp_count FROM pg_tablespace WHERE spcname = 'castle';
+                ASSERT tbsp_count = 1, 'tablespace not found';
+              END $$$$;
+            - --command
+            - |
+              CREATE SCHEMA "tablespace-enabled";
+              CREATE TABLE important (data) TABLESPACE trial AS VALUES ('treasure');
+              CREATE TABLE also_important (data) TABLESPACE castle AS VALUES ('treasure');

testing/kuttl/e2e/tablespace-enabled/01-assert.yaml

@@ -0,0 +1,6 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: psql-connect
+status:
+  succeeded: 1

testing/kuttl/e2e/tablespace-enabled/README.md

@@ -0,0 +1,6 @@
+# Tablespace Enabled
+
+**Note**: This series of tests depends on PGO being deployed with the `TablespaceVolume` feature gate enabled.
+
+00: Start a cluster with tablespace volumes and a configmap `databaseInitSQL` to create tablespaces with the non-superuser as owner
+01: Connect to the db, check that the tablespaces exist, and create tables in the tablespaces -- this would error out if the tablespace didn't exist