Add tablespace functionality

Author: benjaminjb

internal/controller/postgrescluster/instance.go

@@ -1071,6 +1071,7 @@ func (r *Reconciler) reconcileInstance(
 		instanceCertificates *corev1.Secret
 		postgresDataVolume   *corev1.PersistentVolumeClaim
 		postgresWALVolume    *corev1.PersistentVolumeClaim
+		tablespaceVolumes    []*corev1.PersistentVolumeClaim
 	)
 
 	if err == nil {
@@ -1086,11 +1087,14 @@ func (r *Reconciler) reconcileInstance(
 	if err == nil {
 		postgresWALVolume, err = r.reconcilePostgresWALVolume(ctx, cluster, spec, instance, observed, clusterVolumes)
 	}
+	if err == nil {
+		tablespaceVolumes, err = r.reconcileTablespaceVolumes(ctx, cluster, spec, instance, clusterVolumes)
+	}
 	if err == nil {
 		postgres.InstancePod(
 			ctx, cluster, spec,
 			primaryCertificate, replicationCertSecretProjection(clusterReplicationSecret),
-			postgresDataVolume, postgresWALVolume,
+			postgresDataVolume, postgresWALVolume, tablespaceVolumes,
 			&instance.Spec.Template.Spec)
 
 		addPGBackRestToInstancePodSpec(

internal/controller/postgrescluster/postgres.go

@@ -561,6 +561,80 @@ func (r *Reconciler) reconcilePostgresDataVolume(
 	return pvc, err
 }
 
+// TODO think about multiple tablespace volumes
+
+// +kubebuilder:rbac:groups="",resources=persistentvolumeclaims,verbs=create;patch
+
+// reconcileTablespaceVolumes writes the PersistentVolumeClaims for instance's
+// tablespace data volumes.
+func (r *Reconciler) reconcileTablespaceVolumes(
+	ctx context.Context, cluster *v1beta1.PostgresCluster,
+	instanceSpec *v1beta1.PostgresInstanceSetSpec, instance *appsv1.StatefulSet,
+	clusterVolumes []corev1.PersistentVolumeClaim,
+) (tablespaceVolumes []*corev1.PersistentVolumeClaim, err error) {
+
+	if !util.DefaultMutableFeatureGate.Enabled(util.TablespaceVolumes) {
+		return
+	}
+
+	if instanceSpec.TablespaceVolumes == nil {
+		return
+	}
+
+	for _, vol := range instanceSpec.TablespaceVolumes {
+		labelMap := map[string]string{
+			naming.LabelCluster:     cluster.Name,
+			naming.LabelInstanceSet: instanceSpec.Name,
+			naming.LabelInstance:    instance.Name,
+			naming.LabelRole:        "tablespace",
+			naming.LabelData:        vol.Name,
+		}
+
+		var pvc *corev1.PersistentVolumeClaim
+		existingPVCName, err := getPGPVCName(labelMap, clusterVolumes)
+		if err != nil {
+			return nil, errors.WithStack(err)
+		}
+		if existingPVCName != "" {
+			pvc = &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{
+				Namespace: cluster.GetNamespace(),
+				Name:      existingPVCName,
+			}}
+		} else {
+			pvc = &corev1.PersistentVolumeClaim{ObjectMeta: naming.InstanceTablesapceDataVolume(instance, vol.Name)}
+		}
+
+		pvc.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("PersistentVolumeClaim"))
+
+		err = errors.WithStack(r.setControllerReference(cluster, pvc))
+
+		pvc.Annotations = naming.Merge(
+			cluster.Spec.Metadata.GetAnnotationsOrNil(),
+			instanceSpec.Metadata.GetAnnotationsOrNil())
+
+		pvc.Labels = naming.Merge(
+			cluster.Spec.Metadata.GetLabelsOrNil(),
+			instanceSpec.Metadata.GetLabelsOrNil(),
+			labelMap,
+		)
+
+		pvc.Spec = vol.DataVolumeClaimSpec
+
+		if err == nil {
+			err = r.handlePersistentVolumeClaimError(cluster,
+				errors.WithStack(r.apply(ctx, pvc)))
+		}
+
+		if err != nil {
+			return nil, err
+		}
+
+		tablespaceVolumes = append(tablespaceVolumes, pvc)
+	}
+
+	return
+}
+
 // +kubebuilder:rbac:groups="",resources=persistentvolumeclaims,verbs=get
 // +kubebuilder:rbac:groups="",resources=persistentvolumeclaims,verbs=create;delete;patch
 

internal/naming/names.go

@@ -325,6 +325,19 @@ func InstancePostgresDataVolume(instance *appsv1.StatefulSet) metav1.ObjectMeta
 	}
 }
 
+// TODO(benjaminjb): We need to consider shortening the PVC name
+// or at least need to document so that people can avoid too-long names
+// InstanceTablesapceDataVolume returns the ObjectMeta for the tablespace data
+// volume for instance.
+func InstanceTablesapceDataVolume(instance *appsv1.StatefulSet, tablespaceName string) metav1.ObjectMeta {
+	return metav1.ObjectMeta{
+		Namespace: instance.GetNamespace(),
+		Name: instance.GetName() +
+			"-" + tablespaceName +
+			"-tablespace",
+	}
+}
+
 // InstancePostgresWALVolume returns the ObjectMeta for the PostgreSQL WAL
 // volume for instance.
 func InstancePostgresWALVolume(instance *appsv1.StatefulSet) metav1.ObjectMeta {

internal/pgbackrest/reconcile.go

@@ -28,6 +28,7 @@ import (
 	"github.com/crunchydata/postgres-operator/internal/naming"
 	"github.com/crunchydata/postgres-operator/internal/pki"
 	"github.com/crunchydata/postgres-operator/internal/postgres"
+	"github.com/crunchydata/postgres-operator/internal/util"
 	"github.com/crunchydata/postgres-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1"
 )
 
@@ -302,6 +303,14 @@ func addServerContainerAndVolume(
 		postgres.DataVolumeMount().Name: postgres.DataVolumeMount(),
 		postgres.WALVolumeMount().Name:  postgres.WALVolumeMount(),
 	}
+	if util.DefaultMutableFeatureGate.Enabled(util.TablespaceVolumes) {
+		for _, instance := range cluster.Spec.InstanceSets {
+			for _, vol := range instance.TablespaceVolumes {
+				tablespaceVolumeMount := postgres.TablespaceVolumeMount(vol.Name)
+				postgresMounts[postgres.TablespaceVolumeMount(vol.Name).Name] = tablespaceVolumeMount
+			}
+		}
+	}
 	for i := range pod.Volumes {
 		if mount, ok := postgresMounts[pod.Volumes[i].Name]; ok {
 			container.VolumeMounts = append(container.VolumeMounts, mount)

internal/pgbackrest/reconcile_test.go

@@ -30,6 +30,7 @@ import (
 
 	"github.com/crunchydata/postgres-operator/internal/naming"
 	"github.com/crunchydata/postgres-operator/internal/pki"
+	"github.com/crunchydata/postgres-operator/internal/util"
 	"github.com/crunchydata/postgres-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1"
 )
 
@@ -522,6 +523,7 @@ func TestAddServerToInstancePod(t *testing.T) {
 	}
 
 	t.Run("CustomResources", func(t *testing.T) {
+		assert.NilError(t, util.AddAndSetFeatureGates(string(util.TablespaceVolumes+"=false")))
 		cluster := cluster.DeepCopy()
 		cluster.Spec.Backups.PGBackRest.Sidecars = &v1beta1.PGBackRestSidecars{
 			PGBackRest: &v1beta1.Sidecar{
@@ -647,6 +649,105 @@ func TestAddServerToInstancePod(t *testing.T) {
         name: instance-secret-name
 		`))
 	})
+
+	t.Run("AddTablespaces", func(t *testing.T) {
+		assert.NilError(t, util.AddAndSetFeatureGates(string(util.TablespaceVolumes+"=true")))
+		clusterWithTablespaces := cluster.DeepCopy()
+		clusterWithTablespaces.Spec.InstanceSets = []v1beta1.PostgresInstanceSetSpec{
+			{
+				TablespaceVolumes: []v1beta1.TablespaceVolume{
+					{Name: "trial"},
+					{Name: "castle"},
+				},
+			},
+		}
+
+		out := pod.DeepCopy()
+		out.Volumes = append(out.Volumes, corev1.Volume{Name: "trial"}, corev1.Volume{Name: "castle"})
+		AddServerToInstancePod(clusterWithTablespaces, out, "instance-secret-name")
+
+		// Only Containers and Volumes fields have changed.
+		assert.DeepEqual(t, pod, *out, cmpopts.IgnoreFields(pod, "Containers", "Volumes"))
+		assert.Assert(t, marshalMatches(out.Containers, `
+- name: database
+  resources: {}
+- name: other
+  resources: {}
+- command:
+  - pgbackrest
+  - server
+  livenessProbe:
+    exec:
+      command:
+      - pgbackrest
+      - server-ping
+  name: pgbackrest
+  resources: {}
+  securityContext:
+    allowPrivilegeEscalation: false
+    capabilities:
+      drop:
+      - ALL
+    privileged: false
+    readOnlyRootFilesystem: true
+    runAsNonRoot: true
+  volumeMounts:
+  - mountPath: /etc/pgbackrest/server
+    name: pgbackrest-server
+    readOnly: true
+  - mountPath: /pgdata
+    name: postgres-data
+  - mountPath: /pgwal
+    name: postgres-wal
+  - mountPath: /tablespaces/trial
+    name: trial
+  - mountPath: /tablespaces/castle
+    name: castle
+- command:
+  - bash
+  - -ceu
+  - --
+  - |-
+    monitor() {
+    exec {fd}<> <(:)
+    until read -r -t 5 -u "${fd}"; do
+      if
+        [ "${filename}" -nt "/proc/self/fd/${fd}" ] &&
+        pkill -HUP --exact --parent=0 pgbackrest
+      then
+        exec {fd}>&- && exec {fd}<> <(:)
+        stat --dereference --format='Loaded configuration dated %y' "${filename}"
+      elif
+        { [ "${directory}" -nt "/proc/self/fd/${fd}" ] ||
+          [ "${authority}" -nt "/proc/self/fd/${fd}" ]
+        } &&
+        pkill -HUP --exact --parent=0 pgbackrest
+      then
+        exec {fd}>&- && exec {fd}<> <(:)
+        stat --format='Loaded certificates dated %y' "${directory}"
+      fi
+    done
+    }; export directory="$1" authority="$2" filename="$3"; export -f monitor; exec -a "$0" bash -ceu monitor
+  - pgbackrest-config
+  - /etc/pgbackrest/server
+  - /etc/pgbackrest/conf.d/~postgres-operator/tls-ca.crt
+  - /etc/pgbackrest/conf.d/~postgres-operator_server.conf
+  name: pgbackrest-config
+  resources: {}
+  securityContext:
+    allowPrivilegeEscalation: false
+    capabilities:
+      drop:
+      - ALL
+    privileged: false
+    readOnlyRootFilesystem: true
+    runAsNonRoot: true
+  volumeMounts:
+  - mountPath: /etc/pgbackrest/server
+    name: pgbackrest-server
+    readOnly: true
+		`))
+	})
 }
 
 func TestAddServerToRepoPod(t *testing.T) {

internal/postgres/config.go

@@ -22,6 +22,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 
 	"github.com/crunchydata/postgres-operator/internal/naming"
+	"github.com/crunchydata/postgres-operator/internal/util"
 	"github.com/crunchydata/postgres-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1"
 )
 
@@ -62,6 +63,9 @@ safelink() (
 	// dataMountPath is where to mount the main data volume.
 	dataMountPath = "/pgdata"
 
+	// dataMountPath is where to mount the main data volume.
+	tablespaceMountPath = "/tablespaces"
+
 	// walMountPath is where to mount the optional WAL volume.
 	walMountPath = "/pgwal"
 
@@ -201,6 +205,30 @@ func startupCommand(
 	version := fmt.Sprint(cluster.Spec.PostgresVersion)
 	walDir := WALDirectory(cluster, instance)
 
+	// If the user requests tablespaces, we want to create and chmod the directories
+	tablespaceCmd := ""
+	if util.DefaultMutableFeatureGate.Enabled(util.TablespaceVolumes) && instance.TablespaceVolumes != nil {
+		// This command checks if a dir exists and if not, creates it;
+		// if the dir does exist, then we `recreate` it to make sure the owner is correct;
+		// but if the dir exists with the wrong owner and is not writeable, we error.
+		// This is the same behavior we use for the main PGDATA directory.
+		// The path for tablespaces volumes is /tablespaces/NAME/data
+		// -- the `data` path is added so that we can arrange the permissions.
+		checkInstallRecreateCmd := strings.Join([]string{
+			`if [[ ! -e "${tablespace_dir}" || -O "${tablespace_dir}" ]]; then`,
+			`install --directory --mode=0700 "${tablespace_dir}"`,
+			`elif [[ -w "${tablespace_dir}" && -g "${tablespace_dir}" ]]; then`,
+			`recreate "${tablespace_dir}" '0700'`,
+			`else (halt Permissions!); fi ||`,
+			`halt "$(permissions "${tablespace_dir}" ||:)"`,
+		}, "\n")
+
+		for _, tablespace := range instance.TablespaceVolumes {
+			tablespaceCmd = tablespaceCmd + "\ntablespace_dir=/tablespaces/" + tablespace.Name + "/data" + "\n" +
+				checkInstallRecreateCmd
+		}
+	}
+
 	args := []string{version, walDir, naming.PGBackRestPGDataLogPath}
 	script := strings.Join([]string{
 		`declare -r expected_major_version="$1" pgwal_directory="$2" pgbrLog_directory="$3"`,
@@ -287,6 +315,7 @@ func startupCommand(
 			naming.ReplicationCert, naming.ReplicationPrivateKey,
 			naming.ReplicationCACert),
 
+		tablespaceCmd,
 		// When the data directory is empty, there's nothing more to do.
 		`[ -f "${postgres_data_directory}/PG_VERSION" ] || exit 0`,
 

internal/postgres/reconcile.go

@@ -38,6 +38,11 @@ func DataVolumeMount() corev1.VolumeMount {
 	return corev1.VolumeMount{Name: "postgres-data", MountPath: dataMountPath}
 }
 
+// TablespaceVolumeMount returns the name and mount path of the PostgreSQL tablespace data volume.
+func TablespaceVolumeMount(tablespaceName string) corev1.VolumeMount {
+	return corev1.VolumeMount{Name: tablespaceName, MountPath: tablespaceMountPath + "/" + tablespaceName}
+}
+
 // WALVolumeMount returns the name and mount path of the PostgreSQL WAL volume.
 func WALVolumeMount() corev1.VolumeMount {
 	return corev1.VolumeMount{Name: "postgres-wal", MountPath: walMountPath}
@@ -68,6 +73,7 @@ func InstancePod(ctx context.Context,
 	inInstanceSpec *v1beta1.PostgresInstanceSetSpec,
 	inClusterCertificates, inClientCertificates *corev1.SecretProjection,
 	inDataVolume, inWALVolume *corev1.PersistentVolumeClaim,
+	inTablespaceVolumes []*corev1.PersistentVolumeClaim,
 	outInstancePod *corev1.PodSpec,
 ) {
 	certVolumeMount := corev1.VolumeMount{
@@ -218,6 +224,27 @@ func InstancePod(ctx context.Context,
 		downwardAPIVolume,
 	}
 
+	// If `TablespaceVolumes` FeatureGate is enabled, add any tablespace volumes to the pod, and
+	// add volumeMounts to the database and startup containers
+	if util.DefaultMutableFeatureGate.Enabled(util.TablespaceVolumes) &&
+		inTablespaceVolumes != nil {
+		for _, vol := range inTablespaceVolumes {
+			tablespaceVolumeMount := TablespaceVolumeMount(vol.Labels["postgres-operator.crunchydata.com/data"])
+			tablespaceVolume := corev1.Volume{
+				Name: tablespaceVolumeMount.Name,
+				VolumeSource: corev1.VolumeSource{
+					PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{
+						ClaimName: vol.Name,
+						ReadOnly:  false,
+					},
+				},
+			}
+			outInstancePod.Volumes = append(outInstancePod.Volumes, tablespaceVolume)
+			container.VolumeMounts = append(container.VolumeMounts, tablespaceVolumeMount)
+			startup.VolumeMounts = append(startup.VolumeMounts, tablespaceVolumeMount)
+		}
+	}
+
 	if len(inCluster.Spec.Config.Files) != 0 {
 		additionalConfigVolumeMount := AdditionalConfigVolumeMount()
 		additionalConfigVolume := corev1.Volume{Name: additionalConfigVolumeMount.Name}