diff --git a/e2e-tests/tests/scaling/08-assert.yaml b/e2e-tests/tests/scaling/08-assert.yaml index 63ec28ceb..b62d7076e 100644 --- a/e2e-tests/tests/scaling/08-assert.yaml +++ b/e2e-tests/tests/scaling/08-assert.yaml @@ -1,6 +1,6 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert -timeout: 120 +timeout: 200 --- kind: StatefulSet apiVersion: apps/v1 diff --git a/pkg/controller/ps/controller.go b/pkg/controller/ps/controller.go index bc1f5703f..35cafe549 100644 --- a/pkg/controller/ps/controller.go +++ b/pkg/controller/ps/controller.go @@ -736,6 +736,20 @@ func (r *PerconaServerMySQLReconciler) reconcileOrchestrator(ctx context.Context return errors.Wrap(err, "get config map") } + cmData, err := orchestrator.ConfigMapData(cr) + if err != nil { + return errors.Wrap(err, "get ConfigMap data") + } + + configMap := orchestrator.ConfigMap(cr, cmData) + if !reflect.DeepEqual(cmap.Data, cmData) { + if err := k8s.EnsureObjectWithHash(ctx, r.Client, cr, configMap, r.Scheme); err != nil { + return errors.Wrap(err, "reconcile ConfigMap") + } + log.Info("ConfigMap updated", "name", configMap.Name, "data", configMap.Data) + return nil + } + existingNodes := make([]string, 0) if !k8serrors.IsNotFound(err) { cfg, ok := cmap.Data[orchestrator.ConfigFileName] @@ -758,15 +772,6 @@ func (r *PerconaServerMySQLReconciler) reconcileOrchestrator(ctx context.Context } } - cmData, err := orchestrator.ConfigMapData(cr) - if err != nil { - return errors.Wrap(err, "get ConfigMap data") - } - - if err := k8s.EnsureObjectWithHash(ctx, r.Client, cr, orchestrator.ConfigMap(cr, cmData), r.Scheme); err != nil { - return errors.Wrap(err, "reconcile ConfigMap") - } - component := orchestrator.Component(*cr) if err := k8s.EnsureComponent(ctx, r.Client, &component); err != nil { return errors.Wrap(err, "ensure component") @@ -900,7 +905,7 @@ func (r *PerconaServerMySQLReconciler) reconcileReplication(ctx context.Context, sts := &appsv1.StatefulSet{} // no need to set init image since we're just getting obj from API - if err := r.Get(ctx, client.ObjectKeyFromObject(orchestrator.StatefulSet(cr, "", "")), sts); err != nil { + if err := r.Get(ctx, client.ObjectKeyFromObject(orchestrator.StatefulSet(cr, "", "", "")), sts); err != nil { return client.IgnoreNotFound(err) } @@ -1185,7 +1190,7 @@ func (r *PerconaServerMySQLReconciler) cleanupOrchestrator(ctx context.Context, orcExposer := orchestrator.Exposer(*cr) if !cr.OrchestratorEnabled() { - if err := r.Delete(ctx, orchestrator.StatefulSet(cr, "", "")); err != nil && !k8serrors.IsNotFound(err) { + if err := r.Delete(ctx, orchestrator.StatefulSet(cr, "", "", "")); err != nil && !k8serrors.IsNotFound(err) { return errors.Wrap(err, "failed to delete orchestrator statefulset") } diff --git a/pkg/controller/ps/controller_test.go b/pkg/controller/ps/controller_test.go index e594ee734..22f1a89b0 100644 --- a/pkg/controller/ps/controller_test.go +++ b/pkg/controller/ps/controller_test.go @@ -26,6 +26,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" gs "github.com/onsi/gomega/gstruct" + "github.com/percona/percona-server-mysql-operator/pkg/version" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" policyv1 "k8s.io/api/policy/v1" @@ -344,7 +345,7 @@ var _ = Describe("PodDisruptionBudget", Ordered, func() { Context("Check default cluster", Ordered, func() { cr, err := readDefaultCR(crName, ns) - cr.Spec.CRVersion = "0.12.0" + cr.Spec.CRVersion = version.Version() It("should prepare reconciler", func() { r = reconciler() Expect(err).To(Succeed()) @@ -411,6 +412,9 @@ var _ = Describe("PodDisruptionBudget", Ordered, func() { It("should reconcile", func() { _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: crNamespacedName}) Expect(err).NotTo(HaveOccurred()) + // reconcile and a second time cause the orchestrator needs 2 cycles + _, err = r.Reconcile(ctx, ctrl.Request{NamespacedName: crNamespacedName}) + Expect(err).NotTo(HaveOccurred()) }) It("should check PodDisruptionBudget for MySQL", func() { pdb := &policyv1.PodDisruptionBudget{ diff --git a/pkg/orchestrator/component.go b/pkg/orchestrator/component.go index fd1e28584..5ca252db1 100644 --- a/pkg/orchestrator/component.go +++ b/pkg/orchestrator/component.go @@ -4,6 +4,8 @@ import ( "context" "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" "sigs.k8s.io/controller-runtime/pkg/client" apiv1 "github.com/percona/percona-server-mysql-operator/api/v1" @@ -44,10 +46,22 @@ func (c *Component) Object(ctx context.Context, cl client.Client) (client.Object return nil, errors.Wrap(err, "get init image") } + configMap := &corev1.ConfigMap{} + configMapName := client.ObjectKey{Name: ConfigMapName(cr), Namespace: cr.Namespace} + configHash := "" + if err := cl.Get(ctx, configMapName, configMap); err == nil { + configHash, err = k8s.ObjectHash(configMap) + if err != nil { + return nil, errors.Wrap(err, "calculate config map hash") + } + } else if !k8serrors.IsNotFound(err) { + return nil, errors.Wrap(err, "get config map") + } + tlsHash, err := k8s.GetTLSHash(ctx, cl, cr) if err != nil { return nil, errors.Wrapf(err, "get tls hash") } - return StatefulSet(cr, initImage, tlsHash), nil + return StatefulSet(cr, initImage, configHash, tlsHash), nil } diff --git a/pkg/orchestrator/orchestrator.go b/pkg/orchestrator/orchestrator.go index fcfa91f23..08f34cbcd 100644 --- a/pkg/orchestrator/orchestrator.go +++ b/pkg/orchestrator/orchestrator.go @@ -93,10 +93,6 @@ func FQDN(cr *apiv1.PerconaServerMySQL, idx int) string { return fmt.Sprintf("%s.%s.svc", PodName(cr, idx), cr.Namespace) } -func APIHost(cr *apiv1.PerconaServerMySQL) string { - return fmt.Sprintf("http://%s:%d", FQDN(cr, 0), defaultWebPort) -} - // Labels returns labels of orchestrator func Labels(cr *apiv1.PerconaServerMySQL) map[string]string { return util.SSMapMerge(cr.GlobalLabels(), cr.OrchestratorSpec().Labels, MatchLabels(cr)) @@ -106,12 +102,17 @@ func MatchLabels(cr *apiv1.PerconaServerMySQL) map[string]string { return cr.Labels(AppName, naming.ComponentOrchestrator) } -func StatefulSet(cr *apiv1.PerconaServerMySQL, initImage, tlsHash string) *appsv1.StatefulSet { +func StatefulSet(cr *apiv1.PerconaServerMySQL, initImage, configHash, tlsHash string) *appsv1.StatefulSet { selector := MatchLabels(cr) spec := cr.OrchestratorSpec() Replicas := spec.Size - annotations := make(map[string]string, 0) + annotations := make(map[string]string) + if cr.CompareVersion("0.12.0") >= 0 { + if configHash != "" { + annotations[string(naming.AnnotationConfigHash)] = configHash + } + } if tlsHash != "" { annotations[string(naming.AnnotationTLSHash)] = tlsHash } @@ -492,6 +493,14 @@ func orcConfig(cr *apiv1.PerconaServerMySQL) (string, error) { config := make(map[string]interface{}, 0) config["RaftNodes"] = RaftNodes(cr) + + if cr.CompareVersion("0.12.0") >= 0 { + config["RaftEnabledSingleNode"] = false + if cr.Spec.Orchestrator.Size == 1 { + config["RaftEnabledSingleNode"] = true + } + } + configJson, err := json.Marshal(config) if err != nil { return "", errors.Wrap(err, "marshal orchestrator raft nodes to json") @@ -501,7 +510,7 @@ func orcConfig(cr *apiv1.PerconaServerMySQL) (string, error) { } func ConfigMapData(cr *apiv1.PerconaServerMySQL) (map[string]string, error) { - cmData := make(map[string]string, 0) + cmData := make(map[string]string) config, err := orcConfig(cr) if err != nil { diff --git a/pkg/orchestrator/orchestrator_test.go b/pkg/orchestrator/orchestrator_test.go index 71b738ad2..f1aa5ab0f 100644 --- a/pkg/orchestrator/orchestrator_test.go +++ b/pkg/orchestrator/orchestrator_test.go @@ -14,9 +14,12 @@ import ( ) func TestStatefulSet(t *testing.T) { - const ns = "orc-ns" - const initImage = "init-image" - const tlsHash = "tls-hash" + const ( + ns = "orc-ns" + initImage = "init-image" + tlsHash = "tls-hash" + configHash = "config-hash" + ) cr := readDefaultCluster(t, "cluster", ns) if err := cr.CheckNSetDefaults(t.Context(), &platform.ServerVersion{ @@ -28,7 +31,7 @@ func TestStatefulSet(t *testing.T) { t.Run("object meta", func(t *testing.T) { cluster := cr.DeepCopy() - sts := StatefulSet(cluster, initImage, tlsHash) + sts := StatefulSet(cluster, initImage, configHash, tlsHash) assert.NotNil(t, sts) assert.Equal(t, "cluster-orc", sts.Name) @@ -46,7 +49,7 @@ func TestStatefulSet(t *testing.T) { t.Run("defaults", func(t *testing.T) { cluster := cr.DeepCopy() - sts := StatefulSet(cluster, initImage, tlsHash) + sts := StatefulSet(cluster, initImage, configHash, tlsHash) assert.Equal(t, int32(3), *sts.Spec.Replicas) initContainers := sts.Spec.Template.Spec.InitContainers @@ -54,7 +57,8 @@ func TestStatefulSet(t *testing.T) { assert.Equal(t, initImage, initContainers[0].Image) assert.Equal(t, map[string]string{ - "percona.com/last-applied-tls": tlsHash, + "percona.com/configuration-hash": configHash, + "percona.com/last-applied-tls": tlsHash, }, sts.Spec.Template.Annotations) }) @@ -62,19 +66,19 @@ func TestStatefulSet(t *testing.T) { cluster := cr.DeepCopy() cluster.Spec.Orchestrator.TerminationGracePeriodSeconds = nil - sts := StatefulSet(cluster, initImage, tlsHash) + sts := StatefulSet(cluster, initImage, configHash, tlsHash) assert.Equal(t, int64(600), *sts.Spec.Template.Spec.TerminationGracePeriodSeconds) cluster.Spec.Orchestrator.TerminationGracePeriodSeconds = ptr.To(int64(30)) - sts = StatefulSet(cluster, initImage, tlsHash) + sts = StatefulSet(cluster, initImage, configHash, tlsHash) assert.Equal(t, int64(30), *sts.Spec.Template.Spec.TerminationGracePeriodSeconds) }) t.Run("image pull secrets", func(t *testing.T) { cluster := cr.DeepCopy() - sts := StatefulSet(cluster, initImage, tlsHash) + sts := StatefulSet(cluster, initImage, configHash, tlsHash) assert.Equal(t, []corev1.LocalObjectReference(nil), sts.Spec.Template.Spec.ImagePullSecrets) imagePullSecrets := []corev1.LocalObjectReference{ @@ -87,26 +91,26 @@ func TestStatefulSet(t *testing.T) { } cluster.Spec.Orchestrator.ImagePullSecrets = imagePullSecrets - sts = StatefulSet(cluster, initImage, tlsHash) + sts = StatefulSet(cluster, initImage, configHash, tlsHash) assert.Equal(t, imagePullSecrets, sts.Spec.Template.Spec.ImagePullSecrets) }) t.Run("runtime class name", func(t *testing.T) { cluster := cr.DeepCopy() - sts := StatefulSet(cluster, initImage, tlsHash) + sts := StatefulSet(cluster, initImage, configHash, tlsHash) var e *string assert.Equal(t, e, sts.Spec.Template.Spec.RuntimeClassName) const runtimeClassName = "runtimeClassName" cluster.Spec.Orchestrator.RuntimeClassName = ptr.To(runtimeClassName) - sts = StatefulSet(cluster, initImage, tlsHash) + sts = StatefulSet(cluster, initImage, configHash, tlsHash) assert.Equal(t, runtimeClassName, *sts.Spec.Template.Spec.RuntimeClassName) }) t.Run("tolerations", func(t *testing.T) { cluster := cr.DeepCopy() - sts := StatefulSet(cluster, initImage, tlsHash) + sts := StatefulSet(cluster, initImage, configHash, tlsHash) assert.Equal(t, []corev1.Toleration(nil), sts.Spec.Template.Spec.Tolerations) tolerations := []corev1.Toleration{ @@ -120,7 +124,7 @@ func TestStatefulSet(t *testing.T) { } cluster.Spec.Orchestrator.Tolerations = tolerations - sts = StatefulSet(cluster, initImage, tlsHash) + sts = StatefulSet(cluster, initImage, configHash, tlsHash) assert.Equal(t, tolerations, sts.Spec.Template.Spec.Tolerations) }) }