From e3ea532e58f210d9c1bebc7bc448588b7b366bbd Mon Sep 17 00:00:00 2001 From: Alexander Greene Date: Tue, 14 Dec 2021 14:50:56 -0800 Subject: [PATCH 1/2] Introduce olmConfig controller (#2466) This commit introduces a controller for the olmConfig CRD. The olmConfig CRD will be used to configure olm's behavior on cluster. As of today, this CRD introduces the ability for customer to disable copied csvs for operators installed in allNamespace mode. When copied csv are disabled, an event will be created in the operators namespace signaling that it has no copied csvs and that users on the cluster may have difficulty identifying which operators are available in a given namespace. Signed-off-by: Alexander Greene Upstream-repository: operator-lifecycle-manager Upstream-commit: 52f368db86d24cb968f0439131a5788d0bcd5159 --- .../templates/0000_50_olm_02-olmconfig.yaml | 4 + .../pkg/controller/operators/olm/operator.go | 290 +++++++++++++++++- .../test/e2e/csv_e2e_test.go | 254 +++++++++++++++ .../pkg/controller/operators/olm/operator.go | 290 +++++++++++++++++- 4 files changed, 832 insertions(+), 6 deletions(-) create mode 100644 staging/operator-lifecycle-manager/deploy/chart/templates/0000_50_olm_02-olmconfig.yaml diff --git a/staging/operator-lifecycle-manager/deploy/chart/templates/0000_50_olm_02-olmconfig.yaml b/staging/operator-lifecycle-manager/deploy/chart/templates/0000_50_olm_02-olmconfig.yaml new file mode 100644 index 0000000000..ad881b7be7 --- /dev/null +++ b/staging/operator-lifecycle-manager/deploy/chart/templates/0000_50_olm_02-olmconfig.yaml @@ -0,0 +1,4 @@ +apiVersion: operators.coreos.com/v1 +kind: OLMConfig +metadata: + name: cluster diff --git a/staging/operator-lifecycle-manager/pkg/controller/operators/olm/operator.go b/staging/operator-lifecycle-manager/pkg/controller/operators/olm/operator.go index 8e0f175fb9..2ab92f2a65 100644 --- a/staging/operator-lifecycle-manager/pkg/controller/operators/olm/operator.go +++ b/staging/operator-lifecycle-manager/pkg/controller/operators/olm/operator.go @@ -14,9 +14,11 @@ import ( rbacv1 "k8s.io/api/rbac/v1" extinf "k8s.io/apiextensions-apiserver/pkg/client/informers/externalversions" k8serrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/selection" utilclock "k8s.io/apimachinery/pkg/util/clock" utilerrors "k8s.io/apimachinery/pkg/util/errors" utilruntime "k8s.io/apimachinery/pkg/util/runtime" @@ -68,6 +70,7 @@ type Operator struct { copiedCSVLister operatorsv1alpha1listers.ClusterServiceVersionLister ogQueueSet *queueinformer.ResourceQueueSet csvQueueSet *queueinformer.ResourceQueueSet + olmConfigQueue workqueue.RateLimitingInterface csvCopyQueueSet *queueinformer.ResourceQueueSet copiedCSVGCQueueSet *queueinformer.ResourceQueueSet objGCQueueSet *queueinformer.ResourceQueueSet @@ -124,6 +127,7 @@ func newOperatorWithConfig(ctx context.Context, config *operatorConfig) (*Operat client: config.externalClient, ogQueueSet: queueinformer.NewEmptyResourceQueueSet(), csvQueueSet: queueinformer.NewEmptyResourceQueueSet(), + olmConfigQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "olmConfig"), csvCopyQueueSet: queueinformer.NewEmptyResourceQueueSet(), copiedCSVGCQueueSet: queueinformer.NewEmptyResourceQueueSet(), objGCQueueSet: queueinformer.NewEmptyResourceQueueSet(), @@ -433,6 +437,26 @@ func newOperatorWithConfig(ctx context.Context, config *operatorConfig) (*Operat return nil, err } + // Register QueueInformer for olmConfig + olmConfigInformer := externalversions.NewSharedInformerFactoryWithOptions( + op.client, + config.resyncPeriod(), + ).Operators().V1().OLMConfigs().Informer() + olmConfigQueueInformer, err := queueinformer.NewQueueInformer( + ctx, + queueinformer.WithInformer(olmConfigInformer), + queueinformer.WithLogger(op.logger), + queueinformer.WithQueue(op.olmConfigQueue), + queueinformer.WithIndexer(olmConfigInformer.GetIndexer()), + queueinformer.WithSyncer(queueinformer.LegacySyncHandler(op.syncOLMConfig).ToSyncer()), + ) + if err != nil { + return nil, err + } + if err := op.RegisterQueueInformer(olmConfigQueueInformer); err != nil { + return nil, err + } + k8sInformerFactory := informers.NewSharedInformerFactory(op.opClient.KubernetesInterface(), config.resyncPeriod()) clusterRoleInformer := k8sInformerFactory.Rbac().V1().ClusterRoles() op.lister.RbacV1().RegisterClusterRoleLister(clusterRoleInformer.Lister()) @@ -1194,6 +1218,127 @@ func (a *Operator) syncClusterServiceVersion(obj interface{}) (syncError error) return } +func (a *Operator) allNamespaceOperatorGroups() ([]*v1.OperatorGroup, error) { + operatorGroups, err := a.lister.OperatorsV1().OperatorGroupLister().List(labels.Everything()) + if err != nil { + return nil, err + } + + result := []*v1.OperatorGroup{} + for _, operatorGroup := range operatorGroups { + if NewNamespaceSet(operatorGroup.Status.Namespaces).IsAllNamespaces() { + result = append(result, operatorGroup) + } + } + return result, nil +} + +func (a *Operator) syncOLMConfig(obj interface{}) (syncError error) { + a.logger.Info("Processing olmConfig") + olmConfig, ok := obj.(*v1.OLMConfig) + if !ok { + return fmt.Errorf("casting OLMConfig failed") + } + + // Generate an array of allNamespace OperatorGroups + allNSOperatorGroups, err := a.allNamespaceOperatorGroups() + if err != nil { + return err + } + + nonCopiedCSVRequirement, err := labels.NewRequirement(v1alpha1.CopiedLabelKey, selection.DoesNotExist, []string{}) + if err != nil { + return err + } + + csvIsRequeued := false + for _, og := range allNSOperatorGroups { + // Get all copied CSVs owned by this operatorGroup + copiedCSVRequirement, err := labels.NewRequirement(v1alpha1.CopiedLabelKey, selection.Equals, []string{og.GetNamespace()}) + if err != nil { + return err + } + + copiedCSVs, err := a.copiedCSVLister.List(labels.NewSelector().Add(*copiedCSVRequirement)) + if err != nil { + return err + } + + // Filter to unique copies + uniqueCopiedCSVs := map[string]struct{}{} + for _, copiedCSV := range copiedCSVs { + uniqueCopiedCSVs[copiedCSV.GetName()] = struct{}{} + } + + csvs, err := a.lister.OperatorsV1alpha1().ClusterServiceVersionLister().ClusterServiceVersions(og.GetNamespace()).List(labels.NewSelector().Add(*nonCopiedCSVRequirement)) + if err != nil { + return err + } + + for _, csv := range csvs { + // If the correct number of copied CSVs were found, continue + if _, ok := uniqueCopiedCSVs[csv.GetName()]; ok == olmConfig.CopiedCSVsAreEnabled() { + continue + } + + if err := a.csvQueueSet.Requeue(csv.GetNamespace(), csv.GetName()); err != nil { + a.logger.WithError(err).Warn("unable to requeue") + } + csvIsRequeued = true + } + } + + // Update the olmConfig status if it has changed. + condition := getCopiedCSVsCondition(!olmConfig.CopiedCSVsAreEnabled(), csvIsRequeued) + if !isStatusConditionPresentAndAreTypeReasonMessageStatusEqual(olmConfig.Status.Conditions, condition) { + meta.SetStatusCondition(&olmConfig.Status.Conditions, condition) + if _, err := a.client.OperatorsV1().OLMConfigs().UpdateStatus(context.TODO(), olmConfig, metav1.UpdateOptions{}); err != nil { + return err + } + } + + return nil +} + +func isStatusConditionPresentAndAreTypeReasonMessageStatusEqual(conditions []metav1.Condition, condition metav1.Condition) bool { + foundCondition := meta.FindStatusCondition(conditions, condition.Type) + if foundCondition == nil { + return false + } + return foundCondition.Type == condition.Type && + foundCondition.Reason == condition.Reason && + foundCondition.Message == condition.Message && + foundCondition.Status == condition.Status +} + +func getCopiedCSVsCondition(isDisabled, csvIsRequeued bool) metav1.Condition { + condition := metav1.Condition{ + Type: v1.DisabledCopiedCSVsConditionType, + LastTransitionTime: metav1.Now(), + Status: metav1.ConditionFalse, + } + if !isDisabled { + condition.Reason = "CopiedCSVsEnabled" + condition.Message = "Copied CSVs are enabled and present accross the cluster" + if csvIsRequeued { + condition.Message = "Copied CSVs are enabled and at least one copied CSVs is missing" + } + return condition + } + + if csvIsRequeued { + condition.Reason = "CopiedCSVsFound" + condition.Message = "Copied CSVs are disabled and at least one copied CSV was found for an operator installed in AllNamespace mode" + return condition + } + + condition.Status = metav1.ConditionTrue + condition.Reason = "NoCopiedCSVsFound" + condition.Message = "Copied CSVs are disabled and none were found for operators installed in AllNamespace mode" + + return condition +} + func (a *Operator) syncCopyCSV(obj interface{}) (syncError error) { clusterServiceVersion, ok := obj.(*v1alpha1.ClusterServiceVersion) if !ok { @@ -1201,6 +1346,15 @@ func (a *Operator) syncCopyCSV(obj interface{}) (syncError error) { return fmt.Errorf("casting ClusterServiceVersion failed") } + olmConfig, err := a.client.OperatorsV1().OLMConfigs().Get(context.TODO(), "cluster", metav1.GetOptions{}) + if err != nil && !k8serrors.IsNotFound(err) { + return err + } + + if err == nil { + go a.olmConfigQueue.AddAfter(olmConfig, time.Second*5) + } + logger := a.logger.WithFields(logrus.Fields{ "id": queueinformer.NewLoopID(), "csv": clusterServiceVersion.GetName(), @@ -1222,15 +1376,145 @@ func (a *Operator) syncCopyCSV(obj interface{}) (syncError error) { "targetNamespaces": strings.Join(operatorGroup.Status.Namespaces, ","), }).Debug("copying csv to targets") + copiedCSVsAreEnabled, err := a.copiedCSVsAreEnabled() + if err != nil { + return err + } + // Check if we need to do any copying / annotation for the operatorgroup - if err := a.ensureCSVsInNamespaces(clusterServiceVersion, operatorGroup, NewNamespaceSet(operatorGroup.Status.Namespaces)); err != nil { - logger.WithError(err).Info("couldn't copy CSV to target namespaces") - syncError = err + namespaceSet := NewNamespaceSet(operatorGroup.Status.Namespaces) + if copiedCSVsAreEnabled || !namespaceSet.IsAllNamespaces() { + if err := a.ensureCSVsInNamespaces(clusterServiceVersion, operatorGroup, namespaceSet); err != nil { + logger.WithError(err).Info("couldn't copy CSV to target namespaces") + syncError = err + } + + // If the CSV was installed in AllNamespace mode, remove any "CSV Copying Disabled" events + // in which the related object's name, namespace, and uid match the given CSV's. + if namespaceSet.IsAllNamespaces() { + if err := a.deleteCSVCopyingDisabledEvent(clusterServiceVersion); err != nil { + return err + } + } + return + } + + requirement, err := labels.NewRequirement(v1alpha1.CopiedLabelKey, selection.Equals, []string{clusterServiceVersion.Namespace}) + if err != nil { + return err + } + + copiedCSVs, err := a.copiedCSVLister.List(labels.NewSelector().Add(*requirement)) + if err != nil { + return err + } + + for _, copiedCSV := range copiedCSVs { + err := a.client.OperatorsV1alpha1().ClusterServiceVersions(copiedCSV.Namespace).Delete(context.TODO(), copiedCSV.Name, metav1.DeleteOptions{}) + if err != nil && !k8serrors.IsNotFound(err) { + return err + } + } + + if err := a.createCSVCopyingDisabledEvent(clusterServiceVersion); err != nil { + return err } return } +// copiedCSVsAreEnabled determines if csv copying is enabled for OLM. +// +// This method will first attempt to get the "cluster" olmConfig resource, +// if any error other than "IsNotFound" is encountered, false and the error +// will be returned. +// +// If the "cluster" olmConfig resource is found, the value of +// olmConfig.spec.features.disableCopiedCSVs will be returned along with a +// nil error. +// +// If the "cluster" olmConfig resource is not found, true will be returned +// without an error. +func (a *Operator) copiedCSVsAreEnabled() (bool, error) { + olmConfig, err := a.client.OperatorsV1().OLMConfigs().Get(context.TODO(), "cluster", metav1.GetOptions{}) + if err != nil { + // Default to true if olmConfig singleton cannot be found + if k8serrors.IsNotFound(err) { + return true, nil + } + // If there was an error that wasn't an IsNotFound, return the error + return false, err + } + + // If there was no error, return value based on olmConfig singleton + return olmConfig.CopiedCSVsAreEnabled(), nil +} + +func (a *Operator) getCopiedCSVDisabledEventsForCSV(csv *v1alpha1.ClusterServiceVersion) ([]corev1.Event, error) { + result := []corev1.Event{} + if csv == nil { + return result, nil + } + + events, err := a.opClient.KubernetesInterface().CoreV1().Events(csv.GetNamespace()).List(context.TODO(), metav1.ListOptions{}) + if err != nil { + return nil, err + } + + for _, event := range events.Items { + if event.InvolvedObject.Namespace == csv.GetNamespace() && + event.InvolvedObject.Name == csv.GetName() && + event.InvolvedObject.UID == csv.GetUID() && + event.Reason == v1.DisabledCopiedCSVsConditionType { + result = append(result, event) + } + } + + return result, nil +} + +func (a *Operator) deleteCSVCopyingDisabledEvent(csv *v1alpha1.ClusterServiceVersion) error { + events, err := a.getCopiedCSVDisabledEventsForCSV(csv) + if err != nil { + return err + } + + // Remove existing events. + return a.deleteEvents(events) +} + +func (a *Operator) deleteEvents(events []corev1.Event) error { + for _, event := range events { + err := a.opClient.KubernetesInterface().EventsV1().Events(event.GetNamespace()).Delete(context.TODO(), event.GetName(), metav1.DeleteOptions{}) + if err != nil && !k8serrors.IsNotFound(err) { + return err + } + } + return nil +} + +func (a *Operator) createCSVCopyingDisabledEvent(csv *v1alpha1.ClusterServiceVersion) error { + events, err := a.getCopiedCSVDisabledEventsForCSV(csv) + if err != nil { + return err + } + + if len(events) == 1 { + return nil + } + + // Remove existing events. + if len(events) > 1 { + if err := a.deleteEvents(events); err != nil { + return err + } + } + + a.recorder.Eventf(csv, corev1.EventTypeWarning, v1.DisabledCopiedCSVsConditionType, "CSV copying disabled for %s/%s", csv.GetNamespace(), csv.GetName()) + + return nil +} + func (a *Operator) syncGcCsv(obj interface{}) (syncError error) { clusterServiceVersion, ok := obj.(*v1alpha1.ClusterServiceVersion) if !ok { diff --git a/staging/operator-lifecycle-manager/test/e2e/csv_e2e_test.go b/staging/operator-lifecycle-manager/test/e2e/csv_e2e_test.go index 2f7bf6ca87..6d3e95ee8c 100644 --- a/staging/operator-lifecycle-manager/test/e2e/csv_e2e_test.go +++ b/staging/operator-lifecycle-manager/test/e2e/csv_e2e_test.go @@ -18,9 +18,12 @@ import ( apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/api/equality" k8serrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" k8slabels "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/selection" + apitypes "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/diff" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/wait" @@ -4171,6 +4174,253 @@ var _ = Describe("ClusterServiceVersion", func() { }) }) +var _ = Describe("Disabling copied CSVs", func() { + // Define namespace, operatorGroup, and csv upfront + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: genName("csv-toggle-test-"), + }, + } + + operatorGroup := operatorsv1.OperatorGroup{ + ObjectMeta: metav1.ObjectMeta{ + Name: genName("csv-toggle-test-"), + Namespace: ns.GetName(), + }, + } + + csv := operatorsv1alpha1.ClusterServiceVersion{ + ObjectMeta: metav1.ObjectMeta{ + Name: genName("csv-toggle-test-"), + Namespace: ns.GetName(), + }, + Spec: operatorsv1alpha1.ClusterServiceVersionSpec{ + InstallStrategy: newNginxInstallStrategy(genName("csv-toggle-test-"), nil, nil), + InstallModes: []operatorsv1alpha1.InstallMode{ + { + Type: operatorsv1alpha1.InstallModeTypeAllNamespaces, + Supported: true, + }, + }, + }, + } + + When("an operator is installed in AllNamespace mode", func() { + BeforeEach(func() { + Eventually(func() error { + if err := ctx.Ctx().Client().Create(context.TODO(), ns); err != nil && !k8serrors.IsAlreadyExists(err) { + ctx.Ctx().Logf("Unable to create ns: %v", err) + return err + } + + if err := ctx.Ctx().Client().Create(context.TODO(), &operatorGroup); err != nil && !k8serrors.IsAlreadyExists(err) { + ctx.Ctx().Logf("Unable to create og: %v", err) + return err + } + + if err := ctx.Ctx().Client().Create(context.TODO(), &csv); err != nil && !k8serrors.IsAlreadyExists(err) { + ctx.Ctx().Logf("Unable to create csv: %v", err) + return err + } + + return nil + }).Should(Succeed()) + }) + + It("should have Copied CSVs in all other namespaces", func() { + Eventually(func() error { + requirement, err := k8slabels.NewRequirement(operatorsv1alpha1.CopiedLabelKey, selection.Equals, []string{csv.GetNamespace()}) + if err != nil { + return err + } + + var copiedCSVs operatorsv1alpha1.ClusterServiceVersionList + err = ctx.Ctx().Client().List(context.TODO(), &copiedCSVs, &client.ListOptions{ + LabelSelector: k8slabels.NewSelector().Add(*requirement), + }) + if err != nil { + return err + } + + var namespaces corev1.NamespaceList + if err := ctx.Ctx().Client().List(context.TODO(), &namespaces, &client.ListOptions{}); err != nil { + return err + } + + if len(namespaces.Items)-1 != len(copiedCSVs.Items) { + return fmt.Errorf("%d copied CSVs found, expected %d", len(copiedCSVs.Items), len(namespaces.Items)-1) + } + + return nil + }).Should(Succeed()) + }) + }) + + When("Copied CSVs are disabled", func() { + BeforeEach(func() { + Eventually(func() error { + var olmConfig operatorsv1.OLMConfig + if err := ctx.Ctx().Client().Get(context.TODO(), apitypes.NamespacedName{Name: "cluster"}, &olmConfig); err != nil { + ctx.Ctx().Logf("Error getting olmConfig %v", err) + return err + } + + // Exit early if copied CSVs are disabled. + if !olmConfig.CopiedCSVsAreEnabled() { + return nil + } + + olmConfig.Spec = operatorsv1.OLMConfigSpec{ + Features: &operatorsv1.Features{ + DisableCopiedCSVs: getPointer(true), + }, + } + + if err := ctx.Ctx().Client().Update(context.TODO(), &olmConfig); err != nil { + ctx.Ctx().Logf("Error setting olmConfig %v", err) + return err + } + + return nil + }).Should(Succeed()) + }) + + It("should not have any copied CSVs", func() { + Eventually(func() error { + requirement, err := k8slabels.NewRequirement(operatorsv1alpha1.CopiedLabelKey, selection.Equals, []string{csv.GetNamespace()}) + if err != nil { + return err + } + + var copiedCSVs operatorsv1alpha1.ClusterServiceVersionList + err = ctx.Ctx().Client().List(context.TODO(), &copiedCSVs, &client.ListOptions{ + LabelSelector: k8slabels.NewSelector().Add(*requirement), + }) + if err != nil { + return err + } + + if numCSVs := len(copiedCSVs.Items); numCSVs != 0 { + return fmt.Errorf("Found %d copied CSVs, should be 0", numCSVs) + } + return nil + }).Should(Succeed()) + }) + + It("should be reflected in the olmConfig.Status.Condition array that the expected number of copied CSVs exist", func() { + Eventually(func() error { + var olmConfig operatorsv1.OLMConfig + if err := ctx.Ctx().Client().Get(context.TODO(), apitypes.NamespacedName{Name: "cluster"}, &olmConfig); err != nil { + return err + } + + foundCondition := meta.FindStatusCondition(olmConfig.Status.Conditions, operatorsv1.DisabledCopiedCSVsConditionType) + if foundCondition == nil { + return fmt.Errorf("%s condition not found", operatorsv1.DisabledCopiedCSVsConditionType) + } + + expectedCondition := metav1.Condition{ + Reason: "NoCopiedCSVsFound", + Message: "Copied CSVs are disabled and none were found for operators installed in AllNamespace mode", + Status: metav1.ConditionTrue, + } + + if foundCondition.Reason != expectedCondition.Reason || + foundCondition.Message != expectedCondition.Message || + foundCondition.Status != expectedCondition.Status { + return fmt.Errorf("condition does not have expected reason, message, and status. Expected %v, got %v", expectedCondition, foundCondition) + } + + return nil + }).Should(Succeed()) + }) + }) + + When("Copied CSVs are toggled back on", func() { + BeforeEach(func() { + Eventually(func() error { + var olmConfig operatorsv1.OLMConfig + if err := ctx.Ctx().Client().Get(context.TODO(), apitypes.NamespacedName{Name: "cluster"}, &olmConfig); err != nil { + return err + } + + // Exit early if copied CSVs are enabled. + if olmConfig.CopiedCSVsAreEnabled() { + return nil + } + + olmConfig.Spec = operatorsv1.OLMConfigSpec{ + Features: &operatorsv1.Features{ + DisableCopiedCSVs: getPointer(false), + }, + } + + if err := ctx.Ctx().Client().Update(context.TODO(), &olmConfig); err != nil { + return err + } + + return nil + }).Should(Succeed()) + }) + + It("should have copied CSVs in all other Namespaces", func() { + Eventually(func() error { + // find copied csvs... + requirement, err := k8slabels.NewRequirement(operatorsv1alpha1.CopiedLabelKey, selection.Equals, []string{csv.GetNamespace()}) + if err != nil { + return err + } + + var copiedCSVs operatorsv1alpha1.ClusterServiceVersionList + err = ctx.Ctx().Client().List(context.TODO(), &copiedCSVs, &client.ListOptions{ + LabelSelector: k8slabels.NewSelector().Add(*requirement), + }) + if err != nil { + return err + } + + var namespaces corev1.NamespaceList + if err := ctx.Ctx().Client().List(context.TODO(), &namespaces, &client.ListOptions{}); err != nil { + return err + } + + if len(namespaces.Items)-1 != len(copiedCSVs.Items) { + return fmt.Errorf("%d copied CSVs found, expected %d", len(copiedCSVs.Items), len(namespaces.Items)-1) + } + + return nil + }).Should(Succeed()) + }) + + It("should be reflected in the olmConfig.Status.Condition array that the expected number of copied CSVs exist", func() { + Eventually(func() error { + var olmConfig operatorsv1.OLMConfig + if err := ctx.Ctx().Client().Get(context.TODO(), apitypes.NamespacedName{Name: "cluster"}, &olmConfig); err != nil { + return err + } + foundCondition := meta.FindStatusCondition(olmConfig.Status.Conditions, operatorsv1.DisabledCopiedCSVsConditionType) + if foundCondition == nil { + return fmt.Errorf("%s condition not found", operatorsv1.DisabledCopiedCSVsConditionType) + } + + expectedCondition := metav1.Condition{ + Reason: "CopiedCSVsEnabled", + Message: "Copied CSVs are enabled and present accross the cluster", + Status: metav1.ConditionFalse, + } + + if foundCondition.Reason != expectedCondition.Reason || + foundCondition.Message != expectedCondition.Message || + foundCondition.Status != expectedCondition.Status { + return fmt.Errorf("condition does not have expected reason, message, and status. Expected %v, got %v", expectedCondition, foundCondition) + } + + return nil + }).Should(Succeed()) + }) + }) +}) + var singleInstance = int32(1) type cleanupFunc func() @@ -4219,6 +4469,10 @@ func buildCSVCleanupFunc(c operatorclient.ClientInterface, crc versioned.Interfa } } +func getPointer(b bool) *bool { + return &b +} + func createCSV(c operatorclient.ClientInterface, crc versioned.Interface, csv operatorsv1alpha1.ClusterServiceVersion, namespace string, cleanupCRDs, cleanupAPIServices bool) (cleanupFunc, error) { csv.Kind = operatorsv1alpha1.ClusterServiceVersionKind csv.APIVersion = operatorsv1alpha1.SchemeGroupVersion.String() diff --git a/vendor/github.com/operator-framework/operator-lifecycle-manager/pkg/controller/operators/olm/operator.go b/vendor/github.com/operator-framework/operator-lifecycle-manager/pkg/controller/operators/olm/operator.go index 8e0f175fb9..2ab92f2a65 100644 --- a/vendor/github.com/operator-framework/operator-lifecycle-manager/pkg/controller/operators/olm/operator.go +++ b/vendor/github.com/operator-framework/operator-lifecycle-manager/pkg/controller/operators/olm/operator.go @@ -14,9 +14,11 @@ import ( rbacv1 "k8s.io/api/rbac/v1" extinf "k8s.io/apiextensions-apiserver/pkg/client/informers/externalversions" k8serrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/selection" utilclock "k8s.io/apimachinery/pkg/util/clock" utilerrors "k8s.io/apimachinery/pkg/util/errors" utilruntime "k8s.io/apimachinery/pkg/util/runtime" @@ -68,6 +70,7 @@ type Operator struct { copiedCSVLister operatorsv1alpha1listers.ClusterServiceVersionLister ogQueueSet *queueinformer.ResourceQueueSet csvQueueSet *queueinformer.ResourceQueueSet + olmConfigQueue workqueue.RateLimitingInterface csvCopyQueueSet *queueinformer.ResourceQueueSet copiedCSVGCQueueSet *queueinformer.ResourceQueueSet objGCQueueSet *queueinformer.ResourceQueueSet @@ -124,6 +127,7 @@ func newOperatorWithConfig(ctx context.Context, config *operatorConfig) (*Operat client: config.externalClient, ogQueueSet: queueinformer.NewEmptyResourceQueueSet(), csvQueueSet: queueinformer.NewEmptyResourceQueueSet(), + olmConfigQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "olmConfig"), csvCopyQueueSet: queueinformer.NewEmptyResourceQueueSet(), copiedCSVGCQueueSet: queueinformer.NewEmptyResourceQueueSet(), objGCQueueSet: queueinformer.NewEmptyResourceQueueSet(), @@ -433,6 +437,26 @@ func newOperatorWithConfig(ctx context.Context, config *operatorConfig) (*Operat return nil, err } + // Register QueueInformer for olmConfig + olmConfigInformer := externalversions.NewSharedInformerFactoryWithOptions( + op.client, + config.resyncPeriod(), + ).Operators().V1().OLMConfigs().Informer() + olmConfigQueueInformer, err := queueinformer.NewQueueInformer( + ctx, + queueinformer.WithInformer(olmConfigInformer), + queueinformer.WithLogger(op.logger), + queueinformer.WithQueue(op.olmConfigQueue), + queueinformer.WithIndexer(olmConfigInformer.GetIndexer()), + queueinformer.WithSyncer(queueinformer.LegacySyncHandler(op.syncOLMConfig).ToSyncer()), + ) + if err != nil { + return nil, err + } + if err := op.RegisterQueueInformer(olmConfigQueueInformer); err != nil { + return nil, err + } + k8sInformerFactory := informers.NewSharedInformerFactory(op.opClient.KubernetesInterface(), config.resyncPeriod()) clusterRoleInformer := k8sInformerFactory.Rbac().V1().ClusterRoles() op.lister.RbacV1().RegisterClusterRoleLister(clusterRoleInformer.Lister()) @@ -1194,6 +1218,127 @@ func (a *Operator) syncClusterServiceVersion(obj interface{}) (syncError error) return } +func (a *Operator) allNamespaceOperatorGroups() ([]*v1.OperatorGroup, error) { + operatorGroups, err := a.lister.OperatorsV1().OperatorGroupLister().List(labels.Everything()) + if err != nil { + return nil, err + } + + result := []*v1.OperatorGroup{} + for _, operatorGroup := range operatorGroups { + if NewNamespaceSet(operatorGroup.Status.Namespaces).IsAllNamespaces() { + result = append(result, operatorGroup) + } + } + return result, nil +} + +func (a *Operator) syncOLMConfig(obj interface{}) (syncError error) { + a.logger.Info("Processing olmConfig") + olmConfig, ok := obj.(*v1.OLMConfig) + if !ok { + return fmt.Errorf("casting OLMConfig failed") + } + + // Generate an array of allNamespace OperatorGroups + allNSOperatorGroups, err := a.allNamespaceOperatorGroups() + if err != nil { + return err + } + + nonCopiedCSVRequirement, err := labels.NewRequirement(v1alpha1.CopiedLabelKey, selection.DoesNotExist, []string{}) + if err != nil { + return err + } + + csvIsRequeued := false + for _, og := range allNSOperatorGroups { + // Get all copied CSVs owned by this operatorGroup + copiedCSVRequirement, err := labels.NewRequirement(v1alpha1.CopiedLabelKey, selection.Equals, []string{og.GetNamespace()}) + if err != nil { + return err + } + + copiedCSVs, err := a.copiedCSVLister.List(labels.NewSelector().Add(*copiedCSVRequirement)) + if err != nil { + return err + } + + // Filter to unique copies + uniqueCopiedCSVs := map[string]struct{}{} + for _, copiedCSV := range copiedCSVs { + uniqueCopiedCSVs[copiedCSV.GetName()] = struct{}{} + } + + csvs, err := a.lister.OperatorsV1alpha1().ClusterServiceVersionLister().ClusterServiceVersions(og.GetNamespace()).List(labels.NewSelector().Add(*nonCopiedCSVRequirement)) + if err != nil { + return err + } + + for _, csv := range csvs { + // If the correct number of copied CSVs were found, continue + if _, ok := uniqueCopiedCSVs[csv.GetName()]; ok == olmConfig.CopiedCSVsAreEnabled() { + continue + } + + if err := a.csvQueueSet.Requeue(csv.GetNamespace(), csv.GetName()); err != nil { + a.logger.WithError(err).Warn("unable to requeue") + } + csvIsRequeued = true + } + } + + // Update the olmConfig status if it has changed. + condition := getCopiedCSVsCondition(!olmConfig.CopiedCSVsAreEnabled(), csvIsRequeued) + if !isStatusConditionPresentAndAreTypeReasonMessageStatusEqual(olmConfig.Status.Conditions, condition) { + meta.SetStatusCondition(&olmConfig.Status.Conditions, condition) + if _, err := a.client.OperatorsV1().OLMConfigs().UpdateStatus(context.TODO(), olmConfig, metav1.UpdateOptions{}); err != nil { + return err + } + } + + return nil +} + +func isStatusConditionPresentAndAreTypeReasonMessageStatusEqual(conditions []metav1.Condition, condition metav1.Condition) bool { + foundCondition := meta.FindStatusCondition(conditions, condition.Type) + if foundCondition == nil { + return false + } + return foundCondition.Type == condition.Type && + foundCondition.Reason == condition.Reason && + foundCondition.Message == condition.Message && + foundCondition.Status == condition.Status +} + +func getCopiedCSVsCondition(isDisabled, csvIsRequeued bool) metav1.Condition { + condition := metav1.Condition{ + Type: v1.DisabledCopiedCSVsConditionType, + LastTransitionTime: metav1.Now(), + Status: metav1.ConditionFalse, + } + if !isDisabled { + condition.Reason = "CopiedCSVsEnabled" + condition.Message = "Copied CSVs are enabled and present accross the cluster" + if csvIsRequeued { + condition.Message = "Copied CSVs are enabled and at least one copied CSVs is missing" + } + return condition + } + + if csvIsRequeued { + condition.Reason = "CopiedCSVsFound" + condition.Message = "Copied CSVs are disabled and at least one copied CSV was found for an operator installed in AllNamespace mode" + return condition + } + + condition.Status = metav1.ConditionTrue + condition.Reason = "NoCopiedCSVsFound" + condition.Message = "Copied CSVs are disabled and none were found for operators installed in AllNamespace mode" + + return condition +} + func (a *Operator) syncCopyCSV(obj interface{}) (syncError error) { clusterServiceVersion, ok := obj.(*v1alpha1.ClusterServiceVersion) if !ok { @@ -1201,6 +1346,15 @@ func (a *Operator) syncCopyCSV(obj interface{}) (syncError error) { return fmt.Errorf("casting ClusterServiceVersion failed") } + olmConfig, err := a.client.OperatorsV1().OLMConfigs().Get(context.TODO(), "cluster", metav1.GetOptions{}) + if err != nil && !k8serrors.IsNotFound(err) { + return err + } + + if err == nil { + go a.olmConfigQueue.AddAfter(olmConfig, time.Second*5) + } + logger := a.logger.WithFields(logrus.Fields{ "id": queueinformer.NewLoopID(), "csv": clusterServiceVersion.GetName(), @@ -1222,15 +1376,145 @@ func (a *Operator) syncCopyCSV(obj interface{}) (syncError error) { "targetNamespaces": strings.Join(operatorGroup.Status.Namespaces, ","), }).Debug("copying csv to targets") + copiedCSVsAreEnabled, err := a.copiedCSVsAreEnabled() + if err != nil { + return err + } + // Check if we need to do any copying / annotation for the operatorgroup - if err := a.ensureCSVsInNamespaces(clusterServiceVersion, operatorGroup, NewNamespaceSet(operatorGroup.Status.Namespaces)); err != nil { - logger.WithError(err).Info("couldn't copy CSV to target namespaces") - syncError = err + namespaceSet := NewNamespaceSet(operatorGroup.Status.Namespaces) + if copiedCSVsAreEnabled || !namespaceSet.IsAllNamespaces() { + if err := a.ensureCSVsInNamespaces(clusterServiceVersion, operatorGroup, namespaceSet); err != nil { + logger.WithError(err).Info("couldn't copy CSV to target namespaces") + syncError = err + } + + // If the CSV was installed in AllNamespace mode, remove any "CSV Copying Disabled" events + // in which the related object's name, namespace, and uid match the given CSV's. + if namespaceSet.IsAllNamespaces() { + if err := a.deleteCSVCopyingDisabledEvent(clusterServiceVersion); err != nil { + return err + } + } + return + } + + requirement, err := labels.NewRequirement(v1alpha1.CopiedLabelKey, selection.Equals, []string{clusterServiceVersion.Namespace}) + if err != nil { + return err + } + + copiedCSVs, err := a.copiedCSVLister.List(labels.NewSelector().Add(*requirement)) + if err != nil { + return err + } + + for _, copiedCSV := range copiedCSVs { + err := a.client.OperatorsV1alpha1().ClusterServiceVersions(copiedCSV.Namespace).Delete(context.TODO(), copiedCSV.Name, metav1.DeleteOptions{}) + if err != nil && !k8serrors.IsNotFound(err) { + return err + } + } + + if err := a.createCSVCopyingDisabledEvent(clusterServiceVersion); err != nil { + return err } return } +// copiedCSVsAreEnabled determines if csv copying is enabled for OLM. +// +// This method will first attempt to get the "cluster" olmConfig resource, +// if any error other than "IsNotFound" is encountered, false and the error +// will be returned. +// +// If the "cluster" olmConfig resource is found, the value of +// olmConfig.spec.features.disableCopiedCSVs will be returned along with a +// nil error. +// +// If the "cluster" olmConfig resource is not found, true will be returned +// without an error. +func (a *Operator) copiedCSVsAreEnabled() (bool, error) { + olmConfig, err := a.client.OperatorsV1().OLMConfigs().Get(context.TODO(), "cluster", metav1.GetOptions{}) + if err != nil { + // Default to true if olmConfig singleton cannot be found + if k8serrors.IsNotFound(err) { + return true, nil + } + // If there was an error that wasn't an IsNotFound, return the error + return false, err + } + + // If there was no error, return value based on olmConfig singleton + return olmConfig.CopiedCSVsAreEnabled(), nil +} + +func (a *Operator) getCopiedCSVDisabledEventsForCSV(csv *v1alpha1.ClusterServiceVersion) ([]corev1.Event, error) { + result := []corev1.Event{} + if csv == nil { + return result, nil + } + + events, err := a.opClient.KubernetesInterface().CoreV1().Events(csv.GetNamespace()).List(context.TODO(), metav1.ListOptions{}) + if err != nil { + return nil, err + } + + for _, event := range events.Items { + if event.InvolvedObject.Namespace == csv.GetNamespace() && + event.InvolvedObject.Name == csv.GetName() && + event.InvolvedObject.UID == csv.GetUID() && + event.Reason == v1.DisabledCopiedCSVsConditionType { + result = append(result, event) + } + } + + return result, nil +} + +func (a *Operator) deleteCSVCopyingDisabledEvent(csv *v1alpha1.ClusterServiceVersion) error { + events, err := a.getCopiedCSVDisabledEventsForCSV(csv) + if err != nil { + return err + } + + // Remove existing events. + return a.deleteEvents(events) +} + +func (a *Operator) deleteEvents(events []corev1.Event) error { + for _, event := range events { + err := a.opClient.KubernetesInterface().EventsV1().Events(event.GetNamespace()).Delete(context.TODO(), event.GetName(), metav1.DeleteOptions{}) + if err != nil && !k8serrors.IsNotFound(err) { + return err + } + } + return nil +} + +func (a *Operator) createCSVCopyingDisabledEvent(csv *v1alpha1.ClusterServiceVersion) error { + events, err := a.getCopiedCSVDisabledEventsForCSV(csv) + if err != nil { + return err + } + + if len(events) == 1 { + return nil + } + + // Remove existing events. + if len(events) > 1 { + if err := a.deleteEvents(events); err != nil { + return err + } + } + + a.recorder.Eventf(csv, corev1.EventTypeWarning, v1.DisabledCopiedCSVsConditionType, "CSV copying disabled for %s/%s", csv.GetNamespace(), csv.GetName()) + + return nil +} + func (a *Operator) syncGcCsv(obj interface{}) (syncError error) { clusterServiceVersion, ok := obj.(*v1alpha1.ClusterServiceVersion) if !ok { From 9c548f2fdd49fcea2908c18a983d339786277a09 Mon Sep 17 00:00:00 2001 From: timflannagan Date: Thu, 16 Dec 2021 09:23:26 -0500 Subject: [PATCH 2/2] manifests: Add the OLMConfig CRD --- manifests/0000_50_olm_02-olmconfig.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 manifests/0000_50_olm_02-olmconfig.yaml diff --git a/manifests/0000_50_olm_02-olmconfig.yaml b/manifests/0000_50_olm_02-olmconfig.yaml new file mode 100644 index 0000000000..6aa8717863 --- /dev/null +++ b/manifests/0000_50_olm_02-olmconfig.yaml @@ -0,0 +1,8 @@ +apiVersion: operators.coreos.com/v1 +kind: OLMConfig +metadata: + name: cluster + annotations: + include.release.openshift.io/ibm-cloud-managed: "true" + include.release.openshift.io/self-managed-high-availability: "true" + include.release.openshift.io/single-node-developer: "true"