Skip to content

CLOUDP-285107: fix data federation endpoint reconciliation #1929

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Nov 18, 2024
Merged

CLOUDP-285107: fix data federation endpoint reconciliation #1929

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
da6fe5a
pkg/controller/atlasdatafederation: fix recocncile of private endpoints
s-urbaniak Nov 7, 2024
8d28c34
improve test coverage
helderjs Nov 14, 2024
3a0f64a
increment e2e test
helderjs Nov 14, 2024
6718f05
remove programmatic focus
helderjs Nov 15, 2024
335ffd3
fix test cleanup
helderjs Nov 15, 2024
46a504c
fix test cleanup
helderjs Nov 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 44 additions & 1 deletion internal/translation/datafederation/conversion_endpoints.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,27 +1,34 @@
package datafederation

import (
"encoding/json"
"fmt"
"reflect"

"go.mongodb.org/atlas-sdk/v20231115008/admin"

"github.com/mongodb/mongodb-atlas-kubernetes/v2/internal/cmp"
"github.com/mongodb/mongodb-atlas-kubernetes/v2/internal/pointer"
akov2 "github.com/mongodb/mongodb-atlas-kubernetes/v2/pkg/api/v1"
"github.com/mongodb/mongodb-atlas-kubernetes/v2/pkg/controller/customresource"
)

type DatafederationPrivateEndpointEntry struct {
*akov2.DataFederationPE
ProjectID string
}

func NewDatafederationPrivateEndpointEntry(pe *akov2.DataFederationPE, projectID string) *DatafederationPrivateEndpointEntry {
func NewDataFederationPrivateEndpointEntry(projectID string, pe *akov2.DataFederationPE) *DatafederationPrivateEndpointEntry {
if pe == nil {
return nil
}
return &DatafederationPrivateEndpointEntry{DataFederationPE: pe, ProjectID: projectID}
}

func (e *DatafederationPrivateEndpointEntry) EqualsTo(target *DatafederationPrivateEndpointEntry) bool {
return reflect.DeepEqual(e.DataFederationPE, target.DataFederationPE)
}

func endpointsFromAtlas(endpoints []admin.PrivateNetworkEndpointIdEntry, projectID string) ([]*DatafederationPrivateEndpointEntry, error) {
result := make([]*DatafederationPrivateEndpointEntry, 0, len(endpoints))
for _, entry := range endpoints {
Expand Down Expand Up @@ -58,3 +65,39 @@ func endpointToAtlas(ep *DatafederationPrivateEndpointEntry) *admin.PrivateNetwo
Type: pointer.MakePtrOrNil(ep.Type),
}
}

type DataFederationPrivateEndpoint struct {
AKO, Atlas, LastApplied *DatafederationPrivateEndpointEntry
}

func MapDatafederationPrivateEndpoints(projectID string, df *akov2.AtlasDataFederation, atlasEndpoints []*DatafederationPrivateEndpointEntry) (map[string]*DataFederationPrivateEndpoint, error) {
var lastApplied akov2.AtlasDataFederation
if ann, ok := df.GetAnnotations()[customresource.AnnotationLastAppliedConfiguration]; ok {
err := json.Unmarshal([]byte(ann), &lastApplied.Spec)
if err != nil {
return nil, fmt.Errorf("error reading data federation from last applied annotation: %w", err)
}
}

result := make(map[string]*DataFederationPrivateEndpoint)
addEndpoint := func(endpointID string) {
if _, ok := result[endpointID]; !ok {
result[endpointID] = &DataFederationPrivateEndpoint{}
}
}

for _, pe := range atlasEndpoints {
addEndpoint(pe.EndpointID)
result[pe.EndpointID].Atlas = pe
}
for _, pe := range df.Spec.PrivateEndpoints {
addEndpoint(pe.EndpointID)
result[pe.EndpointID].AKO = NewDataFederationPrivateEndpointEntry(projectID, &pe)
}
for _, pe := range lastApplied.Spec.PrivateEndpoints {
addEndpoint(pe.EndpointID)
result[pe.EndpointID].LastApplied = NewDataFederationPrivateEndpointEntry(projectID, &pe)
}

return result, nil
}
204 changes: 202 additions & 2 deletions internal/translation/datafederation/conversion_endpoints_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,16 @@
package datafederation

import (
"reflect"
"testing"

"github.com/google/go-cmp/cmp"
fuzz "github.com/google/gofuzz"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

akov2 "github.com/mongodb/mongodb-atlas-kubernetes/v2/pkg/api/v1"
"github.com/mongodb/mongodb-atlas-kubernetes/v2/pkg/controller/customresource"
)

func TestRoundtrip_DataFederationPE(t *testing.T) {
Expand All @@ -21,10 +25,206 @@ func TestRoundtrip_DataFederationPE(t *testing.T) {
toAtlasResult := endpointToAtlas(fuzzed)
fromAtlasResult := endpointFromAtlas(toAtlasResult, "")

equals := reflect.DeepEqual(fuzzed, fromAtlasResult)
equals := fuzzed.EqualsTo(fromAtlasResult)
if !equals {
t.Log(cmp.Diff(fuzzed, fromAtlasResult))
}
require.True(t, equals)
}
}

func TestMapDatafederationPrivateEndpoints(t *testing.T) {
tests := map[string]struct {
dataFederation *akov2.AtlasDataFederation
endpoints []*DatafederationPrivateEndpointEntry
expectedResult map[string]*DataFederationPrivateEndpoint
expectedErr string
}{
"failed to parse last config applied annotation": {
dataFederation: &akov2.AtlasDataFederation{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
customresource.AnnotationLastAppliedConfiguration: "wrong,",
},
},
},
endpoints: []*DatafederationPrivateEndpointEntry{},
expectedResult: nil,
expectedErr: "error reading data federation from last applied annotation: invalid character 'w' looking for beginning of value",
},
"map without last applied configuration": {
dataFederation: &akov2.AtlasDataFederation{
Spec: akov2.DataFederationSpec{
PrivateEndpoints: []akov2.DataFederationPE{
{
Provider: "AWS",
Type: "DATA_LAKE",
EndpointID: "vpcpe-123456",
},
{
Provider: "AZURE",
Type: "DATA_LAKE",
EndpointID: "azure/resource/id",
},
},
},
},
endpoints: []*DatafederationPrivateEndpointEntry{
{
DataFederationPE: &akov2.DataFederationPE{
Provider: "AWS",
Type: "DATA_LAKE",
EndpointID: "vpcpe-123456",
},
ProjectID: "project-id",
},
{
DataFederationPE: &akov2.DataFederationPE{
Provider: "AZURE",
Type: "DATA_LAKE",
EndpointID: "azure/resource/id",
},
ProjectID: "project-id",
},
},
expectedResult: map[string]*DataFederationPrivateEndpoint{
"vpcpe-123456": {
AKO: &DatafederationPrivateEndpointEntry{
DataFederationPE: &akov2.DataFederationPE{
Provider: "AWS",
Type: "DATA_LAKE",
EndpointID: "vpcpe-123456",
},
ProjectID: "project-id",
},
Atlas: &DatafederationPrivateEndpointEntry{
DataFederationPE: &akov2.DataFederationPE{
Provider: "AWS",
Type: "DATA_LAKE",
EndpointID: "vpcpe-123456",
},
ProjectID: "project-id",
},
LastApplied: nil,
},
"azure/resource/id": {
AKO: &DatafederationPrivateEndpointEntry{
DataFederationPE: &akov2.DataFederationPE{
Provider: "AZURE",
Type: "DATA_LAKE",
EndpointID: "azure/resource/id",
},
ProjectID: "project-id",
},
Atlas: &DatafederationPrivateEndpointEntry{
DataFederationPE: &akov2.DataFederationPE{
Provider: "AZURE",
Type: "DATA_LAKE",
EndpointID: "azure/resource/id",
},
ProjectID: "project-id",
},
LastApplied: nil,
},
},
},
"map with last applied configuration": {
dataFederation: &akov2.AtlasDataFederation{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
customresource.AnnotationLastAppliedConfiguration: "{\"name\":\"\",\"privateEndpoints\":[{\"endpointId\":\"vpcpe-123456\"," +
"\"provider\":\"AWS\",\"type\":\"DATA_LAKE\"}]}",
},
},
Spec: akov2.DataFederationSpec{
PrivateEndpoints: []akov2.DataFederationPE{
{
Provider: "AWS",
Type: "DATA_LAKE",
EndpointID: "vpcpe-123456",
},
{
Provider: "AZURE",
Type: "DATA_LAKE",
EndpointID: "azure/resource/id",
},
},
},
},
endpoints: []*DatafederationPrivateEndpointEntry{
{
DataFederationPE: &akov2.DataFederationPE{
Provider: "AWS",
Type: "DATA_LAKE",
EndpointID: "vpcpe-123456",
},
ProjectID: "project-id",
},
{
DataFederationPE: &akov2.DataFederationPE{
Provider: "AZURE",
Type: "DATA_LAKE",
EndpointID: "azure/resource/id",
},
ProjectID: "project-id",
},
},
expectedResult: map[string]*DataFederationPrivateEndpoint{
"vpcpe-123456": {
AKO: &DatafederationPrivateEndpointEntry{
DataFederationPE: &akov2.DataFederationPE{
Provider: "AWS",
Type: "DATA_LAKE",
EndpointID: "vpcpe-123456",
},
ProjectID: "project-id",
},
Atlas: &DatafederationPrivateEndpointEntry{
DataFederationPE: &akov2.DataFederationPE{
Provider: "AWS",
Type: "DATA_LAKE",
EndpointID: "vpcpe-123456",
},
ProjectID: "project-id",
},
LastApplied: &DatafederationPrivateEndpointEntry{
DataFederationPE: &akov2.DataFederationPE{
Provider: "AWS",
Type: "DATA_LAKE",
EndpointID: "vpcpe-123456",
},
ProjectID: "project-id",
},
},
"azure/resource/id": {
AKO: &DatafederationPrivateEndpointEntry{
DataFederationPE: &akov2.DataFederationPE{
Provider: "AZURE",
Type: "DATA_LAKE",
EndpointID: "azure/resource/id",
},
ProjectID: "project-id",
},
Atlas: &DatafederationPrivateEndpointEntry{
DataFederationPE: &akov2.DataFederationPE{
Provider: "AZURE",
Type: "DATA_LAKE",
EndpointID: "azure/resource/id",
},
ProjectID: "project-id",
},
LastApplied: nil,
},
},
},
}
for name, tt := range tests {
t.Run(name, func(t *testing.T) {
m, err := MapDatafederationPrivateEndpoints("project-id", tt.dataFederation, tt.endpoints)
if err != nil {
assert.EqualError(t, err, tt.expectedErr)
}
assert.Equal(t, tt.expectedResult, m)
})
}
}
7 changes: 4 additions & 3 deletions pkg/controller/atlasdatafederation/datafederation_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,8 +116,8 @@ func (r *AtlasDataFederationReconciler) Reconcile(context context.Context, req c
return result.ReconcileResult(), nil
}

if result = r.ensurePrivateEndpoints(ctx, project, dataFederation, endpointService); !result.IsOk() {
ctx.SetConditionFromResult(api.DataFederationPEReadyType, result)
if result = r.ensurePrivateEndpoints(ctx, endpointService, project, dataFederation); !result.IsOk() {
ctx.SetConditionFromResult(api.DataFederationReadyType, result)
return result.ReconcileResult(), nil
}

Expand Down Expand Up @@ -145,7 +145,7 @@ func (r *AtlasDataFederationReconciler) Reconcile(context context.Context, req c
return r.handleDelete(ctx, log, dataFederation, project, dataFederationService).ReconcileResult(), nil
}

err = customresource.ApplyLastConfigApplied(context, project, r.Client)
err = customresource.ApplyLastConfigApplied(context, dataFederation, r.Client)
if err != nil {
result = workflow.Terminate(workflow.Internal, err.Error())
ctx.SetConditionFromResult(api.DataFederationReadyType, result)
Expand All @@ -154,6 +154,7 @@ func (r *AtlasDataFederationReconciler) Reconcile(context context.Context, req c
return result.ReconcileResult(), nil
}

ctx.SetConditionTrue(api.DataFederationReadyType)
ctx.SetConditionTrue(api.ReadyType)
return workflow.OK().ReconcileResult(), nil
}
Expand Down
Loading
Loading