Skip to content

Commit c7c2f14

Browse files
VanillaSpoonVanillaSpoon
committed
add: Hypershift Functionality
1 parent 51363f6 commit c7c2f14

File tree

5 files changed

+317
-115
lines changed

5 files changed

+317
-115
lines changed

‎controllers/appwrapper_controller.go

Lines changed: 84 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,11 @@ package controllers
1818

1919
import (
2020
"context"
21-
"fmt"
2221
"strings"
2322
"time"
2423

24+
ocmsdk "github.com/openshift-online/ocm-sdk-go"
25+
2526
"github.com/project-codeflare/instascale/pkg/config"
2627
arbv1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1"
2728
corev1 "k8s.io/api/core/v1"
@@ -37,15 +38,19 @@ import (
3738
"sigs.k8s.io/controller-runtime/pkg/log"
3839
)
3940

41+
type MachineType string
42+
4043
// AppWrapperReconciler reconciles a AppWrapper object
4144
type AppWrapperReconciler struct {
4245
client.Client
43-
Scheme *runtime.Scheme
44-
Config config.InstaScaleConfiguration
45-
kubeClient *kubernetes.Clientset
46-
ocmClusterID string
47-
ocmToken string
48-
useMachineSets bool
46+
Scheme *runtime.Scheme
47+
Config config.InstaScaleConfiguration
48+
kubeClient *kubernetes.Clientset
49+
ocmClusterID string
50+
ocmToken string
51+
ocmConnection *ocmsdk.Connection
52+
MachineType MachineType
53+
machineCheck bool
4954
}
5055

5156
var (
@@ -54,9 +59,12 @@ var (
5459
)
5560

5661
const (
57-
namespaceToList = "openshift-machine-api"
58-
minResyncPeriod = 10 * time.Minute
59-
finalizerName = "instascale.codeflare.dev/finalizer"
62+
namespaceToList = "openshift-machine-api"
63+
minResyncPeriod = 10 * time.Minute
64+
finalizerName = "instascale.codeflare.dev/finalizer"
65+
MachineTypeMachineSet MachineType = "MachineSet"
66+
MachineTypeMachinePool MachineType = "MachinePool"
67+
MachineTypeNodePool MachineType = "NodePool"
6068
)
6169

6270
// +kubebuilder:rbac:groups=workload.codeflare.dev,resources=appwrappers,verbs=get;list;watch;create;update;patch;delete
@@ -81,14 +89,17 @@ const (
8189
// - https://pkg.go.dev/sigs.k8s.io/[email protected]/pkg/reconcile
8290
func (r *AppWrapperReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
8391
_ = log.FromContext(ctx)
84-
// todo: Move the getOCMClusterID call out of reconcile loop.
92+
// todo: Move the setMachineType call out of reconcile loop.
8593
// Only reason we are calling it here is that the client is not able to make
8694
// calls until it is started, so SetupWithManager is not working.
87-
if !r.useMachineSets && r.ocmClusterID == "" {
88-
if err := r.getOCMClusterID(ctx); err != nil {
89-
return ctrl.Result{Requeue: true, RequeueAfter: timeFiveSeconds}, err
95+
if r.machineCheck == false && r.MachineType != MachineTypeMachineSet {
96+
if err := r.setMachineType(ctx); err != nil {
97+
return ctrl.Result{}, err
9098
}
9199
}
100+
101+
r.machineCheck = true
102+
92103
var appwrapper arbv1.AppWrapper
93104

94105
if err := r.Get(ctx, req.NamespacedName, &appwrapper); err != nil {
@@ -125,8 +136,14 @@ func (r *AppWrapperReconciler) Reconcile(ctx context.Context, req ctrl.Request)
125136
if status == "Pending" && containsInsufficientCondition(allconditions) {
126137
demandPerInstanceType := r.discoverInstanceTypes(ctx, &appwrapper)
127138
if ocmSecretRef := r.Config.OCMSecretRef; ocmSecretRef != nil {
128-
return r.scaleMachinePool(ctx, &appwrapper, demandPerInstanceType)
139+
switch r.MachineType {
140+
case MachineTypeNodePool:
141+
return r.scaleNodePool(ctx, &appwrapper, demandPerInstanceType)
142+
case MachineTypeMachinePool:
143+
return r.scaleMachinePool(ctx, &appwrapper, demandPerInstanceType)
144+
}
129145
} else {
146+
// use MachineSets
130147
switch strings.ToLower(r.Config.MachineSetsStrategy) {
131148
case "reuse":
132149
return r.reconcileReuseMachineSet(ctx, &appwrapper, demandPerInstanceType)
@@ -146,7 +163,8 @@ func (r *AppWrapperReconciler) finalizeScalingDownMachines(ctx context.Context,
146163
} else {
147164
deletionMessage = "deleted"
148165
}
149-
if r.useMachineSets {
166+
switch r.MachineType {
167+
case MachineTypeMachineSet:
150168
switch strings.ToLower(r.Config.MachineSetsStrategy) {
151169
case "reuse":
152170
matchedAw := r.findExactMatch(ctx, appwrapper)
@@ -158,6 +176,9 @@ func (r *AppWrapperReconciler) finalizeScalingDownMachines(ctx context.Context,
158176
"newAppWrapper", matchedAw,
159177
)
160178
if err := r.swapNodeLabels(ctx, appwrapper, matchedAw); err != nil {
179+
logger.Error(err, "Error swapping node labels for AppWrapper",
180+
"appwrapper", appwrapper,
181+
)
161182
return err
162183
}
163184
} else {
@@ -167,6 +188,9 @@ func (r *AppWrapperReconciler) finalizeScalingDownMachines(ctx context.Context,
167188
"deletionMessage", deletionMessage,
168189
)
169190
if err := r.annotateToDeleteMachine(ctx, appwrapper); err != nil {
191+
logger.Error(err, "Error annotating to delete machine for AppWrapper",
192+
"appwrapper", appwrapper,
193+
)
170194
return err
171195
}
172196
}
@@ -177,19 +201,53 @@ func (r *AppWrapperReconciler) finalizeScalingDownMachines(ctx context.Context,
177201
"deletionMessage", deletionMessage,
178202
)
179203
if err := r.deleteMachineSet(ctx, appwrapper); err != nil {
204+
logger.Error(err, "Error deleting MachineSet for AppWrapper",
205+
"appwrapper", appwrapper)
180206
return err
181207
}
182208
}
183-
} else {
209+
case MachineTypeNodePool:
210+
logger.Info(
211+
"AppWrapper deleted, scaling down nodepool",
212+
"appWrapper", appwrapper,
213+
"deletionMessage", deletionMessage,
214+
)
215+
if _, err := r.deleteNodePool(ctx, appwrapper); err != nil {
216+
logger.Error(err, "Error deleting NodePool for AppWrapper",
217+
"appwrapper", appwrapper)
218+
return err
219+
}
220+
221+
case MachineTypeMachinePool:
184222
logger.Info(
185223
"AppWrapper deleted, scaling down machine pool",
186224
"appWrapper", appwrapper,
187225
"deletionMessage", deletionMessage,
188226
)
189227
if _, err := r.deleteMachinePool(ctx, appwrapper); err != nil {
228+
logger.Error(err, "Error deleting MachinePool for AppWrapper",
229+
"appwrapper", appwrapper)
230+
return err
231+
}
232+
}
233+
return nil
234+
}
235+
236+
func (r *AppWrapperReconciler) setMachineType(ctx context.Context) error {
237+
logger := ctrl.LoggerFrom(ctx)
238+
if r.ocmClusterID == "" {
239+
if err := r.getOCMClusterID(ctx); err != nil {
190240
return err
191241
}
192242
}
243+
hypershiftEnabled, err := r.checkHypershiftEnabled(ctx)
244+
if err != nil {
245+
logger.Error(err, "error checking if hypershift is enabled")
246+
return err
247+
}
248+
if hypershiftEnabled {
249+
r.MachineType = MachineTypeNodePool
250+
}
193251
return nil
194252
}
195253

@@ -206,20 +264,19 @@ func (r *AppWrapperReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Ma
206264
}
207265

208266
maxScaleNodesAllowed = int(r.Config.MaxScaleoutAllowed)
209-
r.useMachineSets = true
267+
r.MachineType = MachineTypeMachineSet // default to MachineSet
210268
if ocmSecretRef := r.Config.OCMSecretRef; ocmSecretRef != nil {
211-
r.useMachineSets = false
212269
if ocmSecret, err := r.getOCMSecret(ctx, ocmSecretRef); err != nil {
213-
return fmt.Errorf("error reading OCM Secret from ref %q: %w", ocmSecretRef, err)
270+
logger.Error(err, "error reading OCM Secret from ref",
271+
"ref", ocmSecretRef)
272+
return err
214273
} else if token := ocmSecret.Data["token"]; len(token) > 0 {
215274
r.ocmToken = string(token)
275+
r.MachineType = MachineTypeMachinePool
216276
} else {
217-
return fmt.Errorf("token is missing from OCM Secret %q", ocmSecretRef)
218-
}
219-
if ok, err := r.machinePoolExists(); err != nil {
277+
logger.Error(err, "token is missing from OCM Secret",
278+
"ref", ocmSecretRef)
220279
return err
221-
} else if ok {
222-
logger.Info("Using machine pools for cluster auto-scaling")
223280
}
224281
}
225282

@@ -235,12 +292,8 @@ func (r *AppWrapperReconciler) getOCMSecret(ctx context.Context, secretRef *core
235292
func (r *AppWrapperReconciler) discoverInstanceTypes(ctx context.Context, aw *arbv1.AppWrapper) map[string]int {
236293
logger := ctrl.LoggerFrom(ctx)
237294
demandMapPerInstanceType := make(map[string]int)
238-
var instanceRequired []string
239-
for k, v := range aw.Labels {
240-
if k == "orderedinstance" {
241-
instanceRequired = strings.Split(v, "_")
242-
}
243-
}
295+
296+
instanceRequired := getInstanceRequired(aw.Labels)
244297

245298
if len(instanceRequired) < 1 {
246299
logger.Info(

‎controllers/machinepools.go

Lines changed: 1 addition & 81 deletions
Original file line numberDiff line numberDiff line change
@@ -2,45 +2,14 @@ package controllers
22

33
import (
44
"context"
5-
"fmt"
65
"strings"
76

8-
ocmsdk "github.com/openshift-online/ocm-sdk-go"
97
cmv1 "github.com/openshift-online/ocm-sdk-go/clustersmgmt/v1"
10-
configv1 "github.com/openshift/api/config/v1"
118
arbv1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1"
129

13-
"k8s.io/apimachinery/pkg/types"
1410
ctrl "sigs.k8s.io/controller-runtime"
1511
)
1612

17-
func (r *AppWrapperReconciler) createOCMConnection() (*ocmsdk.Connection, error) {
18-
logger, err := ocmsdk.NewGoLoggerBuilder().
19-
Debug(false).
20-
Build()
21-
if err != nil {
22-
return nil, fmt.Errorf("can't build logger: %v", err)
23-
}
24-
25-
connection, err := ocmsdk.NewConnectionBuilder().
26-
Logger(logger).
27-
Tokens(r.ocmToken).
28-
Build()
29-
if err != nil {
30-
return nil, fmt.Errorf("can't build connection: %v", err)
31-
}
32-
33-
return connection, nil
34-
}
35-
36-
func hasAwLabel(machinePool *cmv1.MachinePool, aw *arbv1.AppWrapper) bool {
37-
value, ok := machinePool.Labels()[aw.Name]
38-
if ok && value == aw.Name {
39-
return true
40-
}
41-
return false
42-
}
43-
4413
func (r *AppWrapperReconciler) scaleMachinePool(ctx context.Context, aw *arbv1.AppWrapper, demandPerInstanceType map[string]int) (ctrl.Result, error) {
4514
logger := ctrl.LoggerFrom(ctx)
4615
connection, err := r.createOCMConnection()
@@ -61,7 +30,7 @@ func (r *AppWrapperReconciler) scaleMachinePool(ctx context.Context, aw *arbv1.A
6130

6231
numberOfMachines := 0
6332
response.Items().Each(func(machinePool *cmv1.MachinePool) bool {
64-
if machinePool.InstanceType() == userRequestedInstanceType && hasAwLabel(machinePool, aw) {
33+
if machinePool.InstanceType() == userRequestedInstanceType && hasAwLabel(machinePool.Labels(), aw) {
6534
numberOfMachines = machinePool.Replicas()
6635
return false
6736
}
@@ -133,52 +102,3 @@ func (r *AppWrapperReconciler) deleteMachinePool(ctx context.Context, aw *arbv1.
133102
})
134103
return ctrl.Result{Requeue: false}, nil
135104
}
136-
137-
func (r *AppWrapperReconciler) machinePoolExists() (bool, error) {
138-
connection, err := r.createOCMConnection()
139-
if err != nil {
140-
return false, fmt.Errorf("error creating OCM connection: %w", err)
141-
}
142-
defer connection.Close()
143-
144-
machinePools := connection.ClustersMgmt().V1().Clusters().Cluster(r.ocmClusterID).MachinePools()
145-
return machinePools != nil, nil
146-
}
147-
148-
// getOCMClusterID determines the internal clusterID to be used for OCM API calls
149-
func (r *AppWrapperReconciler) getOCMClusterID(ctx context.Context) error {
150-
logger := ctrl.LoggerFrom(ctx)
151-
cv := &configv1.ClusterVersion{}
152-
err := r.Get(ctx, types.NamespacedName{Name: "version"}, cv)
153-
if err != nil {
154-
return fmt.Errorf("can't get clusterversion: %v", err)
155-
}
156-
157-
internalClusterID := string(cv.Spec.ClusterID)
158-
159-
connection, err := r.createOCMConnection()
160-
if err != nil {
161-
logger.Error(err, "Error creating OCM connection")
162-
}
163-
defer connection.Close()
164-
165-
// Get the client for the resource that manages the collection of clusters:
166-
collection := connection.ClustersMgmt().V1().Clusters()
167-
168-
response, err := collection.List().Search(fmt.Sprintf("external_id = '%s'", internalClusterID)).Size(1).Page(1).SendContext(ctx)
169-
if err != nil {
170-
logger.Error(err, "Error getting cluster id")
171-
}
172-
173-
response.Items().Each(func(cluster *cmv1.Cluster) bool {
174-
r.ocmClusterID = cluster.ID()
175-
logger.Info(
176-
"Cluster Info",
177-
"clusterId", cluster.ID(),
178-
"clusterName", cluster.Name(),
179-
"clusterState", cluster.State(),
180-
)
181-
return true
182-
})
183-
return nil
184-
}

0 commit comments

Comments
 (0)