add default quota to AW (#396)

* add default quota to AW

* add default quota

* resolve merge

* Added support to query existing QuotaSubTrees. MCAD checks that there is a label associated with every quota-sub-tree for the AppWrapper; if there is none, then it adds it with 'default' value. This way, AppWrappers submitted without labels can be dispatched using the quota management library. Added tests for this, and fixed 'quota_service' sub tree to have a 'default' node. This code assumes the quota administrator created a leaf node name 'default' that can be used to schedule AppWrappers without quota labels.

* Fixed bug that manifested when running MCAD without quota management. The code would think there is no quota (even without quota management) because the new code wasn't inside the proper scope.

* Renamed the function 'GetTreeNames' to 'GetValidQuotaLabels'

* Default quota labels code fixed. Code preserves the existing labels and updates the Kubernetes object correctly. New tests added to check for AppWrappers with no labels at all and with existing labels but no quota labels.

* relax testing scenario

---------

Co-authored-by: Pedro D. Bello-Maldonado <[email protected]>

Author: asm582

pkg/controller/queuejob/queuejob_controller_ex.go

@@ -1260,11 +1260,46 @@ func (qjm *XController) ScheduleNext() {
 			klog.Infof("[ScheduleNext] XQJ %s with resources %v to be scheduled on aggregated idle resources %v", qj.Name, aggqj, resources)
 
 			if aggqj.LessEqual(resources) && qjm.nodeChecks(qjm.cache.GetUnallocatedHistograms(), qj) {
-				//Now evaluate quota
+				// Now evaluate quota
 				fits := true
 				klog.V(4).Infof("[ScheduleNext] HOL available resourse successful check for %s at %s activeQ=%t Unsched=%t &qj=%p Version=%s Status=%+v due to quota limits", qj.Name, time.Now().Sub(HOLStartTime), qjm.qjqueue.IfExistActiveQ(qj), qjm.qjqueue.IfExistUnschedulableQ(qj), qj, qj.ResourceVersion, qj.Status)
 				if qjm.serverOption.QuotaEnabled {
 					if qjm.quotaManager != nil {
+						// Quota tree design:
+						// - All AppWrappers without quota submission will consume quota from the 'default' node.
+						// - All quota trees in the system should have a 'default' node so AppWrappers without
+						//   quota specification can be dispatched
+						// - If the AppWrapper doesn't have a quota label, then one is added for every tree with the 'default' value
+						// - Depending on how the 'default' node is configured, AppWrappers that don't specify quota could be
+						//   preemptable by default (e.g., 'default' node with 'cpu: 0m' and 'memory: 0Mi' quota and 'hardLimit: false'
+						//   such node borrows quota from other nodes already in the system)
+						apiCacheAWJob, err := qjm.queueJobLister.AppWrappers(qj.Namespace).Get(qj.Name)
+						if err != nil {
+							klog.Errorf("[ScheduleNext] Failed to get AppWrapper from API Cache %v/%v: %v",
+								qj.Namespace, qj.Name, err)
+							continue
+						}
+						allTrees := qjm.quotaManager.GetValidQuotaLabels()
+						newLabels := make(map[string]string)
+						for key, value := range apiCacheAWJob.Labels {
+							newLabels[key] = value
+						}
+						updateLabels := false
+						for _, treeName := range allTrees {
+							if _, quotaSetForAW := newLabels[treeName]; !quotaSetForAW {
+								newLabels[treeName] = "default"
+								updateLabels = true
+							}
+						}
+						if updateLabels {
+							apiCacheAWJob.SetLabels(newLabels)
+							if err := qjm.updateEtcd(apiCacheAWJob, "ScheduleNext - setDefaultQuota"); err == nil {
+								klog.V(3).Infof("[ScheduleNext] Default quota added to AW %v", qj.Name)
+							} else {
+								klog.V(3).Infof("[ScheduleNext] Failed to added default quota to AW %v, skipping dispatch of AW", qj.Name)
+								return
+							}
+						}
 						var msg string
 						var preemptAWs []*arbv1.AppWrapper
 						quotaFits, preemptAWs, msg = qjm.quotaManager.Fits(qj, aggqj, proposedPreemptions)

pkg/controller/quota/quota_manager_interface.go

@@ -1,11 +1,11 @@
 // ------------------------------------------------------ {COPYRIGHT-TOP} ---
 // Copyright 2019, 2021, 2022, 2023 The Multi-Cluster App Dispatcher Authors.
-// 
+//
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
-// 
-//    http://www.apache.org/licenses/LICENSE-2.0
+//
+//	http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
@@ -23,4 +23,5 @@ import (
 type QuotaManagerInterface interface {
 	Fits(aw *arbv1.AppWrapper, resources *clusterstateapi.Resource, proposedPremptions []*arbv1.AppWrapper) (bool, []*arbv1.AppWrapper, string)
 	Release(aw *arbv1.AppWrapper) bool
+	GetValidQuotaLabels() []string
 }

pkg/controller/quota/quotaforestmanager/qm_lib_backend_with_quotasubt_mgr.go

@@ -624,3 +624,6 @@ func (qm *QuotaManager) removeConsumer(consumerID string) {
 		klog.Warningf("Failed to remove consumer %s", consumerID)
 	}
 }
+func (qm *QuotaManager) GetValidQuotaLabels() []string {
+	return qm.quotaManagerBackend.GetTreeNames()
+}

pkg/quotaplugins/quota-forest/quota-manager/quota/core/quotatree.go

@@ -5,7 +5,7 @@ Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
 
-    http://www.apache.org/licenses/LICENSE-2.0
+	http://www.apache.org/licenses/LICENSE-2.0
 
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,

pkg/quotaplugins/quota-forest/quota-manager/quota/quotamanager.go

@@ -529,7 +529,7 @@ func (m *Manager) String() string {
 	defer m.mutex.RUnlock()
 
 	var b bytes.Buffer
-	b.WriteString("QuotaManger: \n")
+	b.WriteString("QuotaManager: \n")
 	b.WriteString("Mode: " + m.GetModeString() + "\n")
 	b.WriteString("\n")
 

pkg/quotaplugins/quota-simple-rest/quota_rest_manager.go

@@ -1,11 +1,13 @@
+//go:build !private
 // +build !private
+
 // ------------------------------------------------------ {COPYRIGHT-TOP} ---
 // Copyright 2022, 2023 The Multi-Cluster App Dispatcher Authors.
-// 
+//
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
-// 
+//
 //    http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
@@ -21,6 +23,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+
 	"github.com/project-codeflare/multi-cluster-app-dispatcher/cmd/kar-controllers/app/options"
 	arbv1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1"
 	listersv1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/client/listers/controller/v1"
@@ -29,26 +32,27 @@ import (
 	"github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/quotaplugins/util"
 
 	"io/ioutil"
-	"k8s.io/client-go/rest"
-	"k8s.io/klog/v2"
 	"math"
 	"net/http"
 	"net/http/httputil"
 	"reflect"
 	"strings"
 	"time"
+
+	"k8s.io/client-go/rest"
+	"k8s.io/klog/v2"
 )
 
 // QuotaManager implements a QuotaManagerInterface.
 type QuotaManager struct {
-	url 			string
-	appwrapperLister 	listersv1.AppWrapperLister
-	preemptionEnabled 	bool
+	url               string
+	appwrapperLister  listersv1.AppWrapperLister
+	preemptionEnabled bool
 }
 
 type QuotaGroup struct {
-	GroupContext string  `json:"groupcontext"`
-	GroupId	     string  `json:"groupid"`
+	GroupContext string `json:"groupcontext"`
+	GroupId      string `json:"groupid"`
 }
 
 type Request struct {
@@ -70,19 +74,17 @@ type QuotaResponse struct {
 }
 
 type TreeNode struct {
-	Allocation	string     `json:"allocation"`
-	Quota		string  `json:"quota"`
-	Name		string   `json:"name"`
-	Hard		bool     `json:"hard"`
-	Children	[]TreeNode   `json:"children"`
-	Parent 		string `json:"parent"`
+	Allocation string     `json:"allocation"`
+	Quota      string     `json:"quota"`
+	Name       string     `json:"name"`
+	Hard       bool       `json:"hard"`
+	Children   []TreeNode `json:"children"`
+	Parent     string     `json:"parent"`
 }
 
-
 // Making sure that PriorityQueue implements SchedulingQueue.
 var _ = quota.QuotaManagerInterface(&QuotaManager{})
 
-
 func parseId(id string) (string, string) {
 	ns := ""
 	n := ""
@@ -115,24 +117,24 @@ func createId(ns string, n string) string {
 }
 
 func NewQuotaManager(dispatchedAWDemands map[string]*clusterstateapi.Resource, dispatchedAWs map[string]*arbv1.AppWrapper,
-			awJobLister listersv1.AppWrapperLister, config *rest.Config,
-				serverOptions *options.ServerOption) (*QuotaManager, error) {
+	awJobLister listersv1.AppWrapperLister, config *rest.Config,
+	serverOptions *options.ServerOption) (*QuotaManager, error) {
 	if serverOptions.QuotaEnabled == false {
 		klog.Infof("[NewQuotaManager] Quota management is not enabled.")
 		return nil, nil
 	}
 
 	qm := &QuotaManager{
-		url:                 serverOptions.QuotaRestURL,
-		appwrapperLister:    awJobLister,
-		preemptionEnabled:   serverOptions.Preemption,
+		url:               serverOptions.QuotaRestURL,
+		appwrapperLister:  awJobLister,
+		preemptionEnabled: serverOptions.Preemption,
 	}
 
 	return qm, nil
 }
 
 // Recrusive call to add names of Tree
-func (qm *QuotaManager) addChildrenNodes(parentNode TreeNode, treeIDs []string) ([]string) {
+func (qm *QuotaManager) addChildrenNodes(parentNode TreeNode, treeIDs []string) []string {
 	if len(parentNode.Children) > 0 {
 		for _, childNode := range parentNode.Children {
 			klog.V(10).Infof("[getQuotaTreeIDs] Quota tree response child node from quota mananger: %s", childNode.Name)
@@ -143,7 +145,7 @@ func (qm *QuotaManager) addChildrenNodes(parentNode TreeNode, treeIDs []string)
 	return treeIDs
 }
 
-func (qm *QuotaManager) getQuotaTreeIDs() ([]string) {
+func (qm *QuotaManager) getQuotaTreeIDs() []string {
 	var treeIDs []string
 	// If a url does not exists then assume fits quota
 	if len(qm.url) < 1 {
@@ -173,7 +175,7 @@ func (qm *QuotaManager) getQuotaTreeIDs() ([]string) {
 		body, err := ioutil.ReadAll(response.Body)
 		if err != nil {
 			klog.Errorf("[getQuotaTreeIDs] Failed to read quota tree from the quota manager body: %s, error=%#v",
-											string(body), err)
+				string(body), err)
 			return treeIDs
 		}
 
@@ -205,24 +207,24 @@ func isValidQuota(quotaGroup QuotaGroup, qmTreeIDs []string) bool {
 	return false
 }
 
-func (qm *QuotaManager) getQuotaDesignation(aw *arbv1.AppWrapper) ([]QuotaGroup) {
+func (qm *QuotaManager) getQuotaDesignation(aw *arbv1.AppWrapper) []QuotaGroup {
 	var groups []QuotaGroup
 
 	// Get list of quota management tree IDs
 	qmTreeIDs := qm.getQuotaTreeIDs()
 	if len(qmTreeIDs) < 1 {
 		klog.Warningf("[getQuotaDesignation] No valid quota management IDs found for AppWrapper Job: %s/%s",
-									aw.Namespace, aw.Name)
+			aw.Namespace, aw.Name)
 	}
 
 	labels := aw.GetLabels()
-	if ( labels != nil) {
+	if labels != nil {
 		keys := reflect.ValueOf(labels).MapKeys()
-		for _,  key := range keys {
+		for _, key := range keys {
 			strkey := key.String()
 			quotaGroup := QuotaGroup{
 				GroupContext: strkey,
-				GroupId: labels[strkey],
+				GroupId:      labels[strkey],
 			}
 			if isValidQuota(quotaGroup, qmTreeIDs) {
 				groups = append(groups, quotaGroup)
@@ -236,7 +238,7 @@ func (qm *QuotaManager) getQuotaDesignation(aw *arbv1.AppWrapper) ([]QuotaGroup)
 		}
 	} else {
 		klog.V(4).Infof("[getQuotaDesignation] AppWrapper: %s/%s does not any context quota labels.",
-										aw.Namespace, aw.Name)
+			aw.Namespace, aw.Name)
 	}
 
 	if len(groups) > 0 {
@@ -246,8 +248,8 @@ func (qm *QuotaManager) getQuotaDesignation(aw *arbv1.AppWrapper) ([]QuotaGroup)
 		klog.V(4).Infof("[getQuotaDesignation] AppWrapper: %s/%s does not have any quota labels, using default.",
 			aw.Namespace, aw.Name)
 		var defaultGroup = QuotaGroup{
-			GroupContext: 	"DEFAULTCONTEXT",
-			GroupId:	"DEFAULT",
+			GroupContext: "DEFAULTCONTEXT",
+			GroupId:      "DEFAULT",
 		}
 		groups = append(groups, defaultGroup)
 	}
@@ -256,7 +258,7 @@ func (qm *QuotaManager) getQuotaDesignation(aw *arbv1.AppWrapper) ([]QuotaGroup)
 }
 
 func (qm *QuotaManager) Fits(aw *arbv1.AppWrapper, awResDemands *clusterstateapi.Resource,
-					proposedPreemptions []*arbv1.AppWrapper) (bool, []*arbv1.AppWrapper, string) {
+	proposedPreemptions []*arbv1.AppWrapper) (bool, []*arbv1.AppWrapper, string) {
 
 	// Handle uninitialized quota manager
 	if len(qm.url) <= 0 {
@@ -271,7 +273,7 @@ func (qm *QuotaManager) Fits(aw *arbv1.AppWrapper, awResDemands *clusterstateapi
 	groups := qm.getQuotaDesignation(aw)
 	preemptable := qm.preemptionEnabled
 	awCPU_Demand := int(math.Trunc(awResDemands.MilliCPU))
-	awMem_Demand := int(math.Trunc(awResDemands.Memory)/1000000)
+	awMem_Demand := int(math.Trunc(awResDemands.Memory) / 1000000)
 	var demand []int
 	demand = append(demand, awCPU_Demand)
 	demand = append(demand, awMem_Demand)
@@ -331,8 +333,7 @@ func (qm *QuotaManager) Fits(aw *arbv1.AppWrapper, awResDemands *clusterstateapi
 	return doesFit, preemptIds, ""
 }
 
-
-func  (qm *QuotaManager) getAppWrappers(preemptIds []string) []*arbv1.AppWrapper{
+func (qm *QuotaManager) getAppWrappers(preemptIds []string) []*arbv1.AppWrapper {
 	var aws []*arbv1.AppWrapper
 	if len(preemptIds) <= 0 {
 		return nil
@@ -411,3 +412,6 @@ func (qm *QuotaManager) Release(aw *arbv1.AppWrapper) bool {
 
 	return released
 }
+func (qm *QuotaManager) GetValidQuotaLabels() []string {
+	return qm.getQuotaTreeIDs()
+}

test/e2e-kuttl/install-quota-subtree.yaml

@@ -1,3 +1,4 @@
+---
 apiVersion: ibm.com/v1
 kind: QuotaSubtree
 metadata:
@@ -54,4 +55,32 @@ spec:
         hardLimit: true
         requests:
           cpu: 900m
-          memory: 300Mi
+          memory: 300Mi
+    - name: default
+      quotas:
+        hardLimit: false
+        requests:
+          cpu: 0m
+          memory: 0Mi
+---
+apiVersion: ibm.com/v1
+kind: QuotaSubtree
+metadata:
+  name: service-root-children
+  namespace: kube-system
+  labels:
+    tree: quota_service
+spec:
+  parent: service-root
+  children:
+    - name: gold
+      quotas:
+        requests:
+          cpu: 1075m
+          memory: 1045Mi
+    - name: default
+      quotas:
+        hardLimit: false
+        requests:
+          cpu: 0m
+          memory: 0Mi

test/e2e-kuttl/quota-errors/03-assert.yaml

@@ -6,7 +6,7 @@ metadata:
   namespace: quota-errors
   labels:
     quota_context: "silver"
-    quota_service: "service-root"
+    quota_service: "gold"
 status:
   state: Running
 ---
@@ -24,4 +24,4 @@ status:
     observedGeneration: 1
     readyReplicas: 1
     replicas: 1
-    updatedReplicas: 1
+    updatedReplicas: 1

test/e2e-kuttl/quota-errors/03-install.yaml

@@ -5,7 +5,7 @@ metadata:
   namespace: quota-errors
   labels:
     quota_context: "silver"
-    quota_service: "service-root"
+    quota_service: "gold"
 spec:
   resources:
     GenericItems:

test/e2e-kuttl/quota-forest/01-assert.yaml

@@ -22,3 +22,11 @@ metadata:
   namespace: kube-system
   labels:
     tree: quota_context
+---
+apiVersion: ibm.com/v1
+kind: QuotaSubtree
+metadata:
+  name: service-root-children
+  namespace: kube-system
+  labels:
+    tree: quota_service