From 4b76483197fbb328ee3e807ada028ece322dbcf6 Mon Sep 17 00:00:00 2001 From: Ivan Zhang Date: Mon, 26 Aug 2019 13:08:07 -0400 Subject: [PATCH 01/11] progress --- pkg/operator/endpoints/deploy.go | 6 ++++++ pkg/operator/workloads/api_workload.go | 1 - pkg/operator/workloads/workflow.go | 6 ++++++ 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/pkg/operator/endpoints/deploy.go b/pkg/operator/endpoints/deploy.go index 74a7d7987e..4c52d0abe7 100644 --- a/pkg/operator/endpoints/deploy.go +++ b/pkg/operator/endpoints/deploy.go @@ -56,6 +56,12 @@ func Deploy(w http.ResponseWriter, r *http.Request) { fullCtxMatch = true } + err = workloads.ValidateDeploy(ctx) + if err != nil { + RespondError(w, err) + return + } + deploymentStatus, err := workloads.GetDeploymentStatus(ctx.App.Name) if err != nil { RespondError(w, err) diff --git a/pkg/operator/workloads/api_workload.go b/pkg/operator/workloads/api_workload.go index caf9b1f473..f0ee8c0d3f 100644 --- a/pkg/operator/workloads/api_workload.go +++ b/pkg/operator/workloads/api_workload.go @@ -87,7 +87,6 @@ func (aw *APIWorkload) Start(ctx *context.Context) error { desiredReplicas := getRequestedReplicasFromDeployment(api, k8sDeloyment, hpa) var deploymentSpec *kapps.Deployment - switch api.ModelFormat { case userconfig.TensorFlowModelFormat: deploymentSpec = tfAPISpec(ctx, api, aw.WorkloadID, desiredReplicas) diff --git a/pkg/operator/workloads/workflow.go b/pkg/operator/workloads/workflow.go index 684a1a70fb..dff52f13cd 100644 --- a/pkg/operator/workloads/workflow.go +++ b/pkg/operator/workloads/workflow.go @@ -17,6 +17,7 @@ limitations under the License. package workloads import ( + "log" "path/filepath" "github.com/cortexlabs/cortex/pkg/consts" @@ -294,3 +295,8 @@ func GetDeploymentStatus(appName string) (resource.DeploymentStatus, error) { } return resource.UpdatedDeploymentStatus, nil } + +func ValidateDeploy(ctx *context.Context) error { + log.Println(config.Kubernetes.ListPods(nil)) + return nil +} From 6ea298de4a9377227d1c6b7642ef1b1dd5c916c4 Mon Sep 17 00:00:00 2001 From: Ivan Zhang Date: Mon, 26 Aug 2019 17:58:22 -0400 Subject: [PATCH 02/11] resource warning --- pkg/lib/k8s/k8s.go | 2 ++ pkg/lib/k8s/node.go | 22 +++++++++++++++++++++ pkg/operator/workloads/errors.go | 13 ++++++++++++- pkg/operator/workloads/workflow.go | 31 ++++++++++++++++++++++++++++-- 4 files changed, 65 insertions(+), 3 deletions(-) create mode 100644 pkg/lib/k8s/node.go diff --git a/pkg/lib/k8s/k8s.go b/pkg/lib/k8s/k8s.go index 46de5e1acb..362931dd59 100644 --- a/pkg/lib/k8s/k8s.go +++ b/pkg/lib/k8s/k8s.go @@ -51,6 +51,7 @@ type Client struct { clientset *kclientset.Clientset dynamicClient kclientdynamic.Interface podClient kclientcore.PodInterface + nodeClient kclientcore.NodeInterface serviceClient kclientcore.ServiceInterface configMapClient kclientcore.ConfigMapInterface deploymentClient kclientapps.DeploymentInterface @@ -87,6 +88,7 @@ func New(namespace string, inCluster bool) (*Client, error) { } client.podClient = client.clientset.CoreV1().Pods(namespace) + client.nodeClient = client.clientset.CoreV1().Nodes() client.serviceClient = client.clientset.CoreV1().Services(namespace) client.configMapClient = client.clientset.CoreV1().ConfigMaps(namespace) client.deploymentClient = client.clientset.AppsV1().Deployments(namespace) diff --git a/pkg/lib/k8s/node.go b/pkg/lib/k8s/node.go new file mode 100644 index 0000000000..2fd54f6faf --- /dev/null +++ b/pkg/lib/k8s/node.go @@ -0,0 +1,22 @@ +package k8s + +import ( + "github.com/cortexlabs/cortex/pkg/lib/errors" + + kcore "k8s.io/api/core/v1" + kmeta "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func (c *Client) ListNodes(opts *kmeta.ListOptions) ([]kcore.Node, error) { + if opts == nil { + opts = &kmeta.ListOptions{} + } + podList, err := c.nodeClient.List(*opts) + if err != nil { + return nil, errors.WithStack(err) + } + for i := range podList.Items { + podList.Items[i].TypeMeta = podTypeMeta + } + return podList.Items, nil +} diff --git a/pkg/operator/workloads/errors.go b/pkg/operator/workloads/errors.go index 9f489376e3..15c116336c 100644 --- a/pkg/operator/workloads/errors.go +++ b/pkg/operator/workloads/errors.go @@ -16,6 +16,8 @@ limitations under the License. package workloads +import "fmt" + type ErrorKind int const ( @@ -25,6 +27,7 @@ const ( ErrLoadBalancerInitializing ErrNotFound ErrAPIInitializing + ErrNoAvailableNodeComputeLimit ) var errorKinds = []string{ @@ -34,9 +37,10 @@ var errorKinds = []string{ "err_load_balancer_initializing", "err_not_found", "err_api_initializing", + "err_no_available_node_compute_limit", } -var _ = [1]int{}[int(ErrAPIInitializing)-(len(errorKinds)-1)] // Ensure list length matches +var _ = [1]int{}[int(ErrNoAvailableNodeComputeLimit)-(len(errorKinds)-1)] // Ensure list length matches func (t ErrorKind) String() string { return errorKinds[t] @@ -115,3 +119,10 @@ func ErrorAPIInitializing() error { message: "api is still initializing", } } + +func ErrorNoAvailableNodeComputeLimit(resource string, req, max int64) error { + return Error{ + Kind: ErrNoAvailableNodeComputeLimit, + message: fmt.Sprintf("there are not available nodes that can satisfy the requested %s resource quantity - requested %d but nodes only have %d", resource, req, max), + } +} diff --git a/pkg/operator/workloads/workflow.go b/pkg/operator/workloads/workflow.go index dff52f13cd..b43073f012 100644 --- a/pkg/operator/workloads/workflow.go +++ b/pkg/operator/workloads/workflow.go @@ -17,7 +17,6 @@ limitations under the License. package workloads import ( - "log" "path/filepath" "github.com/cortexlabs/cortex/pkg/consts" @@ -25,6 +24,7 @@ import ( "github.com/cortexlabs/cortex/pkg/lib/sets/strset" "github.com/cortexlabs/cortex/pkg/operator/api/context" "github.com/cortexlabs/cortex/pkg/operator/api/resource" + "github.com/cortexlabs/cortex/pkg/operator/api/userconfig" "github.com/cortexlabs/cortex/pkg/operator/config" ) @@ -297,6 +297,33 @@ func GetDeploymentStatus(appName string) (resource.DeploymentStatus, error) { } func ValidateDeploy(ctx *context.Context) error { - log.Println(config.Kubernetes.ListPods(nil)) + nodes, err := config.Kubernetes.ListNodes(nil) + if err != nil { + return err + } + + maxCPU, _ := nodes[0].Status.Capacity.Cpu().AsInt64() + maxMem, _ := nodes[0].Status.Capacity.Memory().AsInt64() + maxGPUQuantity, ok := nodes[0].Status.Allocatable["nvidia.com/gpu"] + var maxGPU int64 + if ok { + maxGPU, _ = maxGPUQuantity.AsInt64() + } + for _, api := range ctx.APIs { + cpu, _ := api.Compute.CPU.AsInt64() + if cpu > maxCPU { + return errors.Wrap(ErrorNoAvailableNodeComputeLimit("CPU", cpu, maxCPU), userconfig.Identify(api)) + } + if api.Compute.Mem != nil { + mem, _ := api.Compute.Mem.AsInt64() + if mem > maxMem { + return errors.Wrap(ErrorNoAvailableNodeComputeLimit("Mem", mem, maxMem), userconfig.Identify(api)) + } + } + gpu := api.Compute.GPU + if gpu > maxGPU { + return errors.Wrap(ErrorNoAvailableNodeComputeLimit("GPU", gpu, maxGPU), userconfig.Identify(api)) + } + } return nil } From b3c09b11f107cd4ae772bb90690c33bad1576a96 Mon Sep 17 00:00:00 2001 From: Ivan Zhang Date: Mon, 26 Aug 2019 18:04:12 -0400 Subject: [PATCH 03/11] look for max --- pkg/operator/workloads/workflow.go | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/pkg/operator/workloads/workflow.go b/pkg/operator/workloads/workflow.go index b43073f012..aa473aa348 100644 --- a/pkg/operator/workloads/workflow.go +++ b/pkg/operator/workloads/workflow.go @@ -302,13 +302,28 @@ func ValidateDeploy(ctx *context.Context) error { return err } - maxCPU, _ := nodes[0].Status.Capacity.Cpu().AsInt64() - maxMem, _ := nodes[0].Status.Capacity.Memory().AsInt64() - maxGPUQuantity, ok := nodes[0].Status.Allocatable["nvidia.com/gpu"] - var maxGPU int64 - if ok { - maxGPU, _ = maxGPUQuantity.AsInt64() + var maxCPU, maxMem, maxGPU int64 + for _, node := range nodes { + curCPU, _ := node.Status.Capacity.Cpu().AsInt64() + curMem, _ := node.Status.Capacity.Memory().AsInt64() + var curGPU int64 + if GPUQuantity, ok := node.Status.Allocatable["nvidia.com/gpu"]; ok { + curGPU, _ = GPUQuantity.AsInt64() + } + + if curCPU > maxCPU { + maxCPU = curCPU + } + + if curMem > maxMem { + maxMem = curMem + } + + if curGPU > maxGPU { + maxGPU = curGPU + } } + for _, api := range ctx.APIs { cpu, _ := api.Compute.CPU.AsInt64() if cpu > maxCPU { From fcaeccbddb6db6034232c8528b5d408ef620ea01 Mon Sep 17 00:00:00 2001 From: Ivan Zhang Date: Mon, 26 Aug 2019 18:06:25 -0400 Subject: [PATCH 04/11] add license --- pkg/lib/k8s/node.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/pkg/lib/k8s/node.go b/pkg/lib/k8s/node.go index 2fd54f6faf..9107683d1e 100644 --- a/pkg/lib/k8s/node.go +++ b/pkg/lib/k8s/node.go @@ -1,3 +1,19 @@ +/* +Copyright 2019 Cortex Labs, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package k8s import ( From e3da149c8737a73ebe1a4ed813bc72bc6b69d08a Mon Sep 17 00:00:00 2001 From: Ivan Zhang Date: Tue, 27 Aug 2019 16:24:01 -0400 Subject: [PATCH 05/11] remove --- pkg/lib/k8s/node.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/pkg/lib/k8s/node.go b/pkg/lib/k8s/node.go index 9107683d1e..9a128a787c 100644 --- a/pkg/lib/k8s/node.go +++ b/pkg/lib/k8s/node.go @@ -31,8 +31,5 @@ func (c *Client) ListNodes(opts *kmeta.ListOptions) ([]kcore.Node, error) { if err != nil { return nil, errors.WithStack(err) } - for i := range podList.Items { - podList.Items[i].TypeMeta = podTypeMeta - } return podList.Items, nil } From 3f068593a5ca77af10ee3965498cba241d1914b6 Mon Sep 17 00:00:00 2001 From: Ivan Zhang Date: Tue, 27 Aug 2019 16:24:23 -0400 Subject: [PATCH 06/11] grammar --- pkg/operator/workloads/errors.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/operator/workloads/errors.go b/pkg/operator/workloads/errors.go index 15c116336c..6bbf7f417f 100644 --- a/pkg/operator/workloads/errors.go +++ b/pkg/operator/workloads/errors.go @@ -123,6 +123,6 @@ func ErrorAPIInitializing() error { func ErrorNoAvailableNodeComputeLimit(resource string, req, max int64) error { return Error{ Kind: ErrNoAvailableNodeComputeLimit, - message: fmt.Sprintf("there are not available nodes that can satisfy the requested %s resource quantity - requested %d but nodes only have %d", resource, req, max), + message: fmt.Sprintf("there are no available nodes that can satisfy the requested %s resource quantity - requested %d but nodes only have %d", resource, req, max), } } From a42e7d9f755a24d67cd5729413366cd79ef2d08f Mon Sep 17 00:00:00 2001 From: Ivan Zhang Date: Tue, 27 Aug 2019 16:25:08 -0400 Subject: [PATCH 07/11] grammar --- pkg/operator/workloads/errors.go | 2 +- pkg/operator/workloads/workflow.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/operator/workloads/errors.go b/pkg/operator/workloads/errors.go index 6bbf7f417f..06d1f3c2cf 100644 --- a/pkg/operator/workloads/errors.go +++ b/pkg/operator/workloads/errors.go @@ -123,6 +123,6 @@ func ErrorAPIInitializing() error { func ErrorNoAvailableNodeComputeLimit(resource string, req, max int64) error { return Error{ Kind: ErrNoAvailableNodeComputeLimit, - message: fmt.Sprintf("there are no available nodes that can satisfy the requested %s resource quantity - requested %d but nodes only have %d", resource, req, max), + message: fmt.Sprintf("there are no available nodes that can satisfy the requested %s quantity - requested %d but nodes only have %d", resource, req, max), } } diff --git a/pkg/operator/workloads/workflow.go b/pkg/operator/workloads/workflow.go index aa473aa348..a21b4cce44 100644 --- a/pkg/operator/workloads/workflow.go +++ b/pkg/operator/workloads/workflow.go @@ -332,7 +332,7 @@ func ValidateDeploy(ctx *context.Context) error { if api.Compute.Mem != nil { mem, _ := api.Compute.Mem.AsInt64() if mem > maxMem { - return errors.Wrap(ErrorNoAvailableNodeComputeLimit("Mem", mem, maxMem), userconfig.Identify(api)) + return errors.Wrap(ErrorNoAvailableNodeComputeLimit("Memory", mem, maxMem), userconfig.Identify(api)) } } gpu := api.Compute.GPU From 56fc9fe0c0a0f2af9bfb7a9dd3045fc7aa788ee3 Mon Sep 17 00:00:00 2001 From: Ivan Zhang Date: Wed, 28 Aug 2019 13:38:56 -0400 Subject: [PATCH 08/11] address comments --- pkg/lib/k8s/node.go | 4 ++-- pkg/operator/workloads/errors.go | 4 ++-- pkg/operator/workloads/workflow.go | 31 ++++++++++++++++-------------- 3 files changed, 21 insertions(+), 18 deletions(-) diff --git a/pkg/lib/k8s/node.go b/pkg/lib/k8s/node.go index 9a128a787c..5208d6acce 100644 --- a/pkg/lib/k8s/node.go +++ b/pkg/lib/k8s/node.go @@ -27,9 +27,9 @@ func (c *Client) ListNodes(opts *kmeta.ListOptions) ([]kcore.Node, error) { if opts == nil { opts = &kmeta.ListOptions{} } - podList, err := c.nodeClient.List(*opts) + nodeList, err := c.nodeClient.List(*opts) if err != nil { return nil, errors.WithStack(err) } - return podList.Items, nil + return nodeList.Items, nil } diff --git a/pkg/operator/workloads/errors.go b/pkg/operator/workloads/errors.go index 06d1f3c2cf..4e7282b6fb 100644 --- a/pkg/operator/workloads/errors.go +++ b/pkg/operator/workloads/errors.go @@ -120,9 +120,9 @@ func ErrorAPIInitializing() error { } } -func ErrorNoAvailableNodeComputeLimit(resource string, req, max int64) error { +func ErrorNoAvailableNodeComputeLimit(resource, reqStr, maxStr string) error { return Error{ Kind: ErrNoAvailableNodeComputeLimit, - message: fmt.Sprintf("there are no available nodes that can satisfy the requested %s quantity - requested %d but nodes only have %d", resource, req, max), + message: fmt.Sprintf("there are no available nodes that can satisfy the requested %s quantity - requested %s but nodes only have %s", resource, reqStr, maxStr), } } diff --git a/pkg/operator/workloads/workflow.go b/pkg/operator/workloads/workflow.go index a21b4cce44..d7a097497c 100644 --- a/pkg/operator/workloads/workflow.go +++ b/pkg/operator/workloads/workflow.go @@ -17,8 +17,11 @@ limitations under the License. package workloads import ( + "fmt" "path/filepath" + kresource "k8s.io/apimachinery/pkg/api/resource" + "github.com/cortexlabs/cortex/pkg/consts" "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/lib/sets/strset" @@ -302,21 +305,23 @@ func ValidateDeploy(ctx *context.Context) error { return err } - var maxCPU, maxMem, maxGPU int64 + var maxCPU, maxMem kresource.Quantity + var maxGPU int64 for _, node := range nodes { - curCPU, _ := node.Status.Capacity.Cpu().AsInt64() - curMem, _ := node.Status.Capacity.Memory().AsInt64() + curCPU := node.Status.Capacity.Cpu() + curMem := node.Status.Capacity.Memory() + var curGPU int64 if GPUQuantity, ok := node.Status.Allocatable["nvidia.com/gpu"]; ok { curGPU, _ = GPUQuantity.AsInt64() } - if curCPU > maxCPU { - maxCPU = curCPU + if curCPU != nil && maxCPU.Cmp(*curCPU) == -1 { + maxCPU = *curCPU } - if curMem > maxMem { - maxMem = curMem + if curMem != nil && maxMem.Cmp(*curMem) == -1 { + maxMem = *curMem } if curGPU > maxGPU { @@ -325,19 +330,17 @@ func ValidateDeploy(ctx *context.Context) error { } for _, api := range ctx.APIs { - cpu, _ := api.Compute.CPU.AsInt64() - if cpu > maxCPU { - return errors.Wrap(ErrorNoAvailableNodeComputeLimit("CPU", cpu, maxCPU), userconfig.Identify(api)) + if maxCPU.Cmp(api.Compute.CPU.Quantity) == -1 { + return errors.Wrap(ErrorNoAvailableNodeComputeLimit("CPU", api.Compute.CPU.String(), maxCPU.String()), userconfig.Identify(api)) } if api.Compute.Mem != nil { - mem, _ := api.Compute.Mem.AsInt64() - if mem > maxMem { - return errors.Wrap(ErrorNoAvailableNodeComputeLimit("Memory", mem, maxMem), userconfig.Identify(api)) + if maxMem.Cmp(api.Compute.Mem.Quantity) == -1 { + return errors.Wrap(ErrorNoAvailableNodeComputeLimit("Memory", api.Compute.Mem.String(), maxMem.String()), userconfig.Identify(api)) } } gpu := api.Compute.GPU if gpu > maxGPU { - return errors.Wrap(ErrorNoAvailableNodeComputeLimit("GPU", gpu, maxGPU), userconfig.Identify(api)) + return errors.Wrap(ErrorNoAvailableNodeComputeLimit("GPU", fmt.Sprintf("%d", gpu), fmt.Sprintf("%d", maxGPU)), userconfig.Identify(api)) } } return nil From e0a157814bc6396e49eda2338e07cfd15972fe14 Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Wed, 28 Aug 2019 11:37:40 -0700 Subject: [PATCH 09/11] Update errors.go --- pkg/operator/workloads/errors.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/operator/workloads/errors.go b/pkg/operator/workloads/errors.go index 4e7282b6fb..00e262087b 100644 --- a/pkg/operator/workloads/errors.go +++ b/pkg/operator/workloads/errors.go @@ -123,6 +123,6 @@ func ErrorAPIInitializing() error { func ErrorNoAvailableNodeComputeLimit(resource, reqStr, maxStr string) error { return Error{ Kind: ErrNoAvailableNodeComputeLimit, - message: fmt.Sprintf("there are no available nodes that can satisfy the requested %s quantity - requested %s but nodes only have %s", resource, reqStr, maxStr), + message: fmt.Sprintf("no available nodes can satisfy the requested %s quantity - requested %s %s but nodes only have %s %s", resource, reqStr, resource, maxStr, resource), } } From 5c818d41a5e224fa1c7d71e450fe628ad4e8e419 Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Wed, 28 Aug 2019 11:41:11 -0700 Subject: [PATCH 10/11] Update workflow.go --- pkg/operator/workloads/workflow.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/operator/workloads/workflow.go b/pkg/operator/workloads/workflow.go index d7a097497c..d3d3981f7b 100644 --- a/pkg/operator/workloads/workflow.go +++ b/pkg/operator/workloads/workflow.go @@ -316,11 +316,11 @@ func ValidateDeploy(ctx *context.Context) error { curGPU, _ = GPUQuantity.AsInt64() } - if curCPU != nil && maxCPU.Cmp(*curCPU) == -1 { + if curCPU != nil && maxCPU.Cmp(*curCPU) < 0 { maxCPU = *curCPU } - if curMem != nil && maxMem.Cmp(*curMem) == -1 { + if curMem != nil && maxMem.Cmp(*curMem) < 0 { maxMem = *curMem } @@ -330,11 +330,11 @@ func ValidateDeploy(ctx *context.Context) error { } for _, api := range ctx.APIs { - if maxCPU.Cmp(api.Compute.CPU.Quantity) == -1 { + if maxCPU.Cmp(api.Compute.CPU.Quantity) < 0 { return errors.Wrap(ErrorNoAvailableNodeComputeLimit("CPU", api.Compute.CPU.String(), maxCPU.String()), userconfig.Identify(api)) } if api.Compute.Mem != nil { - if maxMem.Cmp(api.Compute.Mem.Quantity) == -1 { + if maxMem.Cmp(api.Compute.Mem.Quantity) < 0 { return errors.Wrap(ErrorNoAvailableNodeComputeLimit("Memory", api.Compute.Mem.String(), maxMem.String()), userconfig.Identify(api)) } } From 3796c69186bdbe6beefb96a8f5241eed9de2d16a Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Wed, 28 Aug 2019 11:53:54 -0700 Subject: [PATCH 11/11] Update errors.go --- pkg/operator/workloads/errors.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pkg/operator/workloads/errors.go b/pkg/operator/workloads/errors.go index 00e262087b..73c93e712b 100644 --- a/pkg/operator/workloads/errors.go +++ b/pkg/operator/workloads/errors.go @@ -120,9 +120,13 @@ func ErrorAPIInitializing() error { } } -func ErrorNoAvailableNodeComputeLimit(resource, reqStr, maxStr string) error { +func ErrorNoAvailableNodeComputeLimit(resource string, reqStr string, maxStr string) error { + message := fmt.Sprintf("no available nodes can satisfy the requested %s quantity - requested %s %s but nodes only have %s %s", resource, reqStr, resource, maxStr, resource) + if maxStr == "0" { + message = fmt.Sprintf("no available nodes can satisfy the requested %s quantity - requested %s %s but nodes don't have any %s", resource, reqStr, resource, resource) + } return Error{ Kind: ErrNoAvailableNodeComputeLimit, - message: fmt.Sprintf("no available nodes can satisfy the requested %s quantity - requested %s %s but nodes only have %s %s", resource, reqStr, resource, maxStr, resource), + message: message, } }