From 3bd95895fdacfae4e3062d5af9fa6e62e1c433fd Mon Sep 17 00:00:00 2001 From: Robert Lucian Chiriac Date: Sun, 18 Jul 2021 03:08:07 +0300 Subject: [PATCH 1/5] Use IPVS load balancer instead of IP Tables --- manager/generate_eks.py | 9 +++++++ manager/install.sh | 10 +++++++ manager/manifests/kube-proxy.yaml | 42 ++++++++++++++++++++++++++++++ manager/upgrade_kube_proxy_mode.py | 31 ++++++++++++++++++++++ 4 files changed, 92 insertions(+) create mode 100644 manager/manifests/kube-proxy.yaml create mode 100644 manager/upgrade_kube_proxy_mode.py diff --git a/manager/generate_eks.py b/manager/generate_eks.py index aa22c9a9a9..f5d4e84885 100644 --- a/manager/generate_eks.py +++ b/manager/generate_eks.py @@ -67,6 +67,15 @@ def default_nodegroup(cluster_config): "evictionHard": {"memory.available": "200Mi", "nodefs.available": "5%"}, "registryPullQPS": 10, }, + "preBootstrapCommands": [ + "sudo yum install -y ipvsadm", + "sudo modprobe ip_vs", # IP virtual server + "sudo modprobe ip_vs_rr", # round robing load balancer + "sudo modprobe ip_vs_lc", # least connected load balancer + "sudo modprobe ip_vs_wrr", # weighted round robin load balancer + "sudo modprobe ip_vs_sh", # source-hashing load balancer + "sudo modprobe nf_conntrack_ipv4", + ], } diff --git a/manager/install.sh b/manager/install.sh index 5286c4d9ed..4860e946b9 100755 --- a/manager/install.sh +++ b/manager/install.sh @@ -41,6 +41,7 @@ function cluster_up() { echo "✓" echo -n "￮ configuring networking (this will take a few minutes) " + setup_ipvs setup_istio python render_template.py $CORTEX_CLUSTER_CONFIG_FILE manifests/apis.yaml.j2 | kubectl apply -f - >/dev/null echo "✓" @@ -368,6 +369,15 @@ function remove_nodegroups() { echo } +func setup_ipvs() { + kube_proxy_pod=$(kubectl get pod -n kube-system -l k8s-app=kube-proxy -o jsonpath='{.items[*].metadata.name}' | cut -d " " -f1) + kubectl exec -it -n kube-system ${kube_proxy_pod} -- cat /var/lib/kube-proxy-config/config > proxy_config.yaml + python upgrade_kube_proxy_mode.py proxy_config.yaml > proxy_config.yaml + kubectl get configmap -n kube-system kube-proxy -o yaml | yq --arg replace "`cat proxy_config.yaml`" '.data.config=$replace' + kubectl patch ds -n kube-system kube-proxy --patch "$(cat manifests/kube-proxy.yaml)" >/dev/null + kubectl rollout status daemonset kube-proxy -n kube-system --timeout 30m >/dev/null +} + function setup_istio() { if ! grep -q "istio-customgateway-certs" <<< $(kubectl get secret -n istio-system); then WEBSITE=localhost diff --git a/manager/manifests/kube-proxy.yaml b/manager/manifests/kube-proxy.yaml new file mode 100644 index 0000000000..315d7b17ef --- /dev/null +++ b/manager/manifests/kube-proxy.yaml @@ -0,0 +1,42 @@ +# Copyright 2021 Cortex Labs, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is a patch that needs to be applied onto the daemonset that's added by eksctl. + +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: kube-proxy + namespace: kube-system +spec: + selector: + matchLabels: + k8s-app: kube-proxy + template: + spec: + containers: + - name: kube-proxy + command: + - kube-proxy + - --v=22 + - --proxy-mode=ipvs + - --ipvs-scheduler=lc + - --config=/var/lib/kube-proxy/config + env: + - name: KUBE_PROXY_MODE + value: ipvs + updateStrategy: + rollingUpdate: + maxUnavailable: 20% + type: RollingUpdate diff --git a/manager/upgrade_kube_proxy_mode.py b/manager/upgrade_kube_proxy_mode.py new file mode 100644 index 0000000000..3159c9a4df --- /dev/null +++ b/manager/upgrade_kube_proxy_mode.py @@ -0,0 +1,31 @@ +# Copyright 2021 Cortex Labs, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Usage: python create_user.py $KUBE_PROXY_CONFIG.yaml + +import yaml +import sys + +def main(): + kube_proxy_config_file = sys.argv[1] + with open(kube_proxy_config_file, "r") as f: + kube_proxy_config = yaml.safe_load(f) + + kube_proxy_config["mode"] = "ipvs" # IP Virtual Server + kube_proxy_config["ipvs"]["scheduler"] = "lc" # least connected + + print(yaml.dump(kube_proxy_config, indent=2)) + +if __name__ == "__main__": + main() From b9ba8436da1dc1eb8087c89a4c32509b4961f8d9 Mon Sep 17 00:00:00 2001 From: Robert Lucian Chiriac Date: Sun, 18 Jul 2021 03:10:23 +0300 Subject: [PATCH 2/5] Make lint --- manager/generate_eks.py | 10 +++++----- manager/upgrade_kube_proxy_mode.py | 6 ++++-- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/manager/generate_eks.py b/manager/generate_eks.py index f5d4e84885..6d4ccbad76 100644 --- a/manager/generate_eks.py +++ b/manager/generate_eks.py @@ -69,11 +69,11 @@ def default_nodegroup(cluster_config): }, "preBootstrapCommands": [ "sudo yum install -y ipvsadm", - "sudo modprobe ip_vs", # IP virtual server - "sudo modprobe ip_vs_rr", # round robing load balancer - "sudo modprobe ip_vs_lc", # least connected load balancer - "sudo modprobe ip_vs_wrr", # weighted round robin load balancer - "sudo modprobe ip_vs_sh", # source-hashing load balancer + "sudo modprobe ip_vs", # IP virtual server + "sudo modprobe ip_vs_rr", # round robing load balancer + "sudo modprobe ip_vs_lc", # least connected load balancer + "sudo modprobe ip_vs_wrr", # weighted round robin load balancer + "sudo modprobe ip_vs_sh", # source-hashing load balancer "sudo modprobe nf_conntrack_ipv4", ], } diff --git a/manager/upgrade_kube_proxy_mode.py b/manager/upgrade_kube_proxy_mode.py index 3159c9a4df..988c59dd4e 100644 --- a/manager/upgrade_kube_proxy_mode.py +++ b/manager/upgrade_kube_proxy_mode.py @@ -17,15 +17,17 @@ import yaml import sys + def main(): kube_proxy_config_file = sys.argv[1] with open(kube_proxy_config_file, "r") as f: kube_proxy_config = yaml.safe_load(f) - kube_proxy_config["mode"] = "ipvs" # IP Virtual Server - kube_proxy_config["ipvs"]["scheduler"] = "lc" # least connected + kube_proxy_config["mode"] = "ipvs" # IP Virtual Server + kube_proxy_config["ipvs"]["scheduler"] = "lc" # least connected print(yaml.dump(kube_proxy_config, indent=2)) + if __name__ == "__main__": main() From 0f777aa92e16dec49e311d95ff43c91c382eb53f Mon Sep 17 00:00:00 2001 From: Robert Lucian Chiriac Date: Mon, 19 Jul 2021 16:46:57 +0300 Subject: [PATCH 3/5] Change load balancing algorithm (lc->rr) --- manager/manifests/kube-proxy.yaml | 2 +- manager/upgrade_kube_proxy_mode.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/manager/manifests/kube-proxy.yaml b/manager/manifests/kube-proxy.yaml index 315d7b17ef..07e2c7da66 100644 --- a/manager/manifests/kube-proxy.yaml +++ b/manager/manifests/kube-proxy.yaml @@ -31,7 +31,7 @@ spec: - kube-proxy - --v=22 - --proxy-mode=ipvs - - --ipvs-scheduler=lc + - --ipvs-scheduler=rr - --config=/var/lib/kube-proxy/config env: - name: KUBE_PROXY_MODE diff --git a/manager/upgrade_kube_proxy_mode.py b/manager/upgrade_kube_proxy_mode.py index 988c59dd4e..e255354b0e 100644 --- a/manager/upgrade_kube_proxy_mode.py +++ b/manager/upgrade_kube_proxy_mode.py @@ -24,7 +24,7 @@ def main(): kube_proxy_config = yaml.safe_load(f) kube_proxy_config["mode"] = "ipvs" # IP Virtual Server - kube_proxy_config["ipvs"]["scheduler"] = "lc" # least connected + kube_proxy_config["ipvs"]["scheduler"] = "rr" # round robin print(yaml.dump(kube_proxy_config, indent=2)) From 4dbf364d3786047ded907c18c8a14797c8cdaaf6 Mon Sep 17 00:00:00 2001 From: Robert Lucian Chiriac Date: Mon, 19 Jul 2021 20:48:07 +0300 Subject: [PATCH 4/5] Changes to the setup_ipvs function --- manager/install.sh | 16 +++++++++++++--- manager/manifests/kube-proxy.yaml | 2 +- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/manager/install.sh b/manager/install.sh index 4860e946b9..8433a924e7 100755 --- a/manager/install.sh +++ b/manager/install.sh @@ -369,11 +369,21 @@ function remove_nodegroups() { echo } -func setup_ipvs() { +function setup_ipvs() { + # get a random kube-proxy pod + until [ "$(kubectl get pod -n kube-system -l k8s-app=kube-proxy -o jsonpath='{.items[*].metadata.name}' | wc -w)" -ne "0" ]; do sleep 1; done kube_proxy_pod=$(kubectl get pod -n kube-system -l k8s-app=kube-proxy -o jsonpath='{.items[*].metadata.name}' | cut -d " " -f1) + + # export kube-proxy's current config kubectl exec -it -n kube-system ${kube_proxy_pod} -- cat /var/lib/kube-proxy-config/config > proxy_config.yaml - python upgrade_kube_proxy_mode.py proxy_config.yaml > proxy_config.yaml - kubectl get configmap -n kube-system kube-proxy -o yaml | yq --arg replace "`cat proxy_config.yaml`" '.data.config=$replace' + + # upgrade proxy mode from the exported kube-proxy config + python upgrade_kube_proxy_mode.py proxy_config.yaml > upgraded_proxy_config.yaml + + # update kube-proxy's configmap to include the updated configuration + kubectl get configmap -n kube-system kube-proxy -o yaml | yq --arg replace "`cat upgraded_proxy_config.yaml`" '.data.config=$replace' | kubectl apply -f - >/dev/null + + # patch the kube-proxy daemonset kubectl patch ds -n kube-system kube-proxy --patch "$(cat manifests/kube-proxy.yaml)" >/dev/null kubectl rollout status daemonset kube-proxy -n kube-system --timeout 30m >/dev/null } diff --git a/manager/manifests/kube-proxy.yaml b/manager/manifests/kube-proxy.yaml index 07e2c7da66..9238549f21 100644 --- a/manager/manifests/kube-proxy.yaml +++ b/manager/manifests/kube-proxy.yaml @@ -29,7 +29,7 @@ spec: - name: kube-proxy command: - kube-proxy - - --v=22 + - --v=2 - --proxy-mode=ipvs - --ipvs-scheduler=rr - --config=/var/lib/kube-proxy/config From 86fb38e394a76e4a3a84a59989a5eb940b3a9a28 Mon Sep 17 00:00:00 2001 From: Robert Lucian Chiriac Date: Tue, 20 Jul 2021 00:35:35 +0300 Subject: [PATCH 5/5] Address PR comments --- dev/versions.md | 7 +++++++ manager/install.sh | 4 ++-- .../manifests/{kube-proxy.yaml => kube-proxy.patch.yaml} | 0 3 files changed, 9 insertions(+), 2 deletions(-) rename manager/manifests/{kube-proxy.yaml => kube-proxy.patch.yaml} (100%) diff --git a/dev/versions.md b/dev/versions.md index 853702188a..a948237bbf 100644 --- a/dev/versions.md +++ b/dev/versions.md @@ -18,6 +18,13 @@ 1. Update `ami.json` (see release checklist for instructions) 1. See instructions for upgrading the Kubernetes client below +## kube-proxy (IPVS mode) + +1. Before spinning up a Cortex cluster with the new eksctl/kubernetes/eks updates, make sure to have the `setup_ipvs` functional call commented out in the manager. +1. Once the cluster is up, run the `cat /var/lib/kube-proxy-config/config` command on any of the kube-proxy pods of the cluster. Compare the output of that with what the `upgrade_kube_proxy_mode.py` script is applying and make sure it's still applicable, if not, check out the spec of the [KubeProxyConfiguration](https://kubernetes.io/docs/reference/config-api/kube-proxy-config.v1alpha1/) and upgrade `upgrade_kube_proxy_mode.py`. +1. Compare the spec of the `kube-proxy.patch.yaml` patch with the current spec of the kube-proxy daemoset and make sure it's still applicable. You can either inspect the `kube-proxy` command helper by exec-ing into the pod or by looking at the [kube-proxy](https://kubernetes.io/docs/reference/command-line-tools-reference/kube-proxy/) documentation for the respective version of Kubernetes. +1. Once both config map and the daemonset are updated and the kube-proxy pod(s) has/have started, make sure you notice the `Using ipvs Proxier` log. + ## aws-iam-authenticator 1. Find the latest release [here](https://docs.aws.amazon.com/eks/latest/userguide/install-aws-iam-authenticator.html) diff --git a/manager/install.sh b/manager/install.sh index 8433a924e7..64374a2596 100755 --- a/manager/install.sh +++ b/manager/install.sh @@ -371,7 +371,7 @@ function remove_nodegroups() { function setup_ipvs() { # get a random kube-proxy pod - until [ "$(kubectl get pod -n kube-system -l k8s-app=kube-proxy -o jsonpath='{.items[*].metadata.name}' | wc -w)" -ne "0" ]; do sleep 1; done + kubectl rollout status daemonset kube-proxy -n kube-system --timeout 30m >/dev/null kube_proxy_pod=$(kubectl get pod -n kube-system -l k8s-app=kube-proxy -o jsonpath='{.items[*].metadata.name}' | cut -d " " -f1) # export kube-proxy's current config @@ -384,7 +384,7 @@ function setup_ipvs() { kubectl get configmap -n kube-system kube-proxy -o yaml | yq --arg replace "`cat upgraded_proxy_config.yaml`" '.data.config=$replace' | kubectl apply -f - >/dev/null # patch the kube-proxy daemonset - kubectl patch ds -n kube-system kube-proxy --patch "$(cat manifests/kube-proxy.yaml)" >/dev/null + kubectl patch ds -n kube-system kube-proxy --patch "$(cat manifests/kube-proxy.patch.yaml)" >/dev/null kubectl rollout status daemonset kube-proxy -n kube-system --timeout 30m >/dev/null } diff --git a/manager/manifests/kube-proxy.yaml b/manager/manifests/kube-proxy.patch.yaml similarity index 100% rename from manager/manifests/kube-proxy.yaml rename to manager/manifests/kube-proxy.patch.yaml