diff --git a/.trivyignore b/.trivyignore index aaf1192cc..f9f0b0860 100644 --- a/.trivyignore +++ b/.trivyignore @@ -9,11 +9,6 @@ AVD-DS-0002 # initcontainers require privileged access AVD-KSV-0017 -# Sharing the host’s network namespace permits processes in the pod to communicate with -# processes bound to the host’s loopback adapter. -# sgx single-node demo deployment uses hostNetwork: true to be able to use the default setting for PCCS URL from containers -AVD-KSV-0009 - # Do not allow privilege escalation from node proxy # Check whether role permits privilege escalation from node proxy # gpu plugin in kubelet mode requires "nodes/proxy" resource access diff --git a/cmd/sgx_plugin/README.md b/cmd/sgx_plugin/README.md index 4af92133c..4bd281110 100644 --- a/cmd/sgx_plugin/README.md +++ b/cmd/sgx_plugin/README.md @@ -195,8 +195,10 @@ Successfully tagged intel/sgx-sdk-demo:devel #### Deploy the pods The demo runs Intel aesmd (architectural enclaves service daemon) that is responsible -for generating SGX quotes for workloads. It is deployed with `hostNetwork: true` -to allow connections to localhost PCCS. +for generating SGX quotes for workloads. + +**Note**: The PCCS URL must be configured in `sgx_default_qcnl.conf`. The default `localhost` URL +is not available in containers ```bash $ kubectl apply -k 'https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/sgx_aesmd?ref=' @@ -239,5 +241,7 @@ $ kubectl logs ecdsa-quote-intelsgx-demo-job-vtq84 Step4: Call sgx_qe_get_quote:succeed!cert_key_type = 0x5 ``` +Similarly, full SGX DCAP Flow with Quote Generation and Trusted Quote Verification can be deployed using the `sgx_ecdsa_inproc_quote` overlay. Again, the PCCS URL must be set beforehand. + > **Note**: The deployment example above uses [kustomize](https://github.com/kubernetes-sigs/kustomize) > that is available in kubectl since Kubernetes v1.14 release. diff --git a/demo/screencast-sgx.sh b/demo/screencast-sgx.sh index ba4d9070e..aed11c927 100755 --- a/demo/screencast-sgx.sh +++ b/demo/screencast-sgx.sh @@ -27,7 +27,7 @@ cleanup() out 'Cleanup demo artifacts' 20 out 'delete node-feature-discovery deployment:' 20 command 'kubectl delete -k https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/nfd/overlays/node-feature-rules?ref=main || true' 20 - command 'kubectl delete -k https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/nfd/overlays/sgx?ref=main || true' 20 + command 'kubectl delete -k https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/nfd?ref=main || true' 20 out 'delete SGX Device Plugin deployment:' 20 command 'kubectl delete sgxdeviceplugin sgxdeviceplugin-sample || true' 20 out 'delete Intel Device Plugin Operator deployment:' 20 @@ -69,10 +69,10 @@ screen3() clear out "2. Deploy node-feature-discovery for Kubernetes" out "It's used to label SGX capable nodes and register SGX EPC as an extended resource" - command "kubectl apply -k https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/nfd/overlays/sgx?ref=main" + command "kubectl apply -k https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/nfd?ref=main" out "Check its pod is running" command "kubectl wait --for=condition=Ready pod/$(kubectl get --no-headers -l app=nfd-worker -o=jsonpath='{.items[0].metadata.name}' pods -n node-feature-discovery) -n node-feature-discovery" - out "Create NodeFeatureRules for SGX specific labels" + out "Create NodeFeatureRules for SGX specific labels and SGX EPC extended resource" command 'kubectl apply -k https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/nfd/overlays/node-feature-rules?ref=main || true' 20 } @@ -91,8 +91,8 @@ screen5() { clear out "4. Verify node resources" - command "kubectl get nodes -o json | jq .items[].status.allocatable | grep sgx" - command "kubectl get nodes -o json | jq .items[].metadata.labels | grep sgx" + command "kubectl get nodes -o jsonpath='{.items[].status.allocatable}' | jq | grep sgx" + command "kubectl get nodes -o jsonpath='{.items[].metadata.labels}' | jq | grep kubernetes.io\/sgx" out "Both node labels and resources for SGX are in place" } @@ -104,7 +104,10 @@ screen6() command "sudo ctr -n k8s.io i import sgx-aesmd.tar" command "sudo ctr -n k8s.io i import sgx-demo.tar" out "Deploy Intel(R) AESMD" - command "kubectl apply -k https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/sgx_aesmd?ref=main -n sgx-ecdsa-quote" + pushd ../deployments/sgx_aesmd/base + jq --arg pccs_url "$PCCS_URL" '.pccs_url = $pccs_url' sgx_default_qcnl.template > sgx_default_qcnl.conf + command "kubectl apply -k . -n sgx-ecdsa-quote" + popd out "Deploy Intel(R) SGX DCAP ECDSA Quote Generation" command "kubectl apply -k https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/sgx_enclave_apps/overlays/sgx_ecdsa_aesmd_quote?ref=main -n sgx-ecdsa-quote" command "kubectl logs $(kubectl get --no-headers -l job-name=ecdsa-quote-intelsgx-demo-job -o=jsonpath='{.items[0].metadata.name}' pods -n sgx-ecdsa-quote) -n sgx-ecdsa-quote" @@ -117,11 +120,14 @@ screen6() screen7() { clear - out "6. Run Intel(R) SGX DCAP ECDSA Quote Generation (in-proc)" - out "Deploy Intel(R) SGX DCAP ECDSA Quote Generation" - command "kubectl apply -k https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/sgx_enclave_apps/overlays/sgx_ecdsa_inproc_quote?ref=main -n sgx-ecdsa-quote" + out "6. Run Intel(R) SGX DCAP ECDSA Quote Generation (in-proc) and Trusted Quote Verification" + out "Deploy Intel(R) SGX DCAP ECDSA DCAP Flow" + pushd ../deployments/sgx_enclave_apps/overlays/sgx_ecdsa_inproc_quote + jq --arg pccs_url "$PCCS_URL" '.pccs_url = $pccs_url' sgx_default_qcnl.template > sgx_default_qcnl.conf + command "kubectl apply -k . -n sgx-ecdsa-quote" + popd command "kubectl logs $(kubectl get --no-headers -l job-name=inproc-ecdsa-quote-intelsgx-demo-job -o=jsonpath='{.items[0].metadata.name}' pods -n sgx-ecdsa-quote) -n sgx-ecdsa-quote" - out "Intel(R) SGX DCAP QuoteGenerationSample successfully generated a quote using DCAP Quote Provider Library" + out "Intel(R) SGX DCAP QuoteGenerationSample successfully generated and verified a quote using DCAP Quote Provider Library" out "Delete the deployment" command "kubectl delete -k https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/sgx_enclave_apps/overlays/sgx_ecdsa_inproc_quote?ref=main -n sgx-ecdsa-quote" } @@ -134,6 +140,7 @@ screen8() out "* SGX Kubernetes* Device Plugin deployment with an Operator" out "* Intel(R) SGX node resource and feature label registration to Kubernetes*" out "* Intel(R) SGX DCAP ECDSA Quote Generation (out-of-proc and in-proc)" + out "* Intel(R) SGX DCAP ECDSA Trusted Quote Verification" } if [ "$1" == 'play' ] ; then diff --git a/demo/sgx-sdk-demo/Dockerfile b/demo/sgx-sdk-demo/Dockerfile index 18115286d..2f9a31687 100644 --- a/demo/sgx-sdk-demo/Dockerfile +++ b/demo/sgx-sdk-demo/Dockerfile @@ -23,7 +23,7 @@ RUN apt-get update && \ # SGX SDK is installed in /opt/intel directory. WORKDIR /opt/intel -ARG DCAP_VERSION=DCAP_1.17 +ARG DCAP_VERSION=DCAP_1.18 RUN echo "deb [arch=amd64 signed-by=/usr/share/keyrings/intel-sgx.gpg] https://download.01.org/intel-sgx/sgx_repo/ubuntu jammy main" | \ tee -a /etc/apt/sources.list.d/intel-sgx.list \ @@ -32,11 +32,12 @@ RUN echo "deb [arch=amd64 signed-by=/usr/share/keyrings/intel-sgx.gpg] https://d && apt-get update \ && env DEBIAN_FRONTEND=noninteractive apt-get install -y \ libsgx-dcap-ql-dev \ + libsgx-dcap-quote-verify-dev \ libsgx-dcap-default-qpl-dev \ libsgx-quote-ex-dev # Install SGX SDK -ARG SGX_SDK_URL=https://download.01.org/intel-sgx/sgx-linux/2.20/distro/ubuntu22.04-server/sgx_linux_x64_sdk_2.20.100.4.bin +ARG SGX_SDK_URL=https://download.01.org/intel-sgx/sgx-linux/2.21/distro/ubuntu22.04-server/sgx_linux_x64_sdk_2.21.100.1.bin RUN wget ${SGX_SDK_URL} \ && export SGX_SDK_INSTALLER=$(basename $SGX_SDK_URL) \ && chmod +x $SGX_SDK_INSTALLER \ @@ -55,6 +56,12 @@ RUN cd SGXDataCenterAttestationPrimitives/SampleCode/QuoteGenerationSample \ && make \ && cd - +RUN cd SGXDataCenterAttestationPrimitives/SampleCode/QuoteVerificationSample \ + && . /opt/intel/sgxsdk/environment \ + && make HW_RELEASE=1 \ + && sgx_sign sign -key ../QuoteGenerationSample/Enclave/Enclave_private_sample.pem -enclave enclave.so -out enclave.signed.so -config Enclave/Enclave.config.xml \ + && cd - + FROM ubuntu:22.04 RUN apt-get update && \ @@ -72,9 +79,12 @@ RUN echo "deb [arch=amd64 signed-by=/usr/share/keyrings/intel-sgx.gpg] https://d libsgx-enclave-common \ libsgx-urts \ libsgx-quote-ex \ + libsgx-dcap-quote-verify \ + libsgx-ae-qve \ libsgx-dcap-ql \ libsgx-dcap-default-qpl \ && mkdir -p /opt/intel/sgx-sample-app/ \ + && mkdir -p /opt/intel/sgx-quote-verification/ \ && mkdir -p /opt/intel/sgx-quote-generation/ COPY --from=builder /opt/intel/sgxsdk/SampleCode/SampleEnclave/app /opt/intel/sgx-sample-app/sgx-sample-app @@ -83,4 +93,9 @@ COPY --from=builder /opt/intel/sgxsdk/SampleCode/SampleEnclave/enclave.signed.so COPY --from=builder /opt/intel/SGXDataCenterAttestationPrimitives/SampleCode/QuoteGenerationSample/app /opt/intel/sgx-quote-generation/sgx-quote-generation COPY --from=builder /opt/intel/SGXDataCenterAttestationPrimitives/SampleCode/QuoteGenerationSample/enclave.signed.so /opt/intel/sgx-quote-generation/enclave.signed.so +COPY --from=builder /opt/intel/SGXDataCenterAttestationPrimitives/SampleCode/QuoteVerificationSample/app /opt/intel/sgx-quote-verification/sgx-quote-verification +COPY --from=builder /opt/intel/SGXDataCenterAttestationPrimitives/SampleCode/QuoteVerificationSample/enclave.signed.so /opt/intel/sgx-quote-verification/enclave.signed.so + +COPY --chmod=555 run-dcap-flow /opt/intel + ENTRYPOINT /opt/intel/sgx-sample-app/sgx-sample-app diff --git a/demo/sgx-sdk-demo/run-dcap-flow b/demo/sgx-sdk-demo/run-dcap-flow new file mode 100755 index 000000000..891f5a2bf --- /dev/null +++ b/demo/sgx-sdk-demo/run-dcap-flow @@ -0,0 +1,11 @@ +#!/bin/bash + +pushd sgx-quote-generation + +./sgx-quote-generation + +popd + +pushd sgx-quote-verification + +./sgx-quote-verification -quote ../sgx-quote-generation/quote.dat diff --git a/deployments/sgx_aesmd/base/intel-sgx-aesmd.yaml b/deployments/sgx_aesmd/base/intel-sgx-aesmd.yaml index b7daf142f..c37546e46 100644 --- a/deployments/sgx_aesmd/base/intel-sgx-aesmd.yaml +++ b/deployments/sgx_aesmd/base/intel-sgx-aesmd.yaml @@ -15,7 +15,6 @@ spec: annotations: sgx.intel.com/quote-provider: "aesmd" spec: - hostNetwork: true containers: - name: aesmd image: intel/sgx-aesmd-demo:devel diff --git a/deployments/sgx_aesmd/base/sgx_default_qcnl.template b/deployments/sgx_aesmd/base/sgx_default_qcnl.template new file mode 100644 index 000000000..e89940b23 --- /dev/null +++ b/deployments/sgx_aesmd/base/sgx_default_qcnl.template @@ -0,0 +1,5 @@ +{ + "pccs_url": "https://localhost:8081/sgx/certification/v4/", + "use_secure_cert": false, + "pccs_api_version": "3.1" +} diff --git a/deployments/sgx_enclave_apps/overlays/sgx_ecdsa_inproc_quote/add_hostnetwork.yaml b/deployments/sgx_enclave_apps/overlays/sgx_ecdsa_inproc_quote/add_hostnetwork.yaml deleted file mode 100644 index d8accbb0f..000000000 --- a/deployments/sgx_enclave_apps/overlays/sgx_ecdsa_inproc_quote/add_hostnetwork.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: batch/v1 -kind: Job -metadata: - name: intelsgx-demo-job -spec: - template: - spec: - hostNetwork: true - containers: - - name: intelsgx-demo-job-1 - image: intel/sgx-sdk-demo:devel diff --git a/deployments/sgx_enclave_apps/overlays/sgx_ecdsa_inproc_quote/change_workingdir_and_command.json b/deployments/sgx_enclave_apps/overlays/sgx_ecdsa_inproc_quote/change_workingdir_and_command.json new file mode 100644 index 000000000..a9809a1f6 --- /dev/null +++ b/deployments/sgx_enclave_apps/overlays/sgx_ecdsa_inproc_quote/change_workingdir_and_command.json @@ -0,0 +1,5 @@ +[ + {"op": "replace", "path": "/spec/template/spec/containers/0/workingDir", "value": "/opt/intel/"}, + {"op": "replace", "path": "/spec/template/spec/containers/0/command", "value": ["/opt/intel/run-dcap-flow"]}, + {"op": "remove", "path": "/spec/template/spec/containers/0/securityContext/readOnlyRootFilesystem"} +] diff --git a/deployments/sgx_enclave_apps/overlays/sgx_ecdsa_inproc_quote/kustomization.yaml b/deployments/sgx_enclave_apps/overlays/sgx_ecdsa_inproc_quote/kustomization.yaml index 2389e51dc..6a90aa60c 100644 --- a/deployments/sgx_enclave_apps/overlays/sgx_ecdsa_inproc_quote/kustomization.yaml +++ b/deployments/sgx_enclave_apps/overlays/sgx_ecdsa_inproc_quote/kustomization.yaml @@ -10,5 +10,10 @@ configMapGenerator: - sgx_default_qcnl.conf name: sgx-attestation-conf patches: -- path: add_hostnetwork.yaml - path: add_sgx_default_qcnl_conf.yaml +- path: change_workingdir_and_command.json + target: + group: batch + kind: Job + name: intelsgx-demo-job + version: v1 diff --git a/deployments/sgx_enclave_apps/overlays/sgx_ecdsa_inproc_quote/sgx_default_qcnl.template b/deployments/sgx_enclave_apps/overlays/sgx_ecdsa_inproc_quote/sgx_default_qcnl.template new file mode 100644 index 000000000..e89940b23 --- /dev/null +++ b/deployments/sgx_enclave_apps/overlays/sgx_ecdsa_inproc_quote/sgx_default_qcnl.template @@ -0,0 +1,5 @@ +{ + "pccs_url": "https://localhost:8081/sgx/certification/v4/", + "use_secure_cert": false, + "pccs_api_version": "3.1" +} diff --git a/go.mod b/go.mod index 3ba734b61..34d19fa7b 100644 --- a/go.mod +++ b/go.mod @@ -18,13 +18,13 @@ require ( golang.org/x/text v0.12.0 google.golang.org/grpc v1.57.0 gopkg.in/yaml.v2 v2.4.0 - k8s.io/api v0.28.0 - k8s.io/apimachinery v0.28.0 - k8s.io/client-go v0.28.0 - k8s.io/component-base v0.28.0 + k8s.io/api v0.28.1 + k8s.io/apimachinery v0.28.1 + k8s.io/client-go v0.28.1 + k8s.io/component-base v0.28.1 k8s.io/klog/v2 v2.100.1 - k8s.io/kubelet v1.28.0 - k8s.io/kubernetes v1.28.0 + k8s.io/kubelet v1.28.1 + k8s.io/kubernetes v1.28.1 k8s.io/pod-security-admission v0.0.0 k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 sigs.k8s.io/controller-runtime v0.16.0 @@ -113,11 +113,11 @@ require ( gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.28.0 // indirect - k8s.io/apiserver v0.28.0 // indirect + k8s.io/apiserver v0.28.1 // indirect k8s.io/cloud-provider v0.0.0 // indirect - k8s.io/component-helpers v0.28.0 // indirect - k8s.io/controller-manager v0.28.0 // indirect - k8s.io/kms v0.28.0 // indirect + k8s.io/component-helpers v0.28.1 // indirect + k8s.io/controller-manager v0.28.1 // indirect + k8s.io/kms v0.28.1 // indirect k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect k8s.io/kubectl v0.0.0 // indirect sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2 // indirect @@ -126,34 +126,34 @@ require ( ) replace ( - k8s.io/api => k8s.io/api v0.28.0 - k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.28.0 - k8s.io/apimachinery => k8s.io/apimachinery v0.28.0 - k8s.io/apiserver => k8s.io/apiserver v0.28.0 - k8s.io/cli-runtime => k8s.io/cli-runtime v0.28.0 - k8s.io/client-go => k8s.io/client-go v0.28.0 - k8s.io/cloud-provider => k8s.io/cloud-provider v0.28.0 - k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.28.0 - k8s.io/code-generator => k8s.io/code-generator v0.28.0 - k8s.io/component-base => k8s.io/component-base v0.28.0 - k8s.io/component-helpers => k8s.io/component-helpers v0.28.0 - k8s.io/controller-manager => k8s.io/controller-manager v0.28.0 - k8s.io/cri-api => k8s.io/cri-api v0.28.0 - k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.28.0 - k8s.io/dynamic-resource-allocation => k8s.io/dynamic-resource-allocation v0.28.0 - k8s.io/endpointslice => k8s.io/endpointslice v0.28.0 - k8s.io/kms => k8s.io/kms v0.28.0 - k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.28.0 - k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.28.0 - k8s.io/kube-proxy => k8s.io/kube-proxy v0.28.0 - k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.28.0 - k8s.io/kubectl => k8s.io/kubectl v0.28.0 - k8s.io/kubelet => k8s.io/kubelet v0.28.0 - k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.28.0 - k8s.io/metrics => k8s.io/metrics v0.28.0 - k8s.io/mount-utils => k8s.io/mount-utils v0.28.0 - k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.28.0 - k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.28.0 - k8s.io/sample-cli-plugin => k8s.io/sample-cli-plugin v0.28.0 - k8s.io/sample-controller => k8s.io/sample-controller v0.28.0 + k8s.io/api => k8s.io/api v0.28.1 + k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.28.1 + k8s.io/apimachinery => k8s.io/apimachinery v0.28.1 + k8s.io/apiserver => k8s.io/apiserver v0.28.1 + k8s.io/cli-runtime => k8s.io/cli-runtime v0.28.1 + k8s.io/client-go => k8s.io/client-go v0.28.1 + k8s.io/cloud-provider => k8s.io/cloud-provider v0.28.1 + k8s.io/cluster-bootstrap => k8s.io/cluster-bootstrap v0.28.1 + k8s.io/code-generator => k8s.io/code-generator v0.28.1 + k8s.io/component-base => k8s.io/component-base v0.28.1 + k8s.io/component-helpers => k8s.io/component-helpers v0.28.1 + k8s.io/controller-manager => k8s.io/controller-manager v0.28.1 + k8s.io/cri-api => k8s.io/cri-api v0.28.1 + k8s.io/csi-translation-lib => k8s.io/csi-translation-lib v0.28.1 + k8s.io/dynamic-resource-allocation => k8s.io/dynamic-resource-allocation v0.28.1 + k8s.io/endpointslice => k8s.io/endpointslice v0.28.1 + k8s.io/kms => k8s.io/kms v0.28.1 + k8s.io/kube-aggregator => k8s.io/kube-aggregator v0.28.1 + k8s.io/kube-controller-manager => k8s.io/kube-controller-manager v0.28.1 + k8s.io/kube-proxy => k8s.io/kube-proxy v0.28.1 + k8s.io/kube-scheduler => k8s.io/kube-scheduler v0.28.1 + k8s.io/kubectl => k8s.io/kubectl v0.28.1 + k8s.io/kubelet => k8s.io/kubelet v0.28.1 + k8s.io/legacy-cloud-providers => k8s.io/legacy-cloud-providers v0.28.1 + k8s.io/metrics => k8s.io/metrics v0.28.1 + k8s.io/mount-utils => k8s.io/mount-utils v0.28.1 + k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.28.1 + k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.28.1 + k8s.io/sample-cli-plugin => k8s.io/sample-cli-plugin v0.28.1 + k8s.io/sample-controller => k8s.io/sample-controller v0.28.1 ) diff --git a/go.sum b/go.sum index bd3ca766a..98e3f3492 100644 --- a/go.sum +++ b/go.sum @@ -650,38 +650,38 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.28.0 h1:3j3VPWmN9tTDI68NETBWlDiA9qOiGJ7sdKeufehBYsM= -k8s.io/api v0.28.0/go.mod h1:0l8NZJzB0i/etuWnIXcwfIv+xnDOhL3lLW919AWYDuY= -k8s.io/apiextensions-apiserver v0.28.0 h1:CszgmBL8CizEnj4sj7/PtLGey6Na3YgWyGCPONv7E9E= -k8s.io/apiextensions-apiserver v0.28.0/go.mod h1:uRdYiwIuu0SyqJKriKmqEN2jThIJPhVmOWETm8ud1VE= -k8s.io/apimachinery v0.28.0 h1:ScHS2AG16UlYWk63r46oU3D5y54T53cVI5mMJwwqFNA= -k8s.io/apimachinery v0.28.0/go.mod h1:X0xh/chESs2hP9koe+SdIAcXWcQ+RM5hy0ZynB+yEvw= -k8s.io/apiserver v0.28.0 h1:wVh7bK6Xj7hq+5ntInysTeQRAOqqFoKGUOW2yj8DXrY= -k8s.io/apiserver v0.28.0/go.mod h1:MvLmtxhQ0Tb1SZk4hfJBjs8iqr5nhYeaFSaoEcz7Lk4= -k8s.io/client-go v0.28.0 h1:ebcPRDZsCjpj62+cMk1eGNX1QkMdRmQ6lmz5BLoFWeM= -k8s.io/client-go v0.28.0/go.mod h1:0Asy9Xt3U98RypWJmU1ZrRAGKhP6NqDPmptlAzK2kMc= -k8s.io/cloud-provider v0.28.0 h1:BTIW7b757T+VXB5yqJeajPXsNOmeooopUgfzQueiWvk= -k8s.io/cloud-provider v0.28.0/go.mod h1:u0MGqdlutkTmCJyNrCzIMJ+OhrwQE9x5X8mBTN0R7us= -k8s.io/component-base v0.28.0 h1:HQKy1enJrOeJlTlN4a6dU09wtmXaUvThC0irImfqyxI= -k8s.io/component-base v0.28.0/go.mod h1:Yyf3+ZypLfMydVzuLBqJ5V7Kx6WwDr/5cN+dFjw1FNk= -k8s.io/component-helpers v0.28.0 h1:ubHUiEF7H/DOx4471pHHsLlH3EGu8jlEvnld5PS4KdI= -k8s.io/component-helpers v0.28.0/go.mod h1:i7hJ/oFhZImqUWwjLFG/yGkLpJ3KFoirY2DLYIMql6Q= -k8s.io/controller-manager v0.28.0 h1:55rmyzwEOnhAZLsuDdDHwVT2sGzkleFY0SqZFKsLN5U= -k8s.io/controller-manager v0.28.0/go.mod h1:WrABGmrpEWBap27eu533RpW5lBnVT5K+u2oc2bDwcmU= +k8s.io/api v0.28.1 h1:i+0O8k2NPBCPYaMB+uCkseEbawEt/eFaiRqUx8aB108= +k8s.io/api v0.28.1/go.mod h1:uBYwID+66wiL28Kn2tBjBYQdEU0Xk0z5qF8bIBqk/Dg= +k8s.io/apiextensions-apiserver v0.28.1 h1:l2ThkBRjrWpw4f24uq0Da2HaEgqJZ7pcgiEUTKSmQZw= +k8s.io/apiextensions-apiserver v0.28.1/go.mod h1:sVvrI+P4vxh2YBBcm8n2ThjNyzU4BQGilCQ/JAY5kGs= +k8s.io/apimachinery v0.28.1 h1:EJD40og3GizBSV3mkIoXQBsws32okPOy+MkRyzh6nPY= +k8s.io/apimachinery v0.28.1/go.mod h1:X0xh/chESs2hP9koe+SdIAcXWcQ+RM5hy0ZynB+yEvw= +k8s.io/apiserver v0.28.1 h1:dw2/NKauDZCnOUAzIo2hFhtBRUo6gQK832NV8kuDbGM= +k8s.io/apiserver v0.28.1/go.mod h1:d8aizlSRB6yRgJ6PKfDkdwCy2DXt/d1FDR6iJN9kY1w= +k8s.io/client-go v0.28.1 h1:pRhMzB8HyLfVwpngWKE8hDcXRqifh1ga2Z/PU9SXVK8= +k8s.io/client-go v0.28.1/go.mod h1:pEZA3FqOsVkCc07pFVzK076R+P/eXqsgx5zuuRWukNE= +k8s.io/cloud-provider v0.28.1 h1:bR7lIRYBHqxfsOkUsY2hJ7V7vmStxb0wjJJdrID8+7I= +k8s.io/cloud-provider v0.28.1/go.mod h1:7jxsc3c15go606KLXnUq8Cy4nX1R1dxFRgn/czIJp/Q= +k8s.io/component-base v0.28.1 h1:LA4AujMlK2mr0tZbQDZkjWbdhTV5bRyEyAFe0TJxlWg= +k8s.io/component-base v0.28.1/go.mod h1:jI11OyhbX21Qtbav7JkhehyBsIRfnO8oEgoAR12ArIU= +k8s.io/component-helpers v0.28.1 h1:ts/vykhyUmPLhUl/hdLdf+a4BWA0giQ3f25HAIhl+RI= +k8s.io/component-helpers v0.28.1/go.mod h1:rHFPj33uXNbgppg+ilmjJ4oR73prZQNRRmg+utVOAb0= +k8s.io/controller-manager v0.28.1 h1:+md/3DAsdLVoMe3AewhyTxljnPLE/gyshTDZ8sX4Rf0= +k8s.io/controller-manager v0.28.1/go.mod h1:yZ8aOBpMYOBTAI/Jd0qpaUzZUlQigmtRcdYg2VgWKiU= k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/kms v0.28.0 h1:BwJhU9qPcJhHLUcQjtelOSjYti+1/caJLr+4jHbKzTA= -k8s.io/kms v0.28.0/go.mod h1:CNU792ls92v2Ye7Vn1jn+xLqYtUSezDZNVu6PLbJyrU= +k8s.io/kms v0.28.1 h1:QLNTIc0k7Yebkt9yobj9Y9qBoRCMB4dq+pFCxVXVBnY= +k8s.io/kms v0.28.1/go.mod h1:I2TwA8oerDRInHWWBOqSUzv1EJDC1+55FQKYkxaPxh0= k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 h1:LyMgNKD2P8Wn1iAwQU5OhxCKlKJy0sHc+PcDwFB24dQ= k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9/go.mod h1:wZK2AVp1uHCp4VamDVgBP2COHZjqD1T68Rf0CM3YjSM= -k8s.io/kubectl v0.28.0 h1:qhfju0OaU+JGeBlToPeeIg2UJUWP++QwTkpio6nlPKg= -k8s.io/kubectl v0.28.0/go.mod h1:1We+E5nSX3/TVoSQ6y5Bzld5OhTBHZHlKEYl7g/NaTk= -k8s.io/kubelet v0.28.0 h1:H/3JAkLIungVF+WLpqrxhgJ4gzwsbN8VA8LOTYsEX3U= -k8s.io/kubelet v0.28.0/go.mod h1:i8jUg4ltbRusT3ExOhSAeqETuHdoHTZcTT2cPr9RTgc= -k8s.io/kubernetes v1.28.0 h1:p8qq/VoNHnBWinLEi5LO2IvCfzFouN7Jhdz8+L++V+U= -k8s.io/kubernetes v1.28.0/go.mod h1:rBQpjGYlLBV0KuOLw8EG45N5EBCskWiPpi0xy5liHMI= -k8s.io/pod-security-admission v0.28.0 h1:Vz8XTjMAKHQFZv9Q4GdmO59CUtelkPPDRJTy/WTTc3g= -k8s.io/pod-security-admission v0.28.0/go.mod h1:hABVUcP7SRALDvESOK+RYIAWc9uZ5I1eSdcUwsOYTU8= +k8s.io/kubectl v0.28.1 h1:jAq4yKEqQL+fwkWcEsUWxhJ7uIRcOYQraJxx4SyAMTY= +k8s.io/kubectl v0.28.1/go.mod h1:a0nk/lMMeKBulp0lMTJAKbkjZg1ykqfLfz/d6dnv1ak= +k8s.io/kubelet v0.28.1 h1:QRfx+jrzNgkLnMSw/nxGkAN7cjHPO446MDbjPITxLkk= +k8s.io/kubelet v0.28.1/go.mod h1:xYBbbJ0e2Rtb/hv+QFie448lFF81J990ImIptce2AHk= +k8s.io/kubernetes v1.28.1 h1:ZQuukGbpVjSbMypkjNErpbsSHni6RPgoqz+2zDBsuMY= +k8s.io/kubernetes v1.28.1/go.mod h1:rBQpjGYlLBV0KuOLw8EG45N5EBCskWiPpi0xy5liHMI= +k8s.io/pod-security-admission v0.28.1 h1:d3jvo/+C6yDR1wnlX9ot1WvLyJ5R4uachJyxhdn9cW8= +k8s.io/pod-security-admission v0.28.1/go.mod h1:Qm1rSy3l96m6QXGNU/8u+cmdpNdmAeA3OYDinrXhi6U= k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 h1:qY1Ad8PODbnymg2pRbkyMT/ylpTrCM8P2RJ0yroCyIk= k8s.io/utils v0.0.0-20230406110748-d93618cff8a2/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=