diff --git a/.github/workflows/lib-build.yaml b/.github/workflows/lib-build.yaml index 27971dc6b..4409eafb7 100644 --- a/.github/workflows/lib-build.yaml +++ b/.github/workflows/lib-build.yaml @@ -27,6 +27,7 @@ jobs: - intel-iaa-plugin - intel-idxd-config-initcontainer - intel-xpumanager-sidecar + - intel-npu-plugin # # Demo images - crypto-perf diff --git a/README.md b/README.md index 31ddfe29b..8ffb977db 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ Table of Contents * [DSA device plugin](#dsa-device-plugin) * [DLB device plugin](#dlb-device-plugin) * [IAA device plugin](#iaa-device-plugin) + * [NPU device plugin](#npu-device-plugin) * [Device Plugins Operator](#device-plugins-operator) * [XeLink XPU Manager sidecar](#xelink-xpu-manager-sidecar) * [Intel GPU Level-Zero sidecar](#intel-gpu-levelzero) @@ -182,12 +183,17 @@ Balancer accelerator(DLB). The [IAA device plugin](cmd/iaa_plugin/README.md) supports acceleration using the Intel Analytics accelerator(IAA). +### NPU Device Plugin + +The [NPU device plugin](cmd/npu_plugin/README.md) supports acceleration using +the Intel Neural Processing Unit(NPU). + ## Device Plugins Operator To simplify the deployment of the device plugins, a unified device plugins operator is implemented. -Currently the operator has support for the DSA, DLB, FPGA, GPU, IAA, QAT, and +Currently the operator has support for the DSA, DLB, FPGA, GPU, IAA, QAT, NPU, and Intel SGX device plugins. Each device plugin has its own custom resource definition (CRD) and the corresponding controller that watches CRUD operations to those custom resources. @@ -247,6 +253,8 @@ The summary of resources available via plugins in this repository is given in th * [crypto-perf-dpdk-pod-requesting-qat-cy.yaml](deployments/qat_dpdk_app/crypto-perf/crypto-perf-dpdk-pod-requesting-qat-cy.yaml) * `sgx.intel.com` : `epc` * [intelsgx-job.yaml](deployments/sgx_enclave_apps/base/intelsgx-job.yaml) + * `npu.intel.com` : `npu` + * TODO ## Developers diff --git a/build/docker/intel-npu-plugin.Dockerfile b/build/docker/intel-npu-plugin.Dockerfile new file mode 100644 index 000000000..bd457d923 --- /dev/null +++ b/build/docker/intel-npu-plugin.Dockerfile @@ -0,0 +1,69 @@ +## This is a generated file, do not edit directly. Edit build/docker/templates/intel-npu-plugin.Dockerfile.in instead. +## +## Copyright 2022 Intel Corporation. All Rights Reserved. +## +## Licensed under the Apache License, Version 2.0 (the "License"); +## you may not use this file except in compliance with the License. +## You may obtain a copy of the License at +## +## http://www.apache.org/licenses/LICENSE-2.0 +## +## Unless required by applicable law or agreed to in writing, software +## distributed under the License is distributed on an "AS IS" BASIS, +## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +## See the License for the specific language governing permissions and +## limitations under the License. +### +ARG CMD=npu_plugin +## FINAL_BASE can be used to configure the base image of the final image. +## +## This is used in two ways: +## 1) make BUILDER= +## 2) docker build ... -f .Dockerfile +## +## The project default is 1) which sets FINAL_BASE=gcr.io/distroless/static +## (see build-image.sh). +## 2) and the default FINAL_BASE is primarily used to build Redhat Certified Openshift Operator container images that must be UBI based. +## The RedHat build tool does not allow additional image build parameters. +ARG FINAL_BASE=registry.access.redhat.com/ubi9-micro:latest +### +## +## GOLANG_BASE can be used to make the build reproducible by choosing an +## image by its hash: +## GOLANG_BASE=golang@sha256:9d64369fd3c633df71d7465d67d43f63bb31192193e671742fa1c26ebc3a6210 +## +## This is used on release branches before tagging a stable version. +## The main branch defaults to using the latest Golang base image. +ARG GOLANG_BASE=golang:1.24-bookworm +### +FROM ${GOLANG_BASE} AS builder +ARG DIR=/intel-device-plugins-for-kubernetes +ARG GO111MODULE=on +ARG LDFLAGS="all=-w -s" +ARG GOFLAGS="-trimpath" +ARG GCFLAGS="all=-spectre=all -N -l" +ARG ASMFLAGS="all=-spectre=all" +ARG GOLICENSES_VERSION +ARG EP=/usr/local/bin/intel_npu_device_plugin +ARG CMD +WORKDIR ${DIR} +COPY . . +RUN (cd cmd/${CMD}; GO111MODULE=${GO111MODULE} GOFLAGS=${GOFLAGS} CGO_ENABLED=0 go install -gcflags="${GCFLAGS}" -asmflags="${ASMFLAGS}" -ldflags="${LDFLAGS}") && install -D /go/bin/${CMD} /install_root${EP} +RUN install -D ${DIR}/LICENSE /install_root/licenses/intel-device-plugins-for-kubernetes/LICENSE \ + && if [ ! -d "licenses/$CMD" ] ; then \ + GO111MODULE=on GOROOT=$(go env GOROOT) go run github.com/google/go-licenses@${GOLICENSES_VERSION} save "./cmd/$CMD" \ + --save_path /install_root/licenses/$CMD/go-licenses ; \ + else mkdir -p /install_root/licenses/$CMD/go-licenses/ && cd licenses/$CMD && cp -r * /install_root/licenses/$CMD/go-licenses/ ; fi && \ + echo "Verifying installed licenses" && test -e /install_root/licenses/$CMD/go-licenses +### +FROM ${FINAL_BASE} +COPY --from=builder /install_root / +ENTRYPOINT ["/usr/local/bin/intel_npu_device_plugin"] +LABEL vendor='Intel®' +LABEL org.opencontainers.image.source='https://github.com/intel/intel-device-plugins-for-kubernetes' +LABEL maintainer="Intel®" +LABEL version='devel' +LABEL release='1' +LABEL name='intel-npu-plugin' +LABEL summary='Intel® NPU device plugin for Kubernetes' +LABEL description='The NPU device plugin provides access to Intel CPU neural processing unit (NPU) device files' diff --git a/build/docker/templates/intel-npu-plugin.Dockerfile.in b/build/docker/templates/intel-npu-plugin.Dockerfile.in new file mode 100644 index 000000000..1c5f83f93 --- /dev/null +++ b/build/docker/templates/intel-npu-plugin.Dockerfile.in @@ -0,0 +1,8 @@ +#define _ENTRYPOINT_ /usr/local/bin/intel_npu_device_plugin +ARG CMD=npu_plugin + +#include "default_plugin.docker" + +LABEL name='intel-npu-plugin' +LABEL summary='Intel® NPU device plugin for Kubernetes' +LABEL description='The NPU device plugin provides access to Intel CPU neural processing unit (NPU) device files' diff --git a/cmd/npu_plugin/README.md b/cmd/npu_plugin/README.md new file mode 100644 index 000000000..60779a7b0 --- /dev/null +++ b/cmd/npu_plugin/README.md @@ -0,0 +1,90 @@ +# Intel NPU device plugin for Kubernetes + +Table of Contents + +* [Introduction](#introduction) +* [Modes and Configuration Options](#modes-and-configuration-options) +* [Pre-built Images](#pre-built-images) +* [Installation](#installation) + * [Install with NFD](#install-with-nfd) + * [Install with Operator](#install-with-operator) + * [Verify Plugin Registration](#verify-plugin-registration) +* [Testing and Demos](#testing-and-demos) + +## Introduction + +Intel NPU plugin facilitates Kubernetes workload offloading by providing access to Intel CPU neural processing units supported by the host kernel. + +The following CPU families are currently detected by the plugin: +* Core Ultra Series 1 +* Core Ultra Series 2 +* Core Ultra 200V Series + +Intel NPU plugin may register one resource to the Kubernetes cluster: +| Resource | Description | +|:---- |:-------- | +| npu.intel.com/npu | NPU | + +## Modes and Configuration Options + +| Flag | Argument | Default | Meaning | +|:---- |:-------- |:------- |:------- | +| -shared-dev-num | int | 1 | Number of containers that can share the same NPU device | + +The plugin also accepts a number of other arguments (common to all plugins) related to logging. +Please use the -h option to see the complete list of logging related options. + +## Pre-built Images + +[Pre-built images](https://hub.docker.com/r/intel/intel-npu-plugin) +are available on the Docker hub. These images are automatically built and uploaded +to the hub from the latest main branch of this repository. + +Release tagged images of the components are also available on the Docker hub, tagged with their +release version numbers in the format `x.y.z`, corresponding to the branches and releases in this +repository. + +See [the development guide](../../DEVEL.md) for details if you want to deploy a customized version of the plugin. + +## Installation + +There are multiple ways to install Intel NPU plugin to a cluster. The most common methods are described below. + +> **Note**: Replace `` with the desired [release tag](https://github.com/intel/intel-device-plugins-for-kubernetes/tags) or `main` to get `devel` images. + +> **Note**: Add ```--dry-run=client -o yaml``` to the ```kubectl``` commands below to visualize the YAML content being applied. + +### Install with NFD + +Deploy NPU plugin with the help of NFD ([Node Feature Discovery](https://github.com/kubernetes-sigs/node-feature-discovery)). It detects the presence of Intel NPUs and labels them accordingly. GPU plugin's node selector is used to deploy plugin to nodes which have such a NPU label. + +```bash +# Start NFD - if your cluster doesn't have NFD installed yet +$ kubectl apply -k 'https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/nfd?ref=' + +# Create NodeFeatureRules for detecting NPUs on nodes +$ kubectl apply -k 'https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/nfd/overlays/node-feature-rules?ref=' + +# Create NPU plugin daemonset +$ kubectl apply -k 'https://github.com/intel/intel-device-plugins-for-kubernetes/deployments/npu_plugin/overlays/nfd_labeled_nodes?ref=' +``` + +### Install with Operator + +NPU plugin can be installed with the Intel Device Plugin Operator. It allows configuring NPU plugin parameters without kustomizing the deployment files. The general installation is described in the [install documentation](../operator/README.md#installation). + +### Verify Plugin Registration + +You can verify that the plugin has been installed on the expected nodes by searching for the relevant +resource allocation status on the nodes: + +```bash +$ kubectl get nodes -o=jsonpath="{range .items[*]}{.metadata.name}{'\n'}{' npu: '}{.status.allocatable.npu\.intel\.com/npu}{'\n'}" +master + npu: 1 +``` + +## Testing and Demos + +TODO + diff --git a/cmd/npu_plugin/npu_plugin.go b/cmd/npu_plugin/npu_plugin.go new file mode 100644 index 000000000..5ab23e37c --- /dev/null +++ b/cmd/npu_plugin/npu_plugin.go @@ -0,0 +1,212 @@ +// Copyright 2025 Intel Corporation. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "flag" + "fmt" + "os" + "path" + "regexp" + "slices" + "strings" + "time" + + "github.com/pkg/errors" + + "k8s.io/klog/v2" + pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1" + + dpapi "github.com/intel/intel-device-plugins-for-kubernetes/pkg/deviceplugin" +) + +const ( + sysfAccelDirectory = "/sys/class/accel" + devfsAccelDirectory = "/dev/accel" + npuDeviceRE = `^accel[0-9]+$` + vendorString = "0x8086" + + // Device plugin settings. + namespace = "npu.intel.com" + deviceTypeNpu = "npu" + + // Period of device scans. + scanPeriod = 5 * time.Second +) + +var npuIDs = []string{ + "0x7e4c", // Core Ultra Series 1 + "0x643e", // Core Ultra 200V Series + "0xad1d", // Core Ultra Series 2 + "0x7d1d", // Core Ultra Series 2 (H) +} + +type cliOptions struct { + sharedDevNum int +} + +type devicePlugin struct { + npuDeviceReg *regexp.Regexp + + scanTicker *time.Ticker + scanDone chan bool + + sysfsDir string + devfsDir string + + options cliOptions +} + +func newDevicePlugin(sysfsDir, devfsDir string, options cliOptions) *devicePlugin { + dp := &devicePlugin{ + sysfsDir: sysfsDir, + devfsDir: devfsDir, + options: options, + npuDeviceReg: regexp.MustCompile(npuDeviceRE), + scanTicker: time.NewTicker(scanPeriod), + scanDone: make(chan bool, 1), // buffered as we may send to it before Scan starts receiving from it + } + + return dp +} + +func (dp *devicePlugin) Scan(notifier dpapi.Notifier) error { + defer dp.scanTicker.Stop() + + klog.V(1).Infof("NPU (%s) resource share count = %d", deviceTypeNpu, dp.options.sharedDevNum) + + previousCount := 0 + devType := fmt.Sprintf("%s/%s", namespace, deviceTypeNpu) + + for { + devTree, err := dp.scan() + if err != nil { + klog.Errorf("NPU scan failed: %v", err) + return errors.Wrap(err, "NPU scan failed") + } + + count := devTree.DeviceTypeCount(devType) + if count != previousCount { + klog.V(1).Infof("NPU scan update: %d->%d '%s' resources found", previousCount, count, devType) + + previousCount = count + } + + notifier.Notify(devTree) + + select { + case <-dp.scanDone: + return nil + case <-dp.scanTicker.C: + } + } +} + +func (dp *devicePlugin) isCompatibleDevice(name string) bool { + if !dp.npuDeviceReg.MatchString(name) { + klog.V(4).Info("Not compatible device: ", name) + return false + } + + dat, err := os.ReadFile(path.Join(dp.sysfsDir, name, "device/vendor")) + if err != nil { + klog.Warning("Skipping. Can't read vendor file: ", err) + return false + } + + if strings.TrimSpace(string(dat)) != vendorString { + klog.V(4).Info("Non-Intel NPU: ", name) + return false + } + + dat, err = os.ReadFile(path.Join(dp.sysfsDir, name, "device/device")) + if err != nil { + klog.Warning("Skipping. Can't read device file: ", err) + return false + } + + datStr := strings.Split(string(dat), "\n")[0] + if !slices.Contains(npuIDs, datStr) { + klog.Warning("Unknown device id: ", datStr) + return false + } + + return true +} + +func (dp *devicePlugin) scan() (dpapi.DeviceTree, error) { + files, err := os.ReadDir(dp.sysfsDir) + if err != nil { + return nil, errors.Wrap(err, "Can't read sysfs folder") + } + + devTree := dpapi.NewDeviceTree() + + for _, f := range files { + name := f.Name() + + if !dp.isCompatibleDevice(name) { + continue + } + + devPath := path.Join(dp.devfsDir, name) + if _, err = os.Stat(devPath); err != nil { + continue + } + + // even querying metrics requires device to be writable + devSpec := pluginapi.DeviceSpec{ + HostPath: devPath, + ContainerPath: devPath, + Permissions: "rw", + } + + deviceInfo := dpapi.NewDeviceInfo(pluginapi.Healthy, []pluginapi.DeviceSpec{devSpec}, nil, nil, nil, nil) + + for i := 0; i < dp.options.sharedDevNum; i++ { + devID := fmt.Sprintf("%s-%d", name, i) + devTree.AddDevice("npu", devID, deviceInfo) + } + } + + return devTree, nil +} + +func (dp *devicePlugin) Allocate(request *pluginapi.AllocateRequest) (*pluginapi.AllocateResponse, error) { + return nil, &dpapi.UseDefaultMethodError{} +} + +func main() { + var ( + prefix string + opts cliOptions + ) + + flag.StringVar(&prefix, "prefix", "", "Prefix for devfs & sysfs paths") + flag.IntVar(&opts.sharedDevNum, "shared-dev-num", 1, "number of containers sharing the same NPU device") + flag.Parse() + + if opts.sharedDevNum < 1 { + klog.Error("The number of containers sharing the same NPU must greater than zero") + os.Exit(1) + } + + klog.V(1).Infof("NPU device plugin started") + + plugin := newDevicePlugin(prefix+sysfAccelDirectory, prefix+devfsAccelDirectory, opts) + + manager := dpapi.NewManager(namespace, plugin) + manager.Run() +} diff --git a/cmd/npu_plugin/npu_plugin_test.go b/cmd/npu_plugin/npu_plugin_test.go new file mode 100644 index 000000000..d27ac3a67 --- /dev/null +++ b/cmd/npu_plugin/npu_plugin_test.go @@ -0,0 +1,211 @@ +// Copyright 2017-2023 Intel Corporation. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "flag" + "os" + "path" + "testing" + + "github.com/pkg/errors" + "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1" + + dpapi "github.com/intel/intel-device-plugins-for-kubernetes/pkg/deviceplugin" +) + +// mockNotifier implements Notifier interface for NPU. +type mockNotifier struct { + scanDone chan bool + npuCount int +} + +// Notify stops plugin Scan. +func (n *mockNotifier) Notify(newDeviceTree dpapi.DeviceTree) { + n.npuCount = len(newDeviceTree[deviceTypeNpu]) + + n.scanDone <- true +} + +func init() { + _ = flag.Set("v", "4") //Enable debug output +} + +// mockNotifier implements Notifier interface. + +type TestCaseDetails struct { + // test-case environment + sysfsfiles map[string][]byte + name string + sysfsdirs []string + devfsdirs []string + // how plugin should interpret it + options cliOptions + // what the result should be (i915) + expectedDevs int +} + +func createTestFiles(root string, tc TestCaseDetails) (string, string, error) { + sysfs := path.Join(root, "sys") + devfs := path.Join(root, "dev") + + for _, devfsdir := range tc.devfsdirs { + if err := os.MkdirAll(path.Join(devfs, devfsdir), 0750); err != nil { + return "", "", errors.Wrap(err, "Failed to create fake device directory") + } + } + + if err := os.MkdirAll(sysfs, 0750); err != nil { + return "", "", errors.Wrap(err, "Failed to create fake base sysfs directory") + } + + for _, sysfsdir := range tc.sysfsdirs { + if err := os.MkdirAll(path.Join(sysfs, sysfsdir), 0750); err != nil { + return "", "", errors.Wrap(err, "Failed to create fake device directory") + } + } + + for filename, body := range tc.sysfsfiles { + if err := os.WriteFile(path.Join(sysfs, filename), body, 0600); err != nil { + return "", "", errors.Wrap(err, "Failed to create fake vendor file") + } + } + + return sysfs, devfs, nil +} + +func TestNewDevicePlugin(t *testing.T) { + if newDevicePlugin("", "", cliOptions{sharedDevNum: 2}) == nil { + t.Error("Failed to create NPU plugin") + } +} + +func TestAllocate(t *testing.T) { + plugin := newDevicePlugin("", "", cliOptions{sharedDevNum: 2}) + + _, err := plugin.Allocate(&v1beta1.AllocateRequest{}) + if _, ok := err.(*dpapi.UseDefaultMethodError); !ok { + t.Errorf("Unexpected return value: %+v", err) + } +} + +func TestScan(t *testing.T) { + tcases := []TestCaseDetails{ + { + name: "no sysfs mounted", + }, + { + name: "no device installed", + sysfsdirs: []string{"accel0"}, + }, + { + name: "missing dev node", + sysfsdirs: []string{"accel0/device"}, + sysfsfiles: map[string][]byte{ + "accel0/device/vendor": []byte("0x8086"), + }, + }, + { + name: "unknown device", + sysfsdirs: []string{"accel0/device/drm/accel0", "accel0/device/drm/accelD0"}, + sysfsfiles: map[string][]byte{ + "accel0/device/vendor": []byte("0x8086"), + "accel0/device/device": []byte("0xffff"), + }, + devfsdirs: []string{ + "accel0", + }, + expectedDevs: 0, + }, + { + name: "device id with endline", + sysfsdirs: []string{"accel0/device/drm/accel0", "accel0/device/drm/accelD0"}, + sysfsfiles: map[string][]byte{ + "accel0/device/vendor": []byte("0x8086"), + "accel0/device/device": []byte("0x7e4c\n"), + }, + devfsdirs: []string{ + "accel0", + }, + expectedDevs: 1, + }, + { + name: "one device", + sysfsdirs: []string{"accel0/device/drm/accel0", "accel0/device/drm/accelD0"}, + sysfsfiles: map[string][]byte{ + "accel0/device/vendor": []byte("0x8086"), + "accel0/device/device": []byte("0x643e"), + }, + devfsdirs: []string{ + "accel0", + }, + expectedDevs: 1, + }, + { + name: "two devices", + sysfsdirs: []string{ + "accel0/device/drm/accel0", "accel0/device/drm/accelD0", + "accel1/device/drm/accel1", "accel1/device/drm/accelD1", + }, + sysfsfiles: map[string][]byte{ + "accel0/device/vendor": []byte("0x8086"), + "accel0/device/device": []byte("0x643e"), + "accel1/device/vendor": []byte("0x8086"), + "accel1/device/device": []byte("0xad1d"), + }, + devfsdirs: []string{ + "accel0", + "accel1", + }, + expectedDevs: 2, + }, + } + + for _, tc := range tcases { + if tc.options.sharedDevNum == 0 { + tc.options.sharedDevNum = 1 + } + + t.Run(tc.name, func(t *testing.T) { + root, err := os.MkdirTemp("", "test_new_device_plugin") + if err != nil { + t.Fatalf("Can't create temporary directory: %+v", err) + } + // dirs/files need to be removed for the next test + defer os.RemoveAll(root) + + sysfs, devfs, err := createTestFiles(root, tc) + if err != nil { + t.Errorf("Unexpected error: %+v", err) + } + + plugin := newDevicePlugin(sysfs, devfs, tc.options) + + notifier := &mockNotifier{ + scanDone: plugin.scanDone, + } + + err = plugin.Scan(notifier) + // Scans in GPU plugin never fail + if err != nil { + t.Errorf("Unexpected error: %+v", err) + } + if tc.expectedDevs != notifier.npuCount { + t.Errorf("Expected %d, discovered %d devices (NPU)", + tc.expectedDevs, notifier.npuCount) + } + }) + } +} diff --git a/cmd/operator/main.go b/cmd/operator/main.go index 2fca03da5..942f41d74 100644 --- a/cmd/operator/main.go +++ b/cmd/operator/main.go @@ -39,6 +39,7 @@ import ( "github.com/intel/intel-device-plugins-for-kubernetes/pkg/controllers/fpga" "github.com/intel/intel-device-plugins-for-kubernetes/pkg/controllers/gpu" "github.com/intel/intel-device-plugins-for-kubernetes/pkg/controllers/iaa" + "github.com/intel/intel-device-plugins-for-kubernetes/pkg/controllers/npu" "github.com/intel/intel-device-plugins-for-kubernetes/pkg/controllers/qat" "github.com/intel/intel-device-plugins-for-kubernetes/pkg/controllers/sgx" "github.com/intel/intel-device-plugins-for-kubernetes/pkg/fpgacontroller" @@ -65,7 +66,7 @@ type devicePluginControllerAndWebhook map[string](func(ctrl.Manager, string, boo type flagList []string -var supportedDevices = flagList{"dsa", "dlb", "fpga", "gpu", "iaa", "qat", "sgx"} +var supportedDevices = flagList{"dsa", "dlb", "fpga", "gpu", "iaa", "qat", "sgx", "npu"} var devices flagList func (flag *flagList) String() string { @@ -170,6 +171,7 @@ func main() { "iaa": iaa.SetupReconciler, "qat": qat.SetupReconciler, "sgx": sgx.SetupReconciler, + "npu": npu.SetupReconciler, } tlsCfgFuncs := createTLSCfgs(enableHTTP2) diff --git a/deployments/daemonsets.go b/deployments/daemonsets.go index d6f4a6910..eccdc933d 100644 --- a/deployments/daemonsets.go +++ b/deployments/daemonsets.go @@ -71,6 +71,13 @@ func SGXPluginDaemonSet() *apps.DaemonSet { return getDaemonset(contentSGX).DeepCopy() } +//go:embed npu_plugin/base/*plugin*.yaml +var contentNPU []byte + +func NPUPluginDaemonSet() *apps.DaemonSet { + return getDaemonset(contentNPU).DeepCopy() +} + // getDaemonset unmarshalls yaml content into a DaemonSet object. func getDaemonset(content []byte) *apps.DaemonSet { var result apps.DaemonSet diff --git a/deployments/nfd/overlays/node-feature-rules/node-feature-rules.yaml b/deployments/nfd/overlays/node-feature-rules/node-feature-rules.yaml index 2cf52ab7d..ed78525f0 100644 --- a/deployments/nfd/overlays/node-feature-rules/node-feature-rules.yaml +++ b/deployments/nfd/overlays/node-feature-rules/node-feature-rules.yaml @@ -126,3 +126,25 @@ spec: - feature: kernel.config matchExpressions: X86_SGX: {op: Exists} + + - name: "intel.npu" + labels: + "intel.feature.node.kubernetes.io/npu": "true" + matchFeatures: + - feature: pci.device + matchExpressions: + vendor: {op: In, value: ["8086"]} + class: {op: In, value: ["1200"]} + device: { + op: In, + value: ["7e4c","643e","ad1d","7d1d"] + } + matchAny: + - matchFeatures: + - feature: kernel.loadedmodule + matchExpressions: + intel_vpu: {op: Exists} + - matchFeatures: + - feature: kernel.enabledmodule + matchExpressions: + intel_vpu: {op: Exists} diff --git a/deployments/npu_plugin/base/intel-npu-plugin.yaml b/deployments/npu_plugin/base/intel-npu-plugin.yaml new file mode 100644 index 000000000..05f471b0d --- /dev/null +++ b/deployments/npu_plugin/base/intel-npu-plugin.yaml @@ -0,0 +1,67 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: intel-npu-plugin + labels: + app: intel-npu-plugin +spec: + selector: + matchLabels: + app: intel-npu-plugin + updateStrategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 + template: + metadata: + labels: + app: intel-npu-plugin + spec: + containers: + - name: intel-npu-plugin + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + image: intel/intel-npu-plugin:devel + imagePullPolicy: IfNotPresent + securityContext: + seLinuxOptions: + type: "container_device_plugin_t" + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + resources: + requests: + memory: "45Mi" + cpu: 40m + limits: + memory: "90Mi" + cpu: 100m + volumeMounts: + - name: devfs + mountPath: /dev/accel + readOnly: true + - name: sysfsaccel + mountPath: /sys/class/accel + readOnly: true + - name: kubeletsockets + mountPath: /var/lib/kubelet/device-plugins + volumes: + - name: devfs + hostPath: + path: /dev/accel + - name: sysfsaccel + hostPath: + path: /sys/class/accel + - name: kubeletsockets + hostPath: + path: /var/lib/kubelet/device-plugins + nodeSelector: + kubernetes.io/arch: amd64 diff --git a/deployments/npu_plugin/base/kustomization.yaml b/deployments/npu_plugin/base/kustomization.yaml new file mode 100644 index 000000000..aab600ee8 --- /dev/null +++ b/deployments/npu_plugin/base/kustomization.yaml @@ -0,0 +1,2 @@ +resources: + - intel-npu-plugin.yaml diff --git a/deployments/npu_plugin/overlays/nfd_labeled_nodes/add-args.yaml b/deployments/npu_plugin/overlays/nfd_labeled_nodes/add-args.yaml new file mode 100644 index 000000000..f755bb5f3 --- /dev/null +++ b/deployments/npu_plugin/overlays/nfd_labeled_nodes/add-args.yaml @@ -0,0 +1,11 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: intel-npu-plugin +spec: + template: + spec: + containers: + - name: intel-npu-plugin + args: + - "-v=4" diff --git a/deployments/npu_plugin/overlays/nfd_labeled_nodes/add-nodeselector-intel-npu.yaml b/deployments/npu_plugin/overlays/nfd_labeled_nodes/add-nodeselector-intel-npu.yaml new file mode 100644 index 000000000..283213c10 --- /dev/null +++ b/deployments/npu_plugin/overlays/nfd_labeled_nodes/add-nodeselector-intel-npu.yaml @@ -0,0 +1,9 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: intel-npu-plugin +spec: + template: + spec: + nodeSelector: + intel.feature.node.kubernetes.io/npu: "true" diff --git a/deployments/npu_plugin/overlays/nfd_labeled_nodes/kustomization.yaml b/deployments/npu_plugin/overlays/nfd_labeled_nodes/kustomization.yaml new file mode 100644 index 000000000..54c480c36 --- /dev/null +++ b/deployments/npu_plugin/overlays/nfd_labeled_nodes/kustomization.yaml @@ -0,0 +1,5 @@ +resources: + - ../../base +patches: + - path: add-nodeselector-intel-npu.yaml + - path: add-args.yaml diff --git a/deployments/operator/crd/bases/deviceplugin.intel.com_npudeviceplugins.yaml b/deployments/operator/crd/bases/deviceplugin.intel.com_npudeviceplugins.yaml new file mode 100644 index 000000000..45d1fc7ef --- /dev/null +++ b/deployments/operator/crd/bases/deviceplugin.intel.com_npudeviceplugins.yaml @@ -0,0 +1,191 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.17.0 + name: npudeviceplugins.deviceplugin.intel.com +spec: + group: deviceplugin.intel.com + names: + kind: NpuDevicePlugin + listKind: NpuDevicePluginList + plural: npudeviceplugins + singular: npudeviceplugin + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .status.desiredNumberScheduled + name: Desired + type: integer + - jsonPath: .status.numberReady + name: Ready + type: integer + - jsonPath: .spec.nodeSelector + name: Node Selector + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: |- + NpuDevicePlugin is the Schema for the npudeviceplugins API. It represents + the NPU device plugin responsible for advertising Intel NPU hardware resources to + the kubelet. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: NpuDevicePluginSpec defines the desired state of NpuDevicePlugin. + properties: + image: + description: Image is a container image with NPU device plugin executable. + type: string + logLevel: + description: LogLevel sets the plugin's log level. + minimum: 0 + type: integer + nodeSelector: + additionalProperties: + type: string + description: NodeSelector provides a simple way to constrain device + plugin pods to nodes with particular labels. + type: object + sharedDevNum: + description: SharedDevNum is a number of containers that can share + the same NPU device. + minimum: 1 + type: integer + tolerations: + description: Specialized nodes (e.g., with accelerators) can be Tainted + to make sure unwanted pods are not scheduled on them. Tolerations + can be set for the plugin pod to neutralize the Taint. + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + type: object + status: + description: NpuDevicePluginStatus defines the observed state of NpuDevicePlugin. + properties: + controlledDaemonSet: + description: ControlledDaemoSet references the DaemonSet controlled + by the operator. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: |- + If referring to a piece of an object instead of an entire object, this string + should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within a pod, this would take on a value like: + "spec.containers{name}" (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" (container with + index 2 in this pod). This syntax is chosen only to have some well-defined way of + referencing a part of an object. + type: string + kind: + description: |- + Kind of the referent. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + name: + description: |- + Name of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + namespace: + description: |- + Namespace of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ + type: string + resourceVersion: + description: |- + Specific resourceVersion to which this reference is made, if any. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency + type: string + uid: + description: |- + UID of the referent. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids + type: string + type: object + x-kubernetes-map-type: atomic + desiredNumberScheduled: + description: |- + The total number of nodes that should be running the device plugin + pod (including nodes correctly running the device plugin pod). + format: int32 + type: integer + nodeNames: + description: The list of Node names where the device plugin pods are + running. + items: + type: string + type: array + numberReady: + description: |- + The number of nodes that should be running the device plugin pod and have one + or more of the device plugin pod running and ready. + format: int32 + type: integer + required: + - desiredNumberScheduled + - numberReady + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/deployments/operator/crd/kustomization.yaml b/deployments/operator/crd/kustomization.yaml index cbd0d6dfc..2d9a791f5 100644 --- a/deployments/operator/crd/kustomization.yaml +++ b/deployments/operator/crd/kustomization.yaml @@ -9,6 +9,7 @@ resources: - bases/deviceplugin.intel.com_dsadeviceplugins.yaml - bases/deviceplugin.intel.com_iaadeviceplugins.yaml - bases/deviceplugin.intel.com_dlbdeviceplugins.yaml +- bases/deviceplugin.intel.com_npudeviceplugins.yaml - bases/fpga.intel.com_acceleratorfunctions.yaml - bases/fpga.intel.com_fpgaregions.yaml # +kubebuilder:scaffold:crdkustomizeresource diff --git a/deployments/operator/device/npu/kustomization.yaml b/deployments/operator/device/npu/kustomization.yaml new file mode 100644 index 000000000..7419f1b0a --- /dev/null +++ b/deployments/operator/device/npu/kustomization.yaml @@ -0,0 +1,7 @@ +resources: + - ../../default + +patches: +- path: npu.yaml + target: + kind: Deployment diff --git a/deployments/operator/device/npu/npu.yaml b/deployments/operator/device/npu/npu.yaml new file mode 100644 index 000000000..a94ae2f33 --- /dev/null +++ b/deployments/operator/device/npu/npu.yaml @@ -0,0 +1,14 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: intel-deviceplugins-controller-manager + namespace: inteldeviceplugins-system +spec: + template: + spec: + containers: + - args: + - --metrics-bind-address=127.0.0.1:8080 + - --leader-elect + - --devices=npu + name: manager diff --git a/deployments/operator/rbac/role.yaml b/deployments/operator/rbac/role.yaml index f1dfb8ac3..481b57892 100644 --- a/deployments/operator/rbac/role.yaml +++ b/deployments/operator/rbac/role.yaml @@ -64,6 +64,7 @@ rules: - fpgadeviceplugins - gpudeviceplugins - iaadeviceplugins + - npudeviceplugins - qatdeviceplugins - sgxdeviceplugins verbs: @@ -82,6 +83,7 @@ rules: - fpgadeviceplugins/finalizers - gpudeviceplugins/finalizers - iaadeviceplugins/finalizers + - npudeviceplugins/finalizers - qatdeviceplugins/finalizers - sgxdeviceplugins/finalizers verbs: @@ -94,6 +96,7 @@ rules: - fpgadeviceplugins/status - gpudeviceplugins/status - iaadeviceplugins/status + - npudeviceplugins/status - qatdeviceplugins/status - sgxdeviceplugins/status verbs: diff --git a/deployments/operator/samples/deviceplugin_v1_npudeviceplugin.yaml b/deployments/operator/samples/deviceplugin_v1_npudeviceplugin.yaml new file mode 100644 index 000000000..39807a905 --- /dev/null +++ b/deployments/operator/samples/deviceplugin_v1_npudeviceplugin.yaml @@ -0,0 +1,10 @@ +apiVersion: deviceplugin.intel.com/v1 +kind: NpuDevicePlugin +metadata: + name: npudeviceplugin-sample +spec: + image: intel/intel-npu-plugin:0.32.0 + sharedDevNum: 1 + logLevel: 4 + nodeSelector: + intel.feature.node.kubernetes.io/npu: "true" diff --git a/deployments/operator/webhook/manifests.yaml b/deployments/operator/webhook/manifests.yaml index ab2701641..80c8aeaec 100644 --- a/deployments/operator/webhook/manifests.yaml +++ b/deployments/operator/webhook/manifests.yaml @@ -104,6 +104,26 @@ webhooks: resources: - iaadeviceplugins sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /mutate-deviceplugin-intel-com-v1-npudeviceplugin + failurePolicy: Fail + name: mnpudeviceplugin.kb.io + rules: + - apiGroups: + - deviceplugin.intel.com + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - npudeviceplugins + sideEffects: None - admissionReviewVersions: - v1 clientConfig: @@ -290,6 +310,26 @@ webhooks: resources: - iaadeviceplugins sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /validate-deviceplugin-intel-com-v1-npudeviceplugin + failurePolicy: Fail + name: vnpudeviceplugin.kb.io + rules: + - apiGroups: + - deviceplugin.intel.com + apiVersions: + - v1 + operations: + - CREATE + - UPDATE + resources: + - npudeviceplugins + sideEffects: None - admissionReviewVersions: - v1 clientConfig: diff --git a/pkg/apis/deviceplugin/v1/npudeviceplugin_types.go b/pkg/apis/deviceplugin/v1/npudeviceplugin_types.go new file mode 100644 index 000000000..26db49c11 --- /dev/null +++ b/pkg/apis/deviceplugin/v1/npudeviceplugin_types.go @@ -0,0 +1,100 @@ +// Copyright 2025 Intel Corporation. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1 + +import ( + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. + +// NpuDevicePluginSpec defines the desired state of NpuDevicePlugin. +type NpuDevicePluginSpec struct { + // Important: Run "make generate" to regenerate code after modifying this file + + // NodeSelector provides a simple way to constrain device plugin pods to nodes with particular labels. + NodeSelector map[string]string `json:"nodeSelector,omitempty"` + + // Image is a container image with NPU device plugin executable. + Image string `json:"image,omitempty"` + + // Specialized nodes (e.g., with accelerators) can be Tainted to make sure unwanted pods are not scheduled on them. Tolerations can be set for the plugin pod to neutralize the Taint. + Tolerations []v1.Toleration `json:"tolerations,omitempty"` + + // SharedDevNum is a number of containers that can share the same NPU device. + // +kubebuilder:validation:Minimum=1 + SharedDevNum int `json:"sharedDevNum,omitempty"` + + // LogLevel sets the plugin's log level. + // +kubebuilder:validation:Minimum=0 + LogLevel int `json:"logLevel,omitempty"` +} + +// NpuDevicePluginStatus defines the observed state of NpuDevicePlugin. +// TODO(rojkov): consider code deduplication with QatDevicePluginStatus. +type NpuDevicePluginStatus struct { + // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster + // Important: Run "make generate" to regenerate code after modifying this file + + // ControlledDaemoSet references the DaemonSet controlled by the operator. + // +optional + ControlledDaemonSet v1.ObjectReference `json:"controlledDaemonSet,omitempty"` + + // The list of Node names where the device plugin pods are running. + // +optional + NodeNames []string `json:"nodeNames,omitempty"` + + // The total number of nodes that should be running the device plugin + // pod (including nodes correctly running the device plugin pod). + DesiredNumberScheduled int32 `json:"desiredNumberScheduled"` + + // The number of nodes that should be running the device plugin pod and have one + // or more of the device plugin pod running and ready. + NumberReady int32 `json:"numberReady"` +} + +// +kubebuilder:object:root=true +// +kubebuilder:resource:path=npudeviceplugins,scope=Cluster +// +kubebuilder:subresource:status +// +kubebuilder:printcolumn:name="Desired",type=integer,JSONPath=`.status.desiredNumberScheduled` +// +kubebuilder:printcolumn:name="Ready",type=integer,JSONPath=`.status.numberReady` +// +kubebuilder:printcolumn:name="Node Selector",type=string,JSONPath=`.spec.nodeSelector` +// +kubebuilder:printcolumn:name="Age",type=date,JSONPath=`.metadata.creationTimestamp` +// +operator-sdk:csv:customresourcedefinitions:displayName="Intel NPU Device Plugin" + +// NpuDevicePlugin is the Schema for the npudeviceplugins API. It represents +// the NPU device plugin responsible for advertising Intel NPU hardware resources to +// the kubelet. +type NpuDevicePlugin struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Status NpuDevicePluginStatus `json:"status,omitempty"` + Spec NpuDevicePluginSpec `json:"spec,omitempty"` +} + +// +kubebuilder:object:root=true + +// NpuDevicePluginList contains a list of NpuDevicePlugin. +type NpuDevicePluginList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []NpuDevicePlugin `json:"items"` +} + +func init() { + SchemeBuilder.Register(&NpuDevicePlugin{}, &NpuDevicePluginList{}) +} diff --git a/pkg/apis/deviceplugin/v1/npudeviceplugin_webhook.go b/pkg/apis/deviceplugin/v1/npudeviceplugin_webhook.go new file mode 100644 index 000000000..ded3d1f4c --- /dev/null +++ b/pkg/apis/deviceplugin/v1/npudeviceplugin_webhook.go @@ -0,0 +1,42 @@ +// Copyright 2025 Intel Corporation. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v1 + +import ( + ctrl "sigs.k8s.io/controller-runtime" + + "github.com/intel/intel-device-plugins-for-kubernetes/pkg/controllers" +) + +// SetupWebhookWithManager sets up a webhook for GpuDevicePlugin custom resources. +func (r *NpuDevicePlugin) SetupWebhookWithManager(mgr ctrl.Manager) error { + return ctrl.NewWebhookManagedBy(mgr). + For(r). + WithDefaulter(&commonDevicePluginDefaulter{ + defaultImage: "intel/intel-npu-plugin:" + controllers.ImageMinVersion.String(), + }). + WithValidator(&commonDevicePluginValidator{ + expectedImage: "intel-npu-plugin", + expectedVersion: *controllers.ImageMinVersion, + }). + Complete() +} + +// +kubebuilder:webhook:path=/mutate-deviceplugin-intel-com-v1-npudeviceplugin,mutating=true,failurePolicy=fail,groups=deviceplugin.intel.com,resources=npudeviceplugins,verbs=create;update,versions=v1,name=mnpudeviceplugin.kb.io,sideEffects=None,admissionReviewVersions=v1 +// +kubebuilder:webhook:verbs=create;update,path=/validate-deviceplugin-intel-com-v1-npudeviceplugin,mutating=false,failurePolicy=fail,groups=deviceplugin.intel.com,resources=npudeviceplugins,versions=v1,name=vnpudeviceplugin.kb.io,sideEffects=None,admissionReviewVersions=v1 + +func (r *NpuDevicePlugin) validatePlugin(ref *commonDevicePluginValidator) error { + return validatePluginImage(r.Spec.Image, ref.expectedImage, &ref.expectedVersion) +} diff --git a/pkg/apis/deviceplugin/v1/webhook_common.go b/pkg/apis/deviceplugin/v1/webhook_common.go index e175f11e2..9b84d3645 100644 --- a/pkg/apis/deviceplugin/v1/webhook_common.go +++ b/pkg/apis/deviceplugin/v1/webhook_common.go @@ -51,6 +51,8 @@ type commonDevicePluginValidator struct { var _ admission.CustomValidator = &commonDevicePluginValidator{} // Default implements admission.CustomDefaulter so a webhook will be registered for the type. +// +//nolint:gocyclo func (r *commonDevicePluginDefaulter) Default(ctx context.Context, obj runtime.Object) error { logf.FromContext(ctx).Info("default") @@ -86,6 +88,10 @@ func (r *commonDevicePluginDefaulter) Default(ctx context.Context, obj runtime.O if len(v.Spec.Image) == 0 { v.Spec.Image = r.defaultImage } + case *NpuDevicePlugin: + if len(v.Spec.Image) == 0 { + v.Spec.Image = r.defaultImage + } default: return fmt.Errorf("%w: expected an xDevicePlugin object but got %T", errObjType, obj) } @@ -112,6 +118,8 @@ func (r *commonDevicePluginValidator) ValidateCreate(ctx context.Context, obj ru return nil, v.validatePlugin(r) case *SgxDevicePlugin: return nil, v.validatePlugin(r) + case *NpuDevicePlugin: + return nil, v.validatePlugin(r) default: return nil, fmt.Errorf("%w: expected an xDevicePlugin object but got %T", errObjType, obj) } @@ -136,6 +144,8 @@ func (r *commonDevicePluginValidator) ValidateUpdate(ctx context.Context, oldObj return nil, v.validatePlugin(r) case *SgxDevicePlugin: return nil, v.validatePlugin(r) + case *NpuDevicePlugin: + return nil, v.validatePlugin(r) default: return nil, fmt.Errorf("%w: expected an xDevicePlugin object but got %T", errObjType, oldObj) } diff --git a/pkg/apis/deviceplugin/v1/zz_generated.deepcopy.go b/pkg/apis/deviceplugin/v1/zz_generated.deepcopy.go index 64af8322a..027fe8823 100644 --- a/pkg/apis/deviceplugin/v1/zz_generated.deepcopy.go +++ b/pkg/apis/deviceplugin/v1/zz_generated.deepcopy.go @@ -568,6 +568,115 @@ func (in *IaaDevicePluginStatus) DeepCopy() *IaaDevicePluginStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NpuDevicePlugin) DeepCopyInto(out *NpuDevicePlugin) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Status.DeepCopyInto(&out.Status) + in.Spec.DeepCopyInto(&out.Spec) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NpuDevicePlugin. +func (in *NpuDevicePlugin) DeepCopy() *NpuDevicePlugin { + if in == nil { + return nil + } + out := new(NpuDevicePlugin) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NpuDevicePlugin) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NpuDevicePluginList) DeepCopyInto(out *NpuDevicePluginList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]NpuDevicePlugin, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NpuDevicePluginList. +func (in *NpuDevicePluginList) DeepCopy() *NpuDevicePluginList { + if in == nil { + return nil + } + out := new(NpuDevicePluginList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *NpuDevicePluginList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NpuDevicePluginSpec) DeepCopyInto(out *NpuDevicePluginSpec) { + *out = *in + if in.NodeSelector != nil { + in, out := &in.NodeSelector, &out.NodeSelector + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.Tolerations != nil { + in, out := &in.Tolerations, &out.Tolerations + *out = make([]corev1.Toleration, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NpuDevicePluginSpec. +func (in *NpuDevicePluginSpec) DeepCopy() *NpuDevicePluginSpec { + if in == nil { + return nil + } + out := new(NpuDevicePluginSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NpuDevicePluginStatus) DeepCopyInto(out *NpuDevicePluginStatus) { + *out = *in + out.ControlledDaemonSet = in.ControlledDaemonSet + if in.NodeNames != nil { + in, out := &in.NodeNames, &out.NodeNames + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NpuDevicePluginStatus. +func (in *NpuDevicePluginStatus) DeepCopy() *NpuDevicePluginStatus { + if in == nil { + return nil + } + out := new(NpuDevicePluginStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *QatDevicePlugin) DeepCopyInto(out *QatDevicePlugin) { *out = *in diff --git a/pkg/controllers/npu/controller.go b/pkg/controllers/npu/controller.go new file mode 100644 index 000000000..4bfb7fd2b --- /dev/null +++ b/pkg/controllers/npu/controller.go @@ -0,0 +1,182 @@ +// Copyright 2025 Intel Corporation. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package npu contains NPU specific reconciliation logic. +package npu + +import ( + "context" + "reflect" + "strconv" + "strings" + + apps "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/reference" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/intel/intel-device-plugins-for-kubernetes/deployments" + devicepluginv1 "github.com/intel/intel-device-plugins-for-kubernetes/pkg/apis/deviceplugin/v1" + "github.com/intel/intel-device-plugins-for-kubernetes/pkg/controllers" + "github.com/pkg/errors" +) + +const ( + ownerKey = ".metadata.controller.npu" +) + +var defaultNodeSelector = deployments.NPUPluginDaemonSet().Spec.Template.Spec.NodeSelector + +// +kubebuilder:rbac:groups=deviceplugin.intel.com,resources=npudeviceplugins,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=deviceplugin.intel.com,resources=npudeviceplugins/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=deviceplugin.intel.com,resources=npudeviceplugins/finalizers,verbs=update + +// SetupReconciler creates a new reconciler for NpuDevicePlugin objects. +func SetupReconciler(mgr ctrl.Manager, namespace string, withWebhook bool) error { + c := &controller{scheme: mgr.GetScheme(), ns: namespace} + if err := controllers.SetupWithManager(mgr, c, devicepluginv1.GroupVersion.String(), "NpuDevicePlugin", ownerKey); err != nil { + return err + } + + if withWebhook { + return (&devicepluginv1.NpuDevicePlugin{}).SetupWebhookWithManager(mgr) + } + + return nil +} + +type controller struct { + controllers.DefaultServiceAccountFactory + scheme *runtime.Scheme + ns string +} + +func (c *controller) CreateEmptyObject() client.Object { + return &devicepluginv1.NpuDevicePlugin{} +} + +func (c *controller) Upgrade(ctx context.Context, obj client.Object) bool { + dp := obj.(*devicepluginv1.NpuDevicePlugin) + return controllers.UpgradeImages(ctx, &dp.Spec.Image, nil) +} + +func (c *controller) NewDaemonSet(rawObj client.Object) *apps.DaemonSet { + devicePlugin := rawObj.(*devicepluginv1.NpuDevicePlugin) + + daemonSet := deployments.NPUPluginDaemonSet() + daemonSet.Name = controllers.SuffixedName(daemonSet.Name, devicePlugin.Name) + + if len(devicePlugin.Spec.NodeSelector) > 0 { + daemonSet.Spec.Template.Spec.NodeSelector = devicePlugin.Spec.NodeSelector + } + + if devicePlugin.Spec.Tolerations != nil { + daemonSet.Spec.Template.Spec.Tolerations = devicePlugin.Spec.Tolerations + } + + daemonSet.ObjectMeta.Namespace = c.ns + daemonSet.Spec.Template.Spec.Containers[0].Args = getPodArgs(devicePlugin) + daemonSet.Spec.Template.Spec.Containers[0].Image = devicePlugin.Spec.Image + + return daemonSet +} + +func processNodeSelector(ds *apps.DaemonSet, dp *devicepluginv1.NpuDevicePlugin) bool { + if len(dp.Spec.NodeSelector) > 0 { + if !reflect.DeepEqual(ds.Spec.Template.Spec.NodeSelector, dp.Spec.NodeSelector) { + ds.Spec.Template.Spec.NodeSelector = dp.Spec.NodeSelector + + return true + } + } else if !reflect.DeepEqual(ds.Spec.Template.Spec.NodeSelector, defaultNodeSelector) { + ds.Spec.Template.Spec.NodeSelector = defaultNodeSelector + + return true + } + + return false +} + +func (c *controller) UpdateDaemonSet(rawObj client.Object, ds *apps.DaemonSet) (updated bool) { + dp := rawObj.(*devicepluginv1.NpuDevicePlugin) + + if ds.Spec.Template.Spec.Containers[0].Image != dp.Spec.Image { + ds.Spec.Template.Spec.Containers[0].Image = dp.Spec.Image + updated = true + } + + if processNodeSelector(ds, dp) { + updated = true + } + + newargs := getPodArgs(dp) + oldArgString := strings.Join(ds.Spec.Template.Spec.Containers[0].Args, " ") + + if oldArgString != strings.Join(newargs, " ") { + ds.Spec.Template.Spec.Containers[0].Args = newargs + updated = true + } + + if controllers.HasTolerationsChanged(ds.Spec.Template.Spec.Tolerations, dp.Spec.Tolerations) { + ds.Spec.Template.Spec.Tolerations = dp.Spec.Tolerations + updated = true + } + + return updated +} + +func (c *controller) UpdateStatus(rawObj client.Object, ds *apps.DaemonSet, nodeNames []string) (updated bool, err error) { + dp := rawObj.(*devicepluginv1.NpuDevicePlugin) + + dsRef, err := reference.GetReference(c.scheme, ds) + if err != nil { + return false, errors.Wrap(err, "unable to make reference to controlled daemon set") + } + + if dp.Status.ControlledDaemonSet.UID != dsRef.UID { + dp.Status.ControlledDaemonSet = *dsRef + updated = true + } + + if dp.Status.DesiredNumberScheduled != ds.Status.DesiredNumberScheduled { + dp.Status.DesiredNumberScheduled = ds.Status.DesiredNumberScheduled + updated = true + } + + if dp.Status.NumberReady != ds.Status.NumberReady { + dp.Status.NumberReady = ds.Status.NumberReady + updated = true + } + + if strings.Join(dp.Status.NodeNames, ",") != strings.Join(nodeNames, ",") { + dp.Status.NodeNames = nodeNames + updated = true + } + + return updated, nil +} + +func getPodArgs(gdp *devicepluginv1.NpuDevicePlugin) []string { + args := make([]string, 0, 4) + args = append(args, "-v", strconv.Itoa(gdp.Spec.LogLevel)) + + if gdp.Spec.SharedDevNum > 0 { + args = append(args, "-shared-dev-num", strconv.Itoa(gdp.Spec.SharedDevNum)) + } else { + args = append(args, "-shared-dev-num", "1") + } + + return args +} diff --git a/pkg/controllers/npu/controller_test.go b/pkg/controllers/npu/controller_test.go new file mode 100644 index 000000000..4359ba1b0 --- /dev/null +++ b/pkg/controllers/npu/controller_test.go @@ -0,0 +1,241 @@ +// Copyright 2025 Intel Corporation. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package npu contains NPU specific reconciliation logic. +package npu + +import ( + "reflect" + "testing" + + apps "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/diff" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client" + + devicepluginv1 "github.com/intel/intel-device-plugins-for-kubernetes/pkg/apis/deviceplugin/v1" +) + +const appLabel = "intel-npu-plugin" + +// newDaemonSetExpected creates plugin daemonset +// it's copied from the original controller code (before the usage of go:embed). +func (c *controller) newDaemonSetExpected(rawObj client.Object) *apps.DaemonSet { + devicePlugin := rawObj.(*devicepluginv1.NpuDevicePlugin) + + yes := true + no := false + maxUnavailable := intstr.FromInt(1) + maxSurge := intstr.FromInt(0) + + daemonSet := apps.DaemonSet{ + TypeMeta: metav1.TypeMeta{ + Kind: "DaemonSet", + APIVersion: "apps/v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: c.ns, + Name: appLabel + "-" + devicePlugin.Name, + Labels: map[string]string{ + "app": appLabel, + }, + }, + Spec: apps.DaemonSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": appLabel, + }, + }, + UpdateStrategy: apps.DaemonSetUpdateStrategy{ + Type: "RollingUpdate", + RollingUpdate: &apps.RollingUpdateDaemonSet{ + MaxUnavailable: &maxUnavailable, + MaxSurge: &maxSurge, + }, + }, + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": appLabel, + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: appLabel, + Env: []v1.EnvVar{ + { + Name: "NODE_NAME", + ValueFrom: &v1.EnvVarSource{ + FieldRef: &v1.ObjectFieldSelector{ + FieldPath: "spec.nodeName", + }, + }, + }, + }, + Args: getPodArgs(devicePlugin), + Image: devicePlugin.Spec.Image, + ImagePullPolicy: "IfNotPresent", + SecurityContext: &v1.SecurityContext{ + SELinuxOptions: &v1.SELinuxOptions{ + Type: "container_device_plugin_t", + }, + ReadOnlyRootFilesystem: &yes, + AllowPrivilegeEscalation: &no, + Capabilities: &v1.Capabilities{Drop: []v1.Capability{"ALL"}}, + SeccompProfile: &v1.SeccompProfile{Type: v1.SeccompProfileTypeRuntimeDefault}, + }, + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + v1.ResourceMemory: resource.MustParse("90Mi"), + }, + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("40m"), + v1.ResourceMemory: resource.MustParse("45Mi"), + }, + }, + VolumeMounts: []v1.VolumeMount{ + { + Name: "devfs", + MountPath: "/dev/accel", + ReadOnly: true, + }, + { + Name: "sysfsaccel", + MountPath: "/sys/class/accel", + ReadOnly: true, + }, + { + Name: "kubeletsockets", + MountPath: "/var/lib/kubelet/device-plugins", + }, + }, + }, + }, + NodeSelector: map[string]string{"kubernetes.io/arch": "amd64"}, + Volumes: []v1.Volume{ + { + Name: "devfs", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: "/dev/accel", + }, + }, + }, + { + Name: "sysfsaccel", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: "/sys/class/accel", + }, + }, + }, + { + Name: "kubeletsockets", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: "/var/lib/kubelet/device-plugins", + }, + }, + }, + }, + }, + }, + }, + } + + return &daemonSet +} + +func (c *controller) updateDaemonSetExpected(rawObj client.Object, ds *apps.DaemonSet) { + dp := rawObj.(*devicepluginv1.NpuDevicePlugin) + + ds.Spec.Template.Spec.Containers[0].Args = getPodArgs(dp) +} + +// Test that NPU daemonsets created by using go:embed +// are equal to the expected daemonsets. +func TestNewDamonSetNPU(t *testing.T) { + tcases := []struct { + name string + }{ + { + "plugin", + }, + } + + c := &controller{} + + for _, tc := range tcases { + plugin := &devicepluginv1.NpuDevicePlugin{} + + plugin.Name = "new-npu-cr-testing" + + t.Run(tc.name, func(t *testing.T) { + expected := c.newDaemonSetExpected(plugin) + actual := c.NewDaemonSet(plugin) + + if !reflect.DeepEqual(expected, actual) { + t.Errorf("expected and actuall daemonsets differ: %+s", diff.ObjectGoPrintDiff(expected, actual)) + } + }) + } +} + +func TestUpdateDamonSetGPU(t *testing.T) { + tcases := []struct { + name string + sharedBefore int + sharedAfter int + }{ + {"plugin from 1 to 5 shared-dev-num", 1, 5}, + } + + c := &controller{} + + for _, tc := range tcases { + before := &devicepluginv1.NpuDevicePlugin{} + before.Name = "update-npu-cr-testing" + + before.Spec.SharedDevNum = tc.sharedBefore + + after := &devicepluginv1.NpuDevicePlugin{} + after.Name = "update-npu-cr-testing" + + after.Spec.SharedDevNum = tc.sharedAfter + + t.Run(tc.name, func(t *testing.T) { + expected := c.newDaemonSetExpected(before) + actual := c.NewDaemonSet(before) + + if !reflect.DeepEqual(expected, actual) { + t.Errorf("expected and actual daemonsets differ: %+s", diff.ObjectGoPrintDiff(expected, actual)) + } + + updated := c.UpdateDaemonSet(after, actual) + if updated == false { + t.Error("daemonset didn't update while it should have") + } + c.updateDaemonSetExpected(after, expected) + + if !reflect.DeepEqual(expected, actual) { + t.Errorf("updated expected and actual daemonsets differ: %+s", diff.ObjectGoPrintDiff(expected, actual)) + } + }) + } +}