diff --git a/cmd/gpu_plugin/README.md b/cmd/gpu_plugin/README.md index e86a90452..923fa766a 100644 --- a/cmd/gpu_plugin/README.md +++ b/cmd/gpu_plugin/README.md @@ -57,6 +57,8 @@ For workloads on different KMDs, see [KMD and UMD](#kmd-and-umd). | -health-management | - | disabled | Enable health management by requesting data from oneAPI/Level-Zero interface. Requires [GPU Level-Zero](../gpu_levelzero/) sidecar. See [health management](#health-management) | | -wsl | - | disabled | Adapt plugin to run in the WSL environment. Requires [GPU Level-Zero](../gpu_levelzero/) sidecar. | | -shared-dev-num | int | 1 | Number of containers that can share the same GPU device | +| -allow-ids | string | "" | A list of PCI Device IDs that are allowed to be registered as resources. Default is empty (=all registered). Cannot be used together with `deny-ids`. | +| -deny-ids | string | "" | A list of PCI Device IDs that are denied to be registered as resources. Default is empty (=all registered). Cannot be used together with `allow-ids`. | | -allocation-policy | string | none | 3 possible values: balanced, packed, none. For shared-dev-num > 1: _balanced_ mode spreads workloads among GPU devices, _packed_ mode fills one GPU fully before moving to next, and _none_ selects first available device from kubelet. Default is _none_. | The plugin also accepts a number of other arguments (common to all plugins) related to logging. diff --git a/cmd/gpu_plugin/gpu_plugin.go b/cmd/gpu_plugin/gpu_plugin.go index 3f92da730..b191c4832 100644 --- a/cmd/gpu_plugin/gpu_plugin.go +++ b/cmd/gpu_plugin/gpu_plugin.go @@ -69,6 +69,8 @@ const ( type cliOptions struct { preferredAllocationPolicy string + allowIDs string + denyIDs string sharedDevNum int temperatureLimit int enableMonitoring bool @@ -204,6 +206,23 @@ func packedPolicy(req *pluginapi.ContainerPreferredAllocationRequest) []string { return deviceIds } +func validatePCIDeviceIDs(pciIDList string) error { + r := regexp.MustCompile(`^0x[0-9a-f]{4}$`) + + for id := range strings.SplitSeq(pciIDList, ",") { + id = strings.TrimSpace(id) + if id == "" { + return os.ErrNotExist + } + + if !r.MatchString(id) { + return os.ErrInvalid + } + } + + return nil +} + func (dp *devicePlugin) pciAddressForCard(cardPath, cardName string) (string, error) { linkPath, err := os.Readlink(cardPath) if err != nil { @@ -585,6 +604,31 @@ func (dp *devicePlugin) filterOutInvalidCards(files []fs.DirEntry) []fs.DirEntry continue } + allowlist := len(dp.options.allowIDs) > 0 + denylist := len(dp.options.denyIDs) > 0 + + // Skip if the device is either not allowed or denied. + if allowlist || denylist { + pciID, err := pciDeviceIDForCard(path.Join(dp.sysfsDir, f.Name())) + if err != nil { + klog.Warningf("Failed to get PCI ID for device %s: %+v", f.Name(), err) + + continue + } + + if allowlist && !strings.Contains(dp.options.allowIDs, pciID) { + klog.V(4).Infof("Skipping device %s (%s), not in allowlist: %s", f.Name(), pciID, dp.options.allowIDs) + + continue + } + + if denylist && strings.Contains(dp.options.denyIDs, pciID) { + klog.V(4).Infof("Skipping device %s (%s), in denylist: %s", f.Name(), pciID, dp.options.denyIDs) + + continue + } + } + filtered = append(filtered, f) } @@ -710,6 +754,25 @@ func (dp *devicePlugin) Allocate(request *pluginapi.AllocateRequest) (*pluginapi return nil, &dpapi.UseDefaultMethodError{} } +func checkAllowDenyOptions(opts cliOptions) bool { + if len(opts.allowIDs) > 0 && len(opts.denyIDs) > 0 { + klog.Error("Cannot use both allow-ids and deny-ids options at the same time. Please use only one of them.") + return false + } + + if err := validatePCIDeviceIDs(opts.allowIDs); err != nil { + klog.Error("Failed to validate allow-ids: ", err) + return false + } + + if err := validatePCIDeviceIDs(opts.denyIDs); err != nil { + klog.Error("Failed to validate deny-ids: ", err) + return false + } + + return true +} + func main() { var ( prefix string @@ -723,6 +786,9 @@ func main() { flag.IntVar(&opts.sharedDevNum, "shared-dev-num", 1, "number of containers sharing the same GPU device") flag.IntVar(&opts.temperatureLimit, "temp-limit", 100, "temperature limit at which device is marked unhealthy") flag.StringVar(&opts.preferredAllocationPolicy, "allocation-policy", "none", "modes of allocating GPU devices: balanced, packed and none") + flag.StringVar(&opts.allowIDs, "allow-ids", "", "comma-separated list of device IDs to allow (e.g. 0x49c5,0x49c6)") + flag.StringVar(&opts.denyIDs, "deny-ids", "", "comma-separated list of device IDs to deny (e.g. 0x49c5,0x49c6)") + flag.Parse() if opts.sharedDevNum < 1 { @@ -736,6 +802,12 @@ func main() { os.Exit(1) } + if !checkAllowDenyOptions(opts) { + klog.Error("Invalid allow/deny options.") + + os.Exit(1) + } + klog.V(1).Infof("GPU device plugin started with %s preferred allocation policy", opts.preferredAllocationPolicy) plugin := newDevicePlugin(prefix+sysfsDrmDirectory, prefix+devfsDriDirectory, opts) diff --git a/cmd/gpu_plugin/gpu_plugin_test.go b/cmd/gpu_plugin/gpu_plugin_test.go index b3b305314..19fcb6614 100644 --- a/cmd/gpu_plugin/gpu_plugin_test.go +++ b/cmd/gpu_plugin/gpu_plugin_test.go @@ -361,6 +361,93 @@ func TestScan(t *testing.T) { expectedI915Devs: 1, expectedI915Monitors: 1, }, + { + name: "two devices with only one allowed", + sysfsdirs: []string{"card0/device/drm/card0", "card0/device/drm/controlD64", "card1/device/drm/card1"}, + sysfsfiles: map[string][]byte{ + "card0/device/vendor": []byte("0x8086"), + "card0/device/device": []byte("0x1234"), + "card1/device/vendor": []byte("0x8086"), + "card1/device/device": []byte("0x9876"), + }, + symlinkfiles: map[string]string{ + "card0/device/driver": "drivers/xe", + "card1/device/driver": "drivers/i915", + }, + devfsdirs: []string{ + "card0", + "by-path/pci-0000:00:00.0-card", + "by-path/pci-0000:00:00.0-render", + "card1", + "by-path/pci-0000:00:01.0-card", + "by-path/pci-0000:00:01.0-render", + }, + options: cliOptions{enableMonitoring: true, allowIDs: "0x1234"}, + expectedXeDevs: 1, + expectedXeMonitors: 1, + expectedI915Devs: 0, + expectedI915Monitors: 0, + }, + { + name: "three devices with two allowed", + sysfsdirs: []string{"card0/device/drm/card0", "card0/device/drm/controlD64", "card1/device/drm/card1", "card2/device/drm/card2"}, + sysfsfiles: map[string][]byte{ + "card0/device/vendor": []byte("0x8086"), + "card0/device/device": []byte("0x1234"), + "card1/device/vendor": []byte("0x8086"), + "card1/device/device": []byte("0x9876"), + "card2/device/vendor": []byte("0x8086"), + "card2/device/device": []byte("0x0101"), + }, + symlinkfiles: map[string]string{ + "card0/device/driver": "drivers/xe", + "card1/device/driver": "drivers/i915", + "card2/device/driver": "drivers/i915", + }, + devfsdirs: []string{ + "card0", + "by-path/pci-0000:00:00.0-card", + "by-path/pci-0000:00:00.0-render", + "card1", + "by-path/pci-0000:00:01.0-card", + "by-path/pci-0000:00:01.0-render", + "card2", + "by-path/pci-0000:00:02.0-card", + "by-path/pci-0000:00:02.0-render", + }, + options: cliOptions{enableMonitoring: true, allowIDs: "0x1234,0x9876"}, + expectedXeDevs: 1, + expectedXeMonitors: 1, + expectedI915Devs: 1, + expectedI915Monitors: 1, + }, + { + name: "two devices with one denied", + sysfsdirs: []string{"card0/device/drm/card0", "card0/device/drm/controlD64", "card1/device/drm/card1"}, + sysfsfiles: map[string][]byte{ + "card0/device/vendor": []byte("0x8086"), + "card0/device/device": []byte("0x1234"), + "card1/device/vendor": []byte("0x8086"), + "card1/device/device": []byte("0x9876"), + }, + symlinkfiles: map[string]string{ + "card0/device/driver": "drivers/xe", + "card1/device/driver": "drivers/i915", + }, + devfsdirs: []string{ + "card0", + "by-path/pci-0000:00:00.0-card", + "by-path/pci-0000:00:00.0-render", + "card1", + "by-path/pci-0000:00:01.0-card", + "by-path/pci-0000:00:01.0-render", + }, + options: cliOptions{enableMonitoring: true, denyIDs: "0x1234"}, + expectedXeDevs: 0, + expectedXeMonitors: 0, + expectedI915Devs: 1, + expectedI915Monitors: 1, + }, { name: "sriov-1-pf-no-vfs + monitoring", sysfsdirs: []string{"card0/device/drm/card0", "card0/device/drm/controlD64"}, @@ -1048,3 +1135,61 @@ func TestCDIDeviceInclusion(t *testing.T) { t.Error("Invalid count for device (xe)") } } + +func TestParsePCIDeviceIDs(t *testing.T) { + tests := []struct { + name string + input string + wantError bool + }{ + { + name: "valid single ID", + input: "0x1234", + wantError: false, + }, + { + name: "valid multiple IDs", + input: "0x1234,0x5678,0x9abc", + wantError: false, + }, + { + name: "valid IDs with spaces", + input: " 0x1234 , 0x5678 ", + wantError: false, + }, + { + name: "empty string", + input: "", + wantError: true, + }, + { + name: "invalid ID format", + input: "0x1234,abcd", + wantError: true, + }, + { + name: "invalid hex length", + input: "0x123,0x5678", + wantError: true, + }, + { + name: "extra comma", + input: "0x1234,", + wantError: true, + }, + { + name: "capita hex", + input: "0xAA12,", + wantError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := validatePCIDeviceIDs(tt.input) + if (err != nil) != tt.wantError { + t.Errorf("parsePCIDeviceIDs() error = %v, wantError %v", err, tt.wantError) + } + }) + } +} diff --git a/deployments/gpu_plugin/overlays/allowlist-arc/add-args.yaml b/deployments/gpu_plugin/overlays/allowlist-arc/add-args.yaml new file mode 100644 index 000000000..2d3ff4bac --- /dev/null +++ b/deployments/gpu_plugin/overlays/allowlist-arc/add-args.yaml @@ -0,0 +1,12 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: intel-gpu-plugin +spec: + template: + spec: + containers: + - name: intel-gpu-plugin + args: + - "-v=4" + - "-allow-ids=0x56a6,0x56a5,0x56a1,0x56a0,0x5694,0x5693,0x5692,0x5691,0x5690,0x56b3,0x56b2,0x56a4,0x56a3,0x5697,0x5696,0x5695,0x56b1,0x56b0,0x56a2,0x56ba,0x56bc,0x56bd,0x56bb" diff --git a/deployments/gpu_plugin/overlays/allowlist-arc/kustomization.yaml b/deployments/gpu_plugin/overlays/allowlist-arc/kustomization.yaml new file mode 100644 index 000000000..b374ebdad --- /dev/null +++ b/deployments/gpu_plugin/overlays/allowlist-arc/kustomization.yaml @@ -0,0 +1,4 @@ +resources: + - ../../base +patches: + - path: add-args.yaml diff --git a/deployments/operator/crd/bases/deviceplugin.intel.com_gpudeviceplugins.yaml b/deployments/operator/crd/bases/deviceplugin.intel.com_gpudeviceplugins.yaml index 813abc6f4..f1b00a661 100644 --- a/deployments/operator/crd/bases/deviceplugin.intel.com_gpudeviceplugins.yaml +++ b/deployments/operator/crd/bases/deviceplugin.intel.com_gpudeviceplugins.yaml @@ -55,6 +55,20 @@ spec: spec: description: GpuDevicePluginSpec defines the desired state of GpuDevicePlugin. properties: + allowIDs: + description: |- + AllowIDs is a comma-separated list of PCI IDs of GPU devices that should only be advertised by the plugin. + If not set, all devices are advertised. + The list can contain IDs in the form of '0x1234,0x49a4,0x50b4'. + Cannot be used together with DenyIDs. + type: string + denyIDs: + description: |- + DenyIDs is a comma-separated list of PCI IDs of GPU devices that should only be denied by the plugin. + If not set, all devices are advertised. + The list can contain IDs in the form of '0x1234,0x49a4,0x50b4'. + Cannot be used together with AllowIDs. + type: string enableMonitoring: description: |- EnableMonitoring enables the monitoring resource ('i915_monitoring') diff --git a/pkg/apis/deviceplugin/v1/gpudeviceplugin_types.go b/pkg/apis/deviceplugin/v1/gpudeviceplugin_types.go index 01eb4d27f..c97943c98 100644 --- a/pkg/apis/deviceplugin/v1/gpudeviceplugin_types.go +++ b/pkg/apis/deviceplugin/v1/gpudeviceplugin_types.go @@ -34,6 +34,18 @@ type GpuDevicePluginSpec struct { // InitImage is a container image with tools (e.g., GPU NFD source hook) installed on each node. InitImage string `json:"initImage,omitempty"` + // AllowIDs is a comma-separated list of PCI IDs of GPU devices that should only be advertised by the plugin. + // If not set, all devices are advertised. + // The list can contain IDs in the form of '0x1234,0x49a4,0x50b4'. + // Cannot be used together with DenyIDs. + AllowIDs string `json:"allowIDs,omitempty"` + + // DenyIDs is a comma-separated list of PCI IDs of GPU devices that should only be denied by the plugin. + // If not set, all devices are advertised. + // The list can contain IDs in the form of '0x1234,0x49a4,0x50b4'. + // Cannot be used together with AllowIDs. + DenyIDs string `json:"denyIDs,omitempty"` + // PreferredAllocationPolicy sets the mode of allocating GPU devices on a node. // See documentation for detailed description of the policies. Only valid when SharedDevNum > 1 is set. // +kubebuilder:validation:Enum=balanced;packed;none diff --git a/pkg/apis/deviceplugin/v1/gpudeviceplugin_webhook.go b/pkg/apis/deviceplugin/v1/gpudeviceplugin_webhook.go index 64007dc12..34965461d 100644 --- a/pkg/apis/deviceplugin/v1/gpudeviceplugin_webhook.go +++ b/pkg/apis/deviceplugin/v1/gpudeviceplugin_webhook.go @@ -16,14 +16,20 @@ package v1 import ( "fmt" + "regexp" + "strings" ctrl "sigs.k8s.io/controller-runtime" "github.com/intel/intel-device-plugins-for-kubernetes/pkg/controllers" ) +var pciIDRegex regexp.Regexp + // SetupWebhookWithManager sets up a webhook for GpuDevicePlugin custom resources. func (r *GpuDevicePlugin) SetupWebhookWithManager(mgr ctrl.Manager) error { + pciIDRegex = *regexp.MustCompile(`^0x[0-9a-f]{4}$`) + return ctrl.NewWebhookManagedBy(mgr). For(r). WithDefaulter(&commonDevicePluginDefaulter{ @@ -44,5 +50,33 @@ func (r *GpuDevicePlugin) validatePlugin(ref *commonDevicePluginValidator) error return fmt.Errorf("%w: PreferredAllocationPolicy is valid only when setting sharedDevNum > 1", errValidation) } + if r.Spec.AllowIDs != "" { + for id := range strings.SplitSeq(r.Spec.AllowIDs, ",") { + if id == "" { + return fmt.Errorf("%w: Empty PCI Device ID in AllowIDs", errValidation) + } + + if !pciIDRegex.MatchString(id) { + return fmt.Errorf("%w: Invalid PCI Device ID: %s", errValidation, id) + } + } + } + + if r.Spec.DenyIDs != "" { + for id := range strings.SplitSeq(r.Spec.DenyIDs, ",") { + if id == "" { + return fmt.Errorf("%w: Empty PCI Device ID in DenyIDs", errValidation) + } + + if !pciIDRegex.MatchString(id) { + return fmt.Errorf("%w: Invalid PCI Device ID: %s", errValidation, id) + } + } + } + + if len(r.Spec.AllowIDs) > 0 && len(r.Spec.DenyIDs) > 0 { + return fmt.Errorf("%w: AllowIDs and DenyIDs cannot be used together", errValidation) + } + return validatePluginImage(r.Spec.Image, ref.expectedImage, &ref.expectedVersion) } diff --git a/pkg/controllers/gpu/controller.go b/pkg/controllers/gpu/controller.go index 810299a36..bc7b68b54 100644 --- a/pkg/controllers/gpu/controller.go +++ b/pkg/controllers/gpu/controller.go @@ -277,5 +277,13 @@ func getPodArgs(gdp *devicepluginv1.GpuDevicePlugin) []string { args = append(args, "-allocation-policy", "none") } + if gdp.Spec.AllowIDs != "" { + args = append(args, "-allow-ids", gdp.Spec.AllowIDs) + } + + if gdp.Spec.DenyIDs != "" { + args = append(args, "-deny-ids", gdp.Spec.DenyIDs) + } + return args }