gpu: copy nfdhook functionality to gpu-plugin

In order to support v0.14+ NFD and extended resources,
write node labels into a file and store it for NFD's features.d
directory.

The old functionality still exists in the initcontainer, but if it's
used NFD has to be configured to allow it.

Signed-off-by: Tuomas Katila <[email protected]>

Author: tkatila

cmd/gpu_nfdhook/README.md

@@ -11,6 +11,8 @@ Table of Contents
 
 ## Introduction
 
+***NOTE:*** NFD's binary hook support will be turned off by default in the 0.14 release. If one desires to use it then, NFD's default configuration has to be modified to allow binary hooks. Due to the change, GPU binary hook's functionality has been copied to GPU plugin.
+
 This is the [Node Feature Discovery](https://github.com/kubernetes-sigs/node-feature-discovery)
 binary hook implementation for the Intel GPUs. The intel-gpu-initcontainer (which
 is built with the other images) can be used as part of the gpu-plugin deployment

cmd/gpu_nfdhook/main.go

@@ -15,9 +15,7 @@
 package main
 
 import (
-	"os"
-
-	"k8s.io/klog/v2"
+	"github.com/intel/intel-device-plugins-for-kubernetes/cmd/internal/labeler"
 )
 
 const (
@@ -27,13 +25,5 @@ const (
 )
 
 func main() {
-	l := newLabeler(sysfsDRMDirectory, debugfsDRIDirectory)
-
-	err := l.createLabels()
-	if err != nil {
-		klog.Errorf("%+v", err)
-		os.Exit(1)
-	}
-
-	l.printLabels()
+	labeler.CreateAndPrintLabels(sysfsDRMDirectory, debugfsDRIDirectory)
 }

cmd/gpu_plugin/README.md

@@ -52,6 +52,8 @@ backend libraries can offload compute operations to GPU.
 | -resource-manager | - | disabled | Enable fractional resource management, [see also dependencies](#fractional-resources) |
 | -shared-dev-num | int | 1 | Number of containers that can share the same GPU device |
 | -allocation-policy | string | none | 3 possible values: balanced, packed, none. For shared-dev-num > 1: _balanced_ mode spreads workloads among GPU devices, _packed_ mode fills one GPU fully before moving to next, and _none_ selects first available device from kubelet. Default is _none_. Allocation policy does not have an effect when resource manager is enabled. |
+| -disable-resource-label-export | bool | false | Disable resource label export to NFD's features.d directory. |
+| -nfd-resource-filename | string | intel-gpu-resources.txt | Name for the NFD feature file. |
 
 The plugin also accepts a number of other arguments (common to all plugins) related to logging.
 Please use the -h option to see the complete list of logging related options.

cmd/gpu_plugin/gpu_plugin.go

@@ -1,4 +1,4 @@
-// Copyright 2017-2022 Intel Corporation. All Rights Reserved.
+// Copyright 2017-2023 Intel Corporation. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -31,17 +31,20 @@ import (
 	pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
 
 	"github.com/intel/intel-device-plugins-for-kubernetes/cmd/gpu_plugin/rm"
+	"github.com/intel/intel-device-plugins-for-kubernetes/cmd/internal/labeler"
 	"github.com/intel/intel-device-plugins-for-kubernetes/cmd/internal/pluginutils"
 	dpapi "github.com/intel/intel-device-plugins-for-kubernetes/pkg/deviceplugin"
 )
 
 const (
-	sysfsDrmDirectory = "/sys/class/drm"
-	devfsDriDirectory = "/dev/dri"
-	gpuDeviceRE       = `^card[0-9]+$`
-	controlDeviceRE   = `^controlD[0-9]+$`
-	pciAddressRE      = "^[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\\.[0-9a-f]{1}$"
-	vendorString      = "0x8086"
+	sysfsDrmDirectory   = "/sys/class/drm"
+	debugfsDRIDirectory = "/sys/kernel/debug/dri"
+	devfsDriDirectory   = "/dev/dri"
+	nfdFeatureDir       = "/etc/kubernetes/node-feature-discovery/features.d"
+	gpuDeviceRE         = `^card[0-9]+$`
+	controlDeviceRE     = `^controlD[0-9]+$`
+	pciAddressRE        = "^[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\\.[0-9a-f]{1}$"
+	vendorString        = "0x8086"
 
 	// Device plugin settings.
 	namespace  = "gpu.intel.com"
@@ -53,13 +56,18 @@ const (
 
 	// Period of device scans.
 	scanPeriod = 5 * time.Second
+
+	// Labeler's max update interval, 5min.
+	labelerMaxInterval = 5 * 60 * time.Second
 )
 
 type cliOptions struct {
-	preferredAllocationPolicy string
-	sharedDevNum              int
-	enableMonitoring          bool
-	resourceManagement        bool
+	preferredAllocationPolicy  string
+	nfdResourceFileName        string
+	sharedDevNum               int
+	enableMonitoring           bool
+	resourceManagement         bool
+	disableResourceLabelExport bool
 }
 
 type preferredAllocationPolicyFunc func(*pluginapi.ContainerPreferredAllocationRequest) []string
@@ -242,8 +250,9 @@ type devicePlugin struct {
 	controlDeviceReg *regexp.Regexp
 	pciAddressReg    *regexp.Regexp
 
-	scanTicker *time.Ticker
-	scanDone   chan bool
+	scanTicker    *time.Ticker
+	scanDone      chan bool
+	scanResources chan bool
 
 	resMan rm.ResourceManager
 
@@ -255,21 +264,24 @@ type devicePlugin struct {
 	policy  preferredAllocationPolicyFunc
 	options cliOptions
 
-	bypathFound bool
+	bypathFound          bool
+	exportResourceLabels bool
 }
 
 func newDevicePlugin(sysfsDir, devfsDir string, options cliOptions) *devicePlugin {
 	dp := &devicePlugin{
-		sysfsDir:         sysfsDir,
-		devfsDir:         devfsDir,
-		bypathDir:        path.Join(devfsDir, "/by-path"),
-		options:          options,
-		gpuDeviceReg:     regexp.MustCompile(gpuDeviceRE),
-		controlDeviceReg: regexp.MustCompile(controlDeviceRE),
-		pciAddressReg:    regexp.MustCompile(pciAddressRE),
-		scanTicker:       time.NewTicker(scanPeriod),
-		scanDone:         make(chan bool, 1), // buffered as we may send to it before Scan starts receiving from it
-		bypathFound:      true,
+		sysfsDir:             sysfsDir,
+		devfsDir:             devfsDir,
+		bypathDir:            path.Join(devfsDir, "/by-path"),
+		options:              options,
+		gpuDeviceReg:         regexp.MustCompile(gpuDeviceRE),
+		controlDeviceReg:     regexp.MustCompile(controlDeviceRE),
+		pciAddressReg:        regexp.MustCompile(pciAddressRE),
+		scanTicker:           time.NewTicker(scanPeriod),
+		scanDone:             make(chan bool, 1), // buffered as we may send to it before Scan starts receiving from it
+		bypathFound:          true,
+		scanResources:        make(chan bool, 1),
+		exportResourceLabels: !options.disableResourceLabelExport,
 	}
 
 	if options.resourceManagement {
@@ -347,17 +359,26 @@ func (dp *devicePlugin) Scan(notifier dpapi.Notifier) error {
 			klog.Warning("Failed to scan: ", err)
 		}
 
+		countChanged := false
+
 		for name, prev := range previousCount {
 			count := devTree.DeviceTypeCount(name)
 			if count != prev {
 				klog.V(1).Infof("GPU scan update: %d->%d '%s' resources found", prev, count, name)
 
 				previousCount[name] = count
+
+				countChanged = true
 			}
 		}
 
 		notifier.Notify(devTree)
 
+		// Trigger resource scan if it's enabled.
+		if dp.exportResourceLabels && countChanged {
+			dp.scanResources <- true
+		}
+
 		select {
 		case <-dp.scanDone:
 			return nil
@@ -494,6 +515,8 @@ func main() {
 	flag.BoolVar(&opts.resourceManagement, "resource-manager", false, "fractional GPU resource management")
 	flag.IntVar(&opts.sharedDevNum, "shared-dev-num", 1, "number of containers sharing the same GPU device")
 	flag.StringVar(&opts.preferredAllocationPolicy, "allocation-policy", "none", "modes of allocating GPU devices: balanced, packed and none")
+	flag.StringVar(&opts.nfdResourceFileName, "nfd-resource-filename", "intel-gpu-resources.txt", "filename for writing labels under NFD's features.d directory")
+	flag.BoolVar(&opts.disableResourceLabelExport, "disable-resource-label-export", false, "export labels from node's GPUs and store them for NFD")
 	flag.Parse()
 
 	if opts.sharedDevNum < 1 {
@@ -512,9 +535,26 @@ func main() {
 		os.Exit(1)
 	}
 
+	if strings.Contains(opts.nfdResourceFileName, "/") {
+		klog.Errorf("NFD resource filename shouldn't have any slashes (/): %s", opts.nfdResourceFileName)
+		os.Exit(1)
+	}
+
 	klog.V(1).Infof("GPU device plugin started with %s preferred allocation policy", opts.preferredAllocationPolicy)
 
 	plugin := newDevicePlugin(prefix+sysfsDrmDirectory, prefix+devfsDriDirectory, opts)
+
+	if plugin.exportResourceLabels {
+		// Start labeler to export labels file for NFD.
+		nfdFeatureFile := path.Join(nfdFeatureDir, path.Base(opts.nfdResourceFileName))
+
+		klog.V(2).Infof("NFD feature file location: %s", nfdFeatureFile)
+
+		// Labeler catches OS signals and calls os.Exit() after receiving any.
+		go labeler.Run(prefix+sysfsDrmDirectory, prefix+debugfsDRIDirectory, nfdFeatureFile,
+			labelerMaxInterval, plugin.scanResources)
+	}
+
 	manager := dpapi.NewManager(namespace, plugin)
 	manager.Run()
 }

cmd/gpu_plugin/gpu_plugin_test.go

@@ -1,4 +1,4 @@
-// Copyright 2017-2021 Intel Corporation. All Rights Reserved.
+// Copyright 2017-2023 Intel Corporation. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

cmd/gpu_plugin/render-device.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 #
-# Copyright 2021 Intel Corporation.
+# Copyright 2021-2023 Intel Corporation.
 #
 # SPDX-License-Identifier: Apache-2.0
 #

cmd/gpu_plugin/rm/gpu_plugin_resource_manager.go

@@ -1,4 +1,4 @@
-// Copyright 2021 Intel Corporation. All Rights Reserved.
+// Copyright 2021-2023 Intel Corporation. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

cmd/gpu_plugin/rm/gpu_plugin_resource_manager_test.go

@@ -1,4 +1,4 @@
-// Copyright 2021 Intel Corporation. All Rights Reserved.
+// Copyright 2021-2023 Intel Corporation. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

cmd/gpu_nfdhook/labeler.go renamed to cmd/internal/labeler/labeler.go

@@ -1,4 +1,4 @@
-// Copyright 2020-2021 Intel Corporation. All Rights Reserved.
+// Copyright 2020-2023 Intel Corporation. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -12,18 +12,22 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package main
+package labeler
 
 import (
 	"bufio"
 	"fmt"
 	"os"
+	"os/signal"
 	"path"
 	"path/filepath"
+	"reflect"
 	"regexp"
 	"sort"
 	"strconv"
 	"strings"
+	"syscall"
+	"time"
 
 	"github.com/intel/intel-device-plugins-for-kubernetes/cmd/internal/pluginutils"
 	"github.com/pkg/errors"
@@ -58,6 +62,7 @@ type labeler struct {
 
 	sysfsDRMDir   string
 	debugfsDRIDir string
+	labelsChanged bool
 }
 
 func newLabeler(sysfsDRMDir, debugfsDRIDir string) *labeler {
@@ -67,6 +72,7 @@ func newLabeler(sysfsDRMDir, debugfsDRIDir string) *labeler {
 		gpuDeviceReg:     regexp.MustCompile(gpuDeviceRE),
 		controlDeviceReg: regexp.MustCompile(controlDeviceRE),
 		labels:           labelMap{},
+		labelsChanged:    true,
 	}
 }
 
@@ -345,6 +351,10 @@ func (l *labeler) createPCIGroupLabel(gpuNumList []string) string {
 
 // createLabels is the main function of plugin labeler, it creates label-value pairs for the gpus.
 func (l *labeler) createLabels() error {
+	prevLabels := l.labels
+
+	l.labels = labelMap{}
+
 	gpuNameList, err := l.scan()
 	if err != nil {
 		return err
@@ -431,6 +441,8 @@ func (l *labeler) createLabels() error {
 		}
 	}
 
+	l.labelsChanged = !reflect.DeepEqual(prevLabels, l.labels)
+
 	return nil
 }
 
@@ -455,8 +467,95 @@ func createNumaNodeMappingLabel(mapping map[int][]string) string {
 	return strings.Join(parts, "_")
 }
 
-func (l *labeler) printLabels() {
+func (l *labeler) printLabelsToFile(labelFile string) error {
+	f, err := os.OpenFile(labelFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		return fmt.Errorf("failed to open file (%s): %w", labelFile, err)
+	}
+
+	defer f.Close()
+
+	for key, val := range l.labels {
+		if _, err := f.WriteString(key + "=" + val + "\n"); err != nil {
+			return fmt.Errorf("failed to write label (%s=%s) to file: %w", key, val, err)
+		}
+	}
+
+	return nil
+}
+
+func CreateAndPrintLabels(sysfsDRMDir, debugfsDRIDir string) {
+	l := newLabeler(sysfsDRMDir, debugfsDRIDir)
+
+	if err := l.createLabels(); err != nil {
+		klog.Warningf("failed to create labels: %+v", err)
+
+		return
+	}
+
 	for key, val := range l.labels {
 		fmt.Println(key + "=" + val)
 	}
 }
+
+// Gathers node's GPU labels on channel trigger or timeout, and write them to a file.
+// The created label file is deleted on exit (process dying).
+func Run(sysfsDrmDir, debugfsDRIDir, nfdFeatureFile string, updateInterval time.Duration, scanResources chan bool) {
+	l := newLabeler(sysfsDrmDir, debugfsDRIDir)
+
+	interruptChan := make(chan os.Signal, 1)
+	signal.Notify(interruptChan, syscall.SIGTERM, syscall.SIGINT, syscall.SIGHUP, syscall.SIGQUIT)
+
+	klog.V(1).Info("Starting GPU labeler")
+
+	running := true
+
+	for running {
+		timeout := time.After(updateInterval)
+
+		select {
+		case <-timeout:
+		case <-scanResources:
+		case interrupt := <-interruptChan:
+			klog.V(2).Infof("Interrupt %d received", interrupt)
+
+			running = false
+
+			continue
+		}
+
+		klog.V(1).Info("Ext resources scanning")
+
+		err := l.createLabels()
+		if err != nil {
+			klog.Warningf("label creation failed: %+v", err)
+
+			continue
+		}
+
+		if l.labelsChanged {
+			klog.V(1).Info("Writing labels")
+
+			if err := l.printLabelsToFile(nfdFeatureFile); err != nil {
+				klog.Warningf("failed to write labels to file: %+v", err)
+
+				// Reset labels so that next time the labeler runs the writing is retried.
+				l.labels = labelMap{}
+			}
+		}
+	}
+
+	signal.Stop(interruptChan)
+
+	klog.V(2).Info("Removing label file")
+
+	err := os.Remove(nfdFeatureFile)
+	if err != nil {
+		klog.Errorf("Failed to cleanup label file: %+v", err)
+	}
+
+	klog.V(1).Info("Stopping GPU labeler")
+
+	// Close the whole application
+	os.Exit(0)
+}