Merge pull request #939 from hj-johannes-lee/qat-allocation-policy

qat: implement preferredAllocation policies

Author: mythi

cmd/qat_plugin/README.md

@@ -53,6 +53,7 @@ The QAT plugin can take a number of command line arguments, summarised in the fo
 | -kernel-vf-drivers | string | Comma separated VF Device Driver of the QuickAssist Devices in the system. Devices supported: DH895xCC, C62x, C3xxx, 4xxx, C4xxx and D15xx (default: `c6xxvf,4xxxvf`) |
 | -max-num-devices | int | maximum number of QAT devices to be provided to the QuickAssist device plugin (default: `32`) |
 | -mode | string | plugin mode which can be either `dpdk` or `kernel` (default: `dpdk`) |
+| -allocation-policy | string | 2 possible values: balanced and packed. Balanced mode spreads allocated QAT VF resources balanced among QAT PF devices, and packed mode packs one QAT PF device full of QAT VF resources before allocating resources from the next QAT PF. (There is no default.) |
 
 The plugin also accepts a number of other arguments related to logging. Please use the `-h` option to see
 the complete list of logging related options.

cmd/qat_plugin/dpdkdrv/dpdkdrv.go

@@ -17,9 +17,11 @@ package dpdkdrv
 
 import (
 	"bytes"
+	"flag"
 	"fmt"
 	"os"
 	"path/filepath"
+	"sort"
 	"strconv"
 	"strings"
 	"time"
@@ -62,11 +64,54 @@ var qatDeviceDriver = map[string]string{
 	"6f55": "d15xxvf",
 }
 
+// swapBDF returns ["C1:B1:A1", "C2:B2:A2"], when the given parameter is ["A1:B1:C1", "A2:B2:C2"``].
+func swapBDF(devstrings []string) []string {
+	result := make([]string, len(devstrings))
+
+	for n, dev := range devstrings {
+		tmp := strings.Split(dev, ":")
+		result[n] = fmt.Sprintf("%v:%v:%v", tmp[2], tmp[1], tmp[0])
+	}
+
+	return result
+}
+
+type preferredAllocationPolicyFunc func(*pluginapi.ContainerPreferredAllocationRequest) []string
+
+// nonePolicy is used when no policy is specified.
+func nonePolicy(req *pluginapi.ContainerPreferredAllocationRequest) []string {
+	deviceIds := req.AvailableDeviceIDs
+
+	return deviceIds[:req.AllocationSize]
+}
+
+// balancedPolicy is used for allocating QAT devices in balance.
+func balancedPolicy(req *pluginapi.ContainerPreferredAllocationRequest) []string {
+	// make it "FDB" and string sort and change back to "BDF"
+	deviceIds := swapBDF(req.AvailableDeviceIDs)
+	sort.Strings(deviceIds)
+	deviceIds = swapBDF(deviceIds)
+
+	return deviceIds[:req.AllocationSize]
+}
+
+// packedPolicy is used for allocating QAT PF devices one by one.
+func packedPolicy(req *pluginapi.ContainerPreferredAllocationRequest) []string {
+	deviceIds := req.AvailableDeviceIDs
+	sort.Strings(deviceIds)
+	deviceIds = deviceIds[:req.AllocationSize]
+
+	return deviceIds
+}
+
 // DevicePlugin represents vfio based QAT plugin.
 type DevicePlugin struct {
 	scanTicker *time.Ticker
 	scanDone   chan bool
 
+	// Note: If restarting the plugin with a new policy, the allocations for existing pods remain with old policy.
+	policy preferredAllocationPolicyFunc
+
 	pciDriverDir    string
 	pciDeviceDir    string
 	dpdkDriver      string
@@ -75,7 +120,7 @@ type DevicePlugin struct {
 }
 
 // NewDevicePlugin returns new instance of vfio based QAT plugin.
-func NewDevicePlugin(maxDevices int, kernelVfDrivers string, dpdkDriver string) (*DevicePlugin, error) {
+func NewDevicePlugin(maxDevices int, kernelVfDrivers string, dpdkDriver string, preferredAllocationPolicy string) (*DevicePlugin, error) {
 	if !isValidDpdkDeviceDriver(dpdkDriver) {
 		return nil, errors.Errorf("wrong DPDK device driver: %s", dpdkDriver)
 	}
@@ -87,10 +132,42 @@ func NewDevicePlugin(maxDevices int, kernelVfDrivers string, dpdkDriver string)
 		}
 	}
 
-	return newDevicePlugin(pciDriverDirectory, pciDeviceDirectory, maxDevices, kernelDrivers, dpdkDriver), nil
+	allocationPolicyFunc := getAllocationPolicy(preferredAllocationPolicy)
+	if allocationPolicyFunc == nil {
+		return nil, errors.Errorf("wrong allocation policy: %s", preferredAllocationPolicy)
+	}
+
+	return newDevicePlugin(pciDriverDirectory, pciDeviceDirectory, maxDevices, kernelDrivers, dpdkDriver, allocationPolicyFunc), nil
 }
 
-func newDevicePlugin(pciDriverDir, pciDeviceDir string, maxDevices int, kernelVfDrivers []string, dpdkDriver string) *DevicePlugin {
+//getAllocationPolicy returns a func that fits the policy given as a parameter. It returns nonePolicy when the flag is not set, and it returns nil when the policy is not valid value.
+func getAllocationPolicy(preferredAllocationPolicy string) preferredAllocationPolicyFunc {
+	switch {
+	case !isFlagSet("allocation-policy"):
+		return nonePolicy
+	case preferredAllocationPolicy == "packed":
+		return packedPolicy
+	case preferredAllocationPolicy == "balanced":
+		return balancedPolicy
+	default:
+		return nil
+	}
+}
+
+// isFlagSet returns true when the flag that has the same name as the parameter is set.
+func isFlagSet(name string) bool {
+	set := false
+
+	flag.Visit(func(f *flag.Flag) {
+		if f.Name == name {
+			set = true
+		}
+	})
+
+	return set
+}
+
+func newDevicePlugin(pciDriverDir, pciDeviceDir string, maxDevices int, kernelVfDrivers []string, dpdkDriver string, preferredAllocationPolicyFunc preferredAllocationPolicyFunc) *DevicePlugin {
 	return &DevicePlugin{
 		maxDevices:      maxDevices,
 		pciDriverDir:    pciDriverDir,
@@ -99,6 +176,7 @@ func newDevicePlugin(pciDriverDir, pciDeviceDir string, maxDevices int, kernelVf
 		dpdkDriver:      dpdkDriver,
 		scanTicker:      time.NewTicker(scanPeriod),
 		scanDone:        make(chan bool, 1),
+		policy:          preferredAllocationPolicyFunc,
 	}
 }
 
@@ -143,6 +221,31 @@ func (dp *DevicePlugin) Scan(notifier dpapi.Notifier) error {
 	}
 }
 
+// Implement the PreferredAllocator interface.
+func (dp *DevicePlugin) GetPreferredAllocation(rqt *pluginapi.PreferredAllocationRequest) (*pluginapi.PreferredAllocationResponse, error) {
+	response := &pluginapi.PreferredAllocationResponse{}
+
+	for _, req := range rqt.ContainerRequests {
+		// Add a security check here. This should never happen unless there occurs error in kubelet device plugin manager.
+		if req.AllocationSize > int32(len(req.AvailableDeviceIDs)) {
+			var err = errors.Errorf("AllocationSize (%d) is greater than the number of available device IDs (%d)", req.AllocationSize, len(req.AvailableDeviceIDs))
+			return nil, err
+		}
+
+		IDs := dp.policy(req)
+		klog.V(3).Infof("AvailableDeviceIDs: %q", req.AvailableDeviceIDs)
+		klog.V(3).Infof("AllocatedDeviceIDs: %q", IDs)
+
+		resp := &pluginapi.ContainerPreferredAllocationResponse{
+			DeviceIDs: IDs,
+		}
+
+		response.ContainerResponses = append(response.ContainerResponses, resp)
+	}
+
+	return response, nil
+}
+
 func (dp *DevicePlugin) getDpdkDevice(vfBdf string) (string, error) {
 	switch dp.dpdkDriver {
 	case igbUio:

cmd/qat_plugin/dpdkdrv/dpdkdrv_test.go

@@ -18,6 +18,7 @@ import (
 	"flag"
 	"os"
 	"path"
+	"reflect"
 	"testing"
 
 	"github.com/pkg/errors"
@@ -95,7 +96,7 @@ func TestNewDevicePlugin(t *testing.T) {
 	}
 	for _, tt := range tcases {
 		t.Run(tt.name, func(t *testing.T) {
-			_, err := NewDevicePlugin(1, tt.kernelVfDrivers, tt.dpdkDriver)
+			_, err := NewDevicePlugin(1, tt.kernelVfDrivers, tt.dpdkDriver, "")
 
 			if tt.expectedErr && err == nil {
 				t.Errorf("Test case '%s': expected error", tt.name)
@@ -119,6 +120,46 @@ func (n *fakeNotifier) Notify(newDeviceTree dpapi.DeviceTree) {
 	n.scanDone <- true
 }
 
+func TestGetPreferredAllocation(t *testing.T) {
+	rqt := &pluginapi.PreferredAllocationRequest{
+		ContainerRequests: []*pluginapi.ContainerPreferredAllocationRequest{
+			{
+				AvailableDeviceIDs: []string{"0000:03:00.4", "0000:04:00.1", "0000:05:00.3", "0000:05:00.4", "0000:05:00.1", "0000:04:00.0", "0000:04:00.4", "0000:06:00.4", "0000:04:00.2", "0000:03:00.1", "0000:05:00.0", "0000:05:00.2", "0000:04:00.3", "0000:03:00.2", "0000:06:00.0", "0000:06:00.3", "0000:03:00.3", "0000:03:00.0", "0000:06:00.1", "0000:06:00.2"},
+				AllocationSize:     4,
+			},
+		},
+	}
+
+	plugin := newDevicePlugin("", "", 4, []string{""}, "", nonePolicy)
+	response, _ := plugin.GetPreferredAllocation(rqt)
+
+	if !reflect.DeepEqual(response.ContainerResponses[0].DeviceIDs, []string{"0000:03:00.4", "0000:04:00.1", "0000:05:00.3", "0000:05:00.4"}) {
+		t.Error("Unexpected return value for balanced preferred allocation")
+	}
+
+	plugin = newDevicePlugin("", "", 4, []string{""}, "", packedPolicy)
+	response, _ = plugin.GetPreferredAllocation(rqt)
+
+	if !reflect.DeepEqual(response.ContainerResponses[0].DeviceIDs, []string{"0000:03:00.0", "0000:03:00.1", "0000:03:00.2", "0000:03:00.3"}) {
+		t.Error("Unexpected return value for balanced preferred allocation")
+	}
+
+	plugin = newDevicePlugin("", "", 4, []string{""}, "", balancedPolicy)
+	response, _ = plugin.GetPreferredAllocation(rqt)
+
+	if !reflect.DeepEqual(response.ContainerResponses[0].DeviceIDs, []string{"0000:03:00.0", "0000:04:00.0", "0000:05:00.0", "0000:06:00.0"}) {
+		t.Error("Unexpected return value for balanced preferred allocation")
+	}
+
+	rqt.ContainerRequests[0].AllocationSize = 32
+	plugin = newDevicePlugin("", "", 4, []string{""}, "", nil)
+	_, err := plugin.GetPreferredAllocation(rqt)
+
+	if err == nil {
+		t.Error("Unexpected nil value return for err when AllocationSize is greater than the number of available device IDs")
+	}
+}
+
 func TestScan(t *testing.T) {
 	tcases := []struct {
 		name            string
@@ -405,6 +446,7 @@ func TestScan(t *testing.T) {
 				tt.maxDevNum,
 				tt.kernelVfDrivers,
 				tt.dpdkDriver,
+				nil,
 			)
 
 			fN := fakeNotifier{

cmd/qat_plugin/qat_plugin.go

@@ -41,12 +41,13 @@ func main() {
 
 	dpdkDriver := flag.String("dpdk-driver", "vfio-pci", "DPDK Device driver for configuring the QAT device")
 	kernelVfDrivers := flag.String("kernel-vf-drivers", "c6xxvf,4xxxvf", "Comma separated VF Device Driver of the QuickAssist Devices in the system. Devices supported: DH895xCC, C62x, C3xxx, C4xxx, 4xxx, and D15xx")
+	preferredAllocationPolicy := flag.String("allocation-policy", "", "Modes of allocating QAT devices: balanced and packed")
 	maxNumDevices := flag.Int("max-num-devices", 32, "maximum number of QAT devices to be provided to the QuickAssist device plugin")
 	flag.Parse()
 
 	switch *mode {
 	case "dpdk":
-		plugin, err = dpdkdrv.NewDevicePlugin(*maxNumDevices, *kernelVfDrivers, *dpdkDriver)
+		plugin, err = dpdkdrv.NewDevicePlugin(*maxNumDevices, *kernelVfDrivers, *dpdkDriver, *preferredAllocationPolicy)
 	case "kernel":
 		plugin = kerneldrv.NewDevicePlugin()
 	default:

deployments/operator/crd/bases/deviceplugin.intel.com_qatdeviceplugins.yaml

@@ -94,6 +94,14 @@ spec:
                 description: NodeSelector provides a simple way to constrain device
                   plugin pods to nodes with particular labels.
                 type: object
+              preferredAllocationPolicy:
+                description: PreferredAllocationPolicy sets the mode of allocating
+                  QAT devices on a node. See documentation for detailed description
+                  of the policies.
+                enum:
+                - balanced
+                - packed
+                type: string
             type: object
           status:
             description: 'QatDevicePluginStatus defines the observed state of QatDevicePlugin.

pkg/apis/deviceplugin/v1/qatdeviceplugin_types.go

@@ -35,6 +35,10 @@ type QatDevicePluginSpec struct {
 
 	// InitImage is a container image with a script that initialize devices.
 	InitImage string `json:"initImage,omitempty"`
+	// PreferredAllocationPolicy sets the mode of allocating QAT devices on a node.
+	// See documentation for detailed description of the policies.
+	// +kubebuilder:validation:Enum=balanced;packed
+	PreferredAllocationPolicy string `json:"preferredAllocationPolicy,omitempty"`
 
 	// DpdkDriver is a DPDK device driver for configuring the QAT device.
 	// +kubebuilder:validation:Enum=igb_uio;vfio-pci

pkg/controllers/qat/controller.go

@@ -280,5 +280,9 @@ func getPodArgs(qdp *devicepluginv1.QatDevicePlugin) []string {
 		args = append(args, "-max-num-devices", "32")
 	}
 
+	if qdp.Spec.PreferredAllocationPolicy != "" {
+		args = append(args, "-allocation-policy", qdp.Spec.PreferredAllocationPolicy)
+	}
+
 	return args
 }

test/envtest/qatdeviceplugin_controller_test.go

@@ -96,6 +96,7 @@ var _ = Describe("QatDevicePlugin Controller", func() {
 			updatedDpdkDriver := "igb_uio"
 			updatedKernelVfDrivers := "c3xxxvf"
 			updatedMaxNumDevices := 16
+			updatedPreferredAllocationPolicy := "balanced"
 			updatedNodeSelector := map[string]string{"updated-qat-nodeselector": "true"}
 
 			fetched.Spec.Image = updatedImage
@@ -104,6 +105,7 @@ var _ = Describe("QatDevicePlugin Controller", func() {
 			fetched.Spec.DpdkDriver = updatedDpdkDriver
 			fetched.Spec.KernelVfDrivers = []devicepluginv1.KernelVfDriver{devicepluginv1.KernelVfDriver(updatedKernelVfDrivers)}
 			fetched.Spec.MaxNumDevices = updatedMaxNumDevices
+			fetched.Spec.PreferredAllocationPolicy = updatedPreferredAllocationPolicy
 			fetched.Spec.NodeSelector = updatedNodeSelector
 
 			Expect(k8sClient.Update(context.Background(), fetched)).Should(Succeed())
@@ -126,6 +128,8 @@ var _ = Describe("QatDevicePlugin Controller", func() {
 				updatedKernelVfDrivers,
 				"-max-num-devices",
 				strconv.Itoa(updatedMaxNumDevices),
+				"-allocation-policy",
+				updatedPreferredAllocationPolicy,
 			}
 
 			Expect(ds.Spec.Template.Spec.Containers[0].Args).Should(ConsistOf(expectArgs))