Skip to content

Commit 069b9bd

Browse files
mythimythi
committed
qat: 4xxx: split generic resource to compression and crypto
Signed-off-by: Mikko Ylinen <[email protected]>
1 parent 482ed7b commit 069b9bd

File tree

5 files changed

+153
-7
lines changed

5 files changed

+153
-7
lines changed

‎README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -218,7 +218,7 @@ The summary of resources available via plugins in this repository is given in a
218218
| `fpga.intel.com` | custom, see [mappings](cmd/fpga_admissionwebhook/README.md#mappings)| [intelfpga-job.yaml](demo/intelfpga-job.yaml) |
219219
| `gpu.intel.com` | `i915` | [intelgpu-job.yaml](demo/intelgpu-job.yaml) |
220220
| `iaa.intel.com` | `wq-user-[shared or dedicated]` | [iaa-qpl-demo-pod.yaml](demo/iaa-qpl-demo-pod.yaml) |
221-
| `qat.intel.com` | `generic` | [crypto-perf-dpdk-pod-requesting-qat.yaml](deployments/qat_dpdk_app/base/crypto-perf-dpdk-pod-requesting-qat.yaml) |
221+
| `qat.intel.com` | `generic` or `cy`/`dc`/`sym`/`asym` | [crypto-perf-dpdk-pod-requesting-qat.yaml](deployments/qat_dpdk_app/base/crypto-perf-dpdk-pod-requesting-qat.yaml) |
222222
| `sgx.intel.com` | `epc` | [intelsgx-job.yaml](deployments/sgx_enclave_apps/base/intelsgx-job.yaml) |
223223
| `vpu.intel.com` | `hddl` | [intelvpu-job.yaml](demo/intelvpu-job.yaml) |
224224

‎cmd/qat_plugin/dpdkdrv/dpdkdrv.go

Lines changed: 63 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
// Copyright 2017-2021 Intel Corporation. All Rights Reserved.
1+
// Copyright 2017-2022 Intel Corporation. All Rights Reserved.
22
//
33
// Licensed under the Apache License, Version 2.0 (the "License");
44
// you may not use this file except in compliance with the License.
@@ -26,6 +26,7 @@ import (
2626
"strings"
2727
"time"
2828

29+
"github.com/go-ini/ini"
2930
"github.com/pkg/errors"
3031

3132
"k8s.io/klog/v2"
@@ -51,6 +52,9 @@ const (
5152

5253
// Period of device scans.
5354
scanPeriod = 5 * time.Second
55+
56+
// Resource name to use when device capabilities are not available.
57+
defaultCapabilities = "generic"
5458
)
5559

5660
// QAT PCI VF Device ID -> kernel QAT VF device driver mappings.
@@ -352,6 +356,57 @@ func (dp *DevicePlugin) getDpdkMounts(dpdkDeviceName string) []pluginapi.Mount {
352356
}
353357
}
354358

359+
func getDeviceCapabilities(device string) (string, error) {
360+
devID, err := getDeviceID(device)
361+
if err != nil {
362+
return "", errors.Wrapf(err, "cannot determine device capabilities")
363+
}
364+
365+
devicesWithCapabilities := map[string]struct{}{
366+
"4941": {}, // Check QAT Gen4 (4xxx) VF PCI ID only
367+
}
368+
369+
if _, ok := devicesWithCapabilities[devID]; !ok {
370+
return defaultCapabilities, nil
371+
}
372+
373+
pfDev, err := filepath.EvalSymlinks(filepath.Join(device, "physfn"))
374+
if err != nil {
375+
klog.Warningf("failed to get PF device ID for %s: %q", filepath.Base(device), err)
376+
return defaultCapabilities, nil
377+
}
378+
379+
// TODO: check the sysfs state entry first when it lands.
380+
381+
lOpts := ini.LoadOptions{
382+
IgnoreInlineComment: true,
383+
}
384+
385+
devCfgPath := filepath.Join(filepath.Dir(filepath.Join(pfDev, "../../")), "kernel/debug",
386+
fmt.Sprintf("qat_4xxx_%s/dev_cfg", filepath.Base(pfDev)))
387+
388+
devCfg, err := ini.LoadSources(lOpts, devCfgPath)
389+
if err != nil {
390+
klog.Warningf("failed to read dev_cfg for %s: %q", filepath.Base(pfDev), err)
391+
return defaultCapabilities, nil
392+
}
393+
394+
switch devCfg.Section("GENERAL").Key("ServicesEnabled").String() {
395+
case "sym;asym":
396+
return "cy", nil
397+
case "asym;sym":
398+
return "cy", nil
399+
case "dc":
400+
return "dc", nil
401+
case "sym":
402+
return "sym", nil
403+
case "asym":
404+
return "asym", nil
405+
default:
406+
return defaultCapabilities, nil
407+
}
408+
}
409+
355410
func getDeviceID(device string) (string, error) {
356411
devID, err := os.ReadFile(filepath.Join(device, "device"))
357412
if err != nil {
@@ -526,7 +581,12 @@ func (dp *DevicePlugin) scan() (dpapi.DeviceTree, error) {
526581
return nil, err
527582
}
528583

529-
klog.V(1).Infof("Device %s found", vfBdf)
584+
cap, err := getDeviceCapabilities(vfDevice)
585+
if err != nil {
586+
return nil, err
587+
}
588+
589+
klog.V(1).Infof("Device %s with %s capabilities found", vfBdf, cap)
530590

531591
n = n + 1
532592
envs := map[string]string{
@@ -535,7 +595,7 @@ func (dp *DevicePlugin) scan() (dpapi.DeviceTree, error) {
535595

536596
devinfo := dpapi.NewDeviceInfo(pluginapi.Healthy, dp.getDpdkDeviceSpecs(dpdkDeviceName), dp.getDpdkMounts(dpdkDeviceName), envs, nil)
537597

538-
devTree.AddDevice("generic", vfBdf, devinfo)
598+
devTree.AddDevice(cap, vfBdf, devinfo)
539599
}
540600

541601
return devTree, nil

‎cmd/qat_plugin/dpdkdrv/dpdkdrv_test.go

Lines changed: 70 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
// Copyright 2018 Intel Corporation. All Rights Reserved.
1+
// Copyright 2018-2022 Intel Corporation. All Rights Reserved.
22
//
33
// Licensed under the Apache License, Version 2.0 (the "License");
44
// you may not use this file except in compliance with the License.
@@ -428,6 +428,69 @@ func TestScan(t *testing.T) {
428428
maxDevNum: 1,
429429
expectedDevNum: 1,
430430
},
431+
{
432+
name: "vfio-pci DPDKdriver with no kernel bound driver and where vfdevID is equal to qatDevId (4941), PF with dc capabilities",
433+
dpdkDriver: "vfio-pci",
434+
kernelVfDrivers: []string{"4xxxvf"},
435+
dirs: []string{
436+
"sys/bus/pci/drivers/4xxx",
437+
"sys/bus/pci/drivers/vfio-pci",
438+
"sys/devices/pci0000:02/0000:02:00.0",
439+
"sys/kernel/debug/qat_4xxx_0000:02:00.0",
440+
"sys/bus/pci/devices/0000:02:01.0",
441+
},
442+
files: map[string][]byte{
443+
"sys/devices/pci0000:02/0000:02:00.0/device": []byte("0x4940"),
444+
"sys/kernel/debug/qat_4xxx_0000:02:00.0/dev_cfg": []byte("[GENERAL]\nServicesEnabled = dc"),
445+
"sys/bus/pci/devices/0000:02:01.0/device": []byte("0x4941"),
446+
},
447+
symlinks: map[string]string{
448+
"sys/bus/pci/devices/0000:02:01.0/iommu_group": "sys/kernel/iommu_groups/vfiotestfile",
449+
"sys/bus/pci/devices/0000:02:01.0/physfn": "sys/devices/pci0000:02/0000:02:00.0",
450+
"sys/bus/pci/drivers/4xxx/0000:02:00.0": "sys/devices/pci0000:02/0000:02:00.0",
451+
"sys/bus/pci/devices/0000:02:00.0": "sys/devices/pci0000:02/0000:02:00.0",
452+
"sys/devices/pci0000:02/0000:02:00.0/virtfn0": "sys/bus/pci/devices/0000:02:01.0",
453+
},
454+
maxDevNum: 1,
455+
expectedDevNum: 1,
456+
},
457+
{
458+
name: "vfio-pci DPDKdriver with no kernel bound driver and where vfdevID is equal to qatDevId (4941), two PFs with dc and cy capabilities",
459+
dpdkDriver: "vfio-pci",
460+
kernelVfDrivers: []string{"4xxxvf"},
461+
dirs: []string{
462+
"sys/bus/pci/drivers/4xxx",
463+
"sys/bus/pci/drivers/vfio-pci",
464+
"sys/devices/pci0000:02/0000:02:00.0",
465+
"sys/devices/pci0000:03/0000:03:00.0",
466+
"sys/kernel/debug/qat_4xxx_0000:02:00.0",
467+
"sys/kernel/debug/qat_4xxx_0000:03:00.0",
468+
"sys/bus/pci/devices/0000:02:01.0",
469+
"sys/bus/pci/devices/0000:03:01.0",
470+
},
471+
files: map[string][]byte{
472+
"sys/devices/pci0000:02/0000:02:00.0/device": []byte("0x4940"),
473+
"sys/devices/pci0000:03/0000:03:00.0/device": []byte("0x4940"),
474+
"sys/kernel/debug/qat_4xxx_0000:02:00.0/dev_cfg": []byte("[GENERAL]\nServicesEnabled = dc"),
475+
"sys/kernel/debug/qat_4xxx_0000:03:00.0/dev_cfg": []byte("[GENERAL]\nServicesEnabled = sym;asym"),
476+
"sys/bus/pci/devices/0000:02:01.0/device": []byte("0x4941"),
477+
"sys/bus/pci/devices/0000:03:01.0/device": []byte("0x4941"),
478+
},
479+
symlinks: map[string]string{
480+
"sys/bus/pci/devices/0000:02:01.0/iommu_group": "sys/kernel/iommu_groups/vfiotestfile",
481+
"sys/bus/pci/devices/0000:03:01.0/iommu_group": "sys/kernel/iommu_groups/vfiotestfile2",
482+
"sys/bus/pci/devices/0000:02:01.0/physfn": "sys/devices/pci0000:02/0000:02:00.0",
483+
"sys/bus/pci/devices/0000:03:01.0/physfn": "sys/devices/pci0000:03/0000:03:00.0",
484+
"sys/bus/pci/drivers/4xxx/0000:02:00.0": "sys/devices/pci0000:02/0000:02:00.0",
485+
"sys/bus/pci/drivers/4xxx/0000:03:00.0": "sys/devices/pci0000:03/0000:03:00.0",
486+
"sys/bus/pci/devices/0000:02:00.0": "sys/devices/pci0000:02/0000:02:00.0",
487+
"sys/bus/pci/devices/0000:03:00.0": "sys/devices/pci0000:03/0000:03:00.0",
488+
"sys/devices/pci0000:02/0000:02:00.0/virtfn0": "sys/bus/pci/devices/0000:02:01.0",
489+
"sys/devices/pci0000:03/0000:03:00.0/virtfn0": "sys/bus/pci/devices/0000:03:01.0",
490+
},
491+
maxDevNum: 2,
492+
expectedDevNum: 2,
493+
},
431494
}
432495
for _, tt := range tcases {
433496
t.Run(tt.name, func(t *testing.T) {
@@ -461,8 +524,12 @@ func TestScan(t *testing.T) {
461524
if !tt.expectedErr && err != nil {
462525
t.Errorf("got unexpected error: %+v", err)
463526
}
464-
if len(fN.tree["generic"]) != tt.expectedDevNum {
465-
t.Errorf("expected %d, but got %d devices", tt.expectedDevNum, len(fN.tree["generic"]))
527+
devNum := 0
528+
for _, resource := range fN.tree {
529+
devNum = devNum + len(resource)
530+
}
531+
if devNum != tt.expectedDevNum {
532+
t.Errorf("expected %d, but got %d devices", tt.expectedDevNum, devNum)
466533
}
467534

468535
if err = os.RemoveAll(tmpdir); err != nil {

‎deployments/qat_plugin/base/intel-qat-plugin.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,9 @@ spec:
2424
- name: devdir
2525
mountPath: /dev/vfio
2626
readOnly: true
27+
- name: debugfsdir
28+
mountPath: /sys/kernel/debug
29+
readOnly: true
2730
- name: pcidir
2831
mountPath: /sys/bus/pci
2932
- name: kubeletsockets
@@ -32,6 +35,9 @@ spec:
3235
- name: devdir
3336
hostPath:
3437
path: /dev/vfio
38+
- name: debugfsdir
39+
hostPath:
40+
path: /sys/kernel/debug
3541
- name: pcidir
3642
hostPath:
3743
path: /sys/bus/pci

‎pkg/controllers/qat/controller_test.go

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,11 @@ func (c *controller) newDaemonSetExpected(rawObj client.Object) *apps.DaemonSet
8181
MountPath: "/dev/vfio",
8282
ReadOnly: true,
8383
},
84+
{
85+
Name: "debugfsdir",
86+
MountPath: "/sys/kernel/debug",
87+
ReadOnly: true,
88+
},
8489
{
8590
Name: "pcidir",
8691
MountPath: "/sys/bus/pci",
@@ -102,6 +107,14 @@ func (c *controller) newDaemonSetExpected(rawObj client.Object) *apps.DaemonSet
102107
},
103108
},
104109
},
110+
{
111+
Name: "debugfsdir",
112+
VolumeSource: v1.VolumeSource{
113+
HostPath: &v1.HostPathVolumeSource{
114+
Path: "/sys/kernel/debug",
115+
},
116+
},
117+
},
105118
{
106119
Name: "pcidir",
107120
VolumeSource: v1.VolumeSource{

0 commit comments

Comments
 (0)