Skip to content

Commit 80316e9

Browse files
mythimythi
committed
qat: 4xxx: split generic resource to compression and crypto
Signed-off-by: Mikko Ylinen <[email protected]>
1 parent 7598c0d commit 80316e9

File tree

5 files changed

+149
-6
lines changed

5 files changed

+149
-6
lines changed

‎README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -218,7 +218,7 @@ The summary of resources available via plugins in this repository is given in a
218218
| `fpga.intel.com` | custom, see [mappings](cmd/fpga_admissionwebhook/README.md#mappings)| [intelfpga-job.yaml](demo/intelfpga-job.yaml) |
219219
| `gpu.intel.com` | `i915` | [intelgpu-job.yaml](demo/intelgpu-job.yaml) |
220220
| `iaa.intel.com` | `wq-user-[shared or dedicated]` | [iaa-qpl-demo-pod.yaml](demo/iaa-qpl-demo-pod.yaml) |
221-
| `qat.intel.com` | `generic` | [crypto-perf-dpdk-pod-requesting-qat.yaml](deployments/qat_dpdk_app/base/crypto-perf-dpdk-pod-requesting-qat.yaml) |
221+
| `qat.intel.com` | `generic` or `cy`/`dc`/`sym`/`asym` | [crypto-perf-dpdk-pod-requesting-qat.yaml](deployments/qat_dpdk_app/base/crypto-perf-dpdk-pod-requesting-qat.yaml) |
222222
| `sgx.intel.com` | `epc` | [intelsgx-job.yaml](deployments/sgx_enclave_apps/base/intelsgx-job.yaml) |
223223
| `vpu.intel.com` | `hddl` | [intelvpu-job.yaml](demo/intelvpu-job.yaml) |
224224

‎cmd/qat_plugin/dpdkdrv/dpdkdrv.go

Lines changed: 60 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import (
2424
"strings"
2525
"time"
2626

27+
"github.com/go-ini/ini"
2728
"github.com/pkg/errors"
2829

2930
"k8s.io/klog/v2"
@@ -49,6 +50,9 @@ const (
4950

5051
// Period of device scans.
5152
scanPeriod = 5 * time.Second
53+
54+
// Resource name to use when device capabilities are not available.
55+
defaultCapabilities = "generic"
5256
)
5357

5458
// QAT PCI VF Device ID -> kernel QAT VF device driver mappings.
@@ -249,6 +253,57 @@ func (dp *DevicePlugin) getDpdkMounts(dpdkDeviceName string) []pluginapi.Mount {
249253
}
250254
}
251255

256+
func getDeviceCapabilities(device string) string {
257+
devID, err := getDeviceID(device)
258+
if err != nil {
259+
return defaultCapabilities
260+
}
261+
262+
devicesWithCapabilities := map[string]struct{}{
263+
"4941": {},
264+
}
265+
266+
if _, ok := devicesWithCapabilities[devID]; !ok {
267+
return defaultCapabilities
268+
}
269+
270+
pfDev, err := filepath.EvalSymlinks(filepath.Join(device, "physfn"))
271+
if err != nil {
272+
klog.Warningf("failed to get PF device ID for %s: %q", filepath.Base(device), err)
273+
return defaultCapabilities
274+
}
275+
276+
// TODO: check the sysfs state entry first when it lands.
277+
278+
lOpts := ini.LoadOptions{
279+
IgnoreInlineComment: true,
280+
}
281+
282+
devCfgPath := filepath.Join(filepath.Dir(filepath.Join(pfDev, "../../")), "kernel/debug",
283+
fmt.Sprintf("qat_4xxx_%s/dev_cfg", filepath.Base(pfDev)))
284+
285+
devCfg, err := ini.LoadSources(lOpts, devCfgPath)
286+
if err != nil {
287+
klog.Warningf("failed to read dev_cfg for %s: %q", filepath.Base(pfDev), err)
288+
return defaultCapabilities
289+
}
290+
291+
switch devCfg.Section("GENERAL").Key("ServicesEnabled").String() {
292+
case "sym;asym":
293+
return "cy"
294+
case "asym;sym":
295+
return "cy"
296+
case "dc":
297+
return "dc"
298+
case "sym":
299+
return "sym"
300+
case "asym":
301+
return "asym"
302+
default:
303+
return defaultCapabilities
304+
}
305+
}
306+
252307
func getDeviceID(device string) (string, error) {
253308
devID, err := os.ReadFile(filepath.Join(device, "device"))
254309
if err != nil {
@@ -423,16 +478,18 @@ func (dp *DevicePlugin) scan() (dpapi.DeviceTree, error) {
423478
return nil, err
424479
}
425480

426-
klog.V(1).Infof("Device %s found", vfBdf)
427-
428481
n = n + 1
429482
envs := map[string]string{
430483
fmt.Sprintf("%s%d", envVarPrefix, n): vfBdf,
431484
}
432485

486+
cap := getDeviceCapabilities(vfDevice)
487+
488+
klog.V(1).Infof("Device %s with capabilities %s found", vfBdf, cap)
489+
433490
devinfo := dpapi.NewDeviceInfo(pluginapi.Healthy, dp.getDpdkDeviceSpecs(dpdkDeviceName), dp.getDpdkMounts(dpdkDeviceName), envs, nil)
434491

435-
devTree.AddDevice("generic", vfBdf, devinfo)
492+
devTree.AddDevice(cap, vfBdf, devinfo)
436493
}
437494

438495
return devTree, nil

‎cmd/qat_plugin/dpdkdrv/dpdkdrv_test.go

Lines changed: 69 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -387,6 +387,69 @@ func TestScan(t *testing.T) {
387387
maxDevNum: 1,
388388
expectedDevNum: 1,
389389
},
390+
{
391+
name: "vfio-pci DPDKdriver with no kernel bound driver and where vfdevID is equal to qatDevId (4941), PF with dc capabilities",
392+
dpdkDriver: "vfio-pci",
393+
kernelVfDrivers: []string{"4xxxvf"},
394+
dirs: []string{
395+
"sys/bus/pci/drivers/4xxx",
396+
"sys/bus/pci/drivers/vfio-pci",
397+
"sys/devices/pci0000:02/0000:02:00.0",
398+
"sys/kernel/debug/qat_4xxx_0000:02:00.0",
399+
"sys/bus/pci/devices/0000:02:01.0",
400+
},
401+
files: map[string][]byte{
402+
"sys/devices/pci0000:02/0000:02:00.0/device": []byte("0x4940"),
403+
"sys/kernel/debug/qat_4xxx_0000:02:00.0/dev_cfg": []byte("[GENERAL]\nServicesEnabled = dc"),
404+
"sys/bus/pci/devices/0000:02:01.0/device": []byte("0x4941"),
405+
},
406+
symlinks: map[string]string{
407+
"sys/bus/pci/devices/0000:02:01.0/iommu_group": "sys/kernel/iommu_groups/vfiotestfile",
408+
"sys/bus/pci/devices/0000:02:01.0/physfn": "sys/devices/pci0000:02/0000:02:00.0",
409+
"sys/bus/pci/drivers/4xxx/0000:02:00.0": "sys/devices/pci0000:02/0000:02:00.0",
410+
"sys/bus/pci/devices/0000:02:00.0": "sys/devices/pci0000:02/0000:02:00.0",
411+
"sys/devices/pci0000:02/0000:02:00.0/virtfn0": "sys/bus/pci/devices/0000:02:01.0",
412+
},
413+
maxDevNum: 1,
414+
expectedDevNum: 1,
415+
},
416+
{
417+
name: "vfio-pci DPDKdriver with no kernel bound driver and where vfdevID is equal to qatDevId (4941), two PFs with dc and cy capabilities",
418+
dpdkDriver: "vfio-pci",
419+
kernelVfDrivers: []string{"4xxxvf"},
420+
dirs: []string{
421+
"sys/bus/pci/drivers/4xxx",
422+
"sys/bus/pci/drivers/vfio-pci",
423+
"sys/devices/pci0000:02/0000:02:00.0",
424+
"sys/devices/pci0000:03/0000:03:00.0",
425+
"sys/kernel/debug/qat_4xxx_0000:02:00.0",
426+
"sys/kernel/debug/qat_4xxx_0000:03:00.0",
427+
"sys/bus/pci/devices/0000:02:01.0",
428+
"sys/bus/pci/devices/0000:03:01.0",
429+
},
430+
files: map[string][]byte{
431+
"sys/devices/pci0000:02/0000:02:00.0/device": []byte("0x4940"),
432+
"sys/devices/pci0000:03/0000:03:00.0/device": []byte("0x4940"),
433+
"sys/kernel/debug/qat_4xxx_0000:02:00.0/dev_cfg": []byte("[GENERAL]\nServicesEnabled = dc"),
434+
"sys/kernel/debug/qat_4xxx_0000:03:00.0/dev_cfg": []byte("[GENERAL]\nServicesEnabled = sym;asym"),
435+
"sys/bus/pci/devices/0000:02:01.0/device": []byte("0x4941"),
436+
"sys/bus/pci/devices/0000:03:01.0/device": []byte("0x4941"),
437+
},
438+
symlinks: map[string]string{
439+
"sys/bus/pci/devices/0000:02:01.0/iommu_group": "sys/kernel/iommu_groups/vfiotestfile",
440+
"sys/bus/pci/devices/0000:03:01.0/iommu_group": "sys/kernel/iommu_groups/vfiotestfile2",
441+
"sys/bus/pci/devices/0000:02:01.0/physfn": "sys/devices/pci0000:02/0000:02:00.0",
442+
"sys/bus/pci/devices/0000:03:01.0/physfn": "sys/devices/pci0000:03/0000:03:00.0",
443+
"sys/bus/pci/drivers/4xxx/0000:02:00.0": "sys/devices/pci0000:02/0000:02:00.0",
444+
"sys/bus/pci/drivers/4xxx/0000:03:00.0": "sys/devices/pci0000:03/0000:03:00.0",
445+
"sys/bus/pci/devices/0000:02:00.0": "sys/devices/pci0000:02/0000:02:00.0",
446+
"sys/bus/pci/devices/0000:03:00.0": "sys/devices/pci0000:03/0000:03:00.0",
447+
"sys/devices/pci0000:02/0000:02:00.0/virtfn0": "sys/bus/pci/devices/0000:02:01.0",
448+
"sys/devices/pci0000:03/0000:03:00.0/virtfn0": "sys/bus/pci/devices/0000:03:01.0",
449+
},
450+
maxDevNum: 2,
451+
expectedDevNum: 2,
452+
},
390453
}
391454
for _, tt := range tcases {
392455
t.Run(tt.name, func(t *testing.T) {
@@ -419,8 +482,12 @@ func TestScan(t *testing.T) {
419482
if !tt.expectedErr && err != nil {
420483
t.Errorf("got unexpected error: %+v", err)
421484
}
422-
if len(fN.tree["generic"]) != tt.expectedDevNum {
423-
t.Errorf("expected %d, but got %d devices", tt.expectedDevNum, len(fN.tree["generic"]))
485+
devNum := 0
486+
for _, resource := range fN.tree {
487+
devNum = devNum + len(resource)
488+
}
489+
if devNum != tt.expectedDevNum {
490+
t.Errorf("expected %d, but got %d devices", tt.expectedDevNum, devNum)
424491
}
425492

426493
if err = os.RemoveAll(tmpdir); err != nil {

‎deployments/qat_plugin/base/intel-qat-plugin.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,9 @@ spec:
2424
- name: devdir
2525
mountPath: /dev/vfio
2626
readOnly: true
27+
- name: debugfsdir
28+
mountPath: /sys/kernel/debug
29+
readOnly: true
2730
- name: pcidir
2831
mountPath: /sys/bus/pci
2932
- name: kubeletsockets
@@ -32,6 +35,9 @@ spec:
3235
- name: devdir
3336
hostPath:
3437
path: /dev/vfio
38+
- name: debugfsdir
39+
hostPath:
40+
path: /sys/kernel/debug
3541
- name: pcidir
3642
hostPath:
3743
path: /sys/bus/pci

‎pkg/controllers/qat/controller_test.go

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,11 @@ func (c *controller) newDaemonSetExpected(rawObj client.Object) *apps.DaemonSet
8181
MountPath: "/dev/vfio",
8282
ReadOnly: true,
8383
},
84+
{
85+
Name: "debugfsdir",
86+
MountPath: "/sys/kernel/debug",
87+
ReadOnly: true,
88+
},
8489
{
8590
Name: "pcidir",
8691
MountPath: "/sys/bus/pci",
@@ -102,6 +107,14 @@ func (c *controller) newDaemonSetExpected(rawObj client.Object) *apps.DaemonSet
102107
},
103108
},
104109
},
110+
{
111+
Name: "debugfsdir",
112+
VolumeSource: v1.VolumeSource{
113+
HostPath: &v1.HostPathVolumeSource{
114+
Path: "/sys/kernel/debug",
115+
},
116+
},
117+
},
105118
{
106119
Name: "pcidir",
107120
VolumeSource: v1.VolumeSource{

0 commit comments

Comments
 (0)