From 243870a7077a58d7731ba52b78fc6d601ae62a14 Mon Sep 17 00:00:00 2001 From: Ed Bartosh Date: Mon, 19 Oct 2020 19:28:19 +0300 Subject: [PATCH] fpga: reimplement device discovering Reimplemented discovering of the FPGA devices using APIs from pkg/fpga/intel_fpga_linux. The APis are also used in the fpga_tool utility. The API is more advanced and supports SR-IOV among other things. Fixes: #372 Signed-off-by: Ed Bartosh --- cmd/fpga_plugin/dfl.go | 70 +------ cmd/fpga_plugin/dfl_test.go | 295 +++++++++------------------- cmd/fpga_plugin/fpga_plugin.go | 127 ++++-------- cmd/fpga_plugin/fpga_plugin_test.go | 92 +++++---- cmd/fpga_plugin/opae.go | 42 +--- cmd/fpga_plugin/opae_test.go | 229 +++++++++++---------- 6 files changed, 331 insertions(+), 524 deletions(-) diff --git a/cmd/fpga_plugin/dfl.go b/cmd/fpga_plugin/dfl.go index e2a82ba27..a40465d9a 100644 --- a/cmd/fpga_plugin/dfl.go +++ b/cmd/fpga_plugin/dfl.go @@ -15,76 +15,19 @@ package main import ( - "fmt" - "os" - "path" - "path/filepath" "regexp" - "github.com/pkg/errors" + "github.com/intel/intel-device-plugins-for-kubernetes/pkg/fpga" ) const ( dflDeviceRE = `^region[0-9]+$` dflPortRE = `^dfl-port\.[0-9]+$` - dflFmeRE = `^dfl-fme\.[0-9]+$` ) -// getDFLRegion reads FME interface id from /sys/fpga/fpga_region/regionX/dfl-fme.k/dfl-fme-region.n/fpga_region/regionN/compat_id. -func (dp *devicePlugin) getDFLRegion(regionFolder string, fme string) (*region, error) { - compatIDPattern := path.Join(regionFolder, fme, "dfl-fme-region.*", "fpga_region", "region*", "compat_id") - matches, err := filepath.Glob(compatIDPattern) - if err != nil { - return nil, err - } - if len(matches) == 0 { - return nil, fmt.Errorf("no compat_id found with pattern '%s'", compatIDPattern) - } - if len(matches) > 1 { - return nil, fmt.Errorf("compat_id path pattern '%s' matches multiple files", compatIDPattern) - } - - reg, err := dp.getFME(matches[0], fme) - if err != nil { - return nil, err - } - - return reg, nil -} - -func getSysFsInfoDFL(dp *devicePlugin, deviceFolder string, deviceFiles []os.FileInfo, fname string) ([]region, []afu, error) { - var regions []region - var afus []afu - - for _, deviceFile := range deviceFiles { - name := deviceFile.Name() - - if dp.fmeReg.MatchString(name) { - if len(regions) > 0 { - return nil, nil, errors.Errorf("Detected more than one FPGA region for device %s. Only one region per FPGA device is supported", fname) - } - - region, err := dp.getDFLRegion(deviceFolder, name) - if err != nil { - return nil, nil, err - } - regions = append(regions, *region) - } else if dp.portReg.MatchString(name) { - afuPath := path.Join(deviceFolder, name, "afu_id") - afu, err := dp.getAFU(afuPath, name) - if err != nil { - return nil, nil, err - } - afus = append(afus, *afu) - } - } - - return regions, afus, nil -} - // newDevicePluginDFL returns new instance of devicePlugin. func newDevicePluginDFL(sysfsDir string, devfsDir string, mode string) (*devicePlugin, error) { - getDevTree, ignoreAfuIDs, annotationValue, err := getPluginParams(mode) + getDevTree, annotationValue, err := getPluginParams(mode) if err != nil { return nil, err } @@ -97,13 +40,10 @@ func newDevicePluginDFL(sysfsDir string, devfsDir string, mode string) (*deviceP deviceReg: regexp.MustCompile(dflDeviceRE), portReg: regexp.MustCompile(dflPortRE), - fmeReg: regexp.MustCompile(dflFmeRE), - getDevTree: getDevTree, - getSysFsInfo: getSysFsInfoDFL, + getDevTree: getDevTree, + newPort: fpga.NewDflPort, - ignoreEmptyRegions: true, - ignoreAfuIDs: ignoreAfuIDs, - annotationValue: annotationValue, + annotationValue: annotationValue, }, nil } diff --git a/cmd/fpga_plugin/dfl_test.go b/cmd/fpga_plugin/dfl_test.go index c3dc55dee..7bff940ce 100644 --- a/cmd/fpga_plugin/dfl_test.go +++ b/cmd/fpga_plugin/dfl_test.go @@ -19,10 +19,10 @@ import ( "os" "path" "reflect" - "strings" "testing" dpapi "github.com/intel/intel-device-plugins-for-kubernetes/pkg/deviceplugin" + "github.com/intel/intel-device-plugins-for-kubernetes/pkg/fpga" pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1" ) @@ -285,240 +285,139 @@ func TestGetAfuTreeDFL(t *testing.T) { } } +// genNewDFLPort generates newPortFunc function. +func genNewDFLPort(sysFsPrefix, devDir string, sysFsInfo map[string][]string) newPortFunc { + return func(portName string) (fpga.Port, error) { + portPciDev := sysFsInfo[portName][0] + portPciDevSysFs := path.Join(sysFsPrefix, portPciDev) + + fmeName := sysFsInfo[portName][1] + fmePciDev := sysFsInfo[portName][2] + fmePciDevSysFs := path.Join(sysFsPrefix, fmePciDev) + + return &fpga.DflPort{ + Name: portName, + DevPath: path.Join(devDir, portName), + SysFsPath: path.Join(portPciDevSysFs, "fpga_region", "region*", portName), + FME: &fpga.DflFME{ + Name: fmeName, + DevPath: path.Join(devDir, fmeName), + SysFsPath: fmePciDevSysFs, + PCIDevice: &fpga.PCIDevice{SysFsPath: fmePciDevSysFs}, + }, + }, nil + } +} func TestScanFPGAsDFL(t *testing.T) { tmpdir, err := ioutil.TempDir("", "TestScanFPGAsDFL") if err != nil { t.Fatalf("can't create temporary directory: %+v", err) } - sysfs := path.Join(tmpdir, "sys", "class", "fpga_region") - devfs := path.Join(tmpdir, "dev") + sysfs := path.Join(tmpdir, "sys") + dev := path.Join(tmpdir, "dev") tcases := []struct { - name string - devfsdirs []string - sysfsdirs []string - sysfsfiles map[string][]byte - errorContains string - expectedDevTree map[string]map[string]dpapi.DeviceInfo - mode string + name string + mode string + devs []string + sysfsdirs []string + sysfsfiles map[string][]byte + newPort newPortFunc + expectedDevTreeKeys map[string][]string }{ { - name: "No sysfs folder exists", + name: "Valid DFL scan in af mode", mode: afMode, - }, - { - name: "Unexpected extra fme devices in the region", - mode: afMode, - sysfsdirs: []string{ - "region1/dfl-fme.0/dfl-fme-region.1/fpga_region/region1/", - "region1/dfl-fme.1"}, - sysfsfiles: map[string][]byte{ - "region1/dfl-fme.0/dfl-fme-region.1/fpga_region/region1/compat_id": []byte("69528db6eb31577a8c3668f9faa081f6\n"), - }, - devfsdirs: []string{"dfl-fme.0"}, - errorContains: "Detected more than one FPGA region for device region1. Only one region per FPGA device is supported", - }, - { - name: "AFU without ID", - mode: afMode, - sysfsdirs: []string{"region1/dfl-port.0"}, - errorContains: "/sys/class/fpga_region/region1/dfl-port.0/afu_id: no such file or directory", - }, - { - name: "No device node for detected AFU", - mode: afMode, - sysfsdirs: []string{"region1/dfl-port.0"}, - sysfsfiles: map[string][]byte{ - "region1/dfl-port.0/afu_id": []byte("d8424dc4a4a3c413f89e433683f9040b\n"), - }, - errorContains: "/dev/dfl-port.0: no such file or directory", - }, - { - name: "AFU without corresponding FME", - mode: afMode, - sysfsdirs: []string{"region1/dfl-port.0"}, - devfsdirs: []string{"dfl-port.0"}, - sysfsfiles: map[string][]byte{ - "region1/dfl-port.0/afu_id": []byte("d8424dc4a4a3c413f89e433683f9040b\n"), + devs: []string{ + "dfl-fme.0", "dfl-port.0", + "dfl-fme.1", "dfl-port.1", }, - errorContains: "region1: AFU without corresponding FME found", - }, - { - name: "More than one FME per FPGA device", - mode: afMode, sysfsdirs: []string{ - "region1/dfl-fme.0/dfl-fme-region.1/fpga_region/region1/", - "region1/dfl-fme.1/dfl-fme-region.1/fpga_region/region1/", + "class/fpga_region/region0/dfl-port.0", + "class/fpga_region/region1/dfl-port.1", + "class/fpga_region/dir", // this should be skipped by plugin.ScanFPGAs + "devices/pci0000:80/0000:80:01.0/0000:81:00.0/fpga_region/region0/dfl-port.0", + "devices/pci0000:80/0000:80:01.0/0000:81:00.0/fpga_region/region0/dfl-fme.0/dfl-fme-region.0/fpga_region/region0", + "devices/pci0000:40/0000:40:02.0/0000:42:00.0/fpga_region/region1/dfl-port.1", + "devices/pci0000:40/0000:40:02.0/0000:42:00.0/fpga_region/region1/dfl-fme.1/dfl-fme-region.1/fpga_region/region1", }, sysfsfiles: map[string][]byte{ - "region1/dfl-fme.0/dfl-fme-region.1/fpga_region/region1/compat_id": []byte("69528db6eb31577a8c3668f9faa081f6\n"), - "region1/dfl-fme.1/dfl-fme-region.1/fpga_region/region1/compat_id": []byte("69528db6eb31577a8c3668f9faa081f6\n"), - }, - devfsdirs: []string{ - "dfl-fme.0", - "dfl-fme.1", - }, - errorContains: "Detected more than one FPGA region", - }, - { - name: "No regionX/dfl-fme.k/dfl-fme-region.n entry found", - mode: afMode, - sysfsdirs: []string{"region1/dfl-fme.0", "region1/dfl-port.0", "region1/dfl-port.1"}, - errorContains: "no compat_id found with pattern ", - }, - { - name: "Duplicate regionX/dfl-fme.k/dfl-fme-region.n entry found", - mode: afMode, - sysfsdirs: []string{ - "region1/dfl-fme.0/dfl-fme-region.1/fpga_region/region1/", - "region1/dfl-fme.0/dfl-fme-region.2/fpga_region/region1/", - "region1/dfl-port.0"}, - sysfsfiles: map[string][]byte{ - "region1/dfl-fme.0/dfl-fme-region.1/fpga_region/region1/compat_id": []byte("69528db6eb31577a8c3668f9faa081f6\n"), - "region1/dfl-fme.0/dfl-fme-region.2/fpga_region/region1/compat_id": []byte("69528db6eb31577a8c3668f9faa081f6\n"), - }, + "devices/pci0000:80/0000:80:01.0/0000:81:00.0/fpga_region/region0/dfl-port.0/afu_id": []byte("d8424dc4a4a3c413f89e433683f9040b\n"), + "devices/pci0000:40/0000:40:02.0/0000:42:00.0/fpga_region/region1/dfl-port.1/afu_id": []byte("f7df405cbd7acf7222f144b0b93acd18\n"), - errorContains: "/sys/class/fpga_region/region1/dfl-fme.0/dfl-fme-region.*/fpga_region/region*/compat_id' matches multiple files", - }, - { - name: "fme device doesn't exist", - mode: afMode, - sysfsdirs: []string{ - "region1/dfl-fme.0/dfl-fme-region.1/fpga_region/region1/", - "region1/dfl-port.0", - "region1/dfl-port.1"}, - sysfsfiles: map[string][]byte{ - "region1/dfl-fme.0/dfl-fme-region.1/fpga_region/region1/compat_id": []byte("69528db6eb31577a8c3668f9faa081f6\n"), + "devices/pci0000:80/0000:80:01.0/0000:81:00.0/fpga_region/region0/dfl-fme.0/dfl-fme-region.0/fpga_region/region0/compat_id": []byte("69528db6eb31577a8c3668f9faa081f6\n"), + "devices/pci0000:40/0000:40:02.0/0000:42:00.0/fpga_region/region1/dfl-fme.1/dfl-fme-region.1/fpga_region/region1/compat_id": []byte("69528db6eb31577a8c3668f9faa081f6\n"), }, - errorContains: "/dev/dfl-fme.0 doesn't exist", - }, - { - name: "region1/dfl-port.0/afu_id file doesn't exist", - mode: afMode, - sysfsdirs: []string{ - "region1/dfl-fme.0/dfl-fme-region.1/fpga_region/region1/", - "region1/dfl-port.0", - "region1/dfl-port.1"}, - sysfsfiles: map[string][]byte{ - "region1/dfl-fme.0/dfl-fme-region.1/fpga_region/region1/compat_id": []byte("69528db6eb31577a8c3668f9faa081f6\n"), - }, - devfsdirs: []string{"dfl-fme.0"}, - errorContains: "region1/dfl-port.0/afu_id: no such file or directory", - }, - { - name: "region1/dfl-port.0/afu_id is a directory", - mode: afMode, - sysfsdirs: []string{ - "region1/dfl-fme.0/dfl-fme-region.1/fpga_region/region1/", - "region1/dfl-port.0/afu_id", - "region1/dfl-port.1"}, - sysfsfiles: map[string][]byte{ - "region1/dfl-fme.0/dfl-fme-region.1/fpga_region/region1/compat_id": []byte("69528db6eb31577a8c3668f9faa081f6\n"), - }, - devfsdirs: []string{"dfl-fme.0"}, - errorContains: "region1/dfl-port.0/afu_id: is a directory", - }, - { - name: "port device doesn't exist", - mode: afMode, - sysfsdirs: []string{ - "region1/dfl-fme.0/dfl-fme-region.1/fpga_region/region1/", - "region1/dfl-port.0", - "region1/dfl-port.1"}, - sysfsfiles: map[string][]byte{ - "region1/dfl-fme.0/dfl-fme-region.1/fpga_region/region1/compat_id": []byte("69528db6eb31577a8c3668f9faa081f6\n"), - "region1/dfl-port.0/afu_id": []byte("d8424dc4a4a3c413f89e433683f9040b\n"), - }, - devfsdirs: []string{"dfl-fme.0"}, - errorContains: "/dev/dfl-port.0 doesn't exist", - }, - { - name: "working af mode", - mode: afMode, - sysfsdirs: []string{ - "region1/dfl-fme.0/dfl-fme-region.1/fpga_region/region1/", - "region1/dfl-port.0", - "region1/dfl-port.1", - "region2/dfl-fme.1/dfl-fme-region.2/fpga_region/region2/", - "region2/dfl-port.2", - "region2/dfl-port.3", - }, - sysfsfiles: map[string][]byte{ - "region1/dfl-fme.0/dfl-fme-region.1/fpga_region/region1/compat_id": []byte("69528db6eb31577a8c3668f9faa081f6\n"), - "region1/dfl-port.0/afu_id": []byte("d8424dc4a4a3c413f89e433683f9040b\n"), - "region1/dfl-port.1/afu_id": []byte("d8424dc4a4a3c413f89e433683f9040b\n"), - "region2/dfl-fme.1/dfl-fme-region.2/fpga_region/region2/compat_id": []byte("69528db6eb31577a8c3668f9faa081f6\n"), - "region2/dfl-port.2/afu_id": []byte("d8424dc4a4a3c413f89e433683f9040b\n"), - "region2/dfl-port.3/afu_id": []byte("d8424dc4a4a3c413f89e433683f9040b\n"), + newPort: genNewDFLPort(sysfs, dev, + map[string][]string{ + "dfl-port.0": {"devices/pci0000:80/0000:80:01.0/0000:81:00.0", "dfl-fme.0", "devices/pci0000:80/0000:80:01.0/0000:81:00.0"}, + "dfl-port.1": {"devices/pci0000:40/0000:40:02.0/0000:42:00.0", "dfl-fme.1", "devices/pci0000:40/0000:40:02.0/0000:42:00.0"}, + }), + expectedDevTreeKeys: map[string][]string{ + "af-695.d84.aVKNtusxV3qMNmj5-qCB9thCTcSko8QT-J5DNoP5BAs": {"dfl-port.0"}, + "af-695.f7d.aVKNtusxV3qMNmj5-qCB9vffQFy9es9yIvFEsLk6zRg": {"dfl-port.1"}, }, - devfsdirs: []string{"dfl-fme.0", "dfl-port.0", "dfl-port.1", "dfl-fme.1", "dfl-port.2", "dfl-port.3"}, }, { - name: "working region mode", + name: "Valid DFL scan in region mode", mode: regionMode, - sysfsdirs: []string{ - "region1/dfl-fme.0/dfl-fme-region.1/fpga_region/region1/", - "region1/dfl-port.0", - "region1/dfl-port.1", - "region2/dfl-fme.1/dfl-fme-region.2/fpga_region/region2/", - "region2/dfl-port.2", - "region2/dfl-port.3", - }, - sysfsfiles: map[string][]byte{ - "region1/dfl-fme.0/dfl-fme-region.1/fpga_region/region1/compat_id": []byte("69528db6eb31577a8c3668f9faa081f6\n"), - "region1/dfl-port.0/afu_id": []byte("d8424dc4a4a3c413f89e433683f9040b\n"), - "region1/dfl-port.1/afu_id": []byte("d8424dc4a4a3c413f89e433683f9040b\n"), - "region2/dfl-fme.1/dfl-fme-region.2/fpga_region/region2/compat_id": []byte("69528db6eb31577a8c3668f9faa081f6\n"), - "region2/dfl-port.2/afu_id": []byte("d8424dc4a4a3c413f89e433683f9040b\n"), - "region2/dfl-port.3/afu_id": []byte("d8424dc4a4a3c413f89e433683f9040b\n"), + devs: []string{ + "dfl-fme.0", "dfl-port.0", + "dfl-fme.1", "dfl-port.1", }, - devfsdirs: []string{"dfl-fme.0", "dfl-port.0", "dfl-port.1", "dfl-fme.1", "dfl-port.2", "dfl-port.3"}, - }, - { - name: "working regionDevel mode", - mode: regionDevelMode, sysfsdirs: []string{ - "region1/dfl-fme.0/dfl-fme-region.1/fpga_region/region1/", - "region1/dfl-port.0", - "region1/dfl-port.1", - "region2/dfl-fme.1/dfl-fme-region.2/fpga_region/region2/", - "region2/dfl-port.2", - "region2/dfl-port.3", + "class/fpga_region/region0/dfl-port.0", + "class/fpga_region/region1/dfl-port.1", + "class/fpga_region/dir", // this should be skipped by plugin.ScanFPGAs + "devices/pci0000:80/0000:80:01.0/0000:81:00.0/fpga_region/region0/dfl-port.0", + "devices/pci0000:80/0000:80:01.0/0000:81:00.0/fpga_region/region0/dfl-fme.0/dfl-fme-region.0/fpga_region/region0", + "devices/pci0000:40/0000:40:02.0/0000:42:00.0/fpga_region/region1/dfl-port.1", + "devices/pci0000:40/0000:40:02.0/0000:42:00.0/fpga_region/region1/dfl-fme.1/dfl-fme-region.1/fpga_region/region1", }, sysfsfiles: map[string][]byte{ - "region1/dfl-fme.0/dfl-fme-region.1/fpga_region/region1/compat_id": []byte("69528db6eb31577a8c3668f9faa081f6\n"), - "region1/dfl-port.0/afu_id": []byte("d8424dc4a4a3c413f89e433683f9040b\n"), - "region1/dfl-port.1/afu_id": []byte("d8424dc4a4a3c413f89e433683f9040b\n"), - "region2/dfl-fme.1/dfl-fme-region.2/fpga_region/region2/compat_id": []byte("69528db6eb31577a8c3668f9faa081f6\n"), - "region2/dfl-port.2/afu_id": []byte("d8424dc4a4a3c413f89e433683f9040b\n"), - "region2/dfl-port.3/afu_id": []byte("d8424dc4a4a3c413f89e433683f9040b\n"), + "devices/pci0000:80/0000:80:01.0/0000:81:00.0/fpga_region/region0/dfl-port.0/afu_id": []byte("d8424dc4a4a3c413f89e433683f9040b\n"), + "devices/pci0000:40/0000:40:02.0/0000:42:00.0/fpga_region/region1/dfl-port.1/afu_id": []byte("f7df405cbd7acf7222f144b0b93acd18\n"), + + "devices/pci0000:80/0000:80:01.0/0000:81:00.0/fpga_region/region0/dfl-fme.0/dfl-fme-region.0/fpga_region/region0/compat_id": []byte("69528db6eb31577a8c3668f9faa081f6\n"), + "devices/pci0000:40/0000:40:02.0/0000:42:00.0/fpga_region/region1/dfl-fme.1/dfl-fme-region.1/fpga_region/region1/compat_id": []byte("69528db6eb31577a8c3668f9faa081f6\n"), + }, + newPort: genNewDFLPort(sysfs, dev, + map[string][]string{ + "dfl-port.0": {"devices/pci0000:80/0000:80:01.0/0000:81:00.0", "dfl-fme.0", "devices/pci0000:80/0000:80:01.0/0000:81:00.0"}, + "dfl-port.1": {"devices/pci0000:40/0000:40:02.0/0000:42:00.0", "dfl-fme.1", "devices/pci0000:40/0000:40:02.0/0000:42:00.0"}, + }), + expectedDevTreeKeys: map[string][]string{ + "region-69528db6eb31577a8c3668f9faa081f6": {"dfl-fme.0", "dfl-fme.1"}, }, - devfsdirs: []string{"dfl-fme.0", "dfl-port.0", "dfl-port.1", "dfl-fme.1", "dfl-port.2", "dfl-port.3"}, }, } for _, tcase := range tcases { t.Run(tcase.name, func(t *testing.T) { - err := createTestDirs(devfs, sysfs, tcase.devfsdirs, tcase.sysfsdirs, tcase.sysfsfiles) + err := createTestDirs(dev, sysfs, tcase.devs, tcase.sysfsdirs, tcase.sysfsfiles) if err != nil { t.Fatalf("%+v", err) } - plugin, err := newDevicePluginDFL(sysfs, devfs, tcase.mode) + plugin, err := newDevicePluginDFL(path.Join(sysfs, "class", "fpga_region"), dev, tcase.mode) + if err != nil { - t.Errorf("error creating DFL plugin: %+v", err) - return - } - plugin.getDevTree = func(devices []device) dpapi.DeviceTree { - return dpapi.NewDeviceTree() + t.Fatalf("%+v", err) } - _, err = plugin.scanFPGAs() - if tcase.errorContains != "" { - if err == nil || !strings.Contains(err.Error(), tcase.errorContains) { - t.Errorf("expected error '%s', but got '%v'", tcase.errorContains, err) + plugin.newPort = tcase.newPort + + devTree, err := plugin.scanFPGAs() + if err != nil { + t.Errorf("unexpected error: '%+v'", err) + } else { + // Validate devTree + if len(devTree) != len(tcase.expectedDevTreeKeys) { + t.Errorf("unexpected device tree size: %d, expected: %d", len(devTree), len(tcase.expectedDevTreeKeys)) + } + err = validateDevTree(tcase.expectedDevTreeKeys, devTree) + if err != nil { + t.Errorf("device tree validation failed: %+v", err) } - } else if err != nil { - t.Errorf("expected no error, but got '%+v'", err) } err = os.RemoveAll(tmpdir) diff --git a/cmd/fpga_plugin/fpga_plugin.go b/cmd/fpga_plugin/fpga_plugin.go index 4505af36d..1df9be688 100644 --- a/cmd/fpga_plugin/fpga_plugin.go +++ b/cmd/fpga_plugin/fpga_plugin.go @@ -20,9 +20,7 @@ import ( "io/ioutil" "os" "path" - "path/filepath" "regexp" - "strings" "time" "k8s.io/klog" @@ -55,8 +53,8 @@ const ( scanPeriod = 5 * time.Second ) +type newPortFunc func(fname string) (fpga.Port, error) type getDevTreeFunc func(devices []device) dpapi.DeviceTree -type getSysFsInfoFunc func(dp *devicePlugin, deviceFolder string, deviceFiles []os.FileInfo, fname string) ([]region, []afu, error) // getRegionDevelTree returns mapping of region interface IDs to AF ports and FME devices. func getRegionDevelTree(devices []device) dpapi.DeviceTree { @@ -173,14 +171,11 @@ type devicePlugin struct { deviceReg *regexp.Regexp portReg *regexp.Regexp - fmeReg *regexp.Regexp - getDevTree getDevTreeFunc - getSysFsInfo getSysFsInfoFunc + getDevTree getDevTreeFunc + newPort newPortFunc - ignoreAfuIDs bool - ignoreEmptyRegions bool - annotationValue string + annotationValue string scanTicker *time.Ticker scanDone chan bool @@ -245,111 +240,75 @@ func (dp *devicePlugin) Scan(notifier dpapi.Notifier) error { } } -func (dp *devicePlugin) getDevNode(devName string) (string, error) { - devNode := path.Join(dp.devfsDir, devName) - if _, err := os.Stat(devNode); err != nil { - return "", errors.Wrapf(err, "Device %s doesn't exist", devNode) - } - - return devNode, nil -} - -func (dp *devicePlugin) getAFU(fpath string, devName string) (*afu, error) { - var afuID string +func (dp *devicePlugin) getRegions(deviceFiles []os.FileInfo) ([]region, error) { + regions := map[string]region{} + for _, deviceFile := range deviceFiles { + name := deviceFile.Name() + if dp.portReg.MatchString(name) { + port, err := dp.newPort(name) + if err != nil { + return nil, errors.Wrapf(err, "can't get port info for %s", name) + } + fme, err := port.GetFME() + if err != nil { + return nil, errors.Wrapf(err, "can't get FME info for %s", name) + } - if dp.ignoreAfuIDs { - afuID = "unused_afu_id" - } else { - data, err := ioutil.ReadFile(filepath.Clean(fpath)) - if err != nil { - return nil, errors.WithStack(err) + afuInfo := afu{id: port.GetName(), afuID: port.GetAcceleratorTypeUUID(), devNode: port.GetDevPath()} + regionName := fme.GetName() + reg, ok := regions[regionName] + if ok { + reg.afus = append(reg.afus, afuInfo) + } else { + regions[regionName] = region{id: regionName, interfaceID: fme.GetInterfaceUUID(), devNode: fme.GetDevPath(), afus: []afu{afuInfo}} + } } - afuID = strings.TrimSpace(string(data)) - } - - devNode, err := dp.getDevNode(devName) - if err != nil { - return nil, err } - return &afu{ - id: devName, - afuID: afuID, - devNode: devNode, - }, nil -} - -func (dp *devicePlugin) getFME(interfaceIDPath string, devName string) (*region, error) { - data, err := ioutil.ReadFile(filepath.Clean(interfaceIDPath)) - if err != nil { - return nil, errors.WithStack(err) - } - devNode, err := dp.getDevNode(devName) - if err != nil { - return nil, err + result := make([]region, 0, len(regions)) + // Get list of regions from the map + for _, reg := range regions { + result = append(result, reg) } - - return ®ion{ - id: devName, - interfaceID: strings.TrimSpace(string(data)), - devNode: devNode, - }, nil + return result, nil } func (dp *devicePlugin) scanFPGAs() (dpapi.DeviceTree, error) { - klog.V(4).Info("Start new FPGA scan") - - fpgaFiles, err := ioutil.ReadDir(dp.sysfsDir) + files, err := ioutil.ReadDir(dp.sysfsDir) if err != nil { klog.Warningf("Can't read folder %s. Kernel driver not loaded?", dp.sysfsDir) return dp.getDevTree([]device{}), nil } - devices := make([]device, 0, len(fpgaFiles)) - for _, fpgaFile := range fpgaFiles { - fname := fpgaFile.Name() + devices := []device{} + for _, file := range files { + devName := file.Name() - if !dp.deviceReg.MatchString(fname) { + if !dp.deviceReg.MatchString(devName) { continue } - deviceFolder := path.Join(dp.sysfsDir, fname) - deviceFiles, err := ioutil.ReadDir(deviceFolder) + deviceFiles, err := ioutil.ReadDir(path.Join(dp.sysfsDir, devName)) if err != nil { return nil, errors.WithStack(err) } - regions, afus, err := dp.getSysFsInfo(dp, deviceFolder, deviceFiles, fname) + regions, err := dp.getRegions(deviceFiles) if err != nil { return nil, err } - if len(regions) == 0 { - if len(afus) > 0 { - return nil, errors.Errorf("%s: AFU without corresponding FME found", fname) - } - if dp.ignoreEmptyRegions { - continue // not a base DFL region, skipping - } - return nil, errors.Errorf("%s: No regions found", fname) + if len(regions) > 0 { + devices = append(devices, device{name: devName, regions: regions}) } - - // Currently only one region per device is supported. - regions[0].afus = afus - devices = append(devices, device{ - name: fname, - regions: regions, - }) } - return dp.getDevTree(devices), nil } // getPluginParams is a helper function to avoid code duplication. // It's used in newDevicePluginOPAE and newDevicePluginDFL. -func getPluginParams(mode string) (getDevTreeFunc, bool, string, error) { +func getPluginParams(mode string) (getDevTreeFunc, string, error) { var getDevTree getDevTreeFunc - ignoreAfuIDs := false annotationValue := "" switch mode { @@ -357,16 +316,14 @@ func getPluginParams(mode string) (getDevTreeFunc, bool, string, error) { getDevTree = getAfuTree case regionMode: getDevTree = getRegionTree - ignoreAfuIDs = true annotationValue = fmt.Sprintf("%s/%s", namespace, regionMode) case regionDevelMode: getDevTree = getRegionDevelTree - ignoreAfuIDs = true default: - return nil, ignoreAfuIDs, annotationValue, errors.Errorf("Wrong mode: '%s'", mode) + return nil, annotationValue, errors.Errorf("Wrong mode: '%s'", mode) } - return getDevTree, ignoreAfuIDs, annotationValue, nil + return getDevTree, annotationValue, nil } func main() { diff --git a/cmd/fpga_plugin/fpga_plugin_test.go b/cmd/fpga_plugin/fpga_plugin_test.go index a567ddac2..945f3bc51 100644 --- a/cmd/fpga_plugin/fpga_plugin_test.go +++ b/cmd/fpga_plugin/fpga_plugin_test.go @@ -16,6 +16,7 @@ package main import ( "flag" + "fmt" "io/ioutil" "os" "path" @@ -57,6 +58,21 @@ func createTestDirs(devfs, sysfs string, devfsDirs, sysfsDirs []string, sysfsFil return nil } +// validateDevTree is a helper that reduces code complexity to make golangci-lint happy. +func validateDevTree(expectedDevTreeKeys map[string][]string, devTree dpapi.DeviceTree) error { + for resource, devices := range expectedDevTreeKeys { + if _, ok := devTree[resource]; !ok { + return fmt.Errorf("device tree: resource %s missing", resource) + } + for _, device := range devices { + if _, ok := devTree[resource][device]; !ok { + return fmt.Errorf("device tree resource %s: device %s missing", resource, device) + } + } + } + return nil +} + func TestPostAllocate(t *testing.T) { response := new(pluginapi.AllocateResponse) cresp := new(pluginapi.ContainerAllocateResponse) @@ -167,52 +183,59 @@ func (n *fakeNotifier) Notify(newDeviceTree dpapi.DeviceTree) { } func TestScan(t *testing.T) { - root, err := ioutil.TempDir("", "test_new_device_plugin") + root, err := ioutil.TempDir("", "TestScan") if err != nil { t.Fatalf("can't create temporary directory: %+v", err) } defer os.RemoveAll(root) - devfs := path.Join(root, "dev") + sysfs := path.Join(root, "sys") + dev := path.Join(root, "dev") tcases := []struct { - name string - mode string - sysfs string - devfsdirs []string - sysfsdirs []string - sysfsfiles map[string][]byte - expectedErr bool + name string + mode string + devs []string + sysfsdirs []string + sysfsfiles map[string][]byte + newPort newPortFunc }{ { - name: "valid OPAE scan", - mode: afMode, - sysfs: path.Join(root, "sys", "class", "fpga"), + name: "valid OPAE scan in af mode", + mode: afMode, + devs: []string{ + "intel-fpga-fme.0", "intel-fpga-port.0", + "intel-fpga-fme.1", "intel-fpga-port.1", + }, sysfsdirs: []string{ - "intel-fpga-dev.0/intel-fpga-port.0", - "intel-fpga-dev.0/intel-fpga-fme.0/pr", - "intel-fpga-dev.1/intel-fpga-port.1", - "intel-fpga-dev.1/intel-fpga-fme.1/pr", + "class/fpga/intel-fpga-dev.0/intel-fpga-port.0", + "class/fpga/intel-fpga-dev.1/intel-fpga-port.1", + "class/fpga/dir", // this should be skipped by plugin.ScanFPGAs + "devices/pci0000:00/0000:00:03.2/0000:06:00.0/fpga/intel-fpga-dev.0/intel-fpga-port.0", + "devices/pci0000:00/0000:00:03.2/0000:06:00.0/fpga/intel-fpga-dev.0/intel-fpga-fme.0/pr", + "devices/pci0000:40/0000:40:02.0/0000:42:00.0/fpga/intel-fpga-dev.1/intel-fpga-port.1/", + "devices/pci0000:40/0000:40:02.0/0000:42:00.0/fpga/intel-fpga-dev.1/intel-fpga-fme.1/pr", }, sysfsfiles: map[string][]byte{ - "intel-fpga-dev.0/intel-fpga-port.0/afu_id": []byte("d8424dc4a4a3c413f89e433683f9040b\n"), - "intel-fpga-dev.1/intel-fpga-port.1/afu_id": []byte("d8424dc4a4a3c413f89e433683f9040b\n"), - "intel-fpga-dev.0/intel-fpga-fme.0/pr/interface_id": []byte("ce48969398f05f33946d560708be108a\n"), - "intel-fpga-dev.1/intel-fpga-fme.1/pr/interface_id": []byte("ce48969398f05f33946d560708be108a\n"), - }, - devfsdirs: []string{ - "intel-fpga-port.0", "intel-fpga-fme.0", - "intel-fpga-port.1", "intel-fpga-fme.1", + "devices/pci0000:00/0000:00:03.2/0000:06:00.0/fpga/intel-fpga-dev.0/intel-fpga-port.0/afu_id": []byte("d8424dc4a4a3c413f89e433683f9040b\n"), + "devices/pci0000:40/0000:40:02.0/0000:42:00.0/fpga/intel-fpga-dev.1/intel-fpga-port.1/afu_id": []byte("f7df405cbd7acf7222f144b0b93acd18\n"), + + "devices/pci0000:00/0000:00:03.2/0000:06:00.0/fpga/intel-fpga-dev.0/intel-fpga-fme.0/pr/interface_id": []byte("69528db6eb31577a8c3668f9faa081f6\n"), + "devices/pci0000:40/0000:40:02.0/0000:42:00.0/fpga/intel-fpga-dev.1/intel-fpga-fme.1/pr/interface_id": []byte("69528db6eb31577a8c3668f9faa081f6\n"), }, - expectedErr: false, + newPort: genNewIntelFpgaPort(sysfs, dev, + map[string][]string{ + "intel-fpga-port.0": {"devices/pci0000:00/0000:00:03.2/0000:06:00.0", "intel-fpga-fme.0", "devices/pci0000:00/0000:00:03.2/0000:06:00.0"}, + "intel-fpga-port.1": {"devices/pci0000:40/0000:40:02.0/0000:42:00.0", "intel-fpga-fme.1", "devices/pci0000:40/0000:40:02.0/0000:42:00.0"}, + }), }, } for _, tc := range tcases { t.Run(tc.name, func(t *testing.T) { - err = createTestDirs(devfs, tc.sysfs, tc.devfsdirs, tc.sysfsdirs, tc.sysfsfiles) + err := createTestDirs(dev, sysfs, tc.devs, tc.sysfsdirs, tc.sysfsfiles) if err != nil { - t.Fatal(err) + t.Fatalf("%+v", err) } plugin, err := newDevicePlugin(tc.mode, root) @@ -220,20 +243,17 @@ func TestScan(t *testing.T) { t.Fatalf("failed to create a device plugin: %+v", err) } + plugin.newPort = tc.newPort + err = plugin.Scan(&fakeNotifier{plugin.scanDone}) - if tc.expectedErr && err == nil { - t.Error("unexpected success") - } - if !tc.expectedErr && err != nil { + if err != nil { t.Errorf("unexpected error: %+v", err) } - for _, dir := range []string{"sys", "dev"} { - err = os.RemoveAll(path.Join(root, dir)) - if err != nil { - t.Fatalf("Failed to remove fake sysfs directory: %+v", err) - } + err = os.RemoveAll(root) + if err != nil { + t.Fatal(err) } }) } diff --git a/cmd/fpga_plugin/opae.go b/cmd/fpga_plugin/opae.go index 0b8861755..4185edaeb 100644 --- a/cmd/fpga_plugin/opae.go +++ b/cmd/fpga_plugin/opae.go @@ -15,51 +15,19 @@ package main import ( - "os" - "path" "regexp" - "github.com/pkg/errors" + "github.com/intel/intel-device-plugins-for-kubernetes/pkg/fpga" ) const ( opaeDeviceRE = `^intel-fpga-dev.[0-9]+$` opaePortRE = `^intel-fpga-port.[0-9]+$` - opaeFmeRE = `^intel-fpga-fme.[0-9]+$` ) -func getSysFsInfoOPAE(dp *devicePlugin, deviceFolder string, deviceFiles []os.FileInfo, fname string) ([]region, []afu, error) { - var regions []region - var afus []afu - for _, deviceFile := range deviceFiles { - name := deviceFile.Name() - - if dp.fmeReg.MatchString(name) { - if len(regions) > 0 { - return nil, nil, errors.Errorf("Detected more than one FPGA region for device %s. Only one region per FPGA device is supported", fname) - } - interfaceIDFile := path.Join(deviceFolder, name, "pr", "interface_id") - region, err := dp.getFME(interfaceIDFile, name) - if err != nil { - return nil, nil, err - } - regions = append(regions, *region) - } else if dp.portReg.MatchString(name) { - afuPath := path.Join(deviceFolder, name, "afu_id") - afu, err := dp.getAFU(afuPath, name) - if err != nil { - return nil, nil, err - } - afus = append(afus, *afu) - } - } - - return regions, afus, nil -} - // newDevicePlugin returns new instance of devicePlugin. func newDevicePluginOPAE(sysfsDir string, devfsDir string, mode string) (*devicePlugin, error) { - getDevTree, ignoreAfuIDs, annotationValue, err := getPluginParams(mode) + getDevTree, annotationValue, err := getPluginParams(mode) if err != nil { return nil, err } @@ -72,12 +40,10 @@ func newDevicePluginOPAE(sysfsDir string, devfsDir string, mode string) (*device deviceReg: regexp.MustCompile(opaeDeviceRE), portReg: regexp.MustCompile(opaePortRE), - fmeReg: regexp.MustCompile(opaeFmeRE), - getDevTree: getDevTree, - getSysFsInfo: getSysFsInfoOPAE, + getDevTree: getDevTree, + newPort: fpga.NewIntelFpgaPort, annotationValue: annotationValue, - ignoreAfuIDs: ignoreAfuIDs, }, nil } diff --git a/cmd/fpga_plugin/opae_test.go b/cmd/fpga_plugin/opae_test.go index b6a53d3d6..e935a10cc 100644 --- a/cmd/fpga_plugin/opae_test.go +++ b/cmd/fpga_plugin/opae_test.go @@ -19,10 +19,10 @@ import ( "os" "path" "reflect" - "strings" "testing" dpapi "github.com/intel/intel-device-plugins-for-kubernetes/pkg/deviceplugin" + "github.com/intel/intel-device-plugins-for-kubernetes/pkg/fpga" pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1" ) @@ -236,153 +236,178 @@ func TestGetAfuTreeOPAE(t *testing.T) { } } +// genNewIntelFpgaPort generates newPortFunc function. +func genNewIntelFpgaPort(sysFsPrefix, devDir string, sysFsInfo map[string][]string) newPortFunc { + return func(portName string) (fpga.Port, error) { + portPciDev := sysFsInfo[portName][0] + portPciDevSysFs := path.Join(sysFsPrefix, portPciDev) + + fmeName := sysFsInfo[portName][1] + fmePciDev := sysFsInfo[portName][2] + fmePciDevSysFs := path.Join(sysFsPrefix, fmePciDev) + + return &fpga.IntelFpgaPort{ + Name: portName, + DevPath: path.Join(devDir, portName), + SysFsPath: path.Join(portPciDevSysFs, "fpga", "intel-fpga-dev*", portName), + FME: &fpga.IntelFpgaFME{ + Name: fmeName, + DevPath: path.Join(devDir, fmeName), + SysFsPath: fmePciDevSysFs, + PCIDevice: &fpga.PCIDevice{SysFsPath: fmePciDevSysFs}, + }, + }, nil + } +} + func TestScanFPGAsOPAE(t *testing.T) { tmpdir, err := ioutil.TempDir("", "TestScanFPGAsOPAE") if err != nil { t.Fatalf("can't create temporary directory: %+v", err) } - sysfs := path.Join(tmpdir, "sys", "class", "fpga") - devfs := path.Join(tmpdir, "dev") + sysfs := path.Join(tmpdir, "sys") + dev := path.Join(tmpdir, "dev") tcases := []struct { - name string - devfsdirs []string - sysfsdirs []string - sysfsfiles map[string][]byte - errorContains string - expectedDevTree map[string]map[string]dpapi.DeviceInfo - mode string + name string + mode string + devs []string + sysfsdirs []string + sysfsfiles map[string][]byte + newPort newPortFunc + expectedDevTreeKeys map[string][]string }{ { - name: "No sysfs folder exists", + name: "Valid OPAE scan in af mode", mode: afMode, - }, - { - name: "FPGA device without FME and ports", - mode: afMode, - sysfsdirs: []string{"intel-fpga-dev.0", "incorrect-file-name"}, - errorContains: "No regions found", - }, - { - name: "AFU without ID", - mode: afMode, - sysfsdirs: []string{"intel-fpga-dev.0/intel-fpga-port.0"}, - errorContains: "afu_id: no such file or directory", - }, - { - name: "No device node for detected AFU", - mode: afMode, - sysfsdirs: []string{"intel-fpga-dev.0/intel-fpga-port.0"}, - sysfsfiles: map[string][]byte{ - "intel-fpga-dev.0/intel-fpga-port.0/afu_id": []byte("d8424dc4a4a3c413f89e433683f9040b\n"), - }, - errorContains: "/dev/intel-fpga-port.0: no such file or directory", - }, - { - name: "AFU without corresponding FME", - mode: afMode, - sysfsdirs: []string{"intel-fpga-dev.0/intel-fpga-port.0"}, - devfsdirs: []string{"intel-fpga-port.0"}, - sysfsfiles: map[string][]byte{ - "intel-fpga-dev.0/intel-fpga-port.0/afu_id": []byte("d8424dc4a4a3c413f89e433683f9040b\n"), + devs: []string{ + "intel-fpga-fme.0", "intel-fpga-port.0", + "intel-fpga-fme.1", "intel-fpga-port.1", }, - errorContains: "intel-fpga-dev.0: AFU without corresponding FME found", - }, - { - name: "More than one FME per FPGA device", - mode: afMode, sysfsdirs: []string{ - "intel-fpga-dev.0/intel-fpga-fme.0/pr", - "intel-fpga-dev.0/intel-fpga-fme.1/pr", + "class/fpga/intel-fpga-dev.0/intel-fpga-port.0", + "class/fpga/intel-fpga-dev.1/intel-fpga-port.1", + "class/fpga/dir", // this should be skipped by plugin.ScanFPGAs + "devices/pci0000:00/0000:00:03.2/0000:06:00.0/fpga/intel-fpga-dev.0/intel-fpga-port.0", + "devices/pci0000:00/0000:00:03.2/0000:06:00.0/fpga/intel-fpga-dev.0/intel-fpga-fme.0/pr", + "devices/pci0000:40/0000:40:02.0/0000:42:00.0/fpga/intel-fpga-dev.1/intel-fpga-port.1/", + "devices/pci0000:40/0000:40:02.0/0000:42:00.0/fpga/intel-fpga-dev.1/intel-fpga-fme.1/pr", }, sysfsfiles: map[string][]byte{ - "intel-fpga-dev.0/intel-fpga-fme.0/pr/interface_id": []byte("d8424dc4a4a3c413f89e433683f9040b\n"), - "intel-fpga-dev.0/intel-fpga-fme.1/pr/interface_id": []byte("d8424dc4a4a3c413f89e433683f9040b\n"), - }, - devfsdirs: []string{ - "intel-fpga-fme.0", - "intel-fpga-fme.1", + "devices/pci0000:00/0000:00:03.2/0000:06:00.0/fpga/intel-fpga-dev.0/intel-fpga-port.0/afu_id": []byte("d8424dc4a4a3c413f89e433683f9040b\n"), + "devices/pci0000:40/0000:40:02.0/0000:42:00.0/fpga/intel-fpga-dev.1/intel-fpga-port.1/afu_id": []byte("f7df405cbd7acf7222f144b0b93acd18\n"), + + "devices/pci0000:00/0000:00:03.2/0000:06:00.0/fpga/intel-fpga-dev.0/intel-fpga-fme.0/pr/interface_id": []byte("69528db6eb31577a8c3668f9faa081f6\n"), + "devices/pci0000:40/0000:40:02.0/0000:42:00.0/fpga/intel-fpga-dev.1/intel-fpga-fme.1/pr/interface_id": []byte("69528db6eb31577a8c3668f9faa081f6\n"), }, - errorContains: "Detected more than one FPGA region", - }, - { - name: "FME without interface ID", - mode: afMode, - sysfsdirs: []string{"intel-fpga-dev.0/intel-fpga-fme.0"}, - errorContains: "interface_id: no such file or directory", - }, - { - name: "No device node for detected region", - mode: afMode, - sysfsdirs: []string{"intel-fpga-dev.0/intel-fpga-fme.0/pr"}, - sysfsfiles: map[string][]byte{ - "intel-fpga-dev.0/intel-fpga-fme.0/pr/interface_id": []byte("d8424dc4a4a3c413f89e433683f9040b\n"), + newPort: genNewIntelFpgaPort(sysfs, dev, + map[string][]string{ + "intel-fpga-port.0": {"devices/pci0000:00/0000:00:03.2/0000:06:00.0", "intel-fpga-fme.0", "devices/pci0000:00/0000:00:03.2/0000:06:00.0"}, + "intel-fpga-port.1": {"devices/pci0000:40/0000:40:02.0/0000:42:00.0", "intel-fpga-fme.1", "devices/pci0000:40/0000:40:02.0/0000:42:00.0"}, + }), + expectedDevTreeKeys: map[string][]string{ + "af-695.d84.aVKNtusxV3qMNmj5-qCB9thCTcSko8QT-J5DNoP5BAs": {"intel-fpga-port.0"}, + "af-695.f7d.aVKNtusxV3qMNmj5-qCB9vffQFy9es9yIvFEsLk6zRg": {"intel-fpga-port.1"}, }, - errorContains: "/dev/intel-fpga-fme.0: no such file or directory", }, { - name: "No errors expected in af mode", + name: "Valid OPAE scan in af mode (SR-IOV)", mode: afMode, + devs: []string{ + "intel-fpga-fme.0", "intel-fpga-port.0", + "intel-fpga-fme.1", "intel-fpga-port.2", + }, sysfsdirs: []string{ - "intel-fpga-dev.0/intel-fpga-port.0", - "intel-fpga-dev.0/intel-fpga-fme.0/pr", - "intel-fpga-dev.1/intel-fpga-port.1", - "intel-fpga-dev.1/intel-fpga-fme.1/pr", + "class/fpga/intel-fpga-dev.0/intel-fpga-port.0", + "class/fpga/intel-fpga-dev.1/intel-fpga-fme.1", + "class/fpga/intel-fpga-dev.2/intel-fpga-port.2", + "devices/pci0000:00/0000:00:03.2/0000:06:00.0/fpga/intel-fpga-dev.0/intel-fpga-port.0", + "devices/pci0000:00/0000:00:03.2/0000:06:00.0/fpga/intel-fpga-dev.0/intel-fpga-fme.0/pr", + "devices/pci0000:40/0000:40:02.0/0000:42:00.0/fpga/intel-fpga-dev.1/intel-fpga-fme.1/pr", + "devices/pci0000:40/0000:40:02.0/0000:42:00.1/fpga/intel-fpga-dev.2/intel-fpga-port.2", }, sysfsfiles: map[string][]byte{ - "intel-fpga-dev.0/intel-fpga-port.0/afu_id": []byte("d8424dc4a4a3c413f89e433683f9040b\n"), - "intel-fpga-dev.1/intel-fpga-port.1/afu_id": []byte("d8424dc4a4a3c413f89e433683f9040b\n"), - "intel-fpga-dev.0/intel-fpga-fme.0/pr/interface_id": []byte("ce48969398f05f33946d560708be108a\n"), - "intel-fpga-dev.1/intel-fpga-fme.1/pr/interface_id": []byte("ce48969398f05f33946d560708be108a\n"), + "devices/pci0000:00/0000:00:03.2/0000:06:00.0/fpga/intel-fpga-dev.0/intel-fpga-port.0/afu_id": []byte("d8424dc4a4a3c413f89e433683f9040b\n"), + "devices/pci0000:40/0000:40:02.0/0000:42:00.1/fpga/intel-fpga-dev.2/intel-fpga-port.2/afu_id": []byte("f7df405cbd7acf7222f144b0b93acd18\n"), + + "devices/pci0000:00/0000:00:03.2/0000:06:00.0/fpga/intel-fpga-dev.0/intel-fpga-fme.0/pr/interface_id": []byte("69528db6eb31577a8c3668f9faa081f6\n"), + "devices/pci0000:40/0000:40:02.0/0000:42:00.0/fpga/intel-fpga-dev.1/intel-fpga-fme.1/pr/interface_id": []byte("69528db6eb31577a8c3668f9faa081f6\n"), }, - devfsdirs: []string{ - "intel-fpga-port.0", "intel-fpga-fme.0", - "intel-fpga-port.1", "intel-fpga-fme.1", + newPort: genNewIntelFpgaPort(sysfs, dev, + map[string][]string{ + "intel-fpga-port.0": {"devices/pci0000:00/0000:00:03.2/0000:06:00.0", "intel-fpga-fme.0", "devices/pci0000:00/0000:00:03.2/0000:06:00.0"}, + "intel-fpga-port.2": {"devices/pci0000:40/0000:40:02.0/0000:42:00.1", "intel-fpga-fme.1", "devices/pci0000:40/0000:40:02.0/0000:42:00.0"}, + }), + expectedDevTreeKeys: map[string][]string{ + "af-695.d84.aVKNtusxV3qMNmj5-qCB9thCTcSko8QT-J5DNoP5BAs": {"intel-fpga-port.0"}, + "af-695.f7d.aVKNtusxV3qMNmj5-qCB9vffQFy9es9yIvFEsLk6zRg": {"intel-fpga-port.2"}, }, }, { - name: "No errors expected in region mode", + name: "Valid OPAE scan in region mode (SR-IOV)", mode: regionMode, + devs: []string{ + "intel-fpga-fme.0", "intel-fpga-port.0", + "intel-fpga-fme.1", "intel-fpga-port.2", + }, sysfsdirs: []string{ - "intel-fpga-dev.0/intel-fpga-port.0", - "intel-fpga-dev.0/intel-fpga-fme.0/pr", - "intel-fpga-dev.1/intel-fpga-port.1", - "intel-fpga-dev.1/intel-fpga-fme.1/pr", + "class/fpga/intel-fpga-dev.0/intel-fpga-port.0", + "class/fpga/intel-fpga-dev.1/intel-fpga-fme.1", + "class/fpga/intel-fpga-dev.2/intel-fpga-port.2", + "devices/pci0000:00/0000:00:03.2/0000:06:00.0/fpga/intel-fpga-dev.0/intel-fpga-port.0", + "devices/pci0000:00/0000:00:03.2/0000:06:00.0/fpga/intel-fpga-dev.0/intel-fpga-fme.0/pr", + "devices/pci0000:40/0000:40:02.0/0000:42:00.0/fpga/intel-fpga-dev.1/intel-fpga-fme.1/pr", + "devices/pci0000:40/0000:40:02.0/0000:42:00.1/fpga/intel-fpga-dev.2/intel-fpga-port.2", }, sysfsfiles: map[string][]byte{ - "intel-fpga-dev.0/intel-fpga-fme.0/pr/interface_id": []byte("ce48969398f05f33946d560708be108a\n"), - "intel-fpga-dev.1/intel-fpga-fme.1/pr/interface_id": []byte("ce48969398f05f33946d560708be108a\n"), + "devices/pci0000:00/0000:00:03.2/0000:06:00.0/fpga/intel-fpga-dev.0/intel-fpga-port.0/afu_id": []byte("d8424dc4a4a3c413f89e433683f9040b\n"), + "devices/pci0000:40/0000:40:02.0/0000:42:00.1/fpga/intel-fpga-dev.2/intel-fpga-port.2/afu_id": []byte("f7df405cbd7acf7222f144b0b93acd18\n"), + + "devices/pci0000:00/0000:00:03.2/0000:06:00.0/fpga/intel-fpga-dev.0/intel-fpga-fme.0/pr/interface_id": []byte("69528db6eb31577a8c3668f9faa081f6\n"), + "devices/pci0000:40/0000:40:02.0/0000:42:00.0/fpga/intel-fpga-dev.1/intel-fpga-fme.1/pr/interface_id": []byte("69528db6eb31577a8c3668f9faa081f6\n"), }, - devfsdirs: []string{ - "intel-fpga-port.0", "intel-fpga-fme.0", - "intel-fpga-port.1", "intel-fpga-fme.1", + newPort: genNewIntelFpgaPort(sysfs, dev, + map[string][]string{ + "intel-fpga-port.0": {"devices/pci0000:00/0000:00:03.2/0000:06:00.0", "intel-fpga-fme.0", "devices/pci0000:00/0000:00:03.2/0000:06:00.0"}, + "intel-fpga-port.2": {"devices/pci0000:40/0000:40:02.0/0000:42:00.1", "intel-fpga-fme.1", "devices/pci0000:40/0000:40:02.0/0000:42:00.0"}, + }), + expectedDevTreeKeys: map[string][]string{ + "region-69528db6eb31577a8c3668f9faa081f6": {"intel-fpga-fme.0", "intel-fpga-fme.1"}, }, }, + + { + name: "No sysfs device entries", + mode: afMode, + }, } for _, tcase := range tcases { t.Run(tcase.name, func(t *testing.T) { - err := createTestDirs(devfs, sysfs, tcase.devfsdirs, tcase.sysfsdirs, tcase.sysfsfiles) + err := createTestDirs(dev, sysfs, tcase.devs, tcase.sysfsdirs, tcase.sysfsfiles) if err != nil { t.Fatalf("%+v", err) } - plugin, err := newDevicePluginOPAE(sysfs, devfs, tcase.mode) - - plugin.getDevTree = func(devices []device) dpapi.DeviceTree { - return dpapi.NewDeviceTree() - } + plugin, err := newDevicePluginOPAE(path.Join(sysfs, "class", "fpga"), dev, tcase.mode) if err != nil { t.Fatalf("%+v", err) } - _, err = plugin.scanFPGAs() - if tcase.errorContains != "" { - if err == nil || !strings.Contains(err.Error(), tcase.errorContains) { - t.Errorf("expected error '%s', but got '%v'", tcase.errorContains, err) + plugin.newPort = tcase.newPort + + devTree, err := plugin.scanFPGAs() + if err != nil { + t.Errorf("unexpected error: '%+v'", err) + } else { + // Validate devTree + if len(devTree) != len(tcase.expectedDevTreeKeys) { + t.Errorf("unexpected device tree size: %d, expected: %d", len(devTree), len(tcase.expectedDevTreeKeys)) + } + err = validateDevTree(tcase.expectedDevTreeKeys, devTree) + if err != nil { + t.Errorf("device tree validation failed: %+v", err) } - } else if err != nil { - t.Errorf("expected no error, but got '%+v'", err) } err = os.RemoveAll(tmpdir)