gpu: mount by-path directory

oneCCL requires the /dev/dri/by-path folder to be available
to create a mapping between GPUs.

Signed-off-by: Tuomas Katila <[email protected]>

Author: tkatila

cmd/gpu_plugin/gpu_plugin.go

@@ -35,11 +35,12 @@ import (
 )
 
 const (
-	sysfsDrmDirectory = "/sys/class/drm"
-	devfsDriDirectory = "/dev/dri"
-	gpuDeviceRE       = `^card[0-9]+$`
-	controlDeviceRE   = `^controlD[0-9]+$`
-	vendorString      = "0x8086"
+	sysfsDrmDirectory    = "/sys/class/drm"
+	devfsDriDirectory    = "/dev/dri"
+	devfsBypathDirectory = "/dev/dri/by-path"
+	gpuDeviceRE          = `^card[0-9]+$`
+	controlDeviceRE      = `^controlD[0-9]+$`
+	vendorString         = "0x8086"
 
 	// Device plugin settings.
 	namespace  = "gpu.intel.com"
@@ -338,15 +339,18 @@ func (dp *devicePlugin) scan() (dpapi.DeviceTree, error) {
 		}
 
 		if len(nodes) > 0 {
-			deviceInfo := dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, nil, nil, nil)
+			mounts := pluginutils.BypathMountsForPci(
+				pluginutils.ReadPciAddressForCard(path.Join(sysfsDrmDirectory, f.Name())), devfsBypathDirectory)
+
+			deviceInfo := dpapi.NewDeviceInfo(pluginapi.Healthy, nodes, mounts, nil, nil)
 
 			for i := 0; i < dp.options.sharedDevNum; i++ {
 				devID := fmt.Sprintf("%s-%d", f.Name(), i)
 				// Currently only one device type (i915) is supported.
 				// TODO: check model ID to differentiate device models.
 				devTree.AddDevice(deviceType, devID, deviceInfo)
 
-				rmDevInfos[devID] = rm.NewDeviceInfo(nodes, nil, nil)
+				rmDevInfos[devID] = rm.NewDeviceInfo(nodes, mounts, nil)
 			}
 		}
 	}

cmd/internal/pluginutils/bypath.go

@@ -0,0 +1,78 @@
+// Copyright 2023 Intel Corporation. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pluginutils
+
+import (
+	"os"
+	"path"
+	"strings"
+
+	pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
+)
+
+// Returns a slice of by-path Mounts for a pci device with pciAddress.
+// by-path files are searched from the given bypathDir.
+// In the by-path dir, any files that start with "pci-<pci addr>" will be added to mounts.
+func BypathMountsForPci(pciAddress, bypathDir string) []pluginapi.Mount {
+	var mounts []pluginapi.Mount
+
+	if pciAddress == "" {
+		return nil
+	}
+
+	files, err := os.ReadDir(bypathDir)
+	if err != nil {
+		return nil
+	}
+
+	linkPrefix := "pci-" + pciAddress
+
+	for _, f := range files {
+		if strings.HasPrefix(f.Name(), linkPrefix) {
+			absPath := path.Join(bypathDir, f.Name())
+			mounts = append(mounts, pluginapi.Mount{
+				ContainerPath: absPath,
+				HostPath:      absPath,
+				ReadOnly:      true,
+			})
+		}
+	}
+
+	return mounts
+}
+
+// Returns the pci address for a drm card by reading the
+// symbolic link that the /sys/class/drm/cardX points to.
+// ../../devices/pci0000:00/0000:00:02.0/drm/card
+// -------------------------^^^^^^^^^^^^---------.
+func ReadPciAddressForCard(cardPath string) string {
+	linkPath, err := os.Readlink(cardPath)
+	if err != nil {
+		return ""
+	}
+
+	parts := strings.Split(linkPath, "/")
+	l := len(parts)
+
+	if l < 3 {
+		return ""
+	}
+
+	if parts[l-2] != "drm" {
+		return ""
+	}
+
+	return parts[l-3]
+}

cmd/internal/pluginutils/bypath_test.go

@@ -0,0 +1,130 @@
+// Copyright 2023 Intel Corporation. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package pluginutils
+
+import (
+	"os"
+	"path"
+	"path/filepath"
+	"testing"
+
+	"k8s.io/utils/strings/slices"
+)
+
+func createTestFiles(t *testing.T, root, linkFile string, bypathFiles []string) (string, string) {
+	drmPath := path.Join(root, "sys/class/drm/card0")
+	devPath := path.Join(root, "sys", linkFile)
+	byPath := path.Join(root, "by-path")
+
+	if linkFile != "" {
+		if err := os.MkdirAll(filepath.Dir(devPath), os.ModePerm); err != nil {
+			t.Fatal("Couldn't create test dev dir", err)
+		}
+
+		if err := os.MkdirAll(filepath.Dir(drmPath), os.ModePerm); err != nil {
+			t.Fatal("Couldn't create test drm dir", err)
+		}
+
+		if err := os.WriteFile(devPath, []byte{0}, os.ModePerm); err != nil {
+			t.Fatal("Couldn't create card file", err)
+		}
+
+		if err := os.Symlink(devPath, drmPath); err != nil {
+			t.Fatal("Couldn't create symlink between pci path and sysfs drm path")
+		}
+	}
+
+	if len(bypathFiles) > 0 {
+		if err := os.MkdirAll(byPath, os.ModePerm); err != nil {
+			t.Fatal("Mkdir failed:", byPath)
+		}
+
+		for _, f := range bypathFiles {
+			if err := os.WriteFile(path.Join(byPath, f), []byte{1}, os.ModePerm); err != nil {
+				t.Fatal("WriteFile failed:", path.Join(byPath, f))
+			}
+		}
+	}
+
+	return drmPath, byPath
+}
+
+func TestBypath(t *testing.T) {
+	type testData struct {
+		linkpath    string
+		bypathFiles []string
+		mountCount  int
+	}
+
+	tds := []testData{
+		{
+			"00.10.2/00.334.302/0.0.1.00/0-1-2-3-3342/drm/card0",
+			[]string{"pci-0-1-2-3-3342-card", "pci-0-1-2-3-3342-render"},
+			2,
+		},
+		{
+			"00.10.2/00.334.302/0.0.1.00/0-1-2-3-4343/drm/card0",
+			[]string{"pci-0-1-2-3-4444-card", "pci-0-1-2-3-4444-render"},
+			0,
+		},
+		{
+			"00.10.2/00.334.302/0.0.1.00/0-1-2-3-3342/drm",
+			[]string{"pci-0-1-2-3-4444-card", "pci-0-1-2-3-4444-render"},
+			0,
+		},
+		{
+			"",
+			[]string{"pci-0-1-2-3-3342-card", "pci-0-1-2-3-3342-render"},
+			0,
+		},
+		{
+			"00.10.2/00.334.302/0.0.1.00/0-1-2-3-3342/drm/card0",
+			[]string{},
+			0,
+		},
+	}
+
+	for _, td := range tds {
+		root, err := os.MkdirTemp("", "test_bypath_mounting")
+		if err != nil {
+			t.Fatalf("can't create temporary directory: %+v", err)
+		}
+		// dirs/files need to be removed for the next test
+		defer os.RemoveAll(root)
+
+		drmPath, byPath := createTestFiles(t, root, td.linkpath, td.bypathFiles)
+
+		mounts := BypathMountsForPci(ReadPciAddressForCard(drmPath), byPath)
+
+		if len(mounts) != td.mountCount {
+			t.Errorf("Wrong number of mounts %d vs. %d", len(mounts), td.mountCount)
+		}
+
+		absPaths := []string{}
+		for _, link := range td.bypathFiles {
+			absPaths = append(absPaths, path.Join(byPath, link))
+		}
+
+		for _, mount := range mounts {
+			if !slices.Contains(absPaths, mount.ContainerPath) {
+				t.Errorf("containerpath is incorrect: %s", mount.ContainerPath)
+			}
+
+			if !slices.Contains(absPaths, mount.HostPath) {
+				t.Errorf("hostpath is incorrect: %s", mount.HostPath)
+			}
+		}
+	}
+}