xelink support for gpu_fakedev

Author: hsyrja

cmd/gpu_fakedev/README.md

@@ -26,9 +26,8 @@ file, but each new device variant adding feature(s) that have specific
 support in device plugin, could have their own fake device config.
 
 Xelink support: There two ways to generate Xelink info.
-1. Use xelink-topology FULL
-2. Use direct xelink connection line
-     example: "xelink-connections": "0.0-0.1_0.0-1.0_0.0-1.1_0.0-2.0_0.0-2.1_0.0-3.0_0.0-3.1_0.0-4.0_0.0-4.1_0.0-5.0_0.0-5.1_0.0-6.0_0.0-6.1_0.0-7.0_0.0-7.1_0.1-1"
+1. Use xelink-topology FULL: See example: `config/8x2-PVC-xelink.json`
+2. Use direct xelink connection line: See example: `config/2x4-PVC-xelink.json`
 
 ## Potential improvements
 

cmd/gpu_fakedev/configs/2x4-PVC-xelink.json

@@ -0,0 +1,13 @@
+{
+	"Info": "2x 4 tile 4 GiB PVC [Ponte Vecchio] GPUs",
+	"DevCount": 2,
+        "TilesPerDev": 4,
+        "DevsPerNode": 1,
+	"DevMemSize": 4294967296,
+	"Capabilities": {
+		"platform": "fake_PVC",
+                "connections": "0.1-0.0_0.2-0.0_0.3-0.0_1.0-0.0_1.1-0.0_1.2-0.0_1.3-0.0_0.2-0.1_0.3-0.1_1.0-0.1_1.1-0.1_1.2-0.1_1.3-0.1_0.3-0.2_1.0-0.2_1.1-0.2_1.2-0.2_1.3-0.2_1.0-0.3_1.1-0.3_1.2-0.3_1.3-0.3_1.1-1.0_1.2-1.0_1.3-1.0_1.2-1.1_1.3-1.1_1.3-1.2",
+                "connection-topology": "RAW"
+	}
+}
+

cmd/gpu_fakedev/configs/8x2-PVC-xelink.json

@@ -0,0 +1,12 @@
+{
+	"Info": "8x 4 GiB PVC [Ponte Vecchio] GPUs",
+	"DevCount": 8,
+        "TilesPerDev": 2,
+        "DevsPerNode": 2,
+	"DevMemSize": 4294967296,
+	"Capabilities": {
+		"platform": "fake_PVC",
+                "connections": "",
+                "connection-topology": "FULL"
+	}
+}

cmd/gpu_fakedev/gpu_fakedev.go

@@ -1,4 +1,4 @@
-// Copyright 2021-2022 Intel Corporation. All Rights Reserved.
+// Copyright 2021-2023 Intel Corporation. All Rights Reserved.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -39,13 +39,16 @@ import (
 	"errors"
 	"flag"
 	"fmt"
-	"golang.org/x/sys/unix"
 	"io/fs"
 	"log"
 	"math"
 	"os"
+	"strings"
+
+	"golang.org/x/sys/unix"
 )
 
+// https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/
 const (
 	dirMode    = 0775
 	fileMode   = 0644
@@ -56,11 +59,11 @@ const (
 	devfsPath  = "dev"
 	mib        = 1024.0 * 1024.0
 	// null device major, minor on linux.
-	devNullMajor   = 1
-	devNullMinor   = 3
-	devNullType    = unix.S_IFCHR
-	maxNfdLileLen  = 63
-	fullyConnected = "FULL"
+	devNullMajor    = 1
+	devNullMinor    = 3
+	devNullType     = unix.S_IFCHR
+	maxK8sLabelSize = 63
+	fullyConnected  = "FULL"
 )
 
 var verbose bool
@@ -79,6 +82,11 @@ type genOptions struct {
 	devs  int
 }
 
+func min(val1 int, val2 int) int {
+	// TODO replace this with template version of Min(), once Golang standard library gets one
+	return int(math.Min(float64(val1), float64(val2)))
+}
+
 func addSysfsDriTree(root string, opts *genOptions, i int) error {
 	card := cardBase + i
 	base := fmt.Sprintf("%s/class/drm/card%d", root, card)
@@ -240,7 +248,7 @@ func generateDriFiles(opts genOptions) {
 	log.Printf("Generating fake DRI device(s) sysfs, debugfs and devfs content under '%s' & '%s'",
 		sysfsPath, devfsPath)
 
-	makeXelinkSideCar(opts.Capabilities["xelink-topology"], opts.DevCount, opts.TilesPerDev, opts.Capabilities["xelink-topology"])
+	makeXelinkSideCar(opts.Capabilities["connection-topology"], opts.DevCount, opts.TilesPerDev, opts.Capabilities["connections"])
 
 	opts.dirs, opts.files = 0, 0
 	for i := 0; i < opts.DevCount; i++ {
@@ -306,76 +314,57 @@ func getOptions(name string) genOptions {
 	return opts
 }
 
-func makeXelinkSideCar(topology string, gpus int, tiles int, connections string) {
+func makeXelinkSideCar(topology string, gpus, tiles int, connections string) {
 	if topology != fullyConnected {
-		log.Printf("XELINK: generate xelink sidecar label file, using (GPUs: %d, Tiles: %d, Connections: %s)", gpus, tiles, connections)
+		log.Printf("XELINK: generate xelink sidecar label file, using (GPUs: %d, Tiles: %d)", gpus, tiles)
 	} else {
 		log.Printf("XELINK: generate xelink sidecar label file, using (GPUs: %d, Tiles: %d, Topology: %s)", gpus, tiles, topology)
 	}
 
-	smap := ""
-
 	if topology == fullyConnected {
-		// create 2d slice with zeros
-		var cmap = make([][]int, 0)
-
-		var total = gpus * tiles
+		saveSideCarFile(buildConnectionList(gpus, tiles))
+	} else {
+		saveSideCarFile(connections)
+	}
+}
 
-		for y := 0; y < total; y++ {
-			var xmap = make([]int, 0)
+func buildConnectionList(gpus, tiles int) string {
+	mm := 0
 
-			for x := 0; x < total; x++ {
-				if x != y {
-					xmap = append(xmap, 1)
-				} else {
-					xmap = append(xmap, 0)
-				}
-			}
+	var nodes = make([]string, 0)
 
-			cmap = append(cmap, [][]int{xmap}...)
+	for mm < gpus {
+		nn := 0
+		for nn < tiles {
+			nodes = append(nodes, fmt.Sprintf("%d.%d", mm, nn))
+			nn++
 		}
+		mm++
+	}
 
-		// filter double connections
-		for y := 0; y < total; y++ {
-			for x := 0; x < total; x++ {
-				if cmap[y][x] == cmap[x][y] {
-					cmap[x][y] = 0
-				}
-			}
-		}
+	var links = make(map[string]struct{}, 0)
 
-		// make connection string
-		smap = buildConnectionList(gpus, tiles, cmap)
-	} else {
-		smap = connections
-	}
+	var smap = make([]string, 0)
 
-	saveSideCarFile(smap)
-}
+	for _, from := range nodes {
+		for _, to := range nodes {
+			// no self links, TODO ignore in-gpu xelinks
+			if to == from {
+				continue
+			}
 
-func buildConnectionList(gpus int, tiles int, cmap [][]int) string {
-	smap := ""
+			link := fmt.Sprintf("%s-%s", to, from)
 
-	for gpuy := 0; gpuy < gpus; gpuy++ {
-		for tiley := 0; tiley < tiles; tiley++ {
-			y := gpuy*tiles + tiley
+			reverselink := fmt.Sprintf("%s-%s", from, to)
+			if _, exists := links[reverselink]; !exists {
+				links[link] = struct{}{}
 
-			for gpux := 0; gpux < gpus; gpux++ {
-				for tilex := 0; tilex < tiles; tilex++ {
-					x := gpux*tiles + tilex
-					if cmap[y][x] == 1 {
-						smap = fmt.Sprintf("%s_%d.%d-%d.%d", smap, gpuy, tiley, gpux, tilex)
-					}
-				}
+				smap = append(smap, link)
 			}
 		}
 	}
 
-	if smap != "" {
-		smap = smap[1:]
-	}
-
-	return smap
+	return fmt.Sprintf("%s\n", strings.Join(smap, "_"))
 }
 
 func saveSideCarFile(connections string) {
@@ -385,32 +374,36 @@ func saveSideCarFile(connections string) {
 	}
 	defer f.Close()
 
-	if len(connections) > maxNfdLileLen {
-		line := fmt.Sprintf("xpumanager.intel.com/xe-links=%s", connections[:maxNfdLileLen])
+	if len(connections) <= maxK8sLabelSize {
+		line := fmt.Sprintf("xpumanager.intel.com/xe-links=%s", connections)
 		fmt.Println(line)
 
 		if _, err := f.WriteString(line + "\n"); err != nil {
 			panic(err)
 		}
 
-		index := 2
+		return
+	}
+
+	// Write first line without Z prefix
+	line := fmt.Sprintf("xpumanager.intel.com/xe-links=%s", connections[:maxK8sLabelSize])
+	fmt.Println(line)
 
-		for i := maxNfdLileLen; i < len(connections); i += (maxNfdLileLen - 1) {
-			line := fmt.Sprintf("xpumanager.intel.com/xe-links%d=Z%s", index, connections[i:int(math.Min(float64(len(connections)), float64(i+maxNfdLileLen-1)))])
-			fmt.Println(line)
+	if _, err := f.WriteString(line + "\n"); err != nil {
+		panic(err)
+	}
 
-			if _, err := f.WriteString(line + "\n"); err != nil {
-				panic(err)
-			}
-			index++
-		}
-	} else {
-		line := fmt.Sprintf("xpumanager.intel.com/xe-links=%s", connections[:int(math.Min(float64(len(connections)), float64(maxNfdLileLen)))])
+	index := 2
+
+	// Write next lines with Z prefix
+	for i := maxK8sLabelSize; i < len(connections); i += (maxK8sLabelSize - 1) {
+		line := fmt.Sprintf("xpumanager.intel.com/xe-links%d=Z%s", index, connections[i:min(len(connections), i+maxK8sLabelSize-1)])
 		fmt.Println(line)
 
 		if _, err := f.WriteString(line + "\n"); err != nil {
 			panic(err)
 		}
+		index++
 	}
 }