[SYCL] Adding new device descriptors as SYCL extensions. (#2897)

This commit adds new device descriptors as SYCL extensions.
These new device descriptors are detailed in
llvm/sycl/docs/extensions/IntelGPU/IntelGPUDeviceInfo.md.
They will provide low-level details about Intel GPU devices.
This support is only provided for the Level Zero Backend.
New aspects have been added to indicate if the support is available.

Signed-off-by: Gail Lyons <[email protected]>

Author: glyons-intel

sycl/doc/EnvironmentVariables.md

@@ -31,6 +31,7 @@ subject to change. Do not rely on these variables in production code.
 | SYCL_PI_LEVEL_ZERO_BATCH_SIZE | Integer | Sets a preferred number of commands to batch into a command list before executing the command list. A value of 0 causes the batch size to be adjusted dynamically. A value greater than 0 specifies fixed size batching, with the batch size set to the specified value. The default is 0. |
 | SYCL_PARALLEL_FOR_RANGE_ROUNDING_TRACE | Any(\*) | Enables tracing of parallel_for invocations with rounded-up ranges. |
 | SYCL_DISABLE_PARALLEL_FOR_RANGE_ROUNDING | Any(\*) | Disables automatic rounding-up of parallel_for invocation ranges. |
+| SYCL_ENABLE_PCI | Integer | When set to 1, enables obtaining the GPU PCI address when using the Level Zero backend. The default is 0. |
 
 `(*) Note: Any means this environment variable is effective when set to any non-null value.`
 

sycl/doc/extensions/IntelGPU/IntelGPUDeviceInfo.md

@@ -1,6 +1,5 @@
 # SYCL(TM) Proposal: Intel's Extensions for Device Information
 
-**IMPORTANT**: This specification is a draft.
 
 **NOTE**: Khronos(R) is a registered trademark and SYCL(TM) is a trademark of the Khronos Group, Inc.
 
@@ -15,13 +14,14 @@ The Feature Test Macro will be defined as:
     #define SYCL_EXT_INTEL_DEVICE_INFO 1
 
 
-
 # PCI Address #
 
 A new device descriptor will be added which will provide the PCI address in BDF format.  BDF format contains the address as: `domain:bus:device.function`.
 
 This new device descriptor is only available for devices in the Level Zero platform, and the matching aspect is only true for those devices. The DPC++ default behavior is to expose GPU devices through the Level Zero platform.
 
+**Note:** The environment variable SYCL\_ENABLE\_PCI must be set to 1 to obtain the PCI address.
+
 
 ## Device Information Descriptors ##
 

sycl/include/CL/sycl.hpp

@@ -26,6 +26,7 @@
 #include <CL/sycl/device_selector.hpp>
 #include <CL/sycl/event.hpp>
 #include <CL/sycl/exception.hpp>
+#include <CL/sycl/feature_test.hpp>
 #include <CL/sycl/group.hpp>
 #include <CL/sycl/handler.hpp>
 #include <CL/sycl/id.hpp>

sycl/include/CL/sycl/aspects.hpp

@@ -30,7 +30,14 @@ enum class aspect {
   usm_host_allocations,
   usm_shared_allocations,
   usm_restricted_shared_allocations,
-  usm_system_allocator
+  usm_system_allocator,
+  ext_intel_pci_address,
+  ext_intel_gpu_eu_count,
+  ext_intel_gpu_eu_simd_width,
+  ext_intel_gpu_slices,
+  ext_intel_gpu_subslices_per_slice,
+  ext_intel_gpu_eu_count_per_subslice,
+  ext_intel_max_mem_bandwidth
 };
 
 } // namespace sycl

sycl/include/CL/sycl/detail/pi.h

@@ -42,6 +42,7 @@
 #define _PI_CONCAT(a, b) _PI_STRING_HELPER(a.b)
 #define _PI_H_VERSION_STRING                                                   \
   _PI_CONCAT(_PI_H_VERSION_MAJOR, _PI_H_VERSION_MINOR)
+
 // TODO: we need a mapping of PI to OpenCL somewhere, and this can be done
 // elsewhere, e.g. in the pi_opencl, but constants/enums mapping is now
 // done here, for efficiency and simplicity.
@@ -264,7 +265,15 @@ typedef enum {
   PI_DEVICE_INFO_USM_CROSS_SHARED_SUPPORT =
       CL_DEVICE_CROSS_DEVICE_SHARED_MEM_CAPABILITIES_INTEL,
   PI_DEVICE_INFO_USM_SYSTEM_SHARED_SUPPORT =
-      CL_DEVICE_SHARED_SYSTEM_MEM_CAPABILITIES_INTEL
+      CL_DEVICE_SHARED_SYSTEM_MEM_CAPABILITIES_INTEL,
+  // These are Intel-specific extensions.
+  PI_DEVICE_INFO_PCI_ADDRESS = 0x10020,
+  PI_DEVICE_INFO_GPU_EU_COUNT = 0x10021,
+  PI_DEVICE_INFO_GPU_EU_SIMD_WIDTH = 0x10022,
+  PI_DEVICE_INFO_GPU_SLICES = 0x10023,
+  PI_DEVICE_INFO_GPU_SUBSLICES_PER_SLICE = 0x10024,
+  PI_DEVICE_INFO_GPU_EU_COUNT_PER_SUBSLICE = 0x10025,
+  PI_DEVICE_INFO_MAX_MEM_BANDWIDTH = 0x10026
 } _pi_device_info;
 
 typedef enum {

sycl/include/CL/sycl/feature_test.hpp

@@ -0,0 +1,18 @@
+//==---- feature_test.hpp - SYCL Feature Test Definitions -----*- C++ -*---==//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+// ===--------------------------------------------------------------------=== //
+#pragma once
+
+__SYCL_INLINE_NAMESPACE(cl) {
+namespace sycl {
+
+// Feature test macro definitions
+
+#define SYCL_EXT_INTEL_DEVICE_INFO 1
+
+} // namespace sycl
+} // __SYCL_INLINE_NAMESPACE(cl)

sycl/include/CL/sycl/info/device_traits.def

@@ -85,3 +85,10 @@ __SYCL_PARAM_TRAITS_SPEC(device, usm_host_allocations, bool)
 __SYCL_PARAM_TRAITS_SPEC(device, usm_shared_allocations, bool)
 __SYCL_PARAM_TRAITS_SPEC(device, usm_restricted_shared_allocations, bool)
 __SYCL_PARAM_TRAITS_SPEC(device, usm_system_allocator, bool)
+__SYCL_PARAM_TRAITS_SPEC(device, ext_intel_pci_address, string_class)
+__SYCL_PARAM_TRAITS_SPEC(device, ext_intel_gpu_eu_count, pi_uint32)
+__SYCL_PARAM_TRAITS_SPEC(device, ext_intel_gpu_eu_simd_width, pi_uint32)
+__SYCL_PARAM_TRAITS_SPEC(device, ext_intel_gpu_slices, pi_uint32)
+__SYCL_PARAM_TRAITS_SPEC(device, ext_intel_gpu_subslices_per_slice, pi_uint32)
+__SYCL_PARAM_TRAITS_SPEC(device, ext_intel_gpu_eu_count_per_subslice, pi_uint32)
+__SYCL_PARAM_TRAITS_SPEC(device, ext_intel_max_mem_bandwidth, pi_uint64)

sycl/include/CL/sycl/info/info_desc.hpp

@@ -130,7 +130,16 @@ enum class device : cl_device_info {
   usm_host_allocations = PI_USM_HOST_SUPPORT,
   usm_shared_allocations = PI_USM_SINGLE_SHARED_SUPPORT,
   usm_restricted_shared_allocations = PI_USM_CROSS_SHARED_SUPPORT,
-  usm_system_allocator = PI_USM_SYSTEM_SHARED_SUPPORT
+  usm_system_allocator = PI_USM_SYSTEM_SHARED_SUPPORT,
+  // intel extensions
+  ext_intel_pci_address = PI_DEVICE_INFO_PCI_ADDRESS,
+  ext_intel_gpu_eu_count = PI_DEVICE_INFO_GPU_EU_COUNT,
+  ext_intel_gpu_eu_simd_width = PI_DEVICE_INFO_GPU_EU_SIMD_WIDTH,
+  ext_intel_gpu_slices = PI_DEVICE_INFO_GPU_SLICES,
+  ext_intel_gpu_subslices_per_slice = PI_DEVICE_INFO_GPU_SUBSLICES_PER_SLICE,
+  ext_intel_gpu_eu_count_per_subslice =
+      PI_DEVICE_INFO_GPU_EU_COUNT_PER_SUBSLICE,
+  ext_intel_max_mem_bandwidth = PI_DEVICE_INFO_MAX_MEM_BANDWIDTH
 };
 
 enum class device_type : pi_uint64 {

sycl/plugins/cuda/pi_cuda.cpp

@@ -1452,6 +1452,16 @@ pi_result cuda_piDeviceGetInfo(pi_device device, pi_device_info param_name,
     return getInfo(param_value_size, param_value, param_value_size_ret, value);
   }
 
+    // TODO: Investigate if this information is available on CUDA.
+  case PI_DEVICE_INFO_PCI_ADDRESS:
+  case PI_DEVICE_INFO_GPU_EU_COUNT:
+  case PI_DEVICE_INFO_GPU_EU_SIMD_WIDTH:
+  case PI_DEVICE_INFO_GPU_SLICES:
+  case PI_DEVICE_INFO_GPU_SUBSLICES_PER_SLICE:
+  case PI_DEVICE_INFO_GPU_EU_COUNT_PER_SUBSLICE:
+  case PI_DEVICE_INFO_MAX_MEM_BANDWIDTH:
+    return PI_INVALID_VALUE;
+
   default:
     __SYCL_PI_HANDLE_UNKNOWN_PARAM_NAME(param_name);
   }

sycl/plugins/level_zero/pi_level_zero.cpp

@@ -17,10 +17,12 @@
 #include <cstdio>
 #include <cstring>
 #include <memory>
+#include <sstream>
 #include <string>
 #include <thread>
 #include <utility>
 
+#include <level_zero/zes_api.h>
 #include <level_zero/zet_api.h>
 
 #include "usm_allocator.hpp"
@@ -786,6 +788,12 @@ pi_result piPlatformsGet(pi_uint32 NumEntries, pi_platform *Platforms,
     setEnvVar("ZE_ENABLE_PARAMETER_VALIDATION", "1");
   }
 
+  // Enable SYSMAN support for obtaining the PCI address
+  // and maximum memory bandwidth.
+  if (getenv("SYCL_ENABLE_PCI") != nullptr) {
+    setEnvVar("ZES_ENABLE_SYSMAN", "1");
+  }
+
   // TODO: We can still safely recover if something goes wrong during the init.
   // Implement handling segfault using sigaction.
 
@@ -1578,6 +1586,42 @@ pi_result piDeviceGetInfo(pi_device Device, pi_device_info ParamName,
     }
     return ReturnValue(Supported);
   }
+
+    // intel extensions for GPU information
+  case PI_DEVICE_INFO_PCI_ADDRESS: {
+    if (getenv("ZES_ENABLE_SYSMAN") == nullptr) {
+      zePrint("Set SYCL_ENABLE_PCI=1 to obtain PCI data.\n");
+      return PI_INVALID_VALUE;
+    }
+    zes_pci_properties_t ZeDevicePciProperties = {};
+    ZE_CALL(zesDevicePciGetProperties(ZeDevice, &ZeDevicePciProperties));
+    std::stringstream ss;
+    ss << ZeDevicePciProperties.address.domain << ":"
+       << ZeDevicePciProperties.address.bus << ":"
+       << ZeDevicePciProperties.address.device << "."
+       << ZeDevicePciProperties.address.function;
+    return ReturnValue(ss.str().c_str());
+  }
+  case PI_DEVICE_INFO_GPU_EU_COUNT: {
+    pi_uint32 count = Device->ZeDeviceProperties.numEUsPerSubslice *
+                      Device->ZeDeviceProperties.numSubslicesPerSlice *
+                      Device->ZeDeviceProperties.numSlices;
+    return ReturnValue(pi_uint32{count});
+  }
+  case PI_DEVICE_INFO_GPU_EU_SIMD_WIDTH:
+    return ReturnValue(
+        pi_uint32{Device->ZeDeviceProperties.physicalEUSimdWidth});
+  case PI_DEVICE_INFO_GPU_SLICES:
+    return ReturnValue(pi_uint32{Device->ZeDeviceProperties.numSlices});
+  case PI_DEVICE_INFO_GPU_SUBSLICES_PER_SLICE:
+    return ReturnValue(
+        pi_uint32{Device->ZeDeviceProperties.numSubslicesPerSlice});
+  case PI_DEVICE_INFO_GPU_EU_COUNT_PER_SUBSLICE:
+    return ReturnValue(pi_uint32{Device->ZeDeviceProperties.numEUsPerSubslice});
+  case PI_DEVICE_INFO_MAX_MEM_BANDWIDTH:
+    // currently not supported in level zero runtime
+    return PI_INVALID_VALUE;
+
   default:
     zePrint("Unsupported ParamName in piGetDeviceInfo\n");
     zePrint("ParamName=%d(0x%x)\n", ParamName, ParamName);