From 19a5c2a5c22e6e750d4ea3ae83c1248d321ef645 Mon Sep 17 00:00:00 2001 From: Igor Chorazewicz Date: Wed, 22 Jan 2025 23:22:33 +0000 Subject: [PATCH 1/6] [SYCL][UR] Replace leak checker with L0 validation layer functionality This ensures that calls to L0 made through UMF and user code are also tracked. --- .../source/adapters/level_zero/adapter.cpp | 103 ------------------ .../source/adapters/level_zero/common.cpp | 5 +- .../source/adapters/level_zero/common.hpp | 14 ++- 3 files changed, 11 insertions(+), 111 deletions(-) diff --git a/unified-runtime/source/adapters/level_zero/adapter.cpp b/unified-runtime/source/adapters/level_zero/adapter.cpp index 978439ecb9c89..73b0386a52f37 100644 --- a/unified-runtime/source/adapters/level_zero/adapter.cpp +++ b/unified-runtime/source/adapters/level_zero/adapter.cpp @@ -316,18 +316,6 @@ ur_adapter_handle_t_::ur_adapter_handle_t_() } PlatformCache.Compute = [](Result &result) { - static std::once_flag ZeCallCountInitialized; - try { - std::call_once(ZeCallCountInitialized, []() { - if (UrL0LeaksDebug) { - ZeCallCount = new std::map; - } - }); - } catch (...) { - result = exceptionToResult(std::current_exception()); - return; - } - uint32_t UserForcedSysManInit = 0; // Check if the user has disabled the default L0 Env initialization. const int UrSysManEnvInitEnabled = [&UserForcedSysManInit] { @@ -533,97 +521,6 @@ void globalAdapterOnDemandCleanup() { } ur_result_t adapterStateTeardown() { - // Print the balance of various create/destroy native calls. - // The idea is to verify if the number of create(+) and destroy(-) calls are - // matched. - if (ZeCallCount && (UrL0LeaksDebug) != 0) { - bool LeakFound = false; - // clang-format off - // - // The format of this table is such that each row accounts for a - // specific type of objects, and all elements in the raw except the last - // one are allocating objects of that type, while the last element is known - // to deallocate objects of that type. - // - std::vector> CreateDestroySet = { - {"zeContextCreate", "zeContextDestroy"}, - {"zeCommandQueueCreate", "zeCommandQueueDestroy"}, - {"zeModuleCreate", "zeModuleDestroy"}, - {"zeKernelCreate", "zeKernelDestroy"}, - {"zeEventPoolCreate", "zeEventPoolDestroy"}, - {"zeCommandListCreateImmediate", "zeCommandListCreate", "zeCommandListDestroy"}, - {"zeEventCreate", "zeEventDestroy"}, - {"zeFenceCreate", "zeFenceDestroy"}, - {"zeImageCreate","zeImageViewCreateExt", "zeImageDestroy"}, - {"zeSamplerCreate", "zeSamplerDestroy"}, - {"zeMemAllocDevice", "zeMemAllocHost", "zeMemAllocShared", "zeMemFree"}, - }; - - // A sample output aimed below is this: - // ------------------------------------------------------------------------ - // zeContextCreate = 1 \---> zeContextDestroy = 1 - // zeCommandQueueCreate = 1 \---> zeCommandQueueDestroy = 1 - // zeModuleCreate = 1 \---> zeModuleDestroy = 1 - // zeKernelCreate = 1 \---> zeKernelDestroy = 1 - // zeEventPoolCreate = 1 \---> zeEventPoolDestroy = 1 - // zeCommandListCreateImmediate = 1 | - // zeCommandListCreate = 1 \---> zeCommandListDestroy = 1 ---> LEAK = 1 - // zeEventCreate = 2 \---> zeEventDestroy = 2 - // zeFenceCreate = 1 \---> zeFenceDestroy = 1 - // zeImageCreate = 0 \---> zeImageDestroy = 0 - // zeSamplerCreate = 0 \---> zeSamplerDestroy = 0 - // zeMemAllocDevice = 0 | - // zeMemAllocHost = 1 | - // zeMemAllocShared = 0 \---> zeMemFree = 1 - // - // clang-format on - // TODO: use logger to print this messages - std::cerr << "Check balance of create/destroy calls\n"; - std::cerr << "----------------------------------------------------------\n"; - std::stringstream ss; - for (const auto &Row : CreateDestroySet) { - int diff = 0; - for (auto I = Row.begin(); I != Row.end();) { - const char *ZeName = (*I).c_str(); - const auto &ZeCount = (*ZeCallCount)[*I]; - - bool First = (I == Row.begin()); - bool Last = (++I == Row.end()); - - if (Last) { - ss << " \\--->"; - diff -= ZeCount; - } else { - diff += ZeCount; - if (!First) { - ss << " | "; - std::cerr << ss.str() << "\n"; - ss.str(""); - ss.clear(); - } - } - ss << std::setw(30) << std::right << ZeName; - ss << " = "; - ss << std::setw(5) << std::left << ZeCount; - } - - if (diff) { - LeakFound = true; - ss << " ---> LEAK = " << diff; - } - - std::cerr << ss.str() << '\n'; - ss.str(""); - ss.clear(); - } - - ZeCallCount->clear(); - delete ZeCallCount; - ZeCallCount = nullptr; - if (LeakFound) - return UR_RESULT_ERROR_INVALID_MEM_OBJECT; - } - // Due to multiple DLLMain definitions with SYCL, register to cleanup the // Global Adapter after refcnt is 0 #if defined(_WIN32) diff --git a/unified-runtime/source/adapters/level_zero/common.cpp b/unified-runtime/source/adapters/level_zero/common.cpp index 94d864108e68e..997dbf1be616a 100644 --- a/unified-runtime/source/adapters/level_zero/common.cpp +++ b/unified-runtime/source/adapters/level_zero/common.cpp @@ -141,10 +141,7 @@ ze_result_t ZeCall::doCall(ze_result_t ZeResult, const char *ZeName, logger::debug("ZE ---> {}{}", ZeName, ZeArgs); if (ZeResult == ZE_RESULT_SUCCESS) { - if (UrL0LeaksDebug) { - ++(*ZeCallCount)[ZeName]; - } - return ZE_RESULT_SUCCESS; + return ZeResult; } if (TraceError) { diff --git a/unified-runtime/source/adapters/level_zero/common.hpp b/unified-runtime/source/adapters/level_zero/common.hpp index ad77ae8d22aba..a60ba763606d9 100644 --- a/unified-runtime/source/adapters/level_zero/common.hpp +++ b/unified-runtime/source/adapters/level_zero/common.hpp @@ -70,10 +70,20 @@ const int UrL0Debug = [] { return DebugMode; }(); +// This function will ensure compatibility with both Linux and Windows for +// setting environment variables. +bool setEnvVar(const char *name, const char *value); + const int UrL0LeaksDebug = [] { const char *UrRet = std::getenv("UR_L0_LEAKS_DEBUG"); if (!UrRet) return 0; + + if (std::atoi(UrRet)) { + setEnvVar("ZE_ENABLE_VALIDATION_LAYER", "1"); + setEnvVar("ZEL_ENABLE_BASIC_LEAK_CHECKER", "1"); + } + return std::atoi(UrRet); }(); @@ -154,10 +164,6 @@ class ZeCall { const char *ZeArgs, bool TraceError = true); }; -// This function will ensure compatibility with both Linux and Windows for -// setting environment variables. -bool setEnvVar(const char *name, const char *value); - // Returns the ze_structure_type_t to use in .stype of a structured descriptor. // Intentionally not defined; will give an error if no proper specialization template ze_structure_type_t getZeStructureType(); From 061be6dffd8c60fca58893d54f2be797c80abb60 Mon Sep 17 00:00:00 2001 From: Igor Chorazewicz Date: Thu, 27 Feb 2025 00:25:26 +0000 Subject: [PATCH 2/6] [SYCL][E2E] adjust interop-level-zero-buffer-ownership.cpp test Account for the fact that when using API loggin from L0 loader, calls to zeMemFree from the test are also tracked. --- .../Adapters/level_zero/interop-buffer-ownership.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/sycl/test-e2e/Adapters/level_zero/interop-buffer-ownership.cpp b/sycl/test-e2e/Adapters/level_zero/interop-buffer-ownership.cpp index 310a727368d3f..0a2cc2893005f 100644 --- a/sycl/test-e2e/Adapters/level_zero/interop-buffer-ownership.cpp +++ b/sycl/test-e2e/Adapters/level_zero/interop-buffer-ownership.cpp @@ -17,6 +17,7 @@ // Keep ownership // CHECK: zeMemFree +// CHECK: zeMemFree // Account for zeMemFree used to query page sizes by the UMF (only affects v2 L0 // adapter) @@ -31,10 +32,8 @@ // Transfer ownership // CHECK: zeMemFree -// CHECK: zeMemFree - -// No other calls to zeMemFree -// CHECK-NOT: zeMemFree +// For v2 adapter, all calls (even from this test) are logged +// CHECK-OPT: zeMemFree #include "interop-buffer-helpers.hpp" #include From e463ede3288976174f3aff132e3a302279967f09 Mon Sep 17 00:00:00 2001 From: Igor Chorazewicz Date: Wed, 19 Mar 2025 16:40:20 +0000 Subject: [PATCH 3/6] [SYCL][UR][L0 v2] enable API logging through L0 loader --- .../source/adapters/level_zero/adapter.cpp | 21 ++++++++++ .../source/adapters/level_zero/common.cpp | 7 ++++ .../source/adapters/level_zero/common.hpp | 13 ++---- .../source/adapters/level_zero/v2/common.hpp | 40 ++++++------------- 4 files changed, 44 insertions(+), 37 deletions(-) diff --git a/unified-runtime/source/adapters/level_zero/adapter.cpp b/unified-runtime/source/adapters/level_zero/adapter.cpp index 73b0386a52f37..b746d93cf9353 100644 --- a/unified-runtime/source/adapters/level_zero/adapter.cpp +++ b/unified-runtime/source/adapters/level_zero/adapter.cpp @@ -308,6 +308,12 @@ ur_adapter_handle_t_::ur_adapter_handle_t_() if (UrL0Debug & UR_L0_DEBUG_BASIC) { logger.setLegacySink(std::make_unique()); +#ifdef UR_ADAPTER_LEVEL_ZERO_V2 + setEnvVar("ZEL_ENABLE_LOADER_LOGGING", "1"); + setEnvVar("ZEL_LOADER_LOGGING_LEVEL", "trace"); + setEnvVar("ZEL_LOADER_LOG_CONSOLE", "1"); + setEnvVar("ZE_ENABLE_VALIDATION_LAYER", "1"); +#endif }; if (UrL0Debug & UR_L0_DEBUG_VALIDATION) { @@ -315,6 +321,11 @@ ur_adapter_handle_t_::ur_adapter_handle_t_() setEnvVar("ZE_ENABLE_PARAMETER_VALIDATION", "1"); } + if (UrL0LeaksDebug) { + setEnvVar("ZE_ENABLE_VALIDATION_LAYER", "1"); + setEnvVar("ZEL_ENABLE_BASIC_LEAK_CHECKER", "1"); + } + PlatformCache.Compute = [](Result &result) { uint32_t UserForcedSysManInit = 0; // Check if the user has disabled the default L0 Env initialization. @@ -405,6 +416,16 @@ ur_adapter_handle_t_::ur_adapter_handle_t_() loader_version.patch >= 2)) { useInitDrivers = true; } + +#ifdef UR_ADAPTER_LEVEL_ZERO_V2 + if ((loader_version.major == 1 && loader_version.minor < 21) || + (loader_version.major == 1 && loader_version.minor == 21 && + loader_version.patch < 2)) { + logger::warning( + "WARNING: Level Zero Loader version is older than 1.21.2. " + "Please update to the latest version for API logging support.\n"); + } +#endif } if (useInitDrivers) { diff --git a/unified-runtime/source/adapters/level_zero/common.cpp b/unified-runtime/source/adapters/level_zero/common.cpp index 997dbf1be616a..8f9f5e15427bf 100644 --- a/unified-runtime/source/adapters/level_zero/common.cpp +++ b/unified-runtime/source/adapters/level_zero/common.cpp @@ -136,6 +136,12 @@ void zeParseError(ze_result_t ZeError, const char *&ErrorString) { } // switch } +#ifdef UR_ADAPTER_LEVEL_ZERO_V2 +ze_result_t ZeCall::doCall(ze_result_t ZeResult, const char *, const char *, + bool) { + return ZeResult; +} +#else ze_result_t ZeCall::doCall(ze_result_t ZeResult, const char *ZeName, const char *ZeArgs, bool TraceError) { logger::debug("ZE ---> {}{}", ZeName, ZeArgs); @@ -151,6 +157,7 @@ ze_result_t ZeCall::doCall(ze_result_t ZeResult, const char *ZeName, } return ZeResult; } +#endif // Specializations for various L0 structures template <> ze_structure_type_t getZeStructureType() { diff --git a/unified-runtime/source/adapters/level_zero/common.hpp b/unified-runtime/source/adapters/level_zero/common.hpp index a60ba763606d9..9798760ad4aff 100644 --- a/unified-runtime/source/adapters/level_zero/common.hpp +++ b/unified-runtime/source/adapters/level_zero/common.hpp @@ -70,20 +70,11 @@ const int UrL0Debug = [] { return DebugMode; }(); -// This function will ensure compatibility with both Linux and Windows for -// setting environment variables. -bool setEnvVar(const char *name, const char *value); - const int UrL0LeaksDebug = [] { const char *UrRet = std::getenv("UR_L0_LEAKS_DEBUG"); if (!UrRet) return 0; - if (std::atoi(UrRet)) { - setEnvVar("ZE_ENABLE_VALIDATION_LAYER", "1"); - setEnvVar("ZEL_ENABLE_BASIC_LEAK_CHECKER", "1"); - } - return std::atoi(UrRet); }(); @@ -164,6 +155,10 @@ class ZeCall { const char *ZeArgs, bool TraceError = true); }; +// This function will ensure compatibility with both Linux and Windows for +// setting environment variables. +bool setEnvVar(const char *name, const char *value); + // Returns the ze_structure_type_t to use in .stype of a structured descriptor. // Intentionally not defined; will give an error if no proper specialization template ze_structure_type_t getZeStructureType(); diff --git a/unified-runtime/source/adapters/level_zero/v2/common.hpp b/unified-runtime/source/adapters/level_zero/v2/common.hpp index 5f26108207fef..fc3dcd93ca67b 100644 --- a/unified-runtime/source/adapters/level_zero/v2/common.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/common.hpp @@ -15,24 +15,8 @@ #include "../common.hpp" #include "logger/ur_logger.hpp" -namespace { -#define DECLARE_DESTROY_FUNCTION(name) \ - template ze_result_t name##_wrapped(ZeHandleT handle) { \ - return ZE_CALL_NOCHECK_NAME(name, (handle), #name); \ - } - -#define HANDLE_WRAPPER_TYPE(handle, destroy) \ - ze_handle_wrapper> -} // namespace namespace v2 { - -DECLARE_DESTROY_FUNCTION(zeKernelDestroy) -DECLARE_DESTROY_FUNCTION(zeEventDestroy) -DECLARE_DESTROY_FUNCTION(zeEventPoolDestroy) -DECLARE_DESTROY_FUNCTION(zeContextDestroy) -DECLARE_DESTROY_FUNCTION(zeCommandListDestroy) -DECLARE_DESTROY_FUNCTION(zeImageDestroy) namespace raii { template @@ -108,23 +92,23 @@ struct ze_handle_wrapper { bool ownZeHandle; }; -using ze_kernel_handle_t = HANDLE_WRAPPER_TYPE(::ze_kernel_handle_t, - zeKernelDestroy); +using ze_kernel_handle_t = + ze_handle_wrapper<::ze_kernel_handle_t, zeKernelDestroy>; -using ze_event_handle_t = HANDLE_WRAPPER_TYPE(::ze_event_handle_t, - zeEventDestroy); +using ze_event_handle_t = + ze_handle_wrapper<::ze_event_handle_t, zeEventDestroy>; -using ze_event_pool_handle_t = HANDLE_WRAPPER_TYPE(::ze_event_pool_handle_t, - zeEventPoolDestroy); +using ze_event_pool_handle_t = + ze_handle_wrapper<::ze_event_pool_handle_t, zeEventPoolDestroy>; -using ze_context_handle_t = HANDLE_WRAPPER_TYPE(::ze_context_handle_t, - zeContextDestroy); +using ze_context_handle_t = + ze_handle_wrapper<::ze_context_handle_t, zeContextDestroy>; -using ze_command_list_handle_t = HANDLE_WRAPPER_TYPE(::ze_command_list_handle_t, - zeCommandListDestroy); +using ze_command_list_handle_t = + ze_handle_wrapper<::ze_command_list_handle_t, zeCommandListDestroy>; -using ze_image_handle_t = HANDLE_WRAPPER_TYPE(::ze_image_handle_t, - zeImageDestroy); +using ze_image_handle_t = + ze_handle_wrapper<::ze_image_handle_t, zeImageDestroy>; } // namespace raii } // namespace v2 From d4f231682d6d181c95593cdbc0ba978dbef65383 Mon Sep 17 00:00:00 2001 From: Igor Chorazewicz Date: Fri, 14 Mar 2025 17:56:41 +0000 Subject: [PATCH 4/6] Revert "[SYCL][E2E] temporary disable a few tests on V2 adapter" This reverts commit 8f85bbb0392a42e454a2b7aead3f73e271b3d9d2. --- .../test-e2e/Adapters/level_zero/interop-buffer-ownership.cpp | 2 +- sycl/test-e2e/Adapters/level_zero/queue_profiling.cpp | 2 +- sycl/test-e2e/Adapters/level_zero/usm_device_read_only.cpp | 2 +- sycl/test-e2e/Basic/buffer/buffer_create.cpp | 2 +- sycl/test-e2e/KernelAndProgram/disable-caching.cpp | 2 -- sycl/test-e2e/USM/usm_pooling.cpp | 4 ---- 6 files changed, 4 insertions(+), 10 deletions(-) diff --git a/sycl/test-e2e/Adapters/level_zero/interop-buffer-ownership.cpp b/sycl/test-e2e/Adapters/level_zero/interop-buffer-ownership.cpp index 0a2cc2893005f..2cb2e3e790f0b 100644 --- a/sycl/test-e2e/Adapters/level_zero/interop-buffer-ownership.cpp +++ b/sycl/test-e2e/Adapters/level_zero/interop-buffer-ownership.cpp @@ -1,7 +1,7 @@ // REQUIRES: gpu, level_zero, level_zero_dev_kit // L0 adapter incorrectly reports memory leaks because it doesn't take into // account direct calls to L0 API. -// UNSUPPORTED: ze_debug, level_zero_v2_adapter +// UNSUPPORTED: ze_debug // RUN: %{build} %level_zero_options -o %t.out // RUN: env SYCL_PI_LEVEL_ZERO_DISABLE_USM_ALLOCATOR=1 UR_L0_DEBUG=1 %{run} %t.out 2>&1 | FileCheck %s diff --git a/sycl/test-e2e/Adapters/level_zero/queue_profiling.cpp b/sycl/test-e2e/Adapters/level_zero/queue_profiling.cpp index ae9d63952882d..0673d55886c57 100644 --- a/sycl/test-e2e/Adapters/level_zero/queue_profiling.cpp +++ b/sycl/test-e2e/Adapters/level_zero/queue_profiling.cpp @@ -1,5 +1,5 @@ // REQUIRES: gpu, level_zero -// UNSUPPORTED: ze_debug, level_zero_v2_adapter +// UNSUPPORTED: ze_debug // RUN: %{build} -o %t.out // RUN: env UR_L0_DEBUG=-1 %{l0_leak_check} %{run} %t.out 2>&1 | FileCheck --check-prefixes=WITHOUT %s diff --git a/sycl/test-e2e/Adapters/level_zero/usm_device_read_only.cpp b/sycl/test-e2e/Adapters/level_zero/usm_device_read_only.cpp index 335888b24d68f..d848d68e95af9 100644 --- a/sycl/test-e2e/Adapters/level_zero/usm_device_read_only.cpp +++ b/sycl/test-e2e/Adapters/level_zero/usm_device_read_only.cpp @@ -1,5 +1,5 @@ // REQUIRES: gpu, level_zero -// UNSUPPORTED: ze_debug, level_zero_v2_adapter +// UNSUPPORTED: ze_debug // RUN: %{build} -o %t.out // RUN: env SYCL_UR_TRACE=2 UR_L0_DEBUG=1 %{run} %t.out 2>&1 | FileCheck %s diff --git a/sycl/test-e2e/Basic/buffer/buffer_create.cpp b/sycl/test-e2e/Basic/buffer/buffer_create.cpp index 32e82d8e30236..286241eeb0627 100644 --- a/sycl/test-e2e/Basic/buffer/buffer_create.cpp +++ b/sycl/test-e2e/Basic/buffer/buffer_create.cpp @@ -1,7 +1,7 @@ // REQUIRES: gpu,level_zero,level_zero_dev_kit // RUN: %{build} %level_zero_options -o %t.out // RUN: env UR_L0_DEBUG=1 %{run} %t.out 2>&1 | FileCheck %s -// UNSUPPORTED: ze_debug, level_zero_v2_adapter +// UNSUPPORTED: ze_debug #include #include diff --git a/sycl/test-e2e/KernelAndProgram/disable-caching.cpp b/sycl/test-e2e/KernelAndProgram/disable-caching.cpp index 1513afafc8bb9..8a0b15b12311f 100644 --- a/sycl/test-e2e/KernelAndProgram/disable-caching.cpp +++ b/sycl/test-e2e/KernelAndProgram/disable-caching.cpp @@ -1,7 +1,5 @@ // This test ensures created program/kernels are not retained // if and only if caching is disabled. -// UNSUPPORTED: level_zero_v2_adapter -// UNSUPPORTED-INTENDED: bug in L0 loader param validation (fixed in v.1.21.1) // RUN: %{build} -o %t.out // RUN: env ZE_DEBUG=-6 SYCL_UR_TRACE=2 SYCL_CACHE_IN_MEM=0 %{run} %t.out \ diff --git a/sycl/test-e2e/USM/usm_pooling.cpp b/sycl/test-e2e/USM/usm_pooling.cpp index b68270918ccd6..005552b6ce85f 100644 --- a/sycl/test-e2e/USM/usm_pooling.cpp +++ b/sycl/test-e2e/USM/usm_pooling.cpp @@ -1,10 +1,6 @@ // REQUIRES: level_zero // RUN: %{build} -o %t.out -// UNSUPPORTED: level_zero_v2_adapter -// UNSUPPORTED-INTENDED: memory allocation logs are not emitted in -// level_zero_v2_adapter - // Allocate 2 items of 2MB. Free 2. Allocate 3 more of 2MB. // With no pooling: 1,2,3,4,5 allocs lead to ZE call. From 866ca61643ac427ba9b64e9dc81293e61e2fb12c Mon Sep 17 00:00:00 2001 From: Igor Chorazewicz Date: Wed, 30 Apr 2025 19:25:36 +0000 Subject: [PATCH 5/6] [SYCL][E2E] make sure zeInit in test is called after UR is initialized UR internally sets env variables to control behavior or L0 loader (e.g. enabling validation layer). If zeInit in test is called first the vars will have no effect. --- .../Adapters/level_zero/batch_barrier.cpp | 12 ++++++------ .../level_zero/interop-buffer-multi-dim.cpp | 16 ++++++++-------- .../level_zero/interop-buffer-ownership.cpp | 16 ++++++++-------- .../Adapters/level_zero/interop-buffer.cpp | 16 ++++++++-------- .../level_zero/interop-get-native-mem.cpp | 16 ++++++++-------- .../level_zero/interop-image-get-native-mem.cpp | 8 ++++---- .../level_zero/interop-image-ownership.cpp | 2 ++ .../Adapters/level_zero/interop-image.cpp | 12 ++++++------ .../level_zero/interop-keep-ownership.cpp | 8 ++++---- .../Inputs/interop-level-zero-launch-kernel.cpp | 12 ++++++------ sycl/test-e2e/USM/usm_pooling.cpp | 4 ++++ 11 files changed, 64 insertions(+), 58 deletions(-) diff --git a/sycl/test-e2e/Adapters/level_zero/batch_barrier.cpp b/sycl/test-e2e/Adapters/level_zero/batch_barrier.cpp index f5035d548066c..88a51f579dc0b 100644 --- a/sycl/test-e2e/Adapters/level_zero/batch_barrier.cpp +++ b/sycl/test-e2e/Adapters/level_zero/batch_barrier.cpp @@ -21,6 +21,12 @@ void submit_kernel(queue &q) { } int main(int argc, char *argv[]) { + queue q; + + submit_kernel(q); // starts a batch + // CHECK: ---> urEnqueueKernelLaunch + // CHECK-NOT: zeCommandQueueExecuteCommandLists + // Initialize Level Zero driver is required if this test is linked // statically with Level Zero loader, the driver will not be init otherwise. ze_result_t result = zeInit(ZE_INIT_FLAG_GPU_ONLY); @@ -29,12 +35,6 @@ int main(int argc, char *argv[]) { return 1; } - queue q; - - submit_kernel(q); // starts a batch - // CHECK: ---> urEnqueueKernelLaunch - // CHECK-NOT: zeCommandQueueExecuteCommandLists - // continue the batch event barrier = q.ext_oneapi_submit_barrier(); // CHECK: ---> urEnqueueEventsWaitWithBarrierExt diff --git a/sycl/test-e2e/Adapters/level_zero/interop-buffer-multi-dim.cpp b/sycl/test-e2e/Adapters/level_zero/interop-buffer-multi-dim.cpp index 9d276dd6a5eda..73c8a04d2f88f 100644 --- a/sycl/test-e2e/Adapters/level_zero/interop-buffer-multi-dim.cpp +++ b/sycl/test-e2e/Adapters/level_zero/interop-buffer-multi-dim.cpp @@ -16,14 +16,6 @@ using namespace sycl; int main() { #ifdef SYCL_EXT_ONEAPI_BACKEND_LEVEL_ZERO try { - // Initialize Level Zero driver is required if this test is linked - // statically with Level Zero loader, the driver will not be init otherwise. - ze_result_t result = zeInit(ZE_INIT_FLAG_GPU_ONLY); - if (result != ZE_RESULT_SUCCESS) { - std::cout << "zeInit failed\n"; - return 1; - } - platform Plt{gpu_selector_v}; auto Devices = Plt.get_devices(); @@ -33,6 +25,14 @@ int main() { return 0; } + // Initialize Level Zero driver is required if this test is linked + // statically with Level Zero loader, the driver will not be init otherwise. + ze_result_t result = zeInit(ZE_INIT_FLAG_GPU_ONLY); + if (result != ZE_RESULT_SUCCESS) { + std::cout << "zeInit failed\n"; + return 1; + } + device Device = Devices[0]; context Context{Device}; queue Queue{Context, Device}; diff --git a/sycl/test-e2e/Adapters/level_zero/interop-buffer-ownership.cpp b/sycl/test-e2e/Adapters/level_zero/interop-buffer-ownership.cpp index 2cb2e3e790f0b..d94e8e5dab05d 100644 --- a/sycl/test-e2e/Adapters/level_zero/interop-buffer-ownership.cpp +++ b/sycl/test-e2e/Adapters/level_zero/interop-buffer-ownership.cpp @@ -118,14 +118,6 @@ void test_copyback_and_free( int main() { #ifdef SYCL_EXT_ONEAPI_BACKEND_LEVEL_ZERO try { - // Initialize Level Zero driver is required if this test is linked - // statically with Level Zero loader, the driver will not be init otherwise. - ze_result_t result = zeInit(ZE_INIT_FLAG_GPU_ONLY); - if (result != ZE_RESULT_SUCCESS) { - std::cout << "zeInit failed\n"; - return 1; - } - platform Plt{gpu_selector_v}; auto Devices = Plt.get_devices(); @@ -135,6 +127,14 @@ int main() { return 0; } + // Initialize Level Zero driver is required if this test is linked + // statically with Level Zero loader, the driver will not be init otherwise. + ze_result_t result = zeInit(ZE_INIT_FLAG_GPU_ONLY); + if (result != ZE_RESULT_SUCCESS) { + std::cout << "zeInit failed\n"; + return 1; + } + device Dev1 = Devices[0]; context Context1{Dev1}; queue Queue1{Context1, Dev1}; diff --git a/sycl/test-e2e/Adapters/level_zero/interop-buffer.cpp b/sycl/test-e2e/Adapters/level_zero/interop-buffer.cpp index 69d010a6feaf6..8f824ed113ca8 100644 --- a/sycl/test-e2e/Adapters/level_zero/interop-buffer.cpp +++ b/sycl/test-e2e/Adapters/level_zero/interop-buffer.cpp @@ -36,14 +36,6 @@ class DiscreteSelector : public sycl::device_selector { int main() { #ifdef SYCL_EXT_ONEAPI_BACKEND_LEVEL_ZERO try { - // Initialize Level Zero driver is required if this test is linked - // statically with Level Zero loader, the driver will not be init otherwise. - ze_result_t result = zeInit(ZE_INIT_FLAG_GPU_ONLY); - if (result != ZE_RESULT_SUCCESS) { - std::cout << "zeInit failed\n"; - return 1; - } - queue Queue{}; auto Context = Queue.get_info(); @@ -53,6 +45,14 @@ int main() { auto ZeContext = get_native(Context); auto ZeDevice = get_native(Device); + // Initialize Level Zero driver is required if this test is linked + // statically with Level Zero loader, the driver will not be init otherwise. + ze_result_t result = zeInit(ZE_INIT_FLAG_GPU_ONLY); + if (result != ZE_RESULT_SUCCESS) { + std::cout << "zeInit failed\n"; + return 1; + } + ze_host_mem_alloc_desc_t HostDesc = {}; HostDesc.stype = ZE_STRUCTURE_TYPE_HOST_MEM_ALLOC_DESC; HostDesc.pNext = nullptr; diff --git a/sycl/test-e2e/Adapters/level_zero/interop-get-native-mem.cpp b/sycl/test-e2e/Adapters/level_zero/interop-get-native-mem.cpp index 92f5085da23e7..0980a2ee5c956 100644 --- a/sycl/test-e2e/Adapters/level_zero/interop-get-native-mem.cpp +++ b/sycl/test-e2e/Adapters/level_zero/interop-get-native-mem.cpp @@ -23,14 +23,6 @@ constexpr size_t SIZE = 16; int main() { #ifdef SYCL_EXT_ONEAPI_BACKEND_LEVEL_ZERO try { - // Initialize Level Zero driver is required if this test is linked - // statically with Level Zero loader, the driver will not be init otherwise. - ze_result_t result = zeInit(ZE_INIT_FLAG_GPU_ONLY); - if (result != ZE_RESULT_SUCCESS) { - std::cout << "zeInit failed\n"; - return 1; - } - platform Plt{gpu_selector_v}; auto Devices = Plt.get_devices(); @@ -40,6 +32,14 @@ int main() { return 0; } + // Initialize Level Zero driver is required if this test is linked + // statically with Level Zero loader, the driver will not be init otherwise. + ze_result_t result = zeInit(ZE_INIT_FLAG_GPU_ONLY); + if (result != ZE_RESULT_SUCCESS) { + std::cout << "zeInit failed\n"; + return 1; + } + device Dev1 = Devices[0]; context Context1{Dev1}; queue Queue1{Context1, Dev1}; diff --git a/sycl/test-e2e/Adapters/level_zero/interop-image-get-native-mem.cpp b/sycl/test-e2e/Adapters/level_zero/interop-image-get-native-mem.cpp index 1764f2a98ec37..b6a48138c66d8 100644 --- a/sycl/test-e2e/Adapters/level_zero/interop-image-get-native-mem.cpp +++ b/sycl/test-e2e/Adapters/level_zero/interop-image-get-native-mem.cpp @@ -42,6 +42,10 @@ using namespace sycl; int main() { #ifdef SYCL_EXT_ONEAPI_BACKEND_LEVEL_ZERO + constexpr auto BE = sycl::backend::ext_oneapi_level_zero; + sycl::device D = + sycl::ext::oneapi::filter_selector("level_zero:gpu").select_device(); + // Initialize Level Zero driver is required if this test is linked // statically with Level Zero loader, the driver will not be init otherwise. ze_result_t result = zeInit(ZE_INIT_FLAG_GPU_ONLY); @@ -50,10 +54,6 @@ int main() { return 1; } - constexpr auto BE = sycl::backend::ext_oneapi_level_zero; - sycl::device D = - sycl::ext::oneapi::filter_selector("level_zero:gpu").select_device(); - sycl::context Ctx{D}; sycl::queue Q(Ctx, D); auto ZeContext = sycl::get_native(Ctx); diff --git a/sycl/test-e2e/Adapters/level_zero/interop-image-ownership.cpp b/sycl/test-e2e/Adapters/level_zero/interop-image-ownership.cpp index b783ebc71e136..5a81e10f13b41 100644 --- a/sycl/test-e2e/Adapters/level_zero/interop-image-ownership.cpp +++ b/sycl/test-e2e/Adapters/level_zero/interop-image-ownership.cpp @@ -144,6 +144,8 @@ void test(sycl::ext::oneapi::level_zero::ownership Ownership) { int main() { #ifdef SYCL_EXT_ONEAPI_BACKEND_LEVEL_ZERO + platform Plt{gpu_selector_v}; + // Initialize Level Zero driver is required if this test is linked // statically with Level Zero loader, the driver will not be init otherwise. ze_result_t result = zeInit(ZE_INIT_FLAG_GPU_ONLY); diff --git a/sycl/test-e2e/Adapters/level_zero/interop-image.cpp b/sycl/test-e2e/Adapters/level_zero/interop-image.cpp index 5e3bf12a85bfb..2e811d155af24 100644 --- a/sycl/test-e2e/Adapters/level_zero/interop-image.cpp +++ b/sycl/test-e2e/Adapters/level_zero/interop-image.cpp @@ -32,6 +32,12 @@ using namespace sycl; int main() { #ifdef SYCL_EXT_ONEAPI_BACKEND_LEVEL_ZERO + constexpr auto BE = sycl::backend::ext_oneapi_level_zero; + + platform Plt{gpu_selector_v}; + + auto Devices = Plt.get_devices(); + // Initialize Level Zero driver is required if this test is linked // statically with Level Zero loader, the driver will not be init otherwise. ze_result_t result = zeInit(ZE_INIT_FLAG_GPU_ONLY); @@ -40,12 +46,6 @@ int main() { return 1; } - constexpr auto BE = sycl::backend::ext_oneapi_level_zero; - - platform Plt{gpu_selector_v}; - - auto Devices = Plt.get_devices(); - if (Devices.size() < 1) { std::cout << "Devices not found" << std::endl; return 0; diff --git a/sycl/test-e2e/Adapters/level_zero/interop-keep-ownership.cpp b/sycl/test-e2e/Adapters/level_zero/interop-keep-ownership.cpp index 3dfb723394f74..d3d002ab0ed0d 100644 --- a/sycl/test-e2e/Adapters/level_zero/interop-keep-ownership.cpp +++ b/sycl/test-e2e/Adapters/level_zero/interop-keep-ownership.cpp @@ -14,6 +14,10 @@ using namespace sycl; int main() { + // Creat SYCL platform/device + device Device(gpu_selector_v); + platform Platform = Device.get_info(); + // Initialize Level Zero driver is required if this test is linked // statically with Level Zero loader, the driver will not be init otherwise. ze_result_t result = zeInit(ZE_INIT_FLAG_GPU_ONLY); @@ -22,10 +26,6 @@ int main() { return 1; } - // Creat SYCL platform/device - device Device(gpu_selector_v); - platform Platform = Device.get_info(); - // Create native Level-Zero context ze_context_handle_t ZeContext; ze_context_desc_t ZeContextDesc = {ZE_STRUCTURE_TYPE_CONTEXT_DESC, nullptr, diff --git a/sycl/test-e2e/Graph/Inputs/interop-level-zero-launch-kernel.cpp b/sycl/test-e2e/Graph/Inputs/interop-level-zero-launch-kernel.cpp index 76bf7705a0988..94fdff8ef66dc 100644 --- a/sycl/test-e2e/Graph/Inputs/interop-level-zero-launch-kernel.cpp +++ b/sycl/test-e2e/Graph/Inputs/interop-level-zero-launch-kernel.cpp @@ -37,6 +37,12 @@ std::vector loadSpirvFromFile(std::string FileName) { } int main(int, char **argv) { + device Device; + if (!getDevice(Device, backend::ext_oneapi_level_zero)) { + // No suitable device found. + return 0; + } + // Initialize Level Zero driver is required if this test is linked // statically with Level Zero loader, the driver will not be init otherwise. ze_result_t result = zeInit(ZE_INIT_FLAG_GPU_ONLY); @@ -45,12 +51,6 @@ int main(int, char **argv) { return 1; } - device Device; - if (!getDevice(Device, backend::ext_oneapi_level_zero)) { - // No suitable device found. - return 0; - } - std::vector Spirv = loadSpirvFromFile(argv[1]); const sycl::context Context{Device}; diff --git a/sycl/test-e2e/USM/usm_pooling.cpp b/sycl/test-e2e/USM/usm_pooling.cpp index 005552b6ce85f..b68270918ccd6 100644 --- a/sycl/test-e2e/USM/usm_pooling.cpp +++ b/sycl/test-e2e/USM/usm_pooling.cpp @@ -1,6 +1,10 @@ // REQUIRES: level_zero // RUN: %{build} -o %t.out +// UNSUPPORTED: level_zero_v2_adapter +// UNSUPPORTED-INTENDED: memory allocation logs are not emitted in +// level_zero_v2_adapter + // Allocate 2 items of 2MB. Free 2. Allocate 3 more of 2MB. // With no pooling: 1,2,3,4,5 allocs lead to ZE call. From e91ce19dee56384073824d2773256a6c882e1dab Mon Sep 17 00:00:00 2001 From: Igor Chorazewicz Date: Thu, 1 May 2025 21:34:46 +0000 Subject: [PATCH 6/6] [SYCL] Do not preload L0 v2 adapter if it's not requested The adapter sets certain env variables during adapter init. Preloding it will set those variables and result in unexpected behavior for the legacy adater. --- .../ur_win_proxy_loader.cpp | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/sycl/ur_win_proxy_loader/ur_win_proxy_loader.cpp b/sycl/ur_win_proxy_loader/ur_win_proxy_loader.cpp index 3798d81f6a544..236eb17a31420 100644 --- a/sycl/ur_win_proxy_loader/ur_win_proxy_loader.cpp +++ b/sycl/ur_win_proxy_loader/ur_win_proxy_loader.cpp @@ -107,6 +107,27 @@ void *&getDllHandle() { return dllHandle; } +static bool shouldLoadL0V2adapter() { + auto SyclEnv = std::getenv("SYCL_UR_USE_LEVEL_ZERO_V2"); + auto UREvn = std::getenv("UR_LOADER_USE_LEVEL_ZERO_V2"); + + try { + if (SyclEnv && std::stoi(SyclEnv) == 1) { + return true; + } + } catch (...) { + } + + try { + if (UREvn && std::atoi(UREvn) == 1) { + return true; + } + } catch (...) { + } + + return false; +} + /// Load the adapter libraries void preloadLibraries() { // Suppress system errors. @@ -142,7 +163,8 @@ void preloadLibraries() { getDllHandle() = loadAdapter(UR_LIBRARY_NAME(loader)); loadAdapter(UR_LIBRARY_NAME(adapter_opencl)); loadAdapter(UR_LIBRARY_NAME(adapter_level_zero)); - loadAdapter(UR_LIBRARY_NAME(adapter_level_zero_v2)); + if (shouldLoadL0V2adapter()) + loadAdapter(UR_LIBRARY_NAME(adapter_level_zero_v2)); loadAdapter(UR_LIBRARY_NAME(adapter_cuda)); loadAdapter(UR_LIBRARY_NAME(adapter_hip)); loadAdapter(UR_LIBRARY_NAME(adapter_native_cpu));