Skip to content

Commit 10b4e8a

Browse files
againullagainull
authored
[SYCL] Interoperability API to create a context from a Level Zero handler (#2772)
Implement the missing API to create a SYCL context from a Level Zero handler. Also add a test to check the interop API for the Level Zero. This commit also includes changes to properly copy the Level Zero loader and headers to the build directory.
1 parent c70b047 commit 10b4e8a

File tree

12 files changed

+195
-36
lines changed

12 files changed

+195
-36
lines changed

‎sycl/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -332,6 +332,7 @@ set( SYCL_TOOLCHAIN_DEPLOY_COMPONENTS
332332
clang-offload-bundler
333333
file-table-tform
334334
level-zero-loader
335+
level-zero-headers
335336
llc
336337
llvm-ar
337338
llvm-foreach

‎sycl/include/CL/sycl/backend/level_zero.hpp‎

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -52,10 +52,15 @@ struct interop<backend::level_zero, accessor<DataT, Dimensions, AccessMode,
5252
namespace level_zero {
5353

5454
// Implementation of various "make" functions resides in libsycl.so
55-
platform make_platform(pi_native_handle NativeHandle);
56-
device make_device(const platform &Platform, pi_native_handle NativeHandle);
57-
program make_program(const context &Context, pi_native_handle NativeHandle);
58-
queue make_queue(const context &Context, pi_native_handle InteropHandle);
55+
__SYCL_EXPORT platform make_platform(pi_native_handle NativeHandle);
56+
__SYCL_EXPORT device make_device(const platform &Platform,
57+
pi_native_handle NativeHandle);
58+
__SYCL_EXPORT context make_context(const vector_class<device> &DeviceList,
59+
pi_native_handle NativeHandle);
60+
__SYCL_EXPORT program make_program(const context &Context,
61+
pi_native_handle NativeHandle);
62+
__SYCL_EXPORT queue make_queue(const context &Context,
63+
pi_native_handle InteropHandle);
5964

6065
// Construction of SYCL platform.
6166
template <typename T, typename detail::enable_if_t<
@@ -72,6 +77,18 @@ T make(const platform &Platform,
7277
return make_device(Platform, reinterpret_cast<pi_native_handle>(Interop));
7378
}
7479

80+
/// Construction of SYCL context.
81+
/// \param DeviceList is a vector of devices which must be encapsulated by
82+
/// created SYCL context. Provided devices and native context handle must
83+
/// be associated with the same platform.
84+
/// \param Interop is a Level Zero native context handle.
85+
template <typename T, typename std::enable_if<
86+
std::is_same<T, context>::value>::type * = nullptr>
87+
T make(const vector_class<device> &DeviceList,
88+
typename interop<backend::level_zero, T>::type Interop) {
89+
return make_context(DeviceList, detail::pi::cast<pi_native_handle>(Interop));
90+
}
91+
7592
// Construction of SYCL program.
7693
template <typename T, typename detail::enable_if_t<
7794
std::is_same<T, program>::value> * = nullptr>

‎sycl/include/CL/sycl/detail/pi.h‎

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -35,8 +35,8 @@
3535
// pi_device_binary_property_set PropertySetsEnd;
3636
// 2. A number of types needed to define pi_device_binary_property_set added.
3737
//
38-
#define _PI_H_VERSION_MAJOR 1
39-
#define _PI_H_VERSION_MINOR 2
38+
#define _PI_H_VERSION_MAJOR 2
39+
#define _PI_H_VERSION_MINOR 3
4040

4141
#define _PI_STRING_HELPER(a) #a
4242
#define _PI_CONCAT(a, b) _PI_STRING_HELPER(a.b)
@@ -944,11 +944,27 @@ piextContextGetNativeHandle(pi_context context, pi_native_handle *nativeHandle);
944944

945945
/// Creates PI context object from a native handle.
946946
/// NOTE: The created PI object takes ownership of the native handle.
947+
/// NOTE: The number of devices and the list of devices is needed for Level Zero
948+
/// backend because there is no possilibity to query this information from
949+
/// context handle for Level Zero. If backend has API to query a list of devices
950+
/// from the context native handle then these parameters are ignored.
947951
///
948952
/// \param nativeHandle is the native handle to create PI context from.
953+
/// \param numDevices is the number of devices in the context. Parameter is
954+
/// ignored if number of devices can be queried from the context native
955+
/// handle for a backend.
956+
/// \param devices is the list of devices in the context. Parameter is ignored
957+
/// if devices can be queried from the context native handle for a
958+
/// backend.
949959
/// \param context is the PI context created from the native handle.
960+
/// \return PI_SUCCESS if successfully created pi_context from the handle.
961+
/// PI_OUT_OF_HOST_MEMORY if can't allocate memory for the pi_context
962+
/// object. PI_INVALID_VALUE if numDevices == 0 or devices is NULL but
963+
/// backend doesn't have API to query a list of devices from the context
964+
/// native handle. PI_UNKNOWN_ERROR in case of another error.
950965
__SYCL_EXPORT pi_result piextContextCreateWithNativeHandle(
951-
pi_native_handle nativeHandle, pi_context *context);
966+
pi_native_handle nativeHandle, pi_uint32 numDevices,
967+
const pi_device *devices, pi_context *context);
952968

953969
//
954970
// Queue

‎sycl/plugins/cuda/pi_cuda.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1645,6 +1645,8 @@ pi_result cuda_piextContextGetNativeHandle(pi_context context,
16451645
///
16461646
/// \return TBD
16471647
pi_result cuda_piextContextCreateWithNativeHandle(pi_native_handle nativeHandle,
1648+
pi_uint32 num_devices,
1649+
const pi_device *devices,
16481650
pi_context *context) {
16491651
cl::sycl::detail::pi::die(
16501652
"Creation of PI context from native handle not implemented");

‎sycl/plugins/level_zero/CMakeLists.txt‎

Lines changed: 27 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,8 @@ if (NOT DEFINED LEVEL_ZERO_LIBRARY OR NOT DEFINED LEVEL_ZERO_INCLUDE_DIR)
4444
BUILD_BYPRODUCTS ${LEVEL_ZERO_LOADER}
4545
)
4646
ExternalProject_Add_Step(level-zero-loader llvminstall
47-
COMMAND ${CMAKE_COMMAND} -E copy_directory <INSTALL_DIR>/ ${LLVM_BINARY_DIR}
47+
COMMAND ${CMAKE_COMMAND} -E copy_directory <INSTALL_DIR>/lib/ ${LLVM_BINARY_DIR}/lib
48+
COMMAND ${CMAKE_COMMAND} -E copy_directory <INSTALL_DIR>/include/ ${LLVM_BINARY_DIR}/include/sycl
4849
COMMENT "Installing level-zero-loader into the LLVM binary directory"
4950
DEPENDEES install
5051
)
@@ -54,14 +55,35 @@ if (NOT DEFINED LEVEL_ZERO_LIBRARY OR NOT DEFINED LEVEL_ZERO_INCLUDE_DIR)
5455
COMPONENT level-zero-loader
5556
)
5657

57-
list(APPEND SYCL_TOOLCHAIN_DEPLOY_COMPONENTS level-zero-loader)
58+
set(LEVEL_ZERO_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/level_zero_loader_install/include/)
5859
else()
59-
include_directories("${LEVEL_ZERO_INCLUDE_DIR}")
6060
file(GLOB LEVEL_ZERO_LIBRARY_SRC "${LEVEL_ZERO_LIBRARY}*")
61-
file(COPY ${LEVEL_ZERO_LIBRARY_SRC} DESTINATION ${LLVM_LIBRARY_OUTPUT_INTDIR})
62-
add_custom_target(level-zero-loader DEPENDS ${LEVEL_ZERO_LIBRARY} COMMENT "Copying Level Zero Loader ...")
61+
get_filename_component(LEVEL_ZERO_LIB_NAME ${LEVEL_ZERO_LIBRARY} NAME)
62+
add_custom_target(level-zero-loader
63+
DEPENDS
64+
${LLVM_LIBRARY_OUTPUT_INTDIR}/${LEVEL_ZERO_LIB_NAME}
65+
${LLVM_BINARY_DIR}/include/sycl/level_zero
66+
)
67+
add_custom_command(
68+
OUTPUT
69+
${LLVM_LIBRARY_OUTPUT_INTDIR}/${LEVEL_ZERO_LIB_NAME}
70+
${LLVM_BINARY_DIR}/include/sycl/level_zero
71+
COMMENT
72+
"Copying Level Zero loader and headers"
73+
COMMAND
74+
${CMAKE_COMMAND} -E copy ${LEVEL_ZERO_LIBRARY_SRC} ${LLVM_LIBRARY_OUTPUT_INTDIR}
75+
COMMAND
76+
${CMAKE_COMMAND} -E copy_directory ${LEVEL_ZERO_INCLUDE_DIR} ${LLVM_BINARY_DIR}/include/sycl
77+
DEPENDS
78+
${LEVEL_ZERO_LIBRARY}
79+
${LEVEL_ZERO_INCLUDE_DIR}
80+
)
6381
endif()
6482

83+
list(APPEND SYCL_TOOLCHAIN_DEPLOY_COMPONENTS level-zero-loader level-zero-headers)
84+
85+
include_directories("${LEVEL_ZERO_INCLUDE_DIR}")
86+
6587
add_library (LevelZeroLoader-Headers INTERFACE)
6688
add_library (LevelZeroLoader::Headers ALIAS LevelZeroLoader-Headers)
6789
target_include_directories(LevelZeroLoader-Headers

‎sycl/plugins/level_zero/pi_level_zero.cpp‎

Lines changed: 40 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -423,6 +423,20 @@ pi_result _pi_device::initialize() {
423423
return PI_SUCCESS;
424424
}
425425

426+
pi_result _pi_context::initialize() {
427+
// Create the immediate command list to be used for initializations
428+
// Created as synchronous so level-zero performs implicit synchronization and
429+
// there is no need to query for completion in the plugin
430+
ze_command_queue_desc_t ZeCommandQueueDesc = {};
431+
ZeCommandQueueDesc.ordinal = Devices[0]->ZeComputeQueueGroupIndex;
432+
ZeCommandQueueDesc.index = 0;
433+
ZeCommandQueueDesc.mode = ZE_COMMAND_QUEUE_MODE_SYNCHRONOUS;
434+
ZE_CALL(zeCommandListCreateImmediate(ZeContext, Devices[0]->ZeDevice,
435+
&ZeCommandQueueDesc,
436+
&ZeCommandListInit));
437+
return PI_SUCCESS;
438+
}
439+
426440
pi_result
427441
_pi_queue::resetCommandListFenceEntry(ze_command_list_handle_t ZeCommandList,
428442
bool MakeAvailable) {
@@ -1703,29 +1717,19 @@ pi_result piContextCreate(const pi_context_properties *Properties,
17031717

17041718
assert(RetContext);
17051719

1720+
ze_context_desc_t ContextDesc = {ZE_STRUCTURE_TYPE_CONTEXT_DESC, nullptr, 0};
1721+
ze_context_handle_t ZeContext;
1722+
ZE_CALL(zeContextCreate((*Devices)->Platform->ZeDriver, &ContextDesc,
1723+
&ZeContext));
17061724
try {
1707-
*RetContext = new _pi_context(NumDevices, Devices);
1725+
*RetContext = new _pi_context(ZeContext, NumDevices, Devices);
1726+
(*RetContext)->initialize();
17081727
} catch (const std::bad_alloc &) {
17091728
return PI_OUT_OF_HOST_MEMORY;
17101729
} catch (...) {
17111730
return PI_ERROR_UNKNOWN;
17121731
}
17131732

1714-
ze_context_desc_t ContextDesc = {ZE_STRUCTURE_TYPE_CONTEXT_DESC, nullptr, 0};
1715-
ZE_CALL(zeContextCreate((*Devices)->Platform->ZeDriver, &ContextDesc,
1716-
&((*RetContext)->ZeContext)));
1717-
1718-
// Create the immediate command list to be used for initializations
1719-
// Created as synchronous so level-zero performs implicit synchronization and
1720-
// there is no need to query for completion in the plugin
1721-
ze_command_queue_desc_t ZeCommandQueueDesc = {};
1722-
ZeCommandQueueDesc.ordinal = (*Devices)->ZeComputeQueueGroupIndex;
1723-
ZeCommandQueueDesc.index = 0;
1724-
ZeCommandQueueDesc.mode = ZE_COMMAND_QUEUE_MODE_SYNCHRONOUS;
1725-
ZE_CALL(zeCommandListCreateImmediate(
1726-
(*RetContext)->ZeContext, (*Devices)->ZeDevice, &ZeCommandQueueDesc,
1727-
(&(*RetContext)->ZeCommandListInit)));
1728-
17291733
return PI_SUCCESS;
17301734
}
17311735

@@ -1772,8 +1776,26 @@ pi_result piextContextGetNativeHandle(pi_context Context,
17721776
}
17731777

17741778
pi_result piextContextCreateWithNativeHandle(pi_native_handle NativeHandle,
1775-
pi_context *Context) {
1776-
die("piextContextCreateWithNativeHandle: not supported");
1779+
pi_uint32 NumDevices,
1780+
const pi_device *Devices,
1781+
pi_context *RetContext) {
1782+
assert(NativeHandle);
1783+
assert(RetContext);
1784+
1785+
if (!Devices || !NumDevices) {
1786+
return PI_INVALID_VALUE;
1787+
}
1788+
1789+
try {
1790+
*RetContext = new _pi_context(pi_cast<ze_context_handle_t>(NativeHandle),
1791+
NumDevices, Devices);
1792+
(*RetContext)->initialize();
1793+
} catch (const std::bad_alloc &) {
1794+
return PI_OUT_OF_HOST_MEMORY;
1795+
} catch (...) {
1796+
return PI_ERROR_UNKNOWN;
1797+
}
1798+
17771799
return PI_SUCCESS;
17781800
}
17791801

‎sycl/plugins/level_zero/pi_level_zero.hpp‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -192,10 +192,11 @@ struct _pi_device : _pi_object {
192192
};
193193

194194
struct _pi_context : _pi_object {
195-
_pi_context(pi_uint32 NumDevices, const pi_device *Devs)
196-
: Devices{Devs, Devs + NumDevices}, ZeCommandListInit{nullptr},
197-
ZeEventPool{nullptr}, NumEventsAvailableInEventPool{},
198-
NumEventsLiveInEventPool{} {
195+
_pi_context(ze_context_handle_t ZeContext, pi_uint32 NumDevices,
196+
const pi_device *Devs)
197+
: ZeContext{ZeContext}, Devices{Devs, Devs + NumDevices},
198+
ZeCommandListInit{nullptr}, ZeEventPool{nullptr},
199+
NumEventsAvailableInEventPool{}, NumEventsLiveInEventPool{} {
199200
// Create USM allocator context for each pair (device, context).
200201
for (uint32_t I = 0; I < NumDevices; I++) {
201202
pi_device Device = Devs[I];
@@ -207,9 +208,14 @@ struct _pi_context : _pi_object {
207208
std::piecewise_construct, std::make_tuple(Device),
208209
std::make_tuple(std::unique_ptr<SystemMemory>(
209210
new USMDeviceMemoryAlloc(this, Device))));
211+
// NOTE: one must additionally call initialize() to complete
212+
// PI context creation.
210213
}
211214
}
212215

216+
// Initialize the PI context.
217+
pi_result initialize();
218+
213219
// A L0 context handle is primarily used during creation and management of
214220
// resources that may be used by multiple devices.
215221
ze_context_handle_t ZeContext;

‎sycl/plugins/opencl/pi_opencl.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -511,6 +511,8 @@ pi_result piContextCreate(const pi_context_properties *properties,
511511
}
512512

513513
pi_result piextContextCreateWithNativeHandle(pi_native_handle nativeHandle,
514+
pi_uint32 num_devices,
515+
const pi_device *devices,
514516
pi_context *piContext) {
515517
assert(piContext != nullptr);
516518
*piContext = reinterpret_cast<pi_context>(nativeHandle);

‎sycl/source/backend/level_zero.cpp‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,24 @@ __SYCL_EXPORT device make_device(const platform &Platform,
4545
PlatformImpl->getOrMakeDeviceImpl(PiDevice, PlatformImpl));
4646
}
4747

48+
//----------------------------------------------------------------------------
49+
// Implementation of level_zero::make<context>
50+
__SYCL_EXPORT context make_context(const vector_class<device> &DeviceList,
51+
pi_native_handle NativeHandle) {
52+
const auto &Plugin = pi::getPlugin<backend::level_zero>();
53+
// Create PI context first.
54+
pi_context PiContext;
55+
vector_class<pi_device> DeviceHandles;
56+
for (auto Dev : DeviceList) {
57+
DeviceHandles.push_back(detail::getSyclObjImpl(Dev)->getHandleRef());
58+
}
59+
Plugin.call<PiApiKind::piextContextCreateWithNativeHandle>(
60+
NativeHandle, DeviceHandles.size(), DeviceHandles.data(), &PiContext);
61+
// Construct the SYCL context from PI context.
62+
return detail::createSyclObjFromImpl<context>(
63+
std::make_shared<context_impl>(PiContext, async_handler{}, Plugin));
64+
}
65+
4866
//----------------------------------------------------------------------------
4967
// Implementation of level_zero::make<program>
5068
__SYCL_EXPORT program make_program(const context &Context,

‎sycl/source/backend/opencl.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -50,8 +50,8 @@ __SYCL_EXPORT context make_context(pi_native_handle NativeHandle) {
5050
const auto &Plugin = pi::getPlugin<backend::opencl>();
5151
// Create PI context first.
5252
pi::PiContext PiContext;
53-
Plugin.call<PiApiKind::piextContextCreateWithNativeHandle>(NativeHandle,
54-
&PiContext);
53+
Plugin.call<PiApiKind::piextContextCreateWithNativeHandle>(
54+
NativeHandle, 0, nullptr, &PiContext);
5555
// Construct the SYCL context from PI context.
5656
return detail::createSyclObjFromImpl<context>(
5757
std::make_shared<context_impl>(PiContext, async_handler{}, Plugin));

0 commit comments

Comments
 (0)