diff --git a/.github/workflows/ur-build-hw.yml b/.github/workflows/ur-build-hw.yml index 8ad0f45bb35bb..7c912442d6363 100644 --- a/.github/workflows/ur-build-hw.yml +++ b/.github/workflows/ur-build-hw.yml @@ -136,12 +136,18 @@ jobs: run: cmake --install ${{github.workspace}}/build - name: Test adapter specific - run: ctest -C ${{matrix.build_type}} --test-dir ${{github.workspace}}/build --output-on-failure -L "adapter-specific" -E "memcheck" --timeout 600 + env: + ZE_ENABLE_LOADER_DEBUG_TRACE: 1 + ZE_DEBUG: 1 + run: ctest -C ${{matrix.build_type}} --test-dir ${{github.workspace}}/build --output-on-failure -L "adapter-specific" -E "memcheck" --timeout 600 -VV # Don't run adapter specific tests when building multiple adapters if: ${{ matrix.adapter.other_name == '' }} - name: Test adapters - run: env UR_CTS_ADAPTER_PLATFORM="${{matrix.adapter.platform}}" ctest -C ${{matrix.build_type}} --test-dir ${{github.workspace}}/build --output-on-failure -L "conformance" --timeout 600 + env: + ZE_ENABLE_LOADER_DEBUG_TRACE: 1 + ZE_DEBUG: 1 + run: env UR_CTS_ADAPTER_PLATFORM="${{matrix.adapter.platform}}" ctest -C ${{matrix.build_type}} --test-dir ${{github.workspace}}/build --output-on-failure -L "conformance" --timeout 600 -VV - name: Get information about platform if: ${{ always() }} diff --git a/unified-runtime/cmake/FetchLevelZero.cmake b/unified-runtime/cmake/FetchLevelZero.cmake index 6d0e10d10dae4..45e68d52fcbda 100644 --- a/unified-runtime/cmake/FetchLevelZero.cmake +++ b/unified-runtime/cmake/FetchLevelZero.cmake @@ -37,12 +37,13 @@ if (NOT DEFINED LEVEL_ZERO_LIBRARY OR NOT DEFINED LEVEL_ZERO_INCLUDE_DIR) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-c++98-compat-extra-semi") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unknown-warning-option") endif() + set(BUILD_STATIC ON) if (UR_LEVEL_ZERO_LOADER_REPO STREQUAL "") set(UR_LEVEL_ZERO_LOADER_REPO "https://github.com/oneapi-src/level-zero.git") endif() if (UR_LEVEL_ZERO_LOADER_TAG STREQUAL "") - set(UR_LEVEL_ZERO_LOADER_TAG v1.19.2) + set(UR_LEVEL_ZERO_LOADER_TAG v1.21.1) endif() # Disable due to a bug https://github.com/oneapi-src/level-zero/issues/104 diff --git a/unified-runtime/source/adapters/level_zero/CMakeLists.txt b/unified-runtime/source/adapters/level_zero/CMakeLists.txt index 35d1337078f0f..026511c57a347 100644 --- a/unified-runtime/source/adapters/level_zero/CMakeLists.txt +++ b/unified-runtime/source/adapters/level_zero/CMakeLists.txt @@ -55,13 +55,14 @@ if(UR_BUILD_ADAPTER_L0) ) install_ur_library(ur_adapter_level_zero) + target_compile_definitions(ur_adapter_level_zero PUBLIC UR_STATIC_LEVEL_ZERO) if(UR_STATIC_ADAPTER_L0) target_compile_definitions(ur_adapter_level_zero PUBLIC UR_STATIC_ADAPTER_LEVEL_ZERO) # 'utils' target from 'level-zero-loader' includes path which is prefixed # in the source directory, this breaks the installation of 'utils' target. - set_target_properties(utils PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "") - install(TARGETS ur_umf LevelZeroLoader LevelZeroLoader-Headers ComputeRuntimeLevelZero-Headers ze_loader utils + set_target_properties(level_zero_utils PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "") + install(TARGETS ur_umf LevelZeroLoader LevelZeroLoader-Headers ComputeRuntimeLevelZero-Headers ze_loader level_zero_utils EXPORT ${PROJECT_NAME}-targets ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} diff --git a/unified-runtime/source/adapters/level_zero/adapter.cpp b/unified-runtime/source/adapters/level_zero/adapter.cpp index 6267c6c0952a6..4339aed2a2299 100644 --- a/unified-runtime/source/adapters/level_zero/adapter.cpp +++ b/unified-runtime/source/adapters/level_zero/adapter.cpp @@ -300,6 +300,12 @@ ur_adapter_handle_t_::ur_adapter_handle_t_() ZeInitResult = ZE_RESULT_ERROR_UNINITIALIZED; ZesResult = ZE_RESULT_ERROR_UNINITIALIZED; +#ifdef UR_STATIC_LEVEL_ZERO + // Given static linking of the L0 Loader, we must delay the loader's + // destruction of its context until after the UR Adapter is destroyed. + zelSetDelayLoaderContextTeardown(); +#endif + if (UrL0Debug & UR_L0_DEBUG_BASIC) { logger.setLegacySink(std::make_unique()); }; @@ -335,10 +341,12 @@ ur_adapter_handle_t_::ur_adapter_handle_t_() // Dynamically load the new L0 apis separately. // This must be done to avoid attempting to use symbols that do // not exist in older loader runtimes. +#ifndef UR_STATIC_LEVEL_ZERO #ifdef _WIN32 - HMODULE processHandle = GetModuleHandle(NULL); + GlobalAdapter->processHandle = GetModuleHandle(NULL); #else - HMODULE processHandle = nullptr; + GlobalAdapter->processHandle = nullptr; +#endif #endif // initialize level zero only once. @@ -412,9 +420,13 @@ ur_adapter_handle_t_::ur_adapter_handle_t_() } if (useInitDrivers) { +#ifdef UR_STATIC_LEVEL_ZERO + GlobalAdapter->initDriversFunctionPtr = zeInitDrivers; +#else GlobalAdapter->initDriversFunctionPtr = (ze_pfnInitDrivers_t)ur_loader::LibLoader::getFunctionPtr( - processHandle, "zeInitDrivers"); + GlobalAdapter->processHandle, "zeInitDrivers"); +#endif if (GlobalAdapter->initDriversFunctionPtr) { logger::debug("\nzeInitDrivers with flags value of {}\n", static_cast(GlobalAdapter->InitDriversDesc.flags)); @@ -455,14 +467,6 @@ ur_adapter_handle_t_::ur_adapter_handle_t_() return; } - // Dynamically load the new L0 SysMan separate init and new EXP apis - // separately. This must be done to avoid attempting to use symbols that do - // not exist in older loader runtimes. -#ifdef _WIN32 - GlobalAdapter->processHandle = GetModuleHandle(NULL); -#else - GlobalAdapter->processHandle = nullptr; -#endif // Check if the user has enabled the default L0 SysMan initialization. const int UrSysmanZesinitEnable = [&UserForcedSysManInit] { @@ -484,6 +488,11 @@ ur_adapter_handle_t_::ur_adapter_handle_t_() ZesInitNeeded = true; } if (ZesInitNeeded) { +#ifdef UR_STATIC_LEVEL_ZERO + GlobalAdapter->getDeviceByUUIdFunctionPtr = zesDriverGetDeviceByUuidExp; + GlobalAdapter->getSysManDriversFunctionPtr = zesDriverGet; + GlobalAdapter->sysManInitFunctionPtr = zesInit; +#else GlobalAdapter->getDeviceByUUIdFunctionPtr = (zes_pfnDriverGetDeviceByUuidExp_t) ur_loader::LibLoader::getFunctionPtr( @@ -494,6 +503,7 @@ ur_adapter_handle_t_::ur_adapter_handle_t_() GlobalAdapter->sysManInitFunctionPtr = (zes_pfnInit_t)ur_loader::LibLoader::getFunctionPtr( GlobalAdapter->processHandle, "zesInit"); +#endif } if (GlobalAdapter->getDeviceByUUIdFunctionPtr && GlobalAdapter->getSysManDriversFunctionPtr && @@ -668,7 +678,13 @@ ur_result_t urAdapterRelease(ur_adapter_handle_t) { if (GlobalAdapter) { std::lock_guard Lock{GlobalAdapter->Mutex}; if (--GlobalAdapter->RefCount == 0) { - return adapterStateTeardown(); + auto result = adapterStateTeardown(); +#ifdef UR_STATIC_LEVEL_ZERO + // Given static linking of the L0 Loader, we must delay the loader's + // destruction of its context until after the UR Adapter is destroyed. + zelLoaderContextTeardown(); +#endif + return result; } } diff --git a/unified-runtime/source/adapters/level_zero/platform.cpp b/unified-runtime/source/adapters/level_zero/platform.cpp index 8f877c8b2a32e..1651c4829de58 100644 --- a/unified-runtime/source/adapters/level_zero/platform.cpp +++ b/unified-runtime/source/adapters/level_zero/platform.cpp @@ -354,52 +354,67 @@ ur_result_t ur_platform_handle_t_::initialize() { // Check if mutable command list extension is supported and initialize // function pointers. if (MutableCommandListSpecExtensionSupported) { +#ifdef UR_STATIC_LEVEL_ZERO + ZeMutableCmdListExt.zexCommandListGetNextCommandIdExp = + zeCommandListGetNextCommandIdExp; + ZeMutableCmdListExt.zexCommandListUpdateMutableCommandsExp = + zeCommandListUpdateMutableCommandsExp; + ZeMutableCmdListExt.zexCommandListUpdateMutableCommandSignalEventExp = + zeCommandListUpdateMutableCommandSignalEventExp; + ZeMutableCmdListExt.zexCommandListUpdateMutableCommandWaitEventsExp = + zeCommandListUpdateMutableCommandWaitEventsExp; + ZeMutableCmdListExt.zexCommandListUpdateMutableCommandKernelsExp = + zeCommandListUpdateMutableCommandKernelsExp; + ZeMutableCmdListExt.zexCommandListGetNextCommandIdWithKernelsExp = + zeCommandListGetNextCommandIdWithKernelsExp; +#else ZeMutableCmdListExt.zexCommandListGetNextCommandIdExp = (ze_pfnCommandListGetNextCommandIdExp_t) ur_loader::LibLoader::getFunctionPtr( GlobalAdapter->processHandle, "zeCommandListGetNextCommandIdExp"); - ZeMutableCmdListExt.Supported |= - ZeMutableCmdListExt.zexCommandListGetNextCommandIdExp != nullptr; ZeMutableCmdListExt.zexCommandListUpdateMutableCommandsExp = (ze_pfnCommandListUpdateMutableCommandsExp_t) ur_loader::LibLoader::getFunctionPtr( GlobalAdapter->processHandle, "zeCommandListUpdateMutableCommandsExp"); - ZeMutableCmdListExt.Supported |= - ZeMutableCmdListExt.zexCommandListUpdateMutableCommandsExp != nullptr; ZeMutableCmdListExt.zexCommandListUpdateMutableCommandSignalEventExp = (ze_pfnCommandListUpdateMutableCommandSignalEventExp_t) ur_loader::LibLoader::getFunctionPtr( GlobalAdapter->processHandle, "zeCommandListUpdateMutableCommandSignalEventExp"); - ZeMutableCmdListExt.Supported |= - ZeMutableCmdListExt.zexCommandListUpdateMutableCommandSignalEventExp != - nullptr; ZeMutableCmdListExt.zexCommandListUpdateMutableCommandWaitEventsExp = (ze_pfnCommandListUpdateMutableCommandWaitEventsExp_t) ur_loader::LibLoader::getFunctionPtr( GlobalAdapter->processHandle, "zeCommandListUpdateMutableCommandWaitEventsExp"); - ZeMutableCmdListExt.Supported |= - ZeMutableCmdListExt.zexCommandListUpdateMutableCommandWaitEventsExp != - nullptr; ZeMutableCmdListExt.zexCommandListUpdateMutableCommandKernelsExp = (ze_pfnCommandListUpdateMutableCommandKernelsExp_t) ur_loader::LibLoader::getFunctionPtr( GlobalAdapter->processHandle, "zeCommandListUpdateMutableCommandKernelsExp"); - ZeMutableCmdListExt.Supported |= - ZeMutableCmdListExt.zexCommandListUpdateMutableCommandKernelsExp != - nullptr; ZeMutableCmdListExt.zexCommandListGetNextCommandIdWithKernelsExp = (ze_pfnCommandListGetNextCommandIdWithKernelsExp_t) ur_loader::LibLoader::getFunctionPtr( GlobalAdapter->processHandle, "zeCommandListGetNextCommandIdWithKernelsExp"); +#endif + ZeMutableCmdListExt.Supported |= + ZeMutableCmdListExt.zexCommandListGetNextCommandIdExp != nullptr; ZeMutableCmdListExt.Supported |= ZeMutableCmdListExt.zexCommandListGetNextCommandIdWithKernelsExp != nullptr; + ZeMutableCmdListExt.Supported |= + ZeMutableCmdListExt.zexCommandListUpdateMutableCommandKernelsExp != + nullptr; + ZeMutableCmdListExt.Supported |= + ZeMutableCmdListExt.zexCommandListUpdateMutableCommandWaitEventsExp != + nullptr; + ZeMutableCmdListExt.Supported |= + ZeMutableCmdListExt.zexCommandListUpdateMutableCommandSignalEventExp != + nullptr; + ZeMutableCmdListExt.Supported |= + ZeMutableCmdListExt.zexCommandListUpdateMutableCommandsExp != nullptr; ZeMutableCmdListExt.LoaderExtension = true; } else { ZeMutableCmdListExt.Supported |= @@ -455,12 +470,18 @@ ur_result_t ur_platform_handle_t_::initialize() { // Check if ImmediateAppendCommandList is supported and initialize the // function pointer. if (ZeImmediateCommandListAppendExtensionFound) { +#ifdef UR_STATIC_LEVEL_ZERO + ZeCommandListImmediateAppendExt + .zeCommandListImmediateAppendCommandListsExp = + zeCommandListImmediateAppendCommandListsExp; +#else ZeCommandListImmediateAppendExt .zeCommandListImmediateAppendCommandListsExp = (ze_pfnCommandListImmediateAppendCommandListsExp_t) ur_loader::LibLoader::getFunctionPtr( GlobalAdapter->processHandle, "zeCommandListImmediateAppendCommandListsExp"); +#endif ZeCommandListImmediateAppendExt.Supported = ZeCommandListImmediateAppendExt .zeCommandListImmediateAppendCommandListsExp != nullptr; diff --git a/unified-runtime/test/adapters/level_zero/urKernelCreateWithNativeHandle.cpp b/unified-runtime/test/adapters/level_zero/urKernelCreateWithNativeHandle.cpp index faeb6c933c106..616f825bb2c57 100644 --- a/unified-runtime/test/adapters/level_zero/urKernelCreateWithNativeHandle.cpp +++ b/unified-runtime/test/adapters/level_zero/urKernelCreateWithNativeHandle.cpp @@ -34,6 +34,10 @@ TEST_P(urLevelZeroKernelNativeHandleTest, OwnedHandleRelease) { moduleDesc.pBuildFlags = ""; ze_module_handle_t module; + // Initialize Level Zero driver is required if this test is linked statically + // with Level Zero loader, the driver will not be init otherwise. + zeInit(ZE_INIT_FLAG_GPU_ONLY); + ASSERT_EQ( zeModuleCreate(native_context, native_device, &moduleDesc, &module, NULL), ZE_RESULT_SUCCESS); diff --git a/unified-runtime/test/adapters/level_zero/v2/command_list_cache_test.cpp b/unified-runtime/test/adapters/level_zero/v2/command_list_cache_test.cpp index 2019a913a87b8..44428deb801e6 100644 --- a/unified-runtime/test/adapters/level_zero/v2/command_list_cache_test.cpp +++ b/unified-runtime/test/adapters/level_zero/v2/command_list_cache_test.cpp @@ -19,7 +19,14 @@ #include #include -struct CommandListCacheTest : public uur::urContextTest {}; +struct CommandListCacheTest : public uur::urContextTest { + void SetUp() override { + // Initialize Level Zero driver is required if this test is linked + // statically with Level Zero loader, the driver will not be init otherwise. + zeInit(ZE_INIT_FLAG_GPU_ONLY); + urContextTest::SetUp(); + } +}; UUR_INSTANTIATE_DEVICE_TEST_SUITE(CommandListCacheTest); diff --git a/unified-runtime/test/adapters/level_zero/v2/deferred_kernel.cpp b/unified-runtime/test/adapters/level_zero/v2/deferred_kernel.cpp index 2c11dc8e43266..c5a6b5b4fcc64 100644 --- a/unified-runtime/test/adapters/level_zero/v2/deferred_kernel.cpp +++ b/unified-runtime/test/adapters/level_zero/v2/deferred_kernel.cpp @@ -13,6 +13,9 @@ struct urEnqueueKernelLaunchTest : uur::urKernelExecutionTest { void SetUp() override { + // Initialize Level Zero driver is required if this test is linked + // statically with Level Zero loader, the driver will not be init otherwise. + zeInit(ZE_INIT_FLAG_GPU_ONLY); program_name = "fill"; UUR_RETURN_ON_FATAL_FAILURE(urKernelExecutionTest::SetUp()); } diff --git a/unified-runtime/test/adapters/level_zero/v2/event_pool_test.cpp b/unified-runtime/test/adapters/level_zero/v2/event_pool_test.cpp index 7b28381e74def..9dc627d687658 100644 --- a/unified-runtime/test/adapters/level_zero/v2/event_pool_test.cpp +++ b/unified-runtime/test/adapters/level_zero/v2/event_pool_test.cpp @@ -109,6 +109,10 @@ struct EventPoolTest : public uur::urQueueTestWithParam { auto params = getParam(); + // Initialize Level Zero driver is required if this test is linked + // statically with Level Zero loader, the driver will not be init otherwise. + zeInit(ZE_INIT_FLAG_GPU_ONLY); + mockVec.push_back(device); cache = std::unique_ptr(new event_pool_cache(