Update UR to link the Loader as static (#17104)

Migrated from oneapi-src#2667

---------

Signed-off-by: Neil R. Spruit <[email protected]>

Author: nrspruit

cmake/FetchLevelZero.cmake

@@ -37,12 +37,13 @@ if (NOT DEFINED LEVEL_ZERO_LIBRARY OR NOT DEFINED LEVEL_ZERO_INCLUDE_DIR)
         set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-c++98-compat-extra-semi")
         set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unknown-warning-option")
     endif()
+    set(BUILD_STATIC ON)
 
     if (UR_LEVEL_ZERO_LOADER_REPO STREQUAL "")
         set(UR_LEVEL_ZERO_LOADER_REPO "https://github.com/oneapi-src/level-zero.git")
     endif()
     if (UR_LEVEL_ZERO_LOADER_TAG STREQUAL "")
-        set(UR_LEVEL_ZERO_LOADER_TAG v1.19.2)
+        set(UR_LEVEL_ZERO_LOADER_TAG v1.21.1)
     endif()
 
     # Disable due to a bug https://github.com/oneapi-src/level-zero/issues/104

source/adapters/level_zero/CMakeLists.txt

@@ -55,13 +55,14 @@ if(UR_BUILD_ADAPTER_L0)
     )
     install_ur_library(ur_adapter_level_zero)
 
+    target_compile_definitions(ur_adapter_level_zero PUBLIC UR_STATIC_LEVEL_ZERO)
     if(UR_STATIC_ADAPTER_L0)
         target_compile_definitions(ur_adapter_level_zero PUBLIC UR_STATIC_ADAPTER_LEVEL_ZERO)
 
         # 'utils' target from 'level-zero-loader' includes path which is prefixed
         # in the source directory, this breaks the installation of 'utils' target.
-        set_target_properties(utils PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "")
-        install(TARGETS ur_umf LevelZeroLoader LevelZeroLoader-Headers ComputeRuntimeLevelZero-Headers ze_loader utils
+        set_target_properties(level_zero_utils PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "")
+        install(TARGETS ur_umf LevelZeroLoader LevelZeroLoader-Headers ComputeRuntimeLevelZero-Headers ze_loader level_zero_utils
             EXPORT ${PROJECT_NAME}-targets
             ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
             RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}

source/adapters/level_zero/adapter.cpp

@@ -300,6 +300,12 @@ ur_adapter_handle_t_::ur_adapter_handle_t_()
   ZeInitResult = ZE_RESULT_ERROR_UNINITIALIZED;
   ZesResult = ZE_RESULT_ERROR_UNINITIALIZED;
 
+#ifdef UR_STATIC_LEVEL_ZERO
+  // Given static linking of the L0 Loader, we must delay the loader's
+  // destruction of its context until after the UR Adapter is destroyed.
+  zelSetDelayLoaderContextTeardown();
+#endif
+
   if (UrL0Debug & UR_L0_DEBUG_BASIC) {
     logger.setLegacySink(std::make_unique<ur_legacy_sink>());
   };
@@ -335,10 +341,12 @@ ur_adapter_handle_t_::ur_adapter_handle_t_()
     // Dynamically load the new L0 apis separately.
     // This must be done to avoid attempting to use symbols that do
     // not exist in older loader runtimes.
+#ifndef UR_STATIC_LEVEL_ZERO
 #ifdef _WIN32
-    HMODULE processHandle = GetModuleHandle(NULL);
+    GlobalAdapter->processHandle = GetModuleHandle(NULL);
 #else
-    HMODULE processHandle = nullptr;
+    GlobalAdapter->processHandle = nullptr;
+#endif
 #endif
 
     // initialize level zero only once.
@@ -412,9 +420,13 @@ ur_adapter_handle_t_::ur_adapter_handle_t_()
       }
 
       if (useInitDrivers) {
+#ifdef UR_STATIC_LEVEL_ZERO
+        GlobalAdapter->initDriversFunctionPtr = zeInitDrivers;
+#else
         GlobalAdapter->initDriversFunctionPtr =
             (ze_pfnInitDrivers_t)ur_loader::LibLoader::getFunctionPtr(
-                processHandle, "zeInitDrivers");
+                GlobalAdapter->processHandle, "zeInitDrivers");
+#endif
         if (GlobalAdapter->initDriversFunctionPtr) {
           logger::debug("\nzeInitDrivers with flags value of {}\n",
                         static_cast<int>(GlobalAdapter->InitDriversDesc.flags));
@@ -455,14 +467,6 @@ ur_adapter_handle_t_::ur_adapter_handle_t_()
 
       return;
     }
-    // Dynamically load the new L0 SysMan separate init and new EXP apis
-    // separately. This must be done to avoid attempting to use symbols that do
-    // not exist in older loader runtimes.
-#ifdef _WIN32
-    GlobalAdapter->processHandle = GetModuleHandle(NULL);
-#else
-    GlobalAdapter->processHandle = nullptr;
-#endif
 
     // Check if the user has enabled the default L0 SysMan initialization.
     const int UrSysmanZesinitEnable = [&UserForcedSysManInit] {
@@ -484,6 +488,11 @@ ur_adapter_handle_t_::ur_adapter_handle_t_()
       ZesInitNeeded = true;
     }
     if (ZesInitNeeded) {
+#ifdef UR_STATIC_LEVEL_ZERO
+      GlobalAdapter->getDeviceByUUIdFunctionPtr = zesDriverGetDeviceByUuidExp;
+      GlobalAdapter->getSysManDriversFunctionPtr = zesDriverGet;
+      GlobalAdapter->sysManInitFunctionPtr = zesInit;
+#else
       GlobalAdapter->getDeviceByUUIdFunctionPtr =
           (zes_pfnDriverGetDeviceByUuidExp_t)
               ur_loader::LibLoader::getFunctionPtr(
@@ -494,6 +503,7 @@ ur_adapter_handle_t_::ur_adapter_handle_t_()
       GlobalAdapter->sysManInitFunctionPtr =
           (zes_pfnInit_t)ur_loader::LibLoader::getFunctionPtr(
               GlobalAdapter->processHandle, "zesInit");
+#endif
     }
     if (GlobalAdapter->getDeviceByUUIdFunctionPtr &&
         GlobalAdapter->getSysManDriversFunctionPtr &&
@@ -668,7 +678,13 @@ ur_result_t urAdapterRelease(ur_adapter_handle_t) {
   if (GlobalAdapter) {
     std::lock_guard<std::mutex> Lock{GlobalAdapter->Mutex};
     if (--GlobalAdapter->RefCount == 0) {
-      return adapterStateTeardown();
+      auto result = adapterStateTeardown();
+#ifdef UR_STATIC_LEVEL_ZERO
+      // Given static linking of the L0 Loader, we must delay the loader's
+      // destruction of its context until after the UR Adapter is destroyed.
+      zelLoaderContextTeardown();
+#endif
+      return result;
     }
   }
 

source/adapters/level_zero/platform.cpp

@@ -354,52 +354,67 @@ ur_result_t ur_platform_handle_t_::initialize() {
   // Check if mutable command list extension is supported and initialize
   // function pointers.
   if (MutableCommandListSpecExtensionSupported) {
+#ifdef UR_STATIC_LEVEL_ZERO
+    ZeMutableCmdListExt.zexCommandListGetNextCommandIdExp =
+        zeCommandListGetNextCommandIdExp;
+    ZeMutableCmdListExt.zexCommandListUpdateMutableCommandsExp =
+        zeCommandListUpdateMutableCommandsExp;
+    ZeMutableCmdListExt.zexCommandListUpdateMutableCommandSignalEventExp =
+        zeCommandListUpdateMutableCommandSignalEventExp;
+    ZeMutableCmdListExt.zexCommandListUpdateMutableCommandWaitEventsExp =
+        zeCommandListUpdateMutableCommandWaitEventsExp;
+    ZeMutableCmdListExt.zexCommandListUpdateMutableCommandKernelsExp =
+        zeCommandListUpdateMutableCommandKernelsExp;
+    ZeMutableCmdListExt.zexCommandListGetNextCommandIdWithKernelsExp =
+        zeCommandListGetNextCommandIdWithKernelsExp;
+#else
     ZeMutableCmdListExt.zexCommandListGetNextCommandIdExp =
         (ze_pfnCommandListGetNextCommandIdExp_t)
             ur_loader::LibLoader::getFunctionPtr(
                 GlobalAdapter->processHandle,
                 "zeCommandListGetNextCommandIdExp");
-    ZeMutableCmdListExt.Supported |=
-        ZeMutableCmdListExt.zexCommandListGetNextCommandIdExp != nullptr;
     ZeMutableCmdListExt.zexCommandListUpdateMutableCommandsExp =
         (ze_pfnCommandListUpdateMutableCommandsExp_t)
             ur_loader::LibLoader::getFunctionPtr(
                 GlobalAdapter->processHandle,
                 "zeCommandListUpdateMutableCommandsExp");
-    ZeMutableCmdListExt.Supported |=
-        ZeMutableCmdListExt.zexCommandListUpdateMutableCommandsExp != nullptr;
     ZeMutableCmdListExt.zexCommandListUpdateMutableCommandSignalEventExp =
         (ze_pfnCommandListUpdateMutableCommandSignalEventExp_t)
             ur_loader::LibLoader::getFunctionPtr(
                 GlobalAdapter->processHandle,
                 "zeCommandListUpdateMutableCommandSignalEventExp");
-    ZeMutableCmdListExt.Supported |=
-        ZeMutableCmdListExt.zexCommandListUpdateMutableCommandSignalEventExp !=
-        nullptr;
     ZeMutableCmdListExt.zexCommandListUpdateMutableCommandWaitEventsExp =
         (ze_pfnCommandListUpdateMutableCommandWaitEventsExp_t)
             ur_loader::LibLoader::getFunctionPtr(
                 GlobalAdapter->processHandle,
                 "zeCommandListUpdateMutableCommandWaitEventsExp");
-    ZeMutableCmdListExt.Supported |=
-        ZeMutableCmdListExt.zexCommandListUpdateMutableCommandWaitEventsExp !=
-        nullptr;
     ZeMutableCmdListExt.zexCommandListUpdateMutableCommandKernelsExp =
         (ze_pfnCommandListUpdateMutableCommandKernelsExp_t)
             ur_loader::LibLoader::getFunctionPtr(
                 GlobalAdapter->processHandle,
                 "zeCommandListUpdateMutableCommandKernelsExp");
-    ZeMutableCmdListExt.Supported |=
-        ZeMutableCmdListExt.zexCommandListUpdateMutableCommandKernelsExp !=
-        nullptr;
     ZeMutableCmdListExt.zexCommandListGetNextCommandIdWithKernelsExp =
         (ze_pfnCommandListGetNextCommandIdWithKernelsExp_t)
             ur_loader::LibLoader::getFunctionPtr(
                 GlobalAdapter->processHandle,
                 "zeCommandListGetNextCommandIdWithKernelsExp");
+#endif
+    ZeMutableCmdListExt.Supported |=
+        ZeMutableCmdListExt.zexCommandListGetNextCommandIdExp != nullptr;
     ZeMutableCmdListExt.Supported |=
         ZeMutableCmdListExt.zexCommandListGetNextCommandIdWithKernelsExp !=
         nullptr;
+    ZeMutableCmdListExt.Supported |=
+        ZeMutableCmdListExt.zexCommandListUpdateMutableCommandKernelsExp !=
+        nullptr;
+    ZeMutableCmdListExt.Supported |=
+        ZeMutableCmdListExt.zexCommandListUpdateMutableCommandWaitEventsExp !=
+        nullptr;
+    ZeMutableCmdListExt.Supported |=
+        ZeMutableCmdListExt.zexCommandListUpdateMutableCommandSignalEventExp !=
+        nullptr;
+    ZeMutableCmdListExt.Supported |=
+        ZeMutableCmdListExt.zexCommandListUpdateMutableCommandsExp != nullptr;
     ZeMutableCmdListExt.LoaderExtension = true;
   } else {
     ZeMutableCmdListExt.Supported |=
@@ -455,12 +470,18 @@ ur_result_t ur_platform_handle_t_::initialize() {
   // Check if ImmediateAppendCommandList is supported and initialize the
   // function pointer.
   if (ZeImmediateCommandListAppendExtensionFound) {
+#ifdef UR_STATIC_LEVEL_ZERO
+    ZeCommandListImmediateAppendExt
+        .zeCommandListImmediateAppendCommandListsExp =
+        zeCommandListImmediateAppendCommandListsExp;
+#else
     ZeCommandListImmediateAppendExt
         .zeCommandListImmediateAppendCommandListsExp =
         (ze_pfnCommandListImmediateAppendCommandListsExp_t)
             ur_loader::LibLoader::getFunctionPtr(
                 GlobalAdapter->processHandle,
                 "zeCommandListImmediateAppendCommandListsExp");
+#endif
     ZeCommandListImmediateAppendExt.Supported =
         ZeCommandListImmediateAppendExt
             .zeCommandListImmediateAppendCommandListsExp != nullptr;

test/adapters/level_zero/urKernelCreateWithNativeHandle.cpp

@@ -34,6 +34,10 @@ TEST_P(urLevelZeroKernelNativeHandleTest, OwnedHandleRelease) {
   moduleDesc.pBuildFlags = "";
   ze_module_handle_t module;
 
+  // Initialize Level Zero driver is required if this test is linked statically
+  // with Level Zero loader, the driver will not be init otherwise.
+  zeInit(ZE_INIT_FLAG_GPU_ONLY);
+
   ASSERT_EQ(
       zeModuleCreate(native_context, native_device, &moduleDesc, &module, NULL),
       ZE_RESULT_SUCCESS);

test/adapters/level_zero/v2/command_list_cache_test.cpp

@@ -19,7 +19,14 @@
 #include <unordered_set>
 #include <vector>
 
-struct CommandListCacheTest : public uur::urContextTest {};
+struct CommandListCacheTest : public uur::urContextTest {
+  void SetUp() override {
+    // Initialize Level Zero driver is required if this test is linked
+    // statically with Level Zero loader, the driver will not be init otherwise.
+    zeInit(ZE_INIT_FLAG_GPU_ONLY);
+    urContextTest::SetUp();
+  }
+};
 
 UUR_INSTANTIATE_DEVICE_TEST_SUITE(CommandListCacheTest);
 

test/adapters/level_zero/v2/deferred_kernel.cpp

@@ -13,6 +13,9 @@
 
 struct urEnqueueKernelLaunchTest : uur::urKernelExecutionTest {
   void SetUp() override {
+    // Initialize Level Zero driver is required if this test is linked
+    // statically with Level Zero loader, the driver will not be init otherwise.
+    zeInit(ZE_INIT_FLAG_GPU_ONLY);
     program_name = "fill";
     UUR_RETURN_ON_FATAL_FAILURE(urKernelExecutionTest::SetUp());
   }

test/adapters/level_zero/v2/event_pool_test.cpp

@@ -109,6 +109,10 @@ struct EventPoolTest : public uur::urQueueTestWithParam<ProviderParams> {
 
     auto params = getParam();
 
+    // Initialize Level Zero driver is required if this test is linked
+    // statically with Level Zero loader, the driver will not be init otherwise.
+    zeInit(ZE_INIT_FLAG_GPU_ONLY);
+
     mockVec.push_back(device);
 
     cache = std::unique_ptr<event_pool_cache>(new event_pool_cache(