Skip to content

Commit 7dbe8e9

Browse files
javier-inteljavier-intel
committed
Catch model import failure and report the appropriate error
1 parent e992272 commit 7dbe8e9

File tree

4 files changed

+182
-114
lines changed

4 files changed

+182
-114
lines changed

‎onnxruntime/core/providers/openvino/backend_manager.cc

Lines changed: 6 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
#include "core/providers/openvino/ov_interface.h"
2121
#include "core/providers/openvino/ov_versions/capability.h"
2222
#include "core/providers/openvino/qdq_transformations/qdq_stripping.h"
23+
#include "core/providers/openvino/exceptions.h"
2324

2425
namespace onnxruntime {
2526
namespace openvino_ep {
@@ -146,15 +147,13 @@ BackendManager::BackendManager(SessionContext& session_context,
146147
subgraph_context_,
147148
shared_context_,
148149
model_stream);
149-
} catch (const OnnxRuntimeException& ex) {
150-
std::string exception_str = ex.what();
150+
} catch (const ovep_exception& ex) {
151+
#ifndef OPENVINO_DISABLE_NPU_FALLBACK
151152
bool eligible_for_cpu_fallback = device_type.find("NPU") != std::string::npos &&
152153
!session_context_.so_disable_cpu_ep_fallback &&
153154
!subgraph_context_.is_ep_ctx_graph;
154-
#if defined(OPENVINO_DISABLE_NPU_FALLBACK)
155-
eligible_for_cpu_fallback = false;
156-
#else
157155
if (eligible_for_cpu_fallback) {
156+
std::string exception_str = ex.what();
158157
LOGS_DEFAULT(VERBOSE) << exception_str;
159158
LOGS_DEFAULT(WARNING) << "Model compilation failed at OV NPU."
160159
<< "Falling back to OV CPU for execution";
@@ -169,32 +168,10 @@ BackendManager::BackendManager(SessionContext& session_context,
169168
} catch (std::string const& msg) {
170169
ORT_THROW(msg);
171170
}
171+
} else {
172+
ORT_RETHROW;
172173
}
173174
#endif
174-
if (!eligible_for_cpu_fallback) {
175-
if (device_type.find("NPU") != std::string::npos &&
176-
exception_str.find("intel_npu") != std::string::npos) {
177-
// Handle NPU device related errors
178-
#ifndef NDEBUG
179-
ORT_THROW(exception_str + "\nModel needs to be recompiled\n");
180-
#else
181-
std::string error_message = "UNKNOWN NPU ERROR";
182-
std::string error_code = "code 0x0";
183-
std::regex error_message_pattern(R"(\bZE_\w*\b)");
184-
std::regex error_code_pattern("code 0x[0-9a-fA-F]+");
185-
std::smatch matches;
186-
if (std::regex_search(exception_str, matches, error_message_pattern)) {
187-
error_message = matches[0];
188-
}
189-
if (std::regex_search(exception_str, matches, error_code_pattern)) {
190-
error_code = matches[0];
191-
}
192-
throw std::runtime_error(error_message + ", " + error_code + "\nModel needs to be recompiled\n");
193-
#endif
194-
} else {
195-
ORT_THROW(exception_str);
196-
}
197-
}
198175
}
199176
}
200177
if (session_context_.so_context_enable && !subgraph_context_.is_ep_ctx_graph) {

‎onnxruntime/core/providers/openvino/exceptions.h

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
// Copyright (C) Intel Corporation
2+
// Licensed under the MIT License
3+
4+
#pragma once
5+
6+
#include <exception>
7+
#include <regex>
8+
#include <string>
9+
10+
#include "core/common/status.h"
11+
12+
namespace onnxruntime {
13+
namespace openvino_ep {
14+
15+
struct ovep_exception : public std::exception {
16+
enum class type {
17+
compile_model,
18+
import_model,
19+
query_prop,
20+
read_model,
21+
unknown,
22+
};
23+
24+
ovep_exception(const std::string& message,
25+
enum class type type) : message_{message},
26+
type_{type},
27+
error_code_{ze_result_code_from_string(message)},
28+
error_name_{ze_result_name_from_string(message)} {}
29+
//ovep_exception(const std::exception& ex) : message_{ex.what()} {}
30+
31+
const char* what() const noexcept override {
32+
return message_.data();
33+
}
34+
35+
uint32_t get_code() const { return error_code_; }
36+
37+
operator common::Status() const {
38+
common::StatusCategory category_ort{common::ONNXRUNTIME};
39+
40+
if (type_ == type::unknown) {
41+
return {category_ort, common::FAIL, message_};
42+
}
43+
44+
// Newer drivers
45+
if ((type_ == type::import_model) &&
46+
(error_code_ == 0x7800000f /* ZE_RESULT_ERROR_INVALID_NATIVE_BINARY */)) {
47+
std::string message{error_name_ + ", code 0x" + std::to_string(error_code_) + "\nModel needs to be recompiled\n"};
48+
return {category_ort, common::INVALID_GRAPH, message};
49+
}
50+
51+
return Status::OK();
52+
}
53+
54+
protected:
55+
std::string message_;
56+
type type_{type::unknown};
57+
uint32_t error_code_{0};
58+
std::string error_name_;
59+
60+
private:
61+
uint32_t ze_result_code_from_string(const std::string &ov_exception_string) {
62+
uint32_t error_code{0};
63+
std::regex error_code_pattern("code 0x([0-9a-fA-F]+)");
64+
std::smatch matches;
65+
if (std::regex_search(ov_exception_string, matches, error_code_pattern)) {
66+
std::from_chars(&(*matches[1].first), &(*matches[1].second), error_code, 16);
67+
}
68+
// std::string message{error_message + ", code 0x" + std::to_string(error_code) + "\nModel needs to be recompiled\n"};
69+
return error_code;
70+
}
71+
std::string ze_result_name_from_string(const std::string &ov_exception_string) {
72+
std::string error_message = "UNKNOWN NPU ERROR";
73+
std::regex error_message_pattern(R"(\bZE_\w*\b)");
74+
std::smatch matches;
75+
if (std::regex_search(ov_exception_string, matches, error_message_pattern)) {
76+
error_message = matches[0];
77+
}
78+
// std::string message{error_message + ", code 0x" + std::to_string(error_code) + "\nModel needs to be recompiled\n"};
79+
return error_message;
80+
}
81+
};
82+
83+
} // namespace openvino_ep
84+
} // namespace onnxruntime

‎onnxruntime/core/providers/openvino/openvino_execution_provider.cc

Lines changed: 89 additions & 84 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
#include "core/providers/openvino/onnx_ctx_model_helper.h"
1313
#include "core/providers/openvino/ov_versions/capability.h"
1414
#include "core/providers/openvino/qdq_transformations/qdq_stripping.h"
15+
#include "core/providers/openvino/exceptions.h"
1516
#include "core/session/onnxruntime_session_options_config_keys.h"
1617
#include "openvino/core/version.hpp"
1718
#ifdef USE_OVEP_NPU_MEMORY
@@ -94,101 +95,105 @@ common::Status OpenVINOExecutionProvider::Compile(
9495
auto& logger = *GetLogger();
9596
Status status = Status::OK();
9697

97-
if (!fused_nodes.empty()) {
98-
// Assume these properties are constant for all the model subgraphs, otherwise move to SubGraphContext
99-
const auto& graph_body_viewer_0 = fused_nodes[0].filtered_graph.get();
100-
session_context_.onnx_model_path_name = graph_body_viewer_0.ModelPath().string();
101-
session_context_.onnx_opset_version =
102-
graph_body_viewer_0.DomainToVersionMap().at(kOnnxDomain);
103-
}
104-
105-
// Temporary code to read metadata before it moves to the .bin
106-
auto& metadata = shared_context_->shared_weights.metadata;
107-
if (session_context_.so_share_ep_contexts && metadata.empty()) {
108-
// Metadata is always read from model location, this could be a source or epctx model
109-
fs::path metadata_filename = session_context_.onnx_model_path_name.parent_path() / "metadata.bin";
110-
std::ifstream file(metadata_filename, std::ios::binary);
111-
if (file) {
112-
file >> metadata;
98+
try {
99+
if (!fused_nodes.empty()) {
100+
// Assume these properties are constant for all the model subgraphs, otherwise move to SubGraphContext
101+
const auto& graph_body_viewer_0 = fused_nodes[0].filtered_graph.get();
102+
session_context_.onnx_model_path_name = graph_body_viewer_0.ModelPath().string();
103+
session_context_.onnx_opset_version =
104+
graph_body_viewer_0.DomainToVersionMap().at(kOnnxDomain);
113105
}
114-
}
115106

116-
struct OpenVINOEPFunctionState {
117-
AllocateFunc allocate_func = nullptr;
118-
DestroyFunc destroy_func = nullptr;
119-
AllocatorHandle allocator_handle = nullptr;
120-
BackendManager& backend_manager;
121-
};
122-
123-
for (const FusedNodeAndGraph& fused_node_graph : fused_nodes) {
124-
const GraphViewer& graph_body_viewer = fused_node_graph.filtered_graph;
125-
const Node& fused_node = fused_node_graph.fused_node;
126-
127-
NodeComputeInfo compute_info;
128-
129-
// During backend creation, we check if user wants to use precompiled blob onnx model or the original model
130-
// For precompiled blob, directly load the model instead of compiling the model
131-
// For original model, check if the user wants to export a model with pre-compiled blob
132-
133-
auto& backend_manager = backend_managers_.emplace_back(session_context_,
134-
*shared_context_,
135-
fused_node,
136-
graph_body_viewer,
137-
logger,
138-
ep_ctx_handle_);
139-
140-
compute_info.create_state_func =
141-
[&backend_manager](ComputeContext* context, FunctionState* state) {
142-
OpenVINOEPFunctionState* p = new OpenVINOEPFunctionState{
143-
.allocate_func = context->allocate_func,
144-
.destroy_func = context->release_func,
145-
.allocator_handle = context->allocator_handle,
146-
.backend_manager = backend_manager};
147-
*state = static_cast<FunctionState>(p);
148-
return 0;
149-
};
150-
151-
compute_info.compute_func = [](FunctionState state, const OrtApi* /* api */, OrtKernelContext* context) {
152-
auto function_state = static_cast<OpenVINOEPFunctionState*>(state);
153-
try {
154-
function_state->backend_manager.Compute(context);
155-
} catch (const std::exception& ex) {
156-
return common::Status(common::ONNXRUNTIME, common::FAIL, ex.what());
107+
// Temporary code to read metadata before it moves to the .bin
108+
auto& metadata = shared_context_->shared_weights.metadata;
109+
if (session_context_.so_share_ep_contexts && metadata.empty()) {
110+
// Metadata is always read from model location, this could be a source or epctx model
111+
fs::path metadata_filename = session_context_.onnx_model_path_name.parent_path() / "metadata.bin";
112+
std::ifstream file(metadata_filename, std::ios::binary);
113+
if (file) {
114+
file >> metadata;
157115
}
158-
return Status::OK();
116+
}
117+
118+
struct OpenVINOEPFunctionState {
119+
AllocateFunc allocate_func = nullptr;
120+
DestroyFunc destroy_func = nullptr;
121+
AllocatorHandle allocator_handle = nullptr;
122+
BackendManager& backend_manager;
159123
};
160124

161-
compute_info.release_state_func =
162-
[](FunctionState state) {
163-
if (state) {
164-
OpenVINOEPFunctionState* function_state = static_cast<OpenVINOEPFunctionState*>(state);
165-
delete function_state;
166-
}
167-
};
125+
for (const FusedNodeAndGraph& fused_node_graph : fused_nodes) {
126+
const GraphViewer& graph_body_viewer = fused_node_graph.filtered_graph;
127+
const Node& fused_node = fused_node_graph.fused_node;
128+
129+
NodeComputeInfo compute_info;
130+
131+
// During backend creation, we check if user wants to use precompiled blob onnx model or the original model
132+
// For precompiled blob, directly load the model instead of compiling the model
133+
// For original model, check if the user wants to export a model with pre-compiled blob
134+
135+
auto& backend_manager = backend_managers_.emplace_back(session_context_,
136+
*shared_context_,
137+
fused_node,
138+
graph_body_viewer,
139+
logger,
140+
ep_ctx_handle_);
141+
142+
compute_info.create_state_func =
143+
[&backend_manager](ComputeContext* context, FunctionState* state) {
144+
OpenVINOEPFunctionState* p = new OpenVINOEPFunctionState{
145+
.allocate_func = context->allocate_func,
146+
.destroy_func = context->release_func,
147+
.allocator_handle = context->allocator_handle,
148+
.backend_manager = backend_manager};
149+
*state = static_cast<FunctionState>(p);
150+
return 0;
151+
};
152+
153+
compute_info.compute_func = [](FunctionState state, const OrtApi* /* api */, OrtKernelContext* context) {
154+
auto function_state = static_cast<OpenVINOEPFunctionState*>(state);
155+
try {
156+
function_state->backend_manager.Compute(context);
157+
} catch (const std::exception& ex) {
158+
return common::Status(common::ONNXRUNTIME, common::FAIL, ex.what());
159+
}
160+
return Status::OK();
161+
};
168162

169-
node_compute_funcs.push_back(std::move(compute_info));
163+
compute_info.release_state_func =
164+
[](FunctionState state) {
165+
if (state) {
166+
OpenVINOEPFunctionState* function_state = static_cast<OpenVINOEPFunctionState*>(state);
167+
delete function_state;
168+
}
169+
};
170170

171-
if (!status.IsOK()) {
172-
break;
173-
}
174-
}
171+
node_compute_funcs.push_back(std::move(compute_info));
175172

176-
if (session_context_.so_share_ep_contexts) {
177-
fs::path metadata_filename;
178-
if (session_context_.so_context_file_path.empty()) {
179-
metadata_filename = session_context_.onnx_model_path_name.parent_path() / "metadata.bin";
180-
} else {
181-
metadata_filename = session_context_.so_context_file_path.parent_path() / "metadata.bin";
173+
if (!status.IsOK()) {
174+
break;
175+
}
182176
}
183177

184-
// Metadata is generated only for shared contexts
185-
// If saving metadata then save it to the provided path or ose the original model path
186-
// Multiple calls to Compile() will update the metadata and for the last call
187-
// the resulting file will contain the aggregated content
188-
std::ofstream file(metadata_filename, std::ios::binary);
189-
if (file) {
190-
file << metadata;
178+
if (session_context_.so_share_ep_contexts) {
179+
fs::path metadata_filename;
180+
if (session_context_.so_context_file_path.empty()) {
181+
metadata_filename = session_context_.onnx_model_path_name.parent_path() / "metadata.bin";
182+
} else {
183+
metadata_filename = session_context_.so_context_file_path.parent_path() / "metadata.bin";
184+
}
185+
186+
// Metadata is generated only for shared contexts
187+
// If saving metadata then save it to the provided path or ose the original model path
188+
// Multiple calls to Compile() will update the metadata and for the last call
189+
// the resulting file will contain the aggregated content
190+
std::ofstream file(metadata_filename, std::ios::binary);
191+
if (file) {
192+
file << metadata;
193+
}
191194
}
195+
} catch (ovep_exception ex) {
196+
status = ex;
192197
}
193198

194199
return status;

‎onnxruntime/core/providers/openvino/ov_interface.cc

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99
#include "core/providers/openvino/backend_utils.h"
1010
#include "core/providers/openvino/backends/basic_backend.h"
1111
#include "core/providers/openvino/ov_stateful_patch_utils.h"
12+
#include "core/providers/openvino/exceptions.h"
1213

1314
using Exception = ov::Exception;
1415

@@ -203,7 +204,8 @@ OVExeNetwork OVCore::ImportModel(std::istream& model_stream,
203204
OVExeNetwork exe(obj, hw_target);
204205
return exe;
205206
} catch (const Exception& e) {
206-
ORT_THROW(log_tag + " Exception while Loading Network for graph: " + name + e.what());
207+
std::string message = log_tag + " Exception while Loading Network for graph: " + name + e.what();
208+
throw ovep_exception(message, ovep_exception::type::import_model);
207209
} catch (...) {
208210
ORT_THROW(log_tag + " Exception while Loading Network for graph " + name);
209211
}

0 commit comments

Comments
 (0)