Skip to content

OVEP Dynamic WorkloadType support #22779

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Nov 10, 2024
Merged

OVEP Dynamic WorkloadType support #22779

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
8815549
Add support to set workloadType dynamically in OVEP
preetha-intel Nov 8, 2024
65a2576
Fix lint issues
preetha-intel Nov 8, 2024
0171107
Use defined config key names instead of literals inside OVEP
javier-intel Nov 8, 2024
6e88240
Fix inference with EP context embed mode 0
javier-intel Nov 7, 2024
9aa8c12
Fix lint issues
javier-intel Nov 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions onnxruntime/core/providers/openvino/backend_manager.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,11 @@ GlobalContext& BackendManager::GetGlobalContext() {
return global_context_;
}

ov::CompiledModel& BackendManager::GetOVCompiledModel() {
ov::CompiledModel& ov_ptr = concrete_backend_->GetOVCompiledModel();
return (ov_ptr);
}

BackendManager::BackendManager(const GlobalContext& global_context,
const onnxruntime::Node& fused_node,
const onnxruntime::GraphViewer& subgraph,
Expand Down
1 change: 1 addition & 0 deletions onnxruntime/core/providers/openvino/backend_manager.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ class BackendManager {
GlobalContext& GetGlobalContext();
Status ExportCompiledBlobAsEPCtxNode(const onnxruntime::GraphViewer& subgraph,
const logging::Logger& logger);
ov::CompiledModel& GetOVCompiledModel();

private:
std::unique_ptr<ONNX_NAMESPACE::ModelProto> GetModelProtoFromFusedNode(
Expand Down
36 changes: 35 additions & 1 deletion onnxruntime/core/providers/openvino/openvino_execution_provider.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
// Licensed under the MIT License
#include <filesystem>
#include <utility>
#include <string>

#include "core/providers/shared_library/provider_api.h"
#include "core/providers/openvino/openvino_execution_provider.h"
Expand Down Expand Up @@ -150,7 +151,7 @@ common::Status OpenVINOExecutionProvider::Compile(
graph_body_viewer,
*GetLogger(),
ep_ctx_handle_);

backend_manager_ = backend_manager;
compute_info.create_state_func =
[backend_manager](ComputeContext* context, FunctionState* state) {
OpenVINOEPFunctionState* p = new OpenVINOEPFunctionState();
Expand Down Expand Up @@ -198,4 +199,37 @@ std::vector<AllocatorPtr> OpenVINOExecutionProvider::CreatePreferredAllocators()
}
#endif

common::Status OpenVINOExecutionProvider::SetEpDynamicOptions(gsl::span<const char* const> keys,
gsl::span<const char* const> values) {
std::string workload_type = "";
// Ensure the number of keys and values match
if (keys.size() != values.size()) {
return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "Mismatched keys and values sizes.");
}

for (size_t i = 0; i < keys.size(); ++i) {
std::string key = keys[i];
std::string value = values[i];

if (key == "ep.dynamic.workload_type") {
if (value == "Efficient") {
workload_type = "EFFICIENT";
} else if (value == "Default") {
workload_type = "DEFAULT";
} else {
LOGS_DEFAULT(WARNING) << "Unknown workload_type - ignoring " << key << "/" << value;
LOGS_DEFAULT(WARNING) << "Supported types are 'Efficient' and 'Default' \n";
}
if (workload_type != "") {
LOGS_DEFAULT(INFO) << "SetEpDynamicOptions - modifying: " << key << "/" << value;
ov::CompiledModel& ov_compiled_model = backend_manager_->GetOVCompiledModel();
ov_compiled_model.set_property(ov::workload_type(workload_type));
}
} else {
// Handle unknown options
LOGS_DEFAULT(WARNING) << "Unknown key/value pair - ignoring " << key << "/" << value;
}
}
return Status::OK();
}
} // namespace onnxruntime
4 changes: 4 additions & 0 deletions onnxruntime/core/providers/openvino/openvino_execution_provider.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -188,6 +188,9 @@ class OpenVINOExecutionProvider : public IExecutionProvider {
Status Compile(const std::vector<FusedNodeAndGraph>& fused_nodes,
std::vector<NodeComputeInfo>& node_compute_funcs) override;

Status SetEpDynamicOptions(gsl::span<const char* const> /*keys*/,
gsl::span<const char* const> /*values*/) override;

const void* GetExecutionHandle() const noexcept override {
return nullptr;
}
Expand All @@ -197,6 +200,7 @@ class OpenVINOExecutionProvider : public IExecutionProvider {
private:
std::unique_ptr<openvino_ep::GlobalContext> global_context_;
openvino_ep::EPCtxHandler ep_ctx_handle_{};
std::shared_ptr<openvino_ep::BackendManager> backend_manager_;
};

} // namespace onnxruntime
Loading