Configuring Devices

Introduction

The Inference Engine API provides the ability to configure devices via configuration keys and get device specific metrics. The values taken from InferenceEngine::Core::GetConfig are requested by the string name, while the return type is InferenceEngine::Parameter, making users lost on what the actual type stored in this parameter is.

The OpenVINO Runtime API 2.0 solves these issues by introducing properties, which unify metrics and configuration key concepts. Their main advantage is that they have the C++ type:

static constexpr Property<std::string> full_name{"FULL_DEVICE_NAME"};

And the property can be requested from an inference device as:

// 'auto' is automatically deduced as std::string
// since the type is stored in the property
auto full_device_name = core.get_property("CPU", ov::device::full_name);

The snippets below show how to migrate from an Inference Engine device configuration to OpenVINO Runtime API 2.0 steps.

Set configuration values

Inference Engine API:

core.SetConfig({ { CONFIG_KEY(PERF_COUNT), CONFIG_VALUE(YES) } }, "CPU");
auto exec_network = core.LoadNetwork(model, "MULTI", {
    { MULTI_CONFIG_KEY(DEVICE_PRIORITIES), "CPU, GPU" },
    { CONFIG_KEY(PERFORMANCE_HINT), CONFIG_VALUE(THROUGHPUT) },
    { CONFIG_KEY(ENFORCE_BF16), CONFIG_VALUE(NO) } });
// turn CPU off for multi-device execution
exec_network.SetConfig({ { MULTI_CONFIG_KEY(DEVICE_PRIORITIES), "GPU" } });
core.set_config({"PERF_COUNT": "YES"}, "CPU")
exec_network = core.load_network(net, "MULTI", {"DEVICE_PRIORITIES": "CPU, GPU",
                                                "PERFORMANCE_HINT": "THROUGHPUT",
                                                "ENFORCE_BF16": "NO"})
# turn CPU off for multi-device execution
exec_network.set_config({"DEVICE_PRIORITIES": "GPU"})

OpenVINO Runtime API 2.0:

core.set_property("CPU", ov::enable_profiling(true));
// turn CPU off for multi-device execution
compiled_model.set_property(ov::device::priorities("GPU"));
core.set_property(device_name="CPU", properties={"PERF_COUNT": "YES"})
compiled_model = core.compile_model(model=model, device_name="MULTI", config=
    {
        "MULTI_DEVICE_PRIORITIES": "GPU,CPU",
        "PERFORMANCE_HINT": "THROUGHPUT",
        "INFERENCE_PRECISION_HINT": "f32"
    })
# turn CPU off for multi-device execution
compiled_model.set_property(properties={"MULTI_DEVICE_PRIORITIES": "GPU"})

Get information

Inference Engine API:

// a user has to parse std::string after
auto num_streams = core.GetConfig("CPU", CONFIG_KEY(CPU_THROUGHPUT_STREAMS)).as<std::string>();
auto full_device_name = core.GetMetric("CPU", METRIC_KEY(FULL_DEVICE_NAME)).as<std::string>();
std::string perf_model = exec_network.GetConfig(CONFIG_KEY(PERFORMANCE_HINT)).as<std::string>();
auto nireq = exec_network.GetMetric(EXEC_NETWORK_METRIC_KEY(OPTIMAL_NUMBER_OF_INFER_REQUESTS)).as<uint32_t>();
num_streams = core.get_config("CPU", "CPU_THROUGHPUT_STREAMS")
full_device_name = core.get_metric("CPU", "FULL_DEVICE_NAME")
perf_hint = exec_network.get_config("PERFORMANCE_HINT")
nireq = exec_network.get_metric("OPTIMAL_NUMBER_OF_INFER_REQUESTS")

OpenVINO Runtime API 2.0:

// 'auto' is automatically deduced as ov::streams::Num
// since the type is stored in the property
auto num_streams = core.get_property("CPU", ov::streams::num);
// 'auto' is automatically deduced as std::string
// since the type is stored in the property
auto full_device_name = core.get_property("CPU", ov::device::full_name);
ov::hint::PerformanceMode perf_mode = compiled_model.get_property(ov::hint::performance_mode);
// 'auto' is deduced to 'uint32_t'
auto nireq = compiled_model.get_property(ov::optimal_number_of_infer_requests);
num_streams = core.get_property("CPU", "NUM_STREAMS")
full_device_name = core.get_property("CPU", "FULL_DEVICE_NAME")
perf_mode = compiled_model.get_property("PERFORMANCE_HINT")
nireq = compiled_model.get_property("OPTIMAL_NUMBER_OF_INFER_REQUESTS")