Configuring Devices

The Inference Engine API provides the ability to configure devices with configuration keys and obtain device-specific metrics. The values retrived from InferenceEngine::Core::GetConfig are requested by the string name, while the return type is InferenceEngine::Parameter , which results in users not knowing what the actual type is stored in this parameter.

API 2.0 solves these issues by introducing properties, which unify metrics and configuration key concepts. The main advantage is that they have the C++ type:

static constexpr Property<std::string> full_name{"FULL_DEVICE_NAME"};

where the property can be requested from an inference device as:

// 'auto' is automatically deduced as std::string
// since the type is stored in the property
auto full_device_name = core.get_property("CPU", ov::device::full_name);

The snippets in the following sections demonstrate the device configurations for migrating from Inference Engine to API 2.0.

Note

The Inference Engine API is a legacy solution and it is recomended to use API 2.0. If you want to learn more about Inference Engine API, its configuration and how to obtain device-specific metrics from it, check the following article from the 2021.4 version of OpenVINO documentation.

Setting Configuration Values

Inference Engine API

core.SetConfig({ { CONFIG_KEY(PERF_COUNT), CONFIG_VALUE(YES) } }, "CPU");

auto exec_network = core.LoadNetwork(model, "MULTI", {
    { MULTI_CONFIG_KEY(DEVICE_PRIORITIES), "CPU, GPU" },
    { CONFIG_KEY(PERFORMANCE_HINT), CONFIG_VALUE(THROUGHPUT) },
    { CONFIG_KEY(ENFORCE_BF16), CONFIG_VALUE(NO) } });

// turn CPU off for multi-device execution
exec_network.SetConfig({ { MULTI_CONFIG_KEY(DEVICE_PRIORITIES), "GPU" } });

core.set_config({"PERF_COUNT": "YES"}, "CPU")

exec_network = core.load_network(net, "MULTI", {"DEVICE_PRIORITIES": "CPU, GPU",
                                                "PERFORMANCE_HINT": "THROUGHPUT",
                                                "ENFORCE_BF16": "NO"})

# turn CPU off for multi-device execution
exec_network.set_config({"DEVICE_PRIORITIES": "GPU"})

ie_config_t config = {"PERF_COUNT", "YES", NULL};

ie_config_t config_1 = {"DEVICE_PRIORITIES", "CPU, GPU", NULL};
ie_config_t config_2 = {"PERFORMANCE_HINT", "THROUGHPUT", &config_1};
ie_config_t config_3 = {"ENFORCE_BF16", "NO", &config_2};
ie_executable_network_t *exe_network = NULL;

// turn CPU off for multi-device executio
ie_config_t config_param = {"DEVICE_PRIORITIES", "GPU", NULL};
ie_exec_network_set_config(exe_network, &config_param);

API 2.0

core.set_property(device_name="CPU", properties={"PERF_COUNT": "YES"})

compiled_model = core.compile_model(model=model, device_name="MULTI", config=
    {
        "MULTI_DEVICE_PRIORITIES": "GPU,CPU",
        "PERFORMANCE_HINT": "THROUGHPUT",
        "INFERENCE_PRECISION_HINT": "f32"
    })

# turn CPU off for multi-device execution
compiled_model.set_property(properties={"MULTI_DEVICE_PRIORITIES": "GPU"})

core.set_property("CPU", ov::enable_profiling(true));

auto compiled_model = core.compile_model(model, "MULTI",
    ov::device::priorities("GPU", "CPU"),
    ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT),
    ov::hint::inference_precision(ov::element::f32));

// turn CPU off for multi-device execution
compiled_model.set_property(ov::device::priorities("GPU"));

ov_core_set_property(core, "CPU", ov_property_key_enable_profiling, "TRUE");

ov_compiled_model_t* compiled_model = NULL;

// turn CPU off for multi-device execution

Getting Information

Inference Engine API

num_streams = core.get_config("CPU", "CPU_THROUGHPUT_STREAMS")

full_device_name = core.get_metric("CPU", "FULL_DEVICE_NAME")

# turn CPU off for multi-device execution
exec_network.set_config({"DEVICE_PRIORITIES": "GPU"})

nireq = exec_network.get_metric("OPTIMAL_NUMBER_OF_INFER_REQUESTS")

// a user has to parse std::string after
auto num_streams = core.GetConfig("CPU", CONFIG_KEY(CPU_THROUGHPUT_STREAMS)).as<std::string>();

auto full_device_name = core.GetMetric("CPU", METRIC_KEY(FULL_DEVICE_NAME)).as<std::string>();

// turn CPU off for multi-device execution
exec_network.SetConfig({ { MULTI_CONFIG_KEY(DEVICE_PRIORITIES), "GPU" } });

auto nireq = exec_network.GetMetric(EXEC_NETWORK_METRIC_KEY(OPTIMAL_NUMBER_OF_INFER_REQUESTS)).as<uint32_t>();

ie_param_t num_streams;
num_streams.params = NULL;

ie_param_t full_device_name;
full_device_name.params = NULL;

// turn CPU off for multi-device executio
ie_config_t config_param = {"DEVICE_PRIORITIES", "GPU", NULL};
ie_exec_network_set_config(exe_network, &config_param);

ie_param_t nireq;
nireq.params = NULL;
ie_exec_network_get_metric(exe_network, "OPTIMAL_NUMBER_OF_INFER_REQUESTS", &nireq);
ie_param_free(&nireq);

API 2.0

num_streams = core.get_property("CPU", "NUM_STREAMS")

full_device_name = core.get_property("CPU", "FULL_DEVICE_NAME")

perf_mode = compiled_model.get_property("PERFORMANCE_HINT")

nireq = compiled_model.get_property("OPTIMAL_NUMBER_OF_INFER_REQUESTS")

// 'auto' is automatically deduced as ov::streams::Num
// since the type is stored in the property
auto num_streams = core.get_property("CPU", ov::streams::num);

// 'auto' is automatically deduced as std::string
// since the type is stored in the property
auto full_device_name = core.get_property("CPU", ov::device::full_name);

ov::hint::PerformanceMode perf_mode = compiled_model.get_property(ov::hint::performance_mode);

// 'auto' is deduced to 'uint32_t'
auto nireq = compiled_model.get_property(ov::optimal_number_of_infer_requests);

char* num_streams = NULL;
ov_core_get_property(core, "CPU", ov_property_key_num_streams, &num_streams);
ov_free(num_streams);

char* full_device_name = NULL;
ov_core_get_property(core, "CPU", ov_property_key_device_full_name, &full_device_name);
ov_free(full_device_name);

char* perf_mode = NULL;
ov_compiled_model_get_property(compiled_model, ov_property_key_hint_performance_mode, &perf_mode);
ov_free(perf_mode);

char* nireq = NULL;
ov_compiled_model_get_property(compiled_model, ov_property_key_hint_num_requests, &nireq);
ov_free(nireq);