Configuring Devices

Inference Engine API provides the ability to configure devices via configuration keys and get device specific metrics. The values taken from InferenceEngine::Core::GetConfig are requested by the string name, while the return type is InferenceEngine::Parameter, making users lost on what the actual type is stored in this parameter.

API 2.0 solves these issues by introducing properties, which unify metrics and configuration key concepts. The main advantage is that they have the C++ type:

static constexpr Property<std::string> full_name{"FULL_DEVICE_NAME"};

where the property can be requested from an inference device as:

// 'auto' is automatically deduced as std::string
// since the type is stored in the property
auto full_device_name = core.get_property("CPU", ov::device::full_name);

The snippets in the following sections demostrate the device configurations for migrating from Inference Engine to API 2.0.

Setting Configuration Values

Inference Engine API

core.SetConfig({ { CONFIG_KEY(PERF_COUNT), CONFIG_VALUE(YES) } }, "CPU");

auto exec_network = core.LoadNetwork(model, "MULTI", {
    { MULTI_CONFIG_KEY(DEVICE_PRIORITIES), "CPU, GPU" },
    { CONFIG_KEY(PERFORMANCE_HINT), CONFIG_VALUE(THROUGHPUT) },
    { CONFIG_KEY(ENFORCE_BF16), CONFIG_VALUE(NO) } });

// turn CPU off for multi-device execution
exec_network.SetConfig({ { MULTI_CONFIG_KEY(DEVICE_PRIORITIES), "GPU" } });

core.set_config({"PERF_COUNT": "YES"}, "CPU")

exec_network = core.load_network(net, "MULTI", {"DEVICE_PRIORITIES": "CPU, GPU",
                                                "PERFORMANCE_HINT": "THROUGHPUT",
                                                "ENFORCE_BF16": "NO"})

# turn CPU off for multi-device execution
exec_network.set_config({"DEVICE_PRIORITIES": "GPU"})

ie_config_t config = {"PERF_COUNT", "YES", NULL};
ie_core_set_config(core, &config, "CPU");

ie_config_t config_1 = {"DEVICE_PRIORITIES", "CPU, GPU", NULL};
ie_config_t config_2 = {"PERFORMANCE_HINT", "THROUGHPUT", &config_1};
ie_config_t config_3 = {"ENFORCE_BF16", "NO", &config_2};
ie_executable_network_t \*exe_network = NULL;
ie_core_load_network(core, network, "MULTI", &config_3, &exe_network);

// turn CPU off for multi-device executio
ie_config_t config_param = {"DEVICE_PRIORITIES", "GPU", NULL};
ie_exec_network_set_config(exe_network, &config_param);

API 2.0

core.set_property("CPU", ov::enable_profiling(true));

auto compiled_model = core.compile_model(model, "MULTI",
    ov::device::priorities("GPU", "CPU"),
    ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT),
    ov::hint::inference_precision(ov::element::f32));

// turn CPU off for multi-device execution
compiled_model.set_property(ov::device::priorities("GPU"));

core.set_property(device_name="CPU", properties={"PERF_COUNT": "YES"})

compiled_model = core.compile_model(model=model, device_name="MULTI", config=
    {
        "MULTI_DEVICE_PRIORITIES": "GPU,CPU",
        "PERFORMANCE_HINT": "THROUGHPUT",
        "INFERENCE_PRECISION_HINT": "f32"
    })

# turn CPU off for multi-device execution
compiled_model.set_property(properties={"MULTI_DEVICE_PRIORITIES": "GPU"})

ov_core_set_property(core, "CPU", ov_property_key_enable_profiling, "TRUE");

ov_compiled_model_t\* compiled_model = NULL;
ov_core_compile_model(core, model, "MULTI", 6, &compiled_model,
    ov_property_key_device_priorities, "CPU, CPU",
    ov_property_key_hint_performance_mode, "THROUGHPUT",
    ov_property_key_hint_inference_precision, "f32");

// turn CPU off for multi-device execution
ov_compiled_model_set_property(compiled_model, ov_property_key_device_priorities, "GPU");

Getting Information

Inference Engine API

// a user has to parse std::string after
auto num_streams = core.GetConfig("CPU", CONFIG_KEY(CPU_THROUGHPUT_STREAMS)).as<std::string>();

auto full_device_name = core.GetMetric("CPU", METRIC_KEY(FULL_DEVICE_NAME)).as<std::string>();

std::string perf_model = exec_network.GetConfig(CONFIG_KEY(PERFORMANCE_HINT)).as<std::string>();

auto nireq = exec_network.GetMetric(EXEC_NETWORK_METRIC_KEY(OPTIMAL_NUMBER_OF_INFER_REQUESTS)).as<uint32_t>();

num_streams = core.get_config("CPU", "CPU_THROUGHPUT_STREAMS")

full_device_name = core.get_metric("CPU", "FULL_DEVICE_NAME")

perf_hint = exec_network.get_config("PERFORMANCE_HINT")

nireq = exec_network.get_metric("OPTIMAL_NUMBER_OF_INFER_REQUESTS")

ie_param_t num_streams;
num_streams.params = NULL;
ie_core_get_config(core, "CPU", "CPU_THROUGHPUT_STREAMS", &num_streams);
ie_param_free(&num_streams);

ie_param_t full_device_name;
full_device_name.params = NULL;
ie_core_get_metric(core, "CPU", "FULL_DEVICE_NAME", &full_device_name);
ie_param_free(&full_device_name);

ie_param_t perf_model;
perf_model.params = NULL;
ie_exec_network_get_config(exe_network, "PERFORMANCE_HINT", &perf_model);

ie_param_t nireq;
nireq.params = NULL;
ie_exec_network_get_metric(exe_network, "OPTIMAL_NUMBER_OF_INFER_REQUESTS", &nireq);
ie_param_free(&nireq);

API 2.0

// 'auto' is automatically deduced as ov::streams::Num
// since the type is stored in the property
auto num_streams = core.get_property("CPU", ov::streams::num);

// 'auto' is automatically deduced as std::string
// since the type is stored in the property
auto full_device_name = core.get_property("CPU", ov::device::full_name);

ov::hint::PerformanceMode perf_mode = compiled_model.get_property(ov::hint::performance_mode);

// 'auto' is deduced to 'uint32_t'
auto nireq = compiled_model.get_property(ov::optimal_number_of_infer_requests);

num_streams = core.get_property("CPU", "NUM_STREAMS")

full_device_name = core.get_property("CPU", "FULL_DEVICE_NAME")

perf_mode = compiled_model.get_property("PERFORMANCE_HINT")

nireq = compiled_model.get_property("OPTIMAL_NUMBER_OF_INFER_REQUESTS")

char\* num_streams = NULL;
ov_core_get_property(core, "CPU", ov_property_key_num_streams, &num_streams);
ov_free(num_streams);

char\* full_device_name = NULL;
ov_core_get_property(core, "CPU", ov_property_key_device_full_name, &full_device_name);
ov_free(full_device_name);

char\* perf_mode = NULL;
ov_compiled_model_get_property(compiled_model, ov_property_key_hint_performance_mode, &perf_mode);
ov_free(perf_mode);

char\* nireq = NULL;
ov_compiled_model_get_property(compiled_model, ov_property_key_hint_num_requests, &nireq);
ov_free(nireq);