Configuring Devices

The Inference Engine API provides the ability to configure devices with configuration keys and obtain device-specific metrics. The values retrived from InferenceEngine::Core::GetConfig are requested by the string name, while the return type is InferenceEngine::Parameter , which results in users not knowing what the actual type is stored in this parameter.

API 2.0 solves these issues by introducing properties, which unify metrics and configuration key concepts. The main advantage is that they have the C++ type:

static constexpr Property<std::string> full_name{"FULL_DEVICE_NAME"};

where the property can be requested from an inference device as:

// 'auto' is automatically deduced as std::string
// since the type is stored in the property
auto full_device_name = core.get_property("CPU", ov::device::full_name);

The snippets in the following sections demonstrate the device configurations for migrating from Inference Engine to API 2.0.

Note

The Inference Engine API is a legacy solution and it is recomended to use API 2.0. If you want to learn more about Inference Engine API, its configuration and how to obtain device-specific metrics from it, check the following article from the 2021.4 version of OpenVINO documentation.

Setting Configuration Values

Inference Engine API

core.SetConfig({ { CONFIG_KEY(PERF_COUNT), CONFIG_VALUE(YES) } }, "CPU");
auto exec_network = core.LoadNetwork(model, "MULTI", {
    { MULTI_CONFIG_KEY(DEVICE_PRIORITIES), "CPU, GPU" },
    { CONFIG_KEY(PERFORMANCE_HINT), CONFIG_VALUE(THROUGHPUT) },
    { CONFIG_KEY(ENFORCE_BF16), CONFIG_VALUE(NO) } });
// turn CPU off for multi-device execution
exec_network.SetConfig({ { MULTI_CONFIG_KEY(DEVICE_PRIORITIES), "GPU" } });
    core.set_config({"PERF_COUNT": "YES"}, "CPU")
    exec_network = core.load_network(net, "MULTI", {"DEVICE_PRIORITIES": "CPU, GPU",
                                                    "PERFORMANCE_HINT": "THROUGHPUT",
                                                    "ENFORCE_BF16": "NO"})
    # turn CPU off for multi-device execution
    exec_network.set_config({"DEVICE_PRIORITIES": "GPU"})
ie_config_t config = {"PERF_COUNT", "YES", NULL};
ie_config_t config_1 = {"DEVICE_PRIORITIES", "CPU, GPU", NULL};
ie_config_t config_2 = {"PERFORMANCE_HINT", "THROUGHPUT", &config_1};
ie_config_t config_3 = {"ENFORCE_BF16", "NO", &config_2};
ie_executable_network_t *exe_network = NULL;
// turn CPU off for multi-device executio
ie_config_t config_param = {"DEVICE_PRIORITIES", "GPU", NULL};
ie_exec_network_set_config(exe_network, &config_param);

API 2.0

    core.set_property(device_name="CPU", properties={props.enable_profiling: True})
    compiled_model = core.compile_model(model=model, device_name="MULTI", config=
        {
            device.priorities: "GPU,CPU",
            hints.performance_mode: hints.PerformanceMode.THROUGHPUT,
            hints.inference_precision: ov.Type.f32
        })
    # turn CPU off for multi-device execution
    compiled_model.set_property(properties={device.priorities: "GPU"})
core.set_property("CPU", ov::enable_profiling(true));
auto compiled_model = core.compile_model(model, "MULTI",
    ov::device::priorities("GPU", "CPU"),
    ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT),
    ov::hint::inference_precision(ov::element::f32));
// turn CPU off for multi-device execution
compiled_model.set_property(ov::device::priorities("GPU"));
ov_core_set_property(core, "CPU", ov_property_key_enable_profiling, "TRUE");
ov_compiled_model_t* compiled_model = NULL;
// turn CPU off for multi-device execution

Getting Information

Inference Engine API

    num_streams = core.get_config("CPU", "CPU_THROUGHPUT_STREAMS")
    full_device_name = core.get_metric("CPU", "FULL_DEVICE_NAME")
    # turn CPU off for multi-device execution
    exec_network.set_config({"DEVICE_PRIORITIES": "GPU"})
    nireq = exec_network.get_metric("OPTIMAL_NUMBER_OF_INFER_REQUESTS")
// a user has to parse std::string after
auto num_streams = core.GetConfig("CPU", CONFIG_KEY(CPU_THROUGHPUT_STREAMS)).as<std::string>();
auto full_device_name = core.GetMetric("CPU", METRIC_KEY(FULL_DEVICE_NAME)).as<std::string>();
// turn CPU off for multi-device execution
exec_network.SetConfig({ { MULTI_CONFIG_KEY(DEVICE_PRIORITIES), "GPU" } });
auto nireq = exec_network.GetMetric(EXEC_NETWORK_METRIC_KEY(OPTIMAL_NUMBER_OF_INFER_REQUESTS)).as<uint32_t>();
ie_param_t num_streams;
num_streams.params = NULL;
ie_param_t full_device_name;
full_device_name.params = NULL;
// turn CPU off for multi-device executio
ie_config_t config_param = {"DEVICE_PRIORITIES", "GPU", NULL};
ie_exec_network_set_config(exe_network, &config_param);
ie_param_t nireq;
nireq.params = NULL;
ie_exec_network_get_metric(exe_network, "OPTIMAL_NUMBER_OF_INFER_REQUESTS", &nireq);
ie_param_free(&nireq);

API 2.0

    num_streams = core.get_property("CPU", streams.num)
    full_device_name = core.get_property("CPU", device.full_name)
    perf_mode = compiled_model.get_property(hints.performance_mode)
    nireq = compiled_model.get_property(props.optimal_number_of_infer_requests)
// 'auto' is automatically deduced as ov::streams::Num
// since the type is stored in the property
auto num_streams = core.get_property("CPU", ov::streams::num);
// 'auto' is automatically deduced as std::string
// since the type is stored in the property
auto full_device_name = core.get_property("CPU", ov::device::full_name);
ov::hint::PerformanceMode perf_mode = compiled_model.get_property(ov::hint::performance_mode);
// 'auto' is deduced to 'uint32_t'
auto nireq = compiled_model.get_property(ov::optimal_number_of_infer_requests);
char* num_streams = NULL;
ov_core_get_property(core, "CPU", ov_property_key_num_streams, &num_streams);
ov_free(num_streams);
char* full_device_name = NULL;
ov_core_get_property(core, "CPU", ov_property_key_device_full_name, &full_device_name);
ov_free(full_device_name);
char* perf_mode = NULL;
ov_compiled_model_get_property(compiled_model, ov_property_key_hint_performance_mode, &perf_mode);
ov_free(perf_mode);
char* nireq = NULL;
ov_compiled_model_get_property(compiled_model, ov_property_key_hint_num_requests, &nireq);
ov_free(nireq);

Configuration to property mapping

Inference Engine Configuration

API 2.0 C++

API 2.0 Python

CPU_DENORMALS_OPTIMIZATION

ov::intel_cpu::denormals_optimization

props.intel_cpu.denormals_optimization

CPU_SPARSE_WEIGHTS_DECOMPRESSION_RATE

ov::intel_cpu::sparse_weights_decompression_rate

props.intel_cpu.sparse_weights_decompression_rate

GPU_HOST_TASK_PRIORITY

ov::intel_gpu::hint::host_task_priority

props.intel_gpu.hint.host_task_priority

GPU_ENABLE_LOOP_UNROLLING

ov::intel_gpu::enable_loop_unrolling

props.intel_gpu.host_task_priority

GPU_THROUGHPUT_STREAMS

ov::num_streams

props.streams.num

MULTI_DEVICE_PRIORITIES

ov::device::priorities

props.device.priorities

MODEL_PRIORITY

ov::hint::model_priority

props.hint.model_priority

PERFORMANCE_HINT

ov::hint::performance_mode

props.hint.performance_mode

PERFORMANCE_HINT_NUM_REQUESTS

ov::hint::num_requests

props.hint.num_requests

ALLOW_AUTO_BATCHING

ov::hint::allow_auto_batching

props.hint.allow_auto_batching

AUTO_BATCH_DEVICE_CONFIG

ov::device::priorities

props.device.priorities

AUTO_BATCH_TIMEOUT

ov::auto_batch_timeout

props.auto_batch_timeout

CPU_THREADS_NUM

ov::inference_num_threads

props.inference_num_threads

CPU_BIND_THREAD

ov::hint::enable_cpu_pinning

props.hint.enable_cpu_pinning

CPU_THROUGHPUT_STREAMS

ov::num_streams

props.streams.num

PERF_COUNT

ov::enable_profiling

props.enable_profiling

LOG_LEVEL

ov::log::level

props.log.level

DEVICE_ID

ov::device::id

props.device.id

ENFORCE_BF16

ov::hint::inference_precision(ov::element::bf16)

props.hint.inference_precision(openvino.Type.bf16)

CACHE_DIR

ov::cache_dir

props.cache_dir

FORCE_TBB_TERMINATE

ov::force_tbb_terminate

props.force_tbb_terminate

Note

API 2.0 Python “import openvino.properties as props”