Configuring Devices

The Inference Engine API provides the ability to configure devices with configuration keys and obtain device-specific metrics. The values retrived from InferenceEngine::Core::GetConfig are requested by the string name, while the return type is InferenceEngine::Parameter , which results in users not knowing what the actual type is stored in this parameter.

API 2.0 solves these issues by introducing properties, which unify metrics and configuration key concepts. The main advantage is that they have the C++ type:

static constexpr Property<std::string> full_name{"FULL_DEVICE_NAME"};

where the property can be requested from an inference device as:

// 'auto' is automatically deduced as std::string
// since the type is stored in the property
auto full_device_name = core.get_property("CPU", ov::device::full_name);

The snippets in the following sections demonstrate the device configurations for migrating from Inference Engine to API 2.0.

Note

The Inference Engine API is a legacy solution and it is recomended to use API 2.0. If you want to learn more about Inference Engine API, its configuration and how to obtain device-specific metrics from it, check the following article from the 2021.4 version of OpenVINO documentation.

Setting Configuration Values

Inference Engine API

C++

Devices

core.SetConfig({ { CONFIG_KEY(PERF_COUNT), CONFIG_VALUE(YES) } }, "CPU");

Model Loading

auto exec_network = core.LoadNetwork(model, "MULTI", {
    { MULTI_CONFIG_KEY(DEVICE_PRIORITIES), "CPU, GPU" },
    { CONFIG_KEY(PERFORMANCE_HINT), CONFIG_VALUE(THROUGHPUT) },
    { CONFIG_KEY(ENFORCE_BF16), CONFIG_VALUE(NO) } });

Execution

// turn CPU off for multi-device execution
exec_network.SetConfig({ { MULTI_CONFIG_KEY(DEVICE_PRIORITIES), "GPU" } });

Python

Devices

    core.set_config({"PERF_COUNT": "YES"}, "CPU")

Model Loading

    exec_network = core.load_network(net, "MULTI", {"DEVICE_PRIORITIES": "CPU, GPU",
                                                    "PERFORMANCE_HINT": "THROUGHPUT",
                                                    "ENFORCE_BF16": "NO"})

Execution

    # turn CPU off for multi-device execution
    exec_network.set_config({"DEVICE_PRIORITIES": "GPU"})

C

Devices

ie_config_t config = {"PERF_COUNT", "YES", NULL};

Model Loading

ie_config_t config_1 = {"DEVICE_PRIORITIES", "CPU, GPU", NULL};
ie_config_t config_2 = {"PERFORMANCE_HINT", "THROUGHPUT", &config_1};
ie_config_t config_3 = {"ENFORCE_BF16", "NO", &config_2};
ie_executable_network_t *exe_network = NULL;

Execution

// turn CPU off for multi-device executio
ie_config_t config_param = {"DEVICE_PRIORITIES", "GPU", NULL};
ie_exec_network_set_config(exe_network, &config_param);

API 2.0

Python

Devices

    core.set_property(device_name="CPU", properties={props.enable_profiling: True})

Model Loading

    compiled_model = core.compile_model(model=model, device_name="MULTI", config=
        {
            device.priorities: "GPU,CPU",
            hints.performance_mode: hints.PerformanceMode.THROUGHPUT,
            hints.inference_precision: ov.Type.f32
        })

Execution

    # turn CPU off for multi-device execution
    compiled_model.set_property(properties={device.priorities: "GPU"})

C++

Devices

core.set_property("CPU", ov::enable_profiling(true));

Model Loading

auto compiled_model = core.compile_model(model, "MULTI",
    ov::device::priorities("GPU", "CPU"),
    ov::hint::performance_mode(ov::hint::PerformanceMode::THROUGHPUT),
    ov::hint::inference_precision(ov::element::f32));

Execution

// turn CPU off for multi-device execution
compiled_model.set_property(ov::device::priorities("GPU"));

C

Devices

ov_core_set_property(core, "CPU", ov_property_key_enable_profiling, "TRUE");

Model Loading

ov_compiled_model_t* compiled_model = NULL;

Execution

// turn CPU off for multi-device execution

Getting Information

Inference Engine API

Python

Device Configuration

    num_streams = core.get_config("CPU", "CPU_THROUGHPUT_STREAMS")

Device metrics

    full_device_name = core.get_metric("CPU", "FULL_DEVICE_NAME")

Execution config

    # turn CPU off for multi-device execution
    exec_network.set_config({"DEVICE_PRIORITIES": "GPU"})

Execution metrics

    nireq = exec_network.get_metric("OPTIMAL_NUMBER_OF_INFER_REQUESTS")

C++

Device Configuration

// a user has to parse std::string after
auto num_streams = core.GetConfig("CPU", CONFIG_KEY(CPU_THROUGHPUT_STREAMS)).as<std::string>();

Device metrics

auto full_device_name = core.GetMetric("CPU", METRIC_KEY(FULL_DEVICE_NAME)).as<std::string>();

Execution config

// turn CPU off for multi-device execution
exec_network.SetConfig({ { MULTI_CONFIG_KEY(DEVICE_PRIORITIES), "GPU" } });

Execution metrics

auto nireq = exec_network.GetMetric(EXEC_NETWORK_METRIC_KEY(OPTIMAL_NUMBER_OF_INFER_REQUESTS)).as<uint32_t>();

C

Device Configuration

ie_param_t num_streams;
num_streams.params = NULL;

Device metrics

ie_param_t full_device_name;
full_device_name.params = NULL;

Execution config

// turn CPU off for multi-device executio
ie_config_t config_param = {"DEVICE_PRIORITIES", "GPU", NULL};
ie_exec_network_set_config(exe_network, &config_param);

Execution metrics

ie_param_t nireq;
nireq.params = NULL;
ie_exec_network_get_metric(exe_network, "OPTIMAL_NUMBER_OF_INFER_REQUESTS", &nireq);
ie_param_free(&nireq);

API 2.0

Python

Device Configuration

    num_streams = core.get_property("CPU", streams.num)

Device metrics

    full_device_name = core.get_property("CPU", device.full_name)

Execution config

    perf_mode = compiled_model.get_property(hints.performance_mode)

Execution metrics

    nireq = compiled_model.get_property(props.optimal_number_of_infer_requests)

C++

Device Configuration

// 'auto' is automatically deduced as ov::streams::Num
// since the type is stored in the property
auto num_streams = core.get_property("CPU", ov::streams::num);

Device metrics

// 'auto' is automatically deduced as std::string
// since the type is stored in the property
auto full_device_name = core.get_property("CPU", ov::device::full_name);

Execution config

ov::hint::PerformanceMode perf_mode = compiled_model.get_property(ov::hint::performance_mode);

Execution metrics

// 'auto' is deduced to 'uint32_t'
auto nireq = compiled_model.get_property(ov::optimal_number_of_infer_requests);

C

Device Configuration

char* num_streams = NULL;
ov_core_get_property(core, "CPU", ov_property_key_num_streams, &num_streams);
ov_free(num_streams);

Device metrics

char* full_device_name = NULL;
ov_core_get_property(core, "CPU", ov_property_key_device_full_name, &full_device_name);
ov_free(full_device_name);

Execution config

char* perf_mode = NULL;
ov_compiled_model_get_property(compiled_model, ov_property_key_hint_performance_mode, &perf_mode);
ov_free(perf_mode);

Execution metrics

char* nireq = NULL;
ov_compiled_model_get_property(compiled_model, ov_property_key_hint_num_requests, &nireq);
ov_free(nireq);

Configuration to property mapping

Inference Engine Configuration	API 2.0 C++	API 2.0 Python
CPU_DENORMALS_OPTIMIZATION	ov::intel_cpu::denormals_optimization	props.intel_cpu.denormals_optimization
CPU_SPARSE_WEIGHTS_DECOMPRESSION_RATE	ov::intel_cpu::sparse_weights_decompression_rate	props.intel_cpu.sparse_weights_decompression_rate
GPU_HOST_TASK_PRIORITY	ov::intel_gpu::hint::host_task_priority	props.intel_gpu.hint.host_task_priority
GPU_ENABLE_LOOP_UNROLLING	ov::intel_gpu::enable_loop_unrolling	props.intel_gpu.host_task_priority
GPU_THROUGHPUT_STREAMS	ov::num_streams	props.streams.num
MULTI_DEVICE_PRIORITIES	ov::device::priorities	props.device.priorities
MODEL_PRIORITY	ov::hint::model_priority	props.hint.model_priority
PERFORMANCE_HINT	ov::hint::performance_mode	props.hint.performance_mode
PERFORMANCE_HINT_NUM_REQUESTS	ov::hint::num_requests	props.hint.num_requests
ALLOW_AUTO_BATCHING	ov::hint::allow_auto_batching	props.hint.allow_auto_batching
AUTO_BATCH_DEVICE_CONFIG	ov::device::priorities	props.device.priorities
AUTO_BATCH_TIMEOUT	ov::auto_batch_timeout	props.auto_batch_timeout
CPU_THREADS_NUM	ov::inference_num_threads	props.inference_num_threads
CPU_BIND_THREAD	ov::hint::enable_cpu_pinning	props.hint.enable_cpu_pinning
CPU_THROUGHPUT_STREAMS	ov::num_streams	props.streams.num
PERF_COUNT	ov::enable_profiling	props.enable_profiling
LOG_LEVEL	ov::log::level	props.log.level
DEVICE_ID	ov::device::id	props.device.id
ENFORCE_BF16	ov::hint::inference_precision(ov::element::bf16)	props.hint.inference_precision(openvino.Type.bf16)
CACHE_DIR	ov::cache_dir	props.cache_dir
FORCE_TBB_TERMINATE	ov::force_tbb_terminate	props.force_tbb_terminate

Note

API 2.0 Python “import openvino.properties as props”

More references

InferenceEngine Engine Configuration

• src/inference/include/ie/ie_plugin_config.hpp

• src/inference/include/ie/cpu/cpu_config.hpp

• src/inference/include/ie/gpu/gpu_config.hpp

API 2.0 C++ properties

• src/inference/include/openvino/runtime/properties.hpp

• src/inference/include/openvino/runtime/intel_cpu/properties.hpp

• src/inference/include/openvino/runtime/intel_gpu/properties.hpp

API 2.0 Python properties

• src/bindings/python/src/pyopenvino/core/properties/properties.cpp

• src/bindings/python/tests/test_runtime/test_properties.py

Ask questions

• Go to “Forum”, “Support”, “Github”