Device properties

Overview

// enums

enum ov::Affinity;
enum ov::log::Level;
enum ov::hint::PerformanceMode;
enum ov::hint::Priority;
enum ov::device::Type;

// structs

struct ov::streams::Num;
struct ov::device::Priorities;
struct ov::device::Properties;
struct ov::device::UUID;

// global variables

static constexpr Property<std::vector<PropertyName>, PropertyMutability::RO> ov::supported_properties {     "SUPPORTED_PROPERTIES"};
static constexpr Property<std::vector<std::string>, PropertyMutability::RO> ov::available_devices {"AVAILABLE_DEVICES"};
static constexpr Property<std::string, PropertyMutability::RO> ov::model_name {"NETWORK_NAME"};
static constexpr Property<uint32_t, PropertyMutability::RO> ov::optimal_number_of_infer_requests {     "OPTIMAL_NUMBER_OF_INFER_REQUESTS"};
static constexpr Property<bool> ov::enable_profiling {"PERF_COUNT"};
static constexpr Property<std::string> ov::cache_dir {"CACHE_DIR"};
static constexpr Property<std::tuple<unsigned int, unsigned int>, PropertyMutability::RO> ov::range_for_streams {     "RANGE_FOR_STREAMS"};
static constexpr Property<unsigned int, PropertyMutability::RO> ov::optimal_batch_size {"OPTIMAL_BATCH_SIZE"};
static constexpr Property<uint32_t, PropertyMutability::RO> ov::max_batch_size {"MAX_BATCH_SIZE"};
static constexpr Property<uint32_t, PropertyMutability::RW> ov::auto_batch_timeout {"AUTO_BATCH_TIMEOUT"};
static constexpr Property<std::tuple<unsigned int, unsigned int, unsigned int>, PropertyMutability::RO> ov::range_for_async_infer_requests {"RANGE_FOR_ASYNC_INFER_REQUESTS"};
static constexpr Property<bool, PropertyMutability::RW> ov::force_tbb_terminate {"FORCE_TBB_TERMINATE"};
static constexpr Property<streams::Num, PropertyMutability::RW> ov::num_streams {"NUM_STREAMS"};
static constexpr Property<int32_t, PropertyMutability::RW> ov::inference_num_threads {"INFERENCE_NUM_THREADS"};
static constexpr Property<int32_t, PropertyMutability::RW> ov::compilation_num_threads {"COMPILATION_NUM_THREADS"};
static constexpr Property<Affinity> ov::affinity {"AFFINITY"};
static constexpr Property<std::vector<std::string>, PropertyMutability::RO> ov::execution_devices {"EXECUTION_DEVICES"};
static constexpr Property<std::string> ov::device::id {"DEVICE_ID"};
static constexpr Priorities ov::device::priorities {"MULTI_DEVICE_PRIORITIES"};
static constexpr Properties ov::device::properties;
static constexpr Property<std::string, PropertyMutability::RO> ov::device::full_name {"FULL_DEVICE_NAME"};
static constexpr Property<std::string, PropertyMutability::RO> ov::device::architecture {"DEVICE_ARCHITECTURE"};
static constexpr Property<UUID, PropertyMutability::RO> ov::device::uuid {"DEVICE_UUID"};
static constexpr Property<Type, PropertyMutability::RO> ov::device::type {"DEVICE_TYPE"};
static constexpr Property<std::map<element::Type, float>, PropertyMutability::RO> ov::device::gops {"DEVICE_GOPS"};
static constexpr Property<float, PropertyMutability::RO> ov::device::thermal {"DEVICE_THERMAL"};
static constexpr Property<std::vector<std::string>, PropertyMutability::RO> ov::device::capabilities {"OPTIMIZATION_CAPABILITIES"};
static constexpr static const auto ov::device::capability::FP32 = "FP32";
static constexpr static const auto ov::device::capability::BF16 = "BF16";
static constexpr static const auto ov::device::capability::FP16 = "FP16";
static constexpr static const auto ov::device::capability::INT8 = "INT8";
static constexpr static const auto ov::device::capability::INT16 = "INT16";
static constexpr static const auto ov::device::capability::BIN = "BIN";
static constexpr static const auto ov::device::capability::WINOGRAD = "WINOGRAD";
static constexpr static const auto ov::device::capability::EXPORT_IMPORT = "EXPORT_IMPORT";
static constexpr Property<element::Type, PropertyMutability::RW> ov::hint::inference_precision {"INFERENCE_PRECISION_HINT"};
static constexpr Property<Priority> ov::hint::model_priority {"MODEL_PRIORITY"};
static constexpr Property<PerformanceMode> ov::hint::performance_mode {"PERFORMANCE_HINT"};
static constexpr Property<uint32_t> ov::hint::num_requests {"PERFORMANCE_HINT_NUM_REQUESTS"};
static constexpr Property<std::shared_ptr<ov::Model>> ov::hint::model {"MODEL_PTR"};
static constexpr Property<bool, PropertyMutability::RW> ov::hint::allow_auto_batching {"ALLOW_AUTO_BATCHING"};
static constexpr Property<Level> ov::log::level {"LOG_LEVEL"};
static constexpr Property<Num, PropertyMutability::RW> ov::streams::num {"NUM_STREAMS"};
static constexpr Num ov::streams::AUTO {-1};
static constexpr Num ov::streams::NUMA {-2};

Detailed Documentation

Global Variables

static constexpr Property<std::vector<PropertyName>, PropertyMutability::RO> ov::supported_properties {     "SUPPORTED_PROPERTIES"}

Read-only property to get a std::vector<PropertyName> of supported read-only properties. This can be used as a compiled model property as well.

static constexpr Property<std::vector<std::string>, PropertyMutability::RO> ov::available_devices {"AVAILABLE_DEVICES"}

Read-only property to get a std::vector<std::string> of available device IDs.

static constexpr Property<std::string, PropertyMutability::RO> ov::model_name {"NETWORK_NAME"}

Read-only property to get a name of name of a model.

static constexpr Property<uint32_t, PropertyMutability::RO> ov::optimal_number_of_infer_requests {     "OPTIMAL_NUMBER_OF_INFER_REQUESTS"}

Read-only property to get an unsigned integer value of optimal number of compiled model infer requests.

static constexpr Property<bool> ov::enable_profiling {"PERF_COUNT"}

The name for setting performance counters option.

static constexpr Property<std::string> ov::cache_dir {"CACHE_DIR"}

This property defines the directory which will be used to store any data cached by plugins.

The underlying cache structure is not defined and might differ between OpenVINO releases Cached data might be platform / device specific and might be invalid after OpenVINO version change If this property is not specified or value is empty string, then caching is disabled. The property might enable caching for the plugin using the following code:

ie.set_property("GPU", ov::cache_dir("cache/")); // enables cache for GPU plugin

The following code enables caching of compiled network blobs for devices where import/export is supported

ie.set_property(ov::cache_dir("cache/")); // enables models cache
static constexpr Property<std::tuple<unsigned int, unsigned int>, PropertyMutability::RO> ov::range_for_streams {     "RANGE_FOR_STREAMS"}

Read-only property to provide information about a range for streams on platforms where streams are supported.

Property returns a value of std::tuple<unsigned int, unsigned int> type, where:

  • First value is bottom bound.

  • Second value is upper bound.

static constexpr Property<unsigned int, PropertyMutability::RO> ov::optimal_batch_size {"OPTIMAL_BATCH_SIZE"}

Read-only property to query information optimal batch size for the given device and the network.

Property returns a value of unsigned int type, Returns optimal batch size for a given network on the given device. The returned value is aligned to power of 2. Also, ov::hint::model is the required option for this metric since the optimal batch size depends on the model, so if the ov::hint::model is not given, the result of the metric is always 1. For the GPU the metric is queried automatically whenever the OpenVINO performance hint for the throughput is used, so that the result (>1) governs the automatic batching (transparently to the application). The automatic batching can be disabled with ALLOW_AUTO_BATCHING set to NO

static constexpr Property<uint32_t, PropertyMutability::RO> ov::max_batch_size {"MAX_BATCH_SIZE"}

Read-only property to get maximum batch size which does not cause performance degradation due to memory swap impact.

static constexpr Property<uint32_t, PropertyMutability::RW> ov::auto_batch_timeout {"AUTO_BATCH_TIMEOUT"}

Read-write property to set the timeout used to collect the inputs for the auto-batching impact.

static constexpr Property<std::tuple<unsigned int, unsigned int, unsigned int>, PropertyMutability::RO> ov::range_for_async_infer_requests {"RANGE_FOR_ASYNC_INFER_REQUESTS"}

Read-only property to provide a hint for a range for number of async infer requests. If device supports streams, the metric provides range for number of IRs per stream.

Property returns a value of std::tuple<unsigned int, unsigned int, unsigned int> type, where:

  • First value is bottom bound.

  • Second value is upper bound.

  • Third value is step inside this range.

static constexpr Property<bool, PropertyMutability::RW> ov::force_tbb_terminate {"FORCE_TBB_TERMINATE"}

Read-write property to set whether force terminate tbb when ov core destruction value type: boolean.

  • True explicitly terminate tbb when ov core destruction

  • False will not involve additional tbb operations when core destruction

static constexpr Property<streams::Num, PropertyMutability::RW> ov::num_streams {"NUM_STREAMS"}

The number of executor logical partitions.

static constexpr Property<int32_t, PropertyMutability::RW> ov::inference_num_threads {"INFERENCE_NUM_THREADS"}

Maximum number of threads that can be used for inference tasks.

static constexpr Property<int32_t, PropertyMutability::RW> ov::compilation_num_threads {"COMPILATION_NUM_THREADS"}

Maximum number of threads that can be used for compilation tasks.

static constexpr Property<Affinity> ov::affinity {"AFFINITY"}

The name for setting CPU affinity per thread option.

The setting is ignored, if the OpenVINO compiled with OpenMP and any affinity-related OpenMP’s environment variable is set (as affinity is configured explicitly)

static constexpr Property<std::vector<std::string>, PropertyMutability::RO> ov::execution_devices {"EXECUTION_DEVICES"}

The devices that the inference task been executed.

static constexpr Property<std::string> ov::device::id {"DEVICE_ID"}

the property for setting of required device to execute on values: device id starts from “0” - first device, “1” - second device, etc

static constexpr Priorities ov::device::priorities {"MULTI_DEVICE_PRIORITIES"}

Device Priorities config option, with comma-separated devices listed in the desired priority.

static constexpr Properties ov::device::properties

Property to pass set of property values to specified device

Usage Example:

core.compile_model("HETERO"
    ov::device::priorities("GPU", "CPU"),
    ov::device::properties("CPU", ov::enable_profiling(true)),
    ov::device::properties("GPU", ov::enable_profiling(false)));
static constexpr Property<std::string, PropertyMutability::RO> ov::device::full_name {"FULL_DEVICE_NAME"}

Read-only property to get a std::string value representing a full device name.

static constexpr Property<std::string, PropertyMutability::RO> ov::device::architecture {"DEVICE_ARCHITECTURE"}

Read-only property which defines the device architecture.

static constexpr Property<UUID, PropertyMutability::RO> ov::device::uuid {"DEVICE_UUID"}

Read-only property which defines the UUID of the device.

static constexpr Property<Type, PropertyMutability::RO> ov::device::type {"DEVICE_TYPE"}

Read-only property to get a type of device. See Type enum definition for possible return values.

static constexpr Property<std::map<element::Type, float>, PropertyMutability::RO> ov::device::gops {"DEVICE_GOPS"}

Read-only property which defines Giga OPS per second count (GFLOPS or GIOPS) for a set of precisions supported by specified device.

static constexpr Property<float, PropertyMutability::RO> ov::device::thermal {"DEVICE_THERMAL"}

Read-only property to get a float of device thermal.

static constexpr Property<std::vector<std::string>, PropertyMutability::RO> ov::device::capabilities {"OPTIMIZATION_CAPABILITIES"}

Read-only property to get a std::vector<std::string> of capabilities options per device.

static constexpr static const auto ov::device::capability::FP32 = "FP32"

Device supports fp32 inference.

static constexpr static const auto ov::device::capability::BF16 = "BF16"

Device supports bf16 inference.

static constexpr static const auto ov::device::capability::FP16 = "FP16"

Device supports fp16 inference.

static constexpr static const auto ov::device::capability::INT8 = "INT8"

Device supports int8 inference.

static constexpr static const auto ov::device::capability::INT16 = "INT16"

Device supports int16 inference.

static constexpr static const auto ov::device::capability::BIN = "BIN"

Device supports binary inference.

static constexpr static const auto ov::device::capability::WINOGRAD = "WINOGRAD"

Device supports winograd optimization.

static constexpr static const auto ov::device::capability::EXPORT_IMPORT = "EXPORT_IMPORT"

Device supports compiled model export and import.

static constexpr Property<element::Type, PropertyMutability::RW> ov::hint::inference_precision {"INFERENCE_PRECISION_HINT"}

Hint for device to use specified precision for inference.

static constexpr Property<Priority> ov::hint::model_priority {"MODEL_PRIORITY"}

High-level OpenVINO model priority hint Defines what model should be provided with more performant bounded resource first.

static constexpr Property<PerformanceMode> ov::hint::performance_mode {"PERFORMANCE_HINT"}

High-level OpenVINO Performance Hints unlike low-level properties that are individual (per-device), the hints are something that every device accepts and turns into device-specific settings.

static constexpr Property<uint32_t> ov::hint::num_requests {"PERFORMANCE_HINT_NUM_REQUESTS"}

(Optional) property that backs the (above) Performance Hints by giving additional information on how many inference requests the application will be keeping in flight usually this value comes from the actual use-case (e.g. number of video-cameras, or other sources of inputs)

static constexpr Property<std::shared_ptr<ov::Model>> ov::hint::model {"MODEL_PTR"}

This key identifies shared pointer to the ov::Model, required for some properties (ov::max_batch_size and ov::optimal_batch_size)

static constexpr Property<bool, PropertyMutability::RW> ov::hint::allow_auto_batching {"ALLOW_AUTO_BATCHING"}

Special key for auto batching feature configuration. Enabled by default.

static constexpr Property<Level> ov::log::level {"LOG_LEVEL"}

the property for setting desirable log level.

static constexpr Property<Num, PropertyMutability::RW> ov::streams::num {"NUM_STREAMS"}

The number of executor logical partitions.

static constexpr Num ov::streams::AUTO {-1}

Creates bare minimum of streams to improve the performance.

static constexpr Num ov::streams::NUMA {-2}

Creates as many streams as needed to accommodate NUMA and avoid associated penalties.