ie_plugin_config.hpp
Go to the documentation of this file.
1 // Copyright (C) 2018-2019 Intel Corporation
2 // SPDX-License-Identifier: Apache-2.0
3 //
4 
5 /**
6  * @brief a header for advanced hardware related properties for IE plugins
7  * To use in SetConfig() method of plugins
8  * LoadNetwork() method overloads that accept config as parameter
9  *
10  * @file ie_plugin_config.hpp
11  */
12 #pragma once
13 
14 #include <string>
15 #include <tuple>
16 #include <vector>
17 
18 namespace InferenceEngine {
19 
20 namespace Metrics {
21 
22 #ifndef DECLARE_METRIC_KEY_IMPL
23 # define DECLARE_METRIC_KEY_IMPL(...)
24 #endif
25 
26 /**
27  * @def METRIC_KEY(name)
28  * @brief shortcut for defining common Inference Engine metrics
29  */
30 #define METRIC_KEY(name) InferenceEngine::Metrics::METRIC_##name
31 
32 /**
33  * @def EXEC_NETWORK_METRIC_KEY(name)
34  * @brief shortcut for defining common Inference Engine ExecutableNetwork metrics
35  */
36 #define EXEC_NETWORK_METRIC_KEY(name) METRIC_KEY(name)
37 
38 #define DECLARE_METRIC_KEY(name, ...) \
39  static constexpr auto METRIC_##name = #name; \
40  DECLARE_METRIC_KEY_IMPL(name, __VA_ARGS__)
41 
42 #define DECLARE_EXEC_NETWORK_METRIC_KEY(name, ...) DECLARE_METRIC_KEY(name, __VA_ARGS__)
43 
44 /**
45  * @def METRIC_VALUE(name)
46  * @brief shortcut for defining metric values
47  */
48 #define METRIC_VALUE(name) InferenceEngine::Metrics::name
49 #define DECLARE_METRIC_VALUE(name) static constexpr auto name = #name
50 
51 /**
52 * @brief Metric to get a std::vector<std::string> of available device IDs. String value is "AVAILABLE_DEVICES"
53 */
54 DECLARE_METRIC_KEY(AVAILABLE_DEVICES, std::vector<std::string>);
55 
56 /**
57 * @brief Metric to get a std::vector<std::string> of supported metrics. String value is "SUPPORTED_METRICS"
58 * This can be used as an executable network metric as well.
59 *
60 * Each of the returned device metrics can be passed to Core::GetMetric, executable network metrics
61 * can be passed to ExecutableNetwork::GetMetric.
62 *
63 */
64 DECLARE_METRIC_KEY(SUPPORTED_METRICS, std::vector<std::string>);
65 
66 /**
67 * @brief Metric to get a std::vector<std::string> of supported config keys. String value is "SUPPORTED_CONFIG_KEYS"
68 * This can be used as an executable network metric as well.
69 *
70 * Each of the returned device configuration keys can be passed to Core::SetConfig, Core::GetConfig, and Core::LoadNetwork,
71 * configuration keys for executable networks can be passed to ExecutableNetwork::SetConfig and ExecutableNetwork::GetConfig.
72 *
73 */
74 DECLARE_METRIC_KEY(SUPPORTED_CONFIG_KEYS, std::vector<std::string>);
75 
76 /**
77 * @brief Metric to get a std::string value representing a full device name. String value is "FULL_DEVICE_NAME"
78 */
79 DECLARE_METRIC_KEY(FULL_DEVICE_NAME, std::string);
80 
81 /**
82 * @brief Metric to get a std::vector<std::string> of optimization options per device. String value is "OPTIMIZATION_CAPABILITIES"
83 * The possible values:
84 * - "FP32" - device can support FP32 models
85 * - "FP16" - device can support FP16 models
86 * - "INT8" - device can support models with INT8 layers
87 * - "BIN" - device can support models with BIN layers
88 * - "WINOGRAD" - device can support models where convolution implemented via Winograd transformations
89 */
90 DECLARE_METRIC_KEY(OPTIMIZATION_CAPABILITIES, std::vector<std::string>);
91 
92 DECLARE_METRIC_VALUE(FP32);
93 DECLARE_METRIC_VALUE(FP16);
94 DECLARE_METRIC_VALUE(INT8);
95 DECLARE_METRIC_VALUE(BIN);
96 DECLARE_METRIC_VALUE(WINOGRAD);
97 
98 /**
99 * @brief Metric to provide information about a range for streams on platforms where streams are supported.
100 * Metric returns a value of std::tuple<unsigned int, unsigned int> type, where:
101 * - First value is bottom bound.
102 * - Second value is upper bound.
103 * String value for metric name is "RANGE_FOR_STREAMS".
104 */
105 DECLARE_METRIC_KEY(RANGE_FOR_STREAMS, std::tuple<unsigned int, unsigned int>);
106 
107 /**
108 * @brief Metric to provide a hint for a range for number of async infer requests. If device supports streams,
109 * the metric provides range for number of IRs per stream.
110 * Metric returns a value of std::tuple<unsigned int, unsigned int, unsigned int> type, where:
111 * - First value is bottom bound.
112 * - Second value is upper bound.
113 * - Third value is step inside this range.
114 * String value for metric name is "RANGE_FOR_ASYNC_INFER_REQUESTS".
115 */
116 DECLARE_METRIC_KEY(RANGE_FOR_ASYNC_INFER_REQUESTS, std::tuple<unsigned int, unsigned int, unsigned int>);
117 
118 /**
119 * @brief Metric to get an unsigned int value of number of waiting infer request.
120 * String value is "NUMBER_OF_WAITNING_INFER_REQUESTS". This can be used as an executable network metric as well
121 */
122 DECLARE_METRIC_KEY(NUMBER_OF_WAITING_INFER_REQUESTS, unsigned int);
123 
124 /**
125 * @brief Metric to get an unsigned int value of number of infer request in execution stage.
126 * String value is "NUMBER_OF_EXEC_INFER_REQUESTS". This can be used as an executable network metric as well
127 */
128 DECLARE_METRIC_KEY(NUMBER_OF_EXEC_INFER_REQUESTS, unsigned int);
129 
130 /**
131 * @brief Metric to get a name of network. String value is "NETWORK_NAME".
132 */
133 DECLARE_EXEC_NETWORK_METRIC_KEY(NETWORK_NAME, std::string);
134 
135 /**
136  * @brief Metric to get a float of device thermal. String value is "DEVICE_THERMAL"
137  */
138 DECLARE_METRIC_KEY(DEVICE_THERMAL, float);
139 
140 /**
141 * @brief Metric to get an unsigned integer value of optimal number of executable network infer requests.
142 */
143 DECLARE_EXEC_NETWORK_METRIC_KEY(OPTIMAL_NUMBER_OF_INFER_REQUESTS, unsigned int);
144 
145 } // namespace Metrics
146 
147 namespace PluginConfigParams {
148 
149 /**
150  * @def CONFIG_KEY(name)
151  * @brief shortcut for defining configuration keys
152  */
153 #define CONFIG_KEY(name) InferenceEngine::PluginConfigParams::_CONFIG_KEY(name)
154 #define _CONFIG_KEY(name) KEY_##name
155 #define DECLARE_CONFIG_KEY(name) static constexpr auto _CONFIG_KEY(name) = #name
156 
157 /**
158  * @def CONFIG_VALUE(name)
159  * @brief shortcut for defining configuration values
160  */
161 #define CONFIG_VALUE(name) InferenceEngine::PluginConfigParams::name
162 #define DECLARE_CONFIG_VALUE(name) static constexpr auto name = #name
163 
164 /**
165 * @brief generic boolean values
166 */
167 DECLARE_CONFIG_VALUE(YES);
168 DECLARE_CONFIG_VALUE(NO);
169 
170 /**
171 * @brief Limit #threads that are used by Inference Engine for inference on the CPU.
172 */
173 DECLARE_CONFIG_KEY(CPU_THREADS_NUM);
174 
175 /**
176 * @brief The name for setting CPU affinity per thread option.
177 * It is passed to IInferencePlugin::SetConfig(), this option should be used with values:
178 * PluginConfigParams::YES or PluginConfigParams::NO
179 * Ignored, if the OpenVINO compiled with OpenMP threading and any affinity-related OpenMP's
180 * environment variable is set
181 */
182 DECLARE_CONFIG_KEY(CPU_BIND_THREAD);
183 
184 /**
185 * @brief Optimize CPU execution to maximize throughput.
186 * It is passed to IInferencePlugin::SetConfig(), this option should be used with values:
187 * - KEY_CPU_THROUGHPUT_NUMA creates as many streams as needed to accomodate NUMA and avoid associated penalties
188 * - KEY_CPU_THROUGHPUT_AUTO creates bare minimum of streams to improve the performance,
189 * this is the most portable option if you have no insights into how many cores you target machine will have
190 * (and what is the optimal number of streams)
191 * - finally, specifying the positive integer value creates the requested number of streams
192 */
193 DECLARE_CONFIG_VALUE(CPU_THROUGHPUT_NUMA);
194 DECLARE_CONFIG_VALUE(CPU_THROUGHPUT_AUTO);
195 DECLARE_CONFIG_KEY(CPU_THROUGHPUT_STREAMS);
196 
197 /**
198 * @brief Optimize GPU plugin execution to maximize throughput.
199 * It is passed to IInferencePlugin::SetConfig(), this option should be used with values:
200 * - KEY_GPU_THROUGHPUT_AUTO creates bare minimum of streams that might improve performance in some cases,
201 * this option allows to enable throttle hint for opencl queue thus reduce CPU load without significant performance drop
202 * - a positive integer value creates the requested number of streams
203 */
204 DECLARE_CONFIG_VALUE(GPU_THROUGHPUT_AUTO);
205 DECLARE_CONFIG_KEY(GPU_THROUGHPUT_STREAMS);
206 
207 
208 /**
209 * @brief The name for setting performance counters option.
210 * It is passed to IInferencePlugin::SetConfig(), this option should be used with values:
211 * PluginConfigParams::YES or PluginConfigParams::NO
212 */
213 DECLARE_CONFIG_KEY(PERF_COUNT);
214 
215 /**
216 * @brief The key defines dynamic limit of batch processing.
217 * Specified value is applied to all following Infer() calls. Inference Engine processes
218 * min(batch_limit, original_batch_size) first pictures from input blob. For example, if input
219 * blob has sizes 32x3x224x224 after applying plugin.SetConfig({KEY_DYN_BATCH_LIMIT, 10})
220 * Inference Engine primitives processes only beginner subblobs with size 10x3x224x224.
221 * This value can be changed before any Infer() call to specify a new batch limit.
222 *
223 * The paired parameter value should be convertible to integer number. Acceptable values:
224 * -1 - Do not limit batch processing
225 * >0 - Direct value of limit. Batch size to process is min(new batch_limit, original_batch)
226 */
227 DECLARE_CONFIG_KEY(DYN_BATCH_LIMIT);
228 
229 DECLARE_CONFIG_KEY(DYN_BATCH_ENABLED);
230 
231 /**
232 * @brief The key controls threading inside Inference Engine.
233 * It is passed to IInferencePlugin::SetConfig(), this option should be used with values:
234 * PluginConfigParams::YES or PluginConfigParams::NO
235 */
236 DECLARE_CONFIG_KEY(SINGLE_THREAD);
237 
238 /**
239 * @brief This key directs the plugin to load a configuration file.
240 * The value should be a file name with the plugin specific configuration
241 */
242 DECLARE_CONFIG_KEY(CONFIG_FILE);
243 
244 /**
245 * @brief This key enables dumping of the kernels used by the plugin for custom layers.
246 * This option should be used with values: PluginConfigParams::YES or PluginConfigParams::NO (default)
247 */
248 DECLARE_CONFIG_KEY(DUMP_KERNELS);
249 
250 /**
251 * @brief This key controls performance tuning done or used by the plugin.
252 * This option should be used with values: PluginConfigParams::TUNING_CREATE,
253 * PluginConfigParams::TUNING_USE_EXISTING or PluginConfigParams::TUNING_DISABLED (default)
254 */
255 DECLARE_CONFIG_KEY(TUNING_MODE);
256 
257 
258 DECLARE_CONFIG_VALUE(TUNING_CREATE);
259 DECLARE_CONFIG_VALUE(TUNING_USE_EXISTING);
260 DECLARE_CONFIG_VALUE(TUNING_DISABLED);
261 
262 /**
263 * @brief This key defines the tuning data filename to be created/used
264 */
265 DECLARE_CONFIG_KEY(TUNING_FILE);
266 
267 /**
268 * @brief the key for setting desirable log level.
269 * This option should be used with values: PluginConfigParams::LOG_NONE (default),
270 * PluginConfigParams::LOG_WARNING, PluginConfigParams::LOG_INFO, PluginConfigParams::LOG_DEBUG
271 */
272 DECLARE_CONFIG_KEY(LOG_LEVEL);
273 
274 DECLARE_CONFIG_VALUE(LOG_NONE);
275 DECLARE_CONFIG_VALUE(LOG_WARNING);
276 DECLARE_CONFIG_VALUE(LOG_INFO);
277 DECLARE_CONFIG_VALUE(LOG_DEBUG);
278 
279 /**
280 * @brief the key for setting of required device to execute on
281 * values: device id starts from "0" - first device, "1" - second device, etc
282 */
283 DECLARE_CONFIG_KEY(DEVICE_ID);
284 
285 /**
286 * @brief the key for enabling exclusive mode for async requests of different executable networks and the same plugin.
287 * Sometimes it's necessary to avoid oversubscription requests that are sharing the same device in parallel.
288 * E.g. There 2 task executors for CPU device: one - in the Hetero plugin, another - in pure CPU plugin.
289 * Parallel execution both of them might lead to oversubscription and not optimal CPU usage. More efficient
290 * to run the corresponding tasks one by one via single executor.
291 * By default, the option is set to YES for hetero cases, and to NO for conventional (single-plugin) cases
292 * Notice that setting YES disables the CPU streams feature (see another config key in this file)
293 */
294 DECLARE_CONFIG_KEY(EXCLUSIVE_ASYNC_REQUESTS);
295 
296 /**
297  * @brief This key enables dumping of the internal primitive graph.
298  * Should be passed into LoadNetwork method to enable dumping of internal graph of primitives and
299  * corresponding configuration information. Value is a name of output dot file without extension.
300  * Files <dot_file_name>_init.dot and <dot_file_name>_perf.dot will be produced.
301  */
302 DECLARE_CONFIG_KEY(DUMP_EXEC_GRAPH_AS_DOT);
303 
304 } // namespace PluginConfigParams
305 } // namespace InferenceEngine
InferenceEngine
Definition: ie_argmax_layer.hpp:11