Group Common optimization passes#

group ov_transformation_common_api

A set of common optimization passes.

class AddTransformation : public ov::pass::low_precision::EltwiseBaseTransformation #

#include <add.hpp>

AddTransformation propagates dequantization subtraction from one input branch to another and propagates dequantization multiplication from the same branch through Add operation.

For more details about the transformation, refer to AddTransformation page in the OpenVINO Developer Guide.

class AlignQuantizationIntervals : public ov::pass::ModelPass #

#include <align_quantization_intervals.hpp>

AlignQuantizationIntervals transformation marks precision preserved operations subgraph by IntervalsAlignmentAttribute after FakeQuantize operations.

For more details about the transformation, refer to AlignQuantizationIntervals page in the OpenVINO Developer Guide.

class AlignQuantizationParameters : public ov::pass::ModelPass #

#include <align_quantization_parameters.hpp>

AlignQuantizationParameters transformation marks precision preserved operations subgraph by QuantizationAlignmentAttribute attribute after FakeQuantize operations.

For more details about the transformation, refer to AlignQuantizationParameters page in the OpenVINO Developer Guide.

class AvgPoolTransformation : public ov::pass::low_precision::LayerTransformation #

#include <avg_pool.hpp>

AvgPoolTransformation propagates dequantization operations through AvgPool operation.

For more details about the transformation, refer to AvgPoolTransformation page in the OpenVINO Developer Guide.

class BatchToSpaceTransformation : public ov::pass::low_precision::LayerTransformation #

#include <batch_to_space.hpp>

BatchToSpaceTransformation propagates dequantization operations through BatchToSpace operation.

For more details about the transformation, refer to BatchToSpaceTransformation page in the OpenVINO Developer Guide.

class BroadcastTransformation : public ov::pass::low_precision::TransparentBaseTransformation #

#include <broadcast.hpp>

BroadcastTransformation propagates dequantization operations through Broadcast operation.

For more details about the transformation, refer to BroadcastTransformation page in the OpenVINO Developer Guide.

class ClampTransformation : public ov::pass::low_precision::LayerTransformation #

#include <clamp.hpp>

ClampTransformation propagates dequantization operations through Clamp operation.

For more details about the transformation, refer to ClampTransformation page in the OpenVINO Developer Guide.

class CleanupTransformation : public ov::pass::low_precision::LayerTransformation #

#include <cleanup_transformation.hpp>

Base class for cleanup low precision transformation.

Subclassed by ov::pass::low_precision::EliminateFakeQuantizeTransformation, ov::pass::low_precision::FoldConvertTransformation, ov::pass::low_precision::FuseConvertTransformation, ov::pass::low_precision::FuseElementwiseToFakeQuantizeTransformation, ov::pass::low_precision::MultiplyToGroupConvolutionTransformation

class ConcatTransformation : public ov::pass::low_precision::LayerTransformation #

#include <concat.hpp>

ConcatTransformation propagates dequantization operations through Concat operation.

For more details about the transformation, refer to ConcatTransformation page in the OpenVINO Developer Guide.

class ConvertSubtractConstant : public ov::pass::MatcherPass #

#include <convert_subtract_constant.hpp>

ConvertSubtractConstant marks Convert operations on constant subgraph by DISABLED_CONSTANT_FOLDING attribute to prevent constant folding.

For more details about the transformation, refer to ConvertSubtractConstant page in the OpenVINO Developer Guide.

class ConvolutionTransformation : public ov::pass::low_precision::WeightableLayerTransformation #

#include <convolution.hpp>

ConvolutionTransformation propagates dequantization operations through Convolution operation.

For more details about the transformation, refer to ConvolutionTransformation page in the OpenVINO Developer Guide.

Subclassed by ov::pass::low_precision::GroupConvolutionTransformation

class ConvolutionBackpropDataTransformation : public ov::pass::low_precision::WeightableLayerTransformation #

#include <convolution_backprop_data.hpp>

ConvolutionBackpropDataTransformation propagates dequantization operations through ConvolutionBackpropData operation.

For more details about the transformation, refer to ConvolutionBackpropDataTransformation page in the OpenVINO Developer Guide.

template<typename AttributeType, typename OperationType = ov::pass::pattern::op::Label> class CreateAttribute : public ov::pass::low_precision::BaseMatcherPass #

#include <create_attribute.hpp>

CreateAttribute transformation marks OperationType operations by AttributeType attribute.

For more details about the transformation, refer to CreateAttribute page in the OpenVINO Developer Guide.

template<typename AttributeType, typename OperationType> class CreatePrecisionsDependentAttribute : public ov::pass::MatcherPass #

#include <create_precisions_dependent_attribute.hpp>

CreatePrecisionsDependentAttribute transformation marks OperationType operations by PrecisionPreservedAttribute and AttributeType attributes with the same shared part.

For more details about the transformation, refer to CreatePrecisionsDependentAttribute page in the OpenVINO Developer Guide.

class DepthToSpaceTransformation : public ov::pass::low_precision::TransparentBaseTransformation #

#include <depth_to_space.hpp>

DepthToSpaceTransformation propagates dequantization operations through DepthToSpace operation.

For more details about the transformation, refer to DepthToSpaceTransformation page in the OpenVINO Developer Guide.

class EliminateFakeQuantizeTransformation : public ov::pass::low_precision::CleanupTransformation #

#include <eliminate_fake_quantize.hpp>

EliminateFakeQuantizeTransformation removes FakeQuantize operations.

For more details about the transformation, refer to EliminateFakeQuantizeTransformation page in the OpenVINO Developer Guide.

class EltwiseBaseTransformation : public ov::pass::low_precision::LayerTransformation #

#include <eltwise_base_transformation.hpp>

EltwiseBaseTransformation is base class for element-wise LPT transformations.

Subclassed by ov::pass::low_precision::AddTransformation, ov::pass::low_precision::MultiplyPartialTransformation

class FakeQuantizeTransformation : public ov::pass::low_precision::LayerTransformation #

#include <fake_quantize.hpp>

FakeQuantizeTransformation fuses dequantization operations into FakeQuantize operation.

For more details about the transformation, refer to FakeQuantizeTransformation page in the OpenVINO Developer Guide.

class FakeQuantizeDecompositionTransformation : public ov::pass::low_precision::LayerTransformation #

#include <fake_quantize_decomposition.hpp>

FakeQuantizeDecompositionTransformation decomposes FakeQuantize operations to quantize (FakeQuantize with changes output intervals and low precision output type) and dequantize operations.

For more details about the transformation, refer to FakeQuantizeDecompositionTransformation page in the OpenVINO Developer Guide.

class FoldConvertTransformation : public ov::pass::low_precision::CleanupTransformation #

#include <fold_convert.hpp>

FoldConvertTransformation evaluates Convert operation on Subtract constant subgraph. Important notice: this transformation ignores DisableConstantFolding runtime attribute.

For more details about the transformation, refer to FoldConvertTransformation page in the OpenVINO Developer Guide.

class FoldFakeQuantizeTransformation : public ov::pass::low_precision::LayerTransformation #

#include <fold_fake_quantize.hpp>

FoldFakeQuantizeTransformation evaluate FakeQuantize operations.

For more details about the transformation, refer to FoldFakeQuantizeTransformation page in the OpenVINO Developer Guide.

class FuseConvertTransformation : public ov::pass::low_precision::CleanupTransformation #

#include <fuse_convert.hpp>

FuseConvertTransformation fuses Convert operation with Multiply, Subtract or Add operations.

For more details about the transformation, refer to FuseConvertTransformation page in the OpenVINO Developer Guide.

class FuseElementwiseToFakeQuantizeTransformation : public ov::pass::low_precision::CleanupTransformation #

#include <fuse_elementwise_to_fake_quantize.hpp>

Base class for fuse elementwise to FakeQuantize low precision transformation.

Subclassed by ov::pass::low_precision::FuseMultiplyToFakeQuantizeTransformation, ov::pass::low_precision::FuseSubtractToFakeQuantizeTransformation

class FuseMultiplyToFakeQuantizeTransformation : public ov::pass::low_precision::FuseElementwiseToFakeQuantizeTransformation #

#include <fuse_multiply_to_fake_quantize.hpp>

FuseMultiplyToFakeQuantizeTransformation fuses Multiply operation to FakeQuantize.

For more details about the transformation, refer to FuseMultiplyToFakeQuantizeTransformation page in the OpenVINO Developer Guide.

class FuseSubtractToFakeQuantizeTransformation : public ov::pass::low_precision::FuseElementwiseToFakeQuantizeTransformation #

#include <fuse_subtract_to_fake_quantize.hpp>

FuseSubtractToFakeQuantizeTransformation fuses Subtract operation to FakeQuantize.

For more details about the transformation, refer to FuseSubtractToFakeQuantizeTransformation page in the OpenVINO Developer Guide.

class GroupConvolutionTransformation : public ov::pass::low_precision::ConvolutionTransformation #

#include <group_convolution.hpp>

GroupConvolutionTransformation propagates dequantization operations through GroupConvolution operation.

For more details about the transformation, refer to GroupConvolutionTransformation page in the OpenVINO Developer Guide.

class InterpolateTransformation : public ov::pass::low_precision::LayerTransformation #

#include <interpolate.hpp>

InterpolateTransformation propagates dequantization operations through Interpolate operation.

For more details about the transformation, refer to InterpolateTransformation page in the OpenVINO Developer Guide.

class LayerTransformation : public ov::pass::MatcherPass #

#include <layer_transformation.hpp>

Base class for low precision transformation.

class Params#: #include <layer_transformation.hpp>

class PrecisionDetails#: #include <layer_transformation.hpp>

class MarkupAvgPoolPrecisionPreserved : public ov::pass::ModelPass #

#include <markup_avg_pool_precision_preserved.hpp>

MarkupAvgPoolPrecisionPreserved transformation marks AvgPool operations as precision preserved or not.

For more details about the transformation, refer to MarkupAvgPoolPrecisionPreserved page in the OpenVINO Developer Guide.

class MarkupBias : public ov::pass::MatcherPass #

#include <markup_bias.hpp>

MarkupBias transformation marks biases after target layers.

For more details about the transformation, refer to MarkupBias page in the OpenVINO Developer Guide.

class MarkupCanBeQuantized : public ov::pass::ModelPass #

#include <markup_can_be_quantized.hpp>

MarkupCanBeQuantized transformation marks Convolution, ConvolutionBackpropData, GroupConvolution and Concat operations as able to be quantized or not. If an operation is not quantized, then PrecisionsAttribute attribute instance is created with empty precisions.

For more details about the transformation, refer to MarkupCanBeQuantized page in the OpenVINO Developer Guide.

class MarkupPrecisions : public ov::pass::ModelPass #

#include <markup_precisions.hpp>

MarkupPrecisions transformation marks: 1) not supported operations by PrecisionsAttribute attribute with empty precisions, 2) operations with required precisions by PrecisionsAttribute attribute according to the provided restrictions, 3) precision preserved operations by PrecisionPreservedAttribute attribute.

For more details about the transformation, refer to MarkupPrecisions page in the OpenVINO Developer Guide.

class Restriction#

#include <markup_precisions.hpp>

class RestrictionByVersion#: #include <markup_precisions.hpp>

class MarkupQuantizationGranularity : public ov::pass::ModelPass #

#include <markup_quantization_granularity.hpp>

MarkupPerTensorQuantization transformation marks operations as required per-tensor quantization according to the provided restrictions.

For more details about the transformation, refer to MarkupPerTensorQuantization page in the OpenVINO Developer Guide.

class PerTensorQuantization#: #include <markup_quantization_granularity.hpp>

class MatMulTransformation : public ov::pass::low_precision::LayerTransformation #

#include <mat_mul.hpp>

MatMulTransformation propagates dequantization operations through MatMul operation.

For more details about the transformation, refer to MatMulTransformation page in the OpenVINO Developer Guide.

class MaxPoolTransformation : public ov::pass::low_precision::LayerTransformation #

#include <max_pool.hpp>

MaxPoolTransformation propagates dequantization operations through MaxPool operation.

For more details about the transformation, refer to MaxPoolTransformation page in the OpenVINO Developer Guide.

class MultiplyTransformation : public ov::pass::low_precision::WeightableLayerTransformation #

#include <multiply.hpp>

MultiplyTransformation propagates dequantization operations through Multiply operation.

For more details about the transformation, refer to MultiplyTransformation page in the OpenVINO Developer Guide.

class MultiplyPartialTransformation : public ov::pass::low_precision::EltwiseBaseTransformation #

#include <multiply_partial.hpp>

MultiplyPartialTransformation propagates dequantization operations through Multiply operation.

For more details about the transformation, refer to MultiplyPartialTransformation page in the OpenVINO Developer Guide.

class MultiplyToGroupConvolutionTransformation : public ov::pass::low_precision::CleanupTransformation #

#include <multiply_to_group_convolution.hpp>

MultiplyToGroupConvolutionTransformation replace quantized Multiply operations to GroupConvolution to speed up inference.

For more details about the transformation, refer to MultiplyToGroupConvolutionTransformation page in the OpenVINO Developer Guide.

class MVNTransformation : public ov::pass::low_precision::LayerTransformation #

#include <mvn.hpp>

MVNTransformation propagates dequantization operations through MVN operation.

For more details about the transformation, refer to MVNTransformation page in the OpenVINO Developer Guide.

class NormalizeL2Transformation : public ov::pass::low_precision::LayerTransformation #

#include <normalize_l2.hpp>

NormalizeL2Transformation propagates dequantization operations through NormalizeL2 operation.

For more details about the transformation, refer to NormalizeL2Transformation page in the OpenVINO Developer Guide.

class PadTransformation : public ov::pass::low_precision::LayerTransformation #

#include <pad.hpp>

PadTransformation propagates dequantization operations through Pad operation.

For more details about the transformation, refer to PadTransformation page in the OpenVINO Developer Guide.

class PReluTransformation : public ov::pass::low_precision::LayerTransformation #

#include <prelu.hpp>

PReluTransformation propagates dequantization operations through PRelu operation.

For more details about the transformation, refer to PReluTransformation page in the OpenVINO Developer Guide.

class PropagatePrecisions : public ov::pass::ModelPass #

#include <propagate_precisions.hpp>

PropagatePrecisions transformation propagates PrecisionsAttribute attribute instances precision preserved operations.

For more details about the transformation, refer to PropagatePrecisions page in the OpenVINO Developer Guide.

template<class AttributeType> class PropagateSharedValue : public ov::pass::ModelPass #

#include <propagate_shared_value.hpp>

PropagateSharedValue transformation propagates shared value AttributeType attribute instances through precision preserved operations.

For more details about the transformation, refer to PropagateSharedValue page in the OpenVINO Developer Guide.

template<typename AttributeType> class PropagateThroughPrecisionPreserved : public ov::pass::MatcherPass #

#include <propagate_through_precision_preserved.hpp>

PropagateThroughPrecisionPreserved transformation propagates AttributeType attribute instances through precision preserved operations.

For more details about the transformation, refer to PropagateThroughPrecisionPreserved page in the OpenVINO Developer Guide.

template<typename AttributeType> class PropagateToInput : public ov::pass::MatcherPass #

#include <propagate_to_input.hpp>

PropagateToInput transformation propagates AttributeType shared value attribute instances from parent output ports to consumers input ports.

For more details about the transformation, refer to PropagateToInput page in the OpenVINO Developer Guide.

class PullReshapeThroughDequantization : public ov::pass::MatcherPass #

#include <pull_reshape_through_dequantization.hpp>

PullReshapeThroughDequantization propagates dequantization operations through Reshape operations. The transformation is used on constant subgraph weights to prepare a model for the next low precision transformations.

For more details about the transformation, refer to PullReshapeThroughDequantization page in the OpenVINO Developer Guide.

class PullTransposeThroughDequantization : public ov::pass::MatcherPass #

#include <pull_transpose_through_dequantization.hpp>

PullTransposeThroughDequantization propagates dequantization operations through Transpose operations. The transformation is used on constant subgraph weights to prepare a model for the next low precision transformations.

For more details about the transformation, refer to PullTransposeThroughDequantization page in the OpenVINO Developer Guide.

class ReduceBaseTransformation : public ov::pass::low_precision::LayerTransformation #

#include <reduce_base_transformation.hpp>

ReduceBaseTransformation: base class for Reduce*Transformation, detects dequantization operations in front of the Reduce* operation and propagates them through the Reduce* if possible.

Subclassed by ov::pass::low_precision::ReduceMaxTransformation, ov::pass::low_precision::ReduceMeanTransformation, ov::pass::low_precision::ReduceMinTransformation, ov::pass::low_precision::ReduceSumTransformation

class ReduceMaxTransformation : public ov::pass::low_precision::ReduceBaseTransformation #

#include <reduce_max.hpp>

ReduceMaxTransformation propagates dequantization operations through ReduceMax operation.

For more details about the transformation, refer to ReduceMaxTransformation page in the OpenVINO Developer Guide.

class ReduceMeanTransformation : public ov::pass::low_precision::ReduceBaseTransformation #

#include <reduce_mean.hpp>

ReduceMeanTransformation propagates dequantization operations through ReduceMean operation.

For more details about the transformation, refer to ReduceMeanTransformation page in the OpenVINO Developer Guide.

class ReduceMinTransformation : public ov::pass::low_precision::ReduceBaseTransformation #

#include <reduce_min.hpp>

ReduceMinTransformation propagates dequantization operations through ReduceMin operation.

For more details about the transformation, refer to ReduceMinTransformation page in the OpenVINO Developer Guide.

class ReduceSumTransformation : public ov::pass::low_precision::ReduceBaseTransformation #

#include <reduce_sum.hpp>

ReduceSumTransformation propagates dequantization operations through ReduceSum operation.

For more details about the transformation, refer to ReduceSumTransformation page in the OpenVINO Developer Guide.

class ReluTransformation : public ov::pass::low_precision::LayerTransformation #

#include <relu.hpp>

ReluTransformation propagates dequantization operations through Relu operation.

For more details about the transformation, refer to ReluTransformation page in the OpenVINO Developer Guide.

class ReshapeTransformation : public ov::pass::low_precision::LayerTransformation #

#include <reshape.hpp>

ReshapeTransformation propagates dequantization operations through Reshape operation.

For more details about the transformation, refer to ReshapeTransformation page in the OpenVINO Developer Guide.

class AvgPoolPrecisionPreservedAttribute : public ov::PrecisionPreservedAttribute #

#include <avg_pool_precision_preserved_attribute.hpp>

AvgPoolPrecisionPreservedAttribute is utility attribute which is used only during AvgPool operation precision preserved property definition.

For more details about the attribute, refer to AvgPoolPrecisionPreservedAttribute page in the OpenVINO Developer Guide.

class IntervalsAlignmentSharedValue#

#include <intervals_alignment_attribute.hpp>

IntervalsAlignmentSharedValue is used by IntervalsAlignmentAttribute as attribute shared value.

class Interval#: #include <intervals_alignment_attribute.hpp>

class IntervalsAlignmentAttribute : public SharedAttribute<IntervalsAlignmentSharedValue>#

#include <intervals_alignment_attribute.hpp>

IntervalsAlignmentAttribute defines subgraph with the same quantization intervals alignment. FakeQuantize operations are included. The attribute is used by quantization operations.

For more details about the attribute, refer to IntervalsAlignmentAttribute page in the OpenVINO Developer Guide.

class PrecisionPreservedAttribute : public SharedAttribute<bool>#

#include <precision_preserved_attribute.hpp>

PrecisionPreservedAttribute defines the precision preserved operation. If the attribute is absent, then an operation is not precision preserved.

For more details about the attribute, refer to PrecisionPreservedAttribute page in the OpenVINO Developer Guide.

Subclassed by ov::AvgPoolPrecisionPreservedAttribute

class PrecisionsAttribute : public SharedAttribute<std::vector<ov::element::Type>>#

#include <precisions_attribute.hpp>

PrecisionsAttribute defines precision which is required for input/output port or an operation.

For more details about the attribute, refer to PrecisionsAttribute page in the OpenVINO Developer Guide.

class QuantizationAlignmentAttribute : public SharedAttribute<bool>#

#include <quantization_alignment_attribute.hpp>

QuantizationAlignmentAttribute defines subgraph with the same quantization alignment. FakeQuantize operations are not included. The attribute is used by quantization operations.

For more details about the attribute, refer to QuantizationAlignmentAttribute page in the OpenVINO Developer Guide.

class QuantizationGranularityAttribute : public ov::RuntimeAttribute #

#include <quantization_granularity_attribute.hpp>

QuantizationGranularityAttribute defines quantization granularity of operation inputs.

For more details about the attribute, refer to QuantizationGranularityAttribute page in the OpenVINO Developer Guide.

class SharedValueAttribute : public std::enable_shared_from_this<SharedValueAttribute>#

#include <shared_value_attribute.hpp>

SharedValueAttribute type for shared value attributes. The attribute is used for attribute SharedValue value backward propagation.

struct SharedValue : public std::enable_shared_from_this<SharedValue>#: #include <shared_value_attribute.hpp>

class ShuffleChannelsTransformation : public ov::pass::low_precision::LayerTransformation #

#include <shuffle_channels.hpp>

ShuffleChannelsTransformation propagates dequantization operations through ShuffleChannels operation.

For more details about the transformation, refer to ShuffleChannelsTransformation page in the OpenVINO Developer Guide.

class SliceTransformation : public ov::pass::low_precision::LayerTransformation #

#include <slice.hpp>

SliceTransformation propagates dequantization operations through Slice operation.

For more details about the transformation, refer to SliceTransformation page in the OpenVINO Developer Guide.

class SpaceToBatchTransformation : public ov::pass::low_precision::LayerTransformation #

#include <space_to_batch.hpp>

SpaceToBatchTransformation propagates dequantization operations through SpaceToBatch operation.

For more details about the transformation, refer to SpaceToBatchTransformation page in the OpenVINO Developer Guide.

class SplitTransformation : public ov::pass::low_precision::LayerTransformation #

#include <split.hpp>

SplitTransformation propagates dequantization operations through Split operation.

For more details about the transformation, refer to SplitTransformation page in the OpenVINO Developer Guide.

Subclassed by ov::pass::low_precision::VariadicSplitTransformation

class SqueezeTransformation : public ov::pass::low_precision::LayerTransformation #

#include <squeeze.hpp>

SqueezeTransformation propagates dequantization operations through Squeeze operation.

For more details about the transformation, refer to SqueezeTransformation page in the OpenVINO Developer Guide.

class StridedSliceTransformation : public ov::pass::low_precision::LayerTransformation #

#include <strided_slice.hpp>

StridedSliceTransformation propagates dequantization operations through StridedSlice operation.

For more details about the transformation, refer to StridedSliceTransformation page in the OpenVINO Developer Guide.

class SubtractTransformation : public ov::pass::low_precision::LayerTransformation #

#include <subtract.hpp>

SubtractTransformation propagates dequantization operations through Subtract operation.

For more details about the transformation, refer to SubtractTransformation page in the OpenVINO Developer Guide.

class TransformationContext#: #include <transformation_context.hpp>

TransformationContext instance is used to pass model transformation context data between transformations.

class TransparentBaseTransformation : public ov::pass::low_precision::LayerTransformation #

#include <transparent_base_transformation.hpp>

TransparentBaseTransformation is base type for precision preserved operation transformation.

Subclassed by ov::pass::low_precision::BroadcastTransformation, ov::pass::low_precision::DepthToSpaceTransformation

class TransposeTransformation : public ov::pass::low_precision::LayerTransformation #

#include <transpose.hpp>

TransposeTransformation propagates dequantization operations through Transpose operation.

For more details about the transformation, refer to TransposeTransformation page in the OpenVINO Developer Guide.

class UnsqueezeTransformation : public ov::pass::low_precision::LayerTransformation #

#include <unsqueeze.hpp>

UnsqueezeTransformation propagates dequantization operations through Unsqueeze operation.

For more details about the transformation, refer to UnsqueezeTransformation page in the OpenVINO Developer Guide.

template<typename AttributeType, typename ExpectedAttributeType = AttributeType> class UpdateSharedPrecisionPreserved : public ov::pass::MatcherPass #

#include <update_shared_precision_preserved.hpp>

UpdateSharedPrecisionPreserved transformation updates shared AttributeType attribute instance value to true for precision preserved operations if ExpectedAttributeType exist.

For more details about the transformation, refer to UpdateSharedPrecisionPreserved page in the OpenVINO Developer Guide.

class VariadicSplitTransformation : public ov::pass::low_precision::SplitTransformation #

#include <variadic_split.hpp>

VariadicSplitTransformation propagates dequantization operations through VariadicSplit operation.

For more details about the transformation, refer to VariadicSplitTransformation page in the OpenVINO Developer Guide.

class WeightableLayerTransformation : public ov::pass::low_precision::LayerTransformation #

#include <weightable_layer_transformation.hpp>

WeightableLayerTransformation is base type for weightable operation transformation.

Subclassed by ov::pass::low_precision::ConvolutionBackpropDataTransformation, ov::pass::low_precision::ConvolutionTransformation, ov::pass::low_precision::MultiplyTransformation

struct CanBeTransformedParams#: #include <weightable_layer_transformation.hpp>

class InitMasks : public ov::pass::GraphRewrite #: #include <pruning.hpp>

Initialising masks for pruned operations.

class InitConstMask : public ov::pass::MatcherPass #: #include <pruning.hpp>

Check Constant operation values by given dimensions and set masks according to results that are bases on condition lambda function. Works for Constant with floating point type (f16, f32, f64).

class PropagateMasks : public ov::pass::GraphRewrite #: #include <pruning.hpp>

Contains several MatcherPasses that initialize and propagate masks from Constant operation to the network output.

class ShrinkWeights : public ov::pass::ModelPass #: #include <pruning.hpp>

Based on masks in Constant operation it inserts Gather operations to shrink them. After this pass execution ConstantFolding is required.

class Pruning : public ov::pass::ModelPass #: #include <pruning.hpp>

This is just a sequence of passes that performs pruning transformations pipeline.

class AdaptivePoolToReduce : public ov::pass::MatcherPass #: #include <adaptive_pool_to_reduce.hpp>

AdaptivePoolToReduce transformation replaces AdaptiveXXXPool with ReduceXXX when possible.

class AddFakeQuantizeFusion : public ov::pass::MatcherPass #

#include <add_fake_quantize_fusion.hpp>

AddFakeQuantizeFusion transformation replaces following graph: Add->FakeQuantize to a single FakeQuantize Restrictions:

second input to Add is a Constant

class AUGRUCellFusion : public ov::pass::MatcherPass #

#include <augru_cell_fusion.hpp>

AUGRUCellFusion transformation replaces a sequence of operations with AUGRUCell op.

Supported activations: 1st is Sigmoid, 2nd is Tanh Clip attribute is not supported. Linear_before_reset attribute is not supported. Supported weights format: ‘rzh’

class BatchToSpaceFusion : public ov::pass::MatcherPass #

#include <batch_to_space_fusion.hpp>

BatchToSpaceFusion transformation replaces following graph: Transpose (or Reshape) -> DepthToSpace -> StridedSlice -> Transpose (or Reshape) to BatchToSpace Restrictions:

input rank must be 4
Transpose permutation must be [1, 0, 2, 3]
DepthToSpaceMode must be BLOCKS_FIRST

class BinarizeWeights : public ov::pass::MatcherPass #

#include <binarize_weights.hpp>

This transformation converts weights to -1/+1 form and applies normalization factors to output low/high and after Convolution. For example, following graph.

    .... ....  out_low  out_high           weights ..    ..  out_low out_high
      |    |      |        |                  |     |    |      |     |
     +--------------------------+           +--------------------------+
     | FakeQuantize (levels==2) |           | FakeQuantize (levels==2) |
     |     (on activations)     |           |       (on weights)       |
     +--------------------------+           +--------------------------+
                   |                                      |
                   |                                      |
                   -----------------    -------------------
                                   |    |
                                   v    v
                              +-------------+
                              | Convolution |
                              +-------------+
                                     |
                                     v

is transformed to:

             normalized normalized
    .... ....  out_low   out_high
      |    |      |         |
     +--------------------------+           +--------------------------+
     | FakeQuantize (levels==2) |           |         Constant         |
     |     (on activations)     |           | (with converted weights) |
     +--------------------------+           +--------------------------+
                   |                                      |
                   |                                      |
                   -----------------    -------------------
                                   |    |
                                   v    v
                              +-------------+
                              | Convolution |
                              +-------------+
                                     |
                                     v
                              +------------+     +---------------------------------------------------------------+
                              |  Multiply  | <---| Constant (normalization factor coming from FQ on activations) |
                              +------------+     +---------------------------------------------------------------+
                                     |
                                     v
                              +------------+     +-----------------------------------------------------------+
                              |  Multiply  | <---| Constant (normalization factor coming from FQ on weights) |
                              +------------+     +------------------------------------------------------------
                                     |
                                     v

Normalization factors are chosen based output_high value. If it’s zero - norm factor is equal to output_low and output_high otherwise

class BroadcastElementwiseFusion : public ov::pass::MatcherPass #: #include <broadcast_elementwise_fusion.hpp>

Removing Broadcast OP before ElementWise if output shape of Broadcast are equal neighboring input shape of ElementWise.

class BroadcastTransition : public ov::pass::MatcherPass #: #include <broadcast_transition.hpp>

BroadcastTransition transformation moves broadcast through binary eltwise operation.

class ClampFusion : public ov::pass::MatcherPass #

#include <clamp_fusion.hpp>

ClampFusion transformation replaces following graph: Maximum->Minimum to Clamp Restrictions:

one of the parameters to Maximum is a scalar constant
one of the parameters to Minimum is a scalar constant

class CompressFloatConstantsImpl : public ov::pass::MatcherPass #

#include <compress_float_constants.hpp>

CompressFloatConstantsImpl transformation replaces FP32/FP64 Constants with FP16 ones.

Public Functions

CompressFloatConstantsImpl(bool postponed = false)#

Transformation constructor.

Parameters:: postponed – If true then the transformation won’t compress the constants keeping them in the original type but still will insert Converts. This is a special mode of operation that requires another transformation to apply a real compression on constants. Constants eligible for postponed compression are marked with a special rt_info tag.

class CompressFloatConstants : public ov::pass::GraphRewrite #

#include <compress_float_constants.hpp>

CompressFloatConstants transformation replaces FP32/FP64 Constants with FP16 ones.

Public Functions

inline CompressFloatConstants(bool postponed = false)#

Transformation constructor.

Parameters:: postponed – Postponed compression, see ov::pass::CompressFloatConstantsImpl for details.

class ConcatFusion : public ov::pass::MatcherPass #: #include <concat_fusion.hpp>

ConcatFusion transformation fuses sequence of Concats.

class ReplaceConcatReduceByMinOrMax : public ov::pass::MatcherPass #: #include <concat_reduce_fusion.hpp>

ReplaceConcatReduceByMinOrMax transformation replaces Concat with 2 inputs and ReduceMin/Max by a single Minimum/Maximum with 2 inputs and inserts squeeze in case when Reduce has keep_dims = false.

class PullSqueezeThroughEltwise : public ov::pass::MatcherPass #: #include <concat_reduce_fusion.hpp>

PullSqueezeThroughEltwise transformation propagates Squeeze up through binary elementwise operations:

class ConcatReduceFusion : public ov::pass::GraphRewrite #

#include <concat_reduce_fusion.hpp>

ConcatReduceFusion pass replaces the following graph:

          +---------------+            +---------------+
          │               │            |               |
          │     input     │            |     input     |
          │               │            |               |
          +---------------+            +----------------
                  |                            |
                  |                            |
                  \                            /
                   \                          /
                    \                        /
                     \                      /
                      \                    /
                       \                  /
                        \                /
                         \              /
                          \            /
                         +---------------+
                         |               |
                         |     Concat    |
                         |               |
                         +----------------
                                 |
                                 v
                         +---------------+
                         |               |
                         |   ReduceMin/  |
                         |   ReduceMax   |
                         +----------------

by a single Minimum/Maximum with 2 inputs and tries to eliminate Squeeze/Unsqueeze layers before and after Min/Max.

class ConcatToBroadcast : public ov::pass::MatcherPass #: #include <concat_to_broadcast.hpp>

ConcatToBroadcast transformation replaces Concat, having multiple inputs from the same output, with a Broadcast node.

class ConvToBinaryConv : public ov::pass::MatcherPass #

#include <conv_to_binary_conv.hpp>

This transformation converts Convolution to BinaryConvolution under following conditions:

first input to Convolution is a FakeQuantize with levels==2 with output low,high being either (0, 1) or (-1, 1)

second input (weights) is a constant with values -1 or 1 The transformation also converts weights to binary Constant (with ‘u1’ type) For example, when output_low is equal to 0 and output_high is equal to 1, following graph

 .... ....  out_low   out_high
   |    |      |         |
  +--------------------------+           +-------------------------------------+
  | FakeQuantize (levels==2) |           |               Constant              |
  |     (on activations)     |           | (weights containing -1 or 1 values) |
  +--------------------------+           +-------------------------------------+
                |                                      |
                |                                      |
                -----------------    -------------------
                                |    |
                                v    v
                           +-------------+
                           | Convolution |
                           +-------------+
                                  |
                                  v

is transformed to:

 .... ....  out_low   out_high
   |    |      |         |
  +--------------------------+           +---------------------------------+
  | FakeQuantize (levels==2) |           |     Constant (with u1 type)     |
  |     (on activations)     |           | (with u1 type - binary weights) |
  +--------------------------+           +---------------------------------+
                |                                      |
                |                                      |
                -----------------    -------------------
                                |    |
                                v    v
                        +-------------------+
                        | BinaryConvolution |
                        +-------------------+
                                  |
                                  v
                           +------------+     +----------------------------------------------------+
                           |            |     |                   Constant                         |
                           |     Add    | <---|          (weights from original graph,             |
                           |            |     |  sum-reduced over [1,..., len(weights.shape)] axes |
                           +------------+     +----------------------------------------------------+
                                  |
                                  v
                           +------------+     +-----+
                           |  Multiply  | <---| 0.5 |
                           +------------+     +-----+
                                  |
                                  v

class ConvertNmsGatherPathToUnsigned : public ov::pass::GraphRewrite #: #include <convert_nms_gather_path_to_unsigned.hpp>

Converts Gather indices to unsigned if indices are from NMS selected indices output. NMS returns -1 for not selected boxes, old version of Gather fill corresponding output for such indices with zero. But new Gather-8 has support of negative indices indicating counting from the end. In order to keep such behaviour (until dynamism is not supported) instead of -1 new Gather-8 will accept UINT32_MAX which is always outside of the bounds and corresponding output for such indices in gather always will be filled with zeros.

class ConvertQuantizeDequantize : public ov::pass::MatcherPass #

#include <convert_quantize_dequantize.hpp>

ConvertQuantizeDequantize transformation replaces following graph: FakeQuantize->Convert->Convert->Subtract->Multiply with a single FakeQuantize. Restrictions:

quantized data type must be i8 or u8
’levels’ attribute to FakeQuantize must be equal to 256
(output_low, output_high) must be (-128, 127) or (0, 256) (depends on sign of quantized data type)
’zero_point’ and ‘scale’ must be broadcastable to FakeQuantize’s output

class ConvertU4WeightsZeroPointToScalar : public ov::pass::MatcherPass #: #include <convert_u4_weights_zero_point_to_scalar.hpp>

Converts U4 weights zero point to scalar if all values are equal.

class ConvolutionToGroupConvolutionFusion : public ov::pass::MatcherPass #

#include <convolution_to_group_convolution_fusion.hpp>

ConvolutionToGroupConvolutionFusion transformation replaces following graph: Split (or VariadicSplit) / \ Conv … Conv \ / \ / Concat.

to GroupConvolution

class DepthToSpaceFusion : public ov::pass::MatcherPass #

#include <depth_to_space_fusion.hpp>

DepthToSpaceFusion transformation detects Reshape-Transpose-Reshape pattern and tries to fuse it into a single DepthToSpace layer.

DepthToSpaceFusion transformation is optional and disabled by default. The transformation can be enabled with callback using setCallback method. See the example below.

Callback example:

// This callback enables DepthToSpaceFusion transformation
auto callback = [](const std::shared_ptr<const ov::Node> & node) -> bool {
    return ov::as_type_ptr<const ov::opset3::DepthToSpace>(node) != nullptr;
};

auto p = ov::pass::DepthToSpaceFusion();
p.setCallback(callback);
p.run_on_function(f);

class DilatedConvolutionConverter : public ov::pass::MatcherPass #

#include <dilated_convolution_converter.hpp>

DilatedConvolutionConverter transformation replaces following graph: SpaceToBatch -> Convolution(GroupConvolution) -> BatchToSpace to a single Convolution(GroupConvolution) node with updated pads and dilations Restrictions:

pads in SpaceToBatch must have 0 on first and second position

class DisableRandomUniformConstantFolding : public ov::pass::MatcherPass #: #include <disable_random_uniform_constant_folding.hpp>

Disables ConstantFolding for RandomUniform operation. It is required as RandomUniform should generate new sequence each run.

class DivideFusion : public ov::pass::MatcherPass #: #include <divide_fusion.hpp>

DivideFusion transformation replaces a sub-graph Pow(y, -1) * x or x * Pow(y, -1) with Divide(x,y)

class DropoutWithRandomUniformReplacer : public ov::pass::MatcherPass #

#include <dropout_with_random_uniform_replacer.hpp>

This transformation replaces possible Dropout block (in inference mode) with RandomUniform to Broadcast of half-ones in a sub-graph.

Dropout block: RandomUniform ——-—> Add —> Floor /\ /\ /\ | | | Const(0) Const(1) Const(1) min_val max_val

Resulted block: Broadcast —-—> Add —> Floor /\ /\ | | Const(0.5) Const(1)

class EliminateLoopInputsOutputs : public ov::pass::MatcherPass #: #include <eliminate_loop_inputs_outputs.hpp>

EliminateLoopInputsOutputs transformation manages Loop inputs/outputs. Check if Loop input is not changed in body input node -> Loop input -> body parameter -> body result -> Loop output -> output node than: 1) reconnect input node -> output node directly 2) update Loop input description from merged to invariant.

class EliminateUnsqueezeGather : public ov::pass::MatcherPass #: #include <eliminate_unsqueeze_gather.hpp>

Remove Unsqueeze + Gather pair, if Gather gathers data by dimension that was previously added by Unsqueeze.

class EliminateGatherUnsqueeze : public ov::pass::MatcherPass #: #include <eliminate_unsqueeze_gather.hpp>

Matches Gather ->[Binary Operation]-> Unsqueeze If axis for Gather and Unsqueeze is the same and Gather indices are scalar Unsqueeze is being removed and indices become 1D. Must be executed after SharedOpOptimization — It is possible to have multiple similar Unsqueeze operations after Gather, so they must be optimized beforehand.

class EnableShapeOfConstantFolding : public ov::pass::MatcherPass #: #include <enable_shapeof_constant_folding.hpp>

This transformation enables constfoldability for ShapeOf nodes that was disabled by DisableShapeOfConstantFolding.

class FakeQuantizeMulFusion : public ov::pass::MatcherPass #: #include <fq_mul_fusion.hpp>

This transformation looks for a FQ + Mul pair in the graph and moves the Mul operation above the FQ node. The last two inputs of FQ are multiplied by the value that was originally below the FQ node.

class FakeQuantizeReshapeFusion : public ov::pass::MatcherPass #: #include <fq_reshape_fusion.hpp>

This transformation looks for a FQ + Reshape pair in the graph and moves the Reshape operation above the FQ node. Shapes of limit inputs are updated following FQ broadcasting semantics.

class RoPEFusion : public ov::pass::GraphRewrite #: #include <fuse_rotary_positional_embeddings.hpp>

Fuses special sub-graph into an internal Rotary Positional Embedding operation.

class FusedNamesCleanup : public ov::pass::ModelPass #: #include <fused_names_cleanup.hpp>

FusedNamesCleanup removes fused_names attribute.

class GeluFusionWithErfOne : public ov::pass::MatcherPass #: #include <gelu_fusion.hpp>

GeluFusion transformation replaces a sub-graph (0.5 * x) * (1 + erf(x / sqrt(2))) with a Gelu op.

class GeluFusionWithErfTwo : public ov::pass::MatcherPass #: #include <gelu_fusion.hpp>

GeluFusion transformation replaces a sub-graph 0.5 * (x * (1 + erf(x / sqrt(2)))) with a Gelu op.

class GeluFusionWithErfThree : public ov::pass::MatcherPass #: #include <gelu_fusion.hpp>

GeluFusion transformation replaces a sub-graph x * (0.5 * (1 + erf(x / sqrt(2)))) with a Gelu op.

class GeluFusionWithErfFour : public ov::pass::MatcherPass #: #include <gelu_fusion.hpp>

GeluFusion transformation replaces a sub-graph x * (0.5 + 0.5 * erf(x * (1 / sqrt(2)))) with a Gelu op.

class GeluFusionWithTanh : public ov::pass::MatcherPass #: #include <gelu_fusion.hpp>

GeluFusion transformation replaces a sub-graph x * (0.5 * (1 + tanh([sqrt(2 / pi)] * [x + 0.044715^3]))) with a Gelu (Tanh) op.

class GeluFusionWithTanhNoPower : public ov::pass::MatcherPass #: #include <gelu_fusion.hpp>

GeluFusion transformation replaces a sub-graph x * 0.5 * (1 + tanh((x * 0.044715 * x + 1) * x * sqrt(2 / pi))) with a Gelu (Tanh) op.

class GeluFusion : public ov::pass::GraphRewrite #: #include <gelu_fusion.hpp>

GeluFusion transformation replaces various sub-graphs with a Gelu op.

class GRUCellFusion : public ov::pass::MatcherPass #

#include <gru_cell_fusion.hpp>

GRUCellFusion transformation replaces a sequence of operations with GRUCell op.

If BiasAdds are not present in the pattern, then Constants with zero values will be created to match the specification.

Supported activations: Relu, Sigmoid, Tanh Clip attribute is not supported. Linear_before_reset attribute is not supported. Supported weights formats: zr, rz

class HSigmoidFusionWithReluDiv : public ov::pass::MatcherPass #: #include <hsigmoid_fusion.hpp>

HSigmoidFusion transformation replaces a sub-graph ((min(Relu(x + 3), 6)) / 6) with a HSigmoid op.

class HSigmoidFusionWithReluMul : public ov::pass::MatcherPass #: #include <hsigmoid_fusion.hpp>

HSigmoidFusion transformation replaces a sub-graph ((min(Relu(x + 3), 6)) * const(1/6)) with a HSigmoid op.

class HSigmoidFusionWithoutRelu : public ov::pass::MatcherPass #: #include <hsigmoid_fusion.hpp>

HSigmoidFusion transformation replaces a sub-graph (min(max(x + 3, 0), 6) / 6) with a HSigmoid op.

class HSigmoidFusionWithClampMul : public ov::pass::MatcherPass #: #include <hsigmoid_fusion.hpp>

HSigmoidFusion transformation replaces a sub-graph (Clamp(x + 3, 0, 6) * const(1/6)) with a HSigmoid op.

class HSigmoidFusionWithClampDiv : public ov::pass::MatcherPass #: #include <hsigmoid_fusion.hpp>

HSigmoidFusion transformation replaces a sub-graph (Clamp(x + 3, 0, 6) * / 6) with a HSigmoid op.

class HSigmoidFusion : public ov::pass::GraphRewrite #: #include <hsigmoid_fusion.hpp>

HSigmoidFusion transformation replaces various sub-graphs with a HSigmoid op.

class HSwishFusionWithReluDiv : public ov::pass::MatcherPass #: #include <hswish_fusion.hpp>

HSwishFusion transformation replaces a sub-graph (x * (min(Relu(x + 3), 6))) / 6 with a HSwish op.

class HSwishFusionWithReluMul : public ov::pass::MatcherPass #: #include <hswish_fusion.hpp>

HSwishFusion transformation replaces a sub-graph (x * (min(Relu(x + 3), 6)) * const(1/6) with a HSwish op.

class HSwishFusionWithHSigmoid : public ov::pass::MatcherPass #: #include <hswish_fusion.hpp>

HSwishFusion transformation replaces a sub-graph x * HSigmoid(x) with a HSwish op.

class HSwishFusionWithClamp : public ov::pass::MatcherPass #: #include <hswish_fusion.hpp>

HSwishFusion transformation replaces a sub-graph (Clamp(x + 3, 0, 6) * x) with a HSwish * 6.

class HSwishFusion : public ov::pass::GraphRewrite #: #include <hswish_fusion.hpp>

HSwishFusion transformation replaces various sub-graphs with a HSwish op.

class InterpolateSequenceFusion : public ov::pass::MatcherPass #: #include <interpolate_sequence_fusion.hpp>

InterpolateSequenceFusion transformation replaces a sequence of operations to Interpolate op.

class LeakyReluFusion : public ov::pass::MatcherPass #: #include <leaky_relu_fusion.hpp>

LeakyReluFusion transformation replaces following graph: Multiply->Maximum to LeakyRelu.

class LinOpSequenceFusion : public ov::pass::GraphRewrite #: #include <lin_op_sequence_fusion.hpp>

LinOpSequenceFusion transformation fuses linear operation sequence.

class LSTMCellFusionWithJointWeights : public ov::pass::MatcherPass #: #include <lstm_cell_fusion.hpp>

LSTMCellFusionWithJointWeights transformation converts a sequence of operations with merged weights into LSTMCell op.

class LSTMCellFusionWithSplitWeights : public ov::pass::MatcherPass #: #include <lstm_cell_fusion.hpp>

LSTMCellFusionWithSplitWeights transformation converts a sequence of operations with split weights into LSTMCell op.

class LSTMCellFusion : public ov::pass::GraphRewrite #: #include <lstm_cell_fusion.hpp>

LSTMCellFusion transformation replaces various sub-graphs with a LSTMCell op.

class MarkPrecisionSensitiveShapeOfSubgraphs : public ov::pass::ModelPass #

#include <mark_precision_sensitive_shapeof_subgraphs.hpp>

MarkPrecisionSensitiveShapeOfSubgraphs marks entirely all shape subgraphs starting from precision-sensitive inputs and ending at the ShapeOf node as disabled for FP16 compression.

Subclassed by ov::pass::MarkDividesInShapeSubgraphs, ov::pass::MarkPrecisionSensitiveConstants, ov::pass::MarkShapeOfSubgraphs

class MarkShapeOfSubgraphs : public ov::pass::MarkPrecisionSensitiveShapeOfSubgraphs #: #include <mark_precision_sensitive_shapeof_subgraphs.hpp>

MarkShapeOfSubgraphs marks shape subgraphs. Information whether the node belongs to the shape path or to the data path is needed during evaluate and CF.

class MarkPrecisionSensitiveConstants : public ov::pass::MarkPrecisionSensitiveShapeOfSubgraphs #: #include <mark_precision_sensitive_shapeof_subgraphs.hpp>

MarkPrecisionSensitiveConstants marks the constants inside of all shape subgraphs starting from precision-sensitive inputs and ending at the ShapeOf node as disabled for FP16 compression.

class MarkDividesInShapeSubgraphs : public ov::pass::MarkPrecisionSensitiveShapeOfSubgraphs #: #include <mark_precision_sensitive_shapeof_subgraphs.hpp>

MarkDividesInShapeSubgraphs marks the Divide layers inside of all shape subgraphs starting from precision-sensitive input and ending at the ShapeOf node as disabled for ConvertDivide transformation.

class MarkRopeInputsToKeepInMixedPrecision : public ov::pass::MatcherPass #

#include <mark_rope_input_to_keep_in_mixed_precision.hpp>

This transformation markups the 2nd/3rd inputs of Rope with FP32 to mantian accuracy. +—-—+ +—-—+ +—-—+ |intput1| |input2 | |input3 | |(orig) | |(fp32) | |(fp32) | +—|—+ +—|—+ +—|—+.


+—+————	————+—+

| | | ROPE | +—————————-—+

class MatMulMultiplyFusion : public ov::pass::MatcherPass #

#include <matmul_multiply_fusion.hpp>

MatMulMultiplyFusion transformation matches following graph:

    +----------+            +----------+
    |    A     |            |    B     |
    +----------+            +----------+
         |                       |
         -----------    ----------
                   |    |
                   v    v
                 +--------+
                 | MatMul |
                 +--------+
                     |
                     v
                +----------+     +----------+
                | Multiply |<----| Constant |
                +----------+     +----------+

and replaces with:

                      +-------+   +----------+
                      |   B   |   | Constant |
                      +-------+   +----------+
                           |            |
                           ------  ------
                                |  |
                                v  v
    +----------+            +----------+
    |    A     |            | Multiply |
    +----------+            +----------+
         |                       |
         -----------    ----------
                   |    |
                   v    v
                 +--------+
                 | MatMul |
                 +--------+

class MishFusion : public ov::pass::MatcherPass #: #include <mish_fusion.hpp>

MishFusion transformation replaces group of operations: x * tanh(log(exp(x) + 1)) to Mish op.

class MultiplyConvolutionFusion : public ov::pass::MatcherPass #

#include <mul_conv_fusion.hpp>

Multiply->Convolution fusion replaces following graph:

+—-—+ +——-—+ | Input

| | Constant | +—-—+ +——-—+ | |

| | v v +——-

—+ +———+ | Multiply | | Weights | +——-—+ +———+ | |

| | v v +————-

—+ | Convolution Op | +————-—+

to following:

                      +---------+   +----------+
                      | Weights |   | Constant |
                      +---------+   +----------+
                           |            |
                           ------  ------
                                |  |
                                v  v
     +-------+              +----------+
     | Input |              | Multiply |
     +-------+              +----------+
         |                       |
         -----------    ----------
                   |    |
                   v    v
              +----------------+
              | Convolution Op |
              +----------------+

where ‘Convolution Op’ is one of:

Convolution
ConvolutionBackpropData
GroupConvolution
GroupConvolutionBackpropData

Restrictions:

weights’ shape is static
if the constant input to Multiply has the same rank as weights, the constant first dimension has to be 1
constant input to Multiply has to be broadcastable to weights when ‘Convolution Op’ is either Convolution or GroupConvolution
shape of a constant input to Multiply has to be in one of following forms: (1), (1, 1, …, 1), (C, 1, …, 1), (1, C, 1, …, 1) when ‘Convolution Op’ is either ConvolutionBackpropData or GroupConvolutionBackpropData

class MulFakeQuantizeFusion : public ov::pass::MatcherPass #

#include <mul_fake_quantize_fusion.hpp>

MulFakeQuantizeFusion transformation replaces following graph: Mul->FakeQuantize to a single FakeQuantize Restrictions:

second input to Mul is a Constant

class MVNFusionWithoutConstants : public ov::pass::MatcherPass #: #include <mvn_fusion.hpp>

MVNFusion transformation replaces group of operations: (x - ReduceMean(x, axes)) / (Sqrt(ReduceMean((x - ReduceMean(x, axes)) ^ 2)) + eps) to MVN op.

class MVNFusionWithConstantsInside : public ov::pass::MatcherPass #: #include <mvn_fusion.hpp>

MVNFusion transformation replaces group of operations: gamma * (x - ReduceMean(x, axes)) / (Sqrt(ReduceMean((x - ReduceMean(x, axes)) ^ 2)) + eps) + beta to MVN op.

class MVNFusion : public ov::pass::GraphRewrite #: #include <mvn_fusion.hpp>

MVNFusion transformation replaces various sub-graphs with a MVN op.

class NearestNeighborUpsamplingFusion : public ov::pass::MatcherPass #: #include <nearest_neighbor_upsampling_fusion.hpp>

NearestNeighborUpsamplingFusion transformation fuses subgraph that uses the simpler operations, as ShapeOf, StridedSlice, Concat, Reshape, Mul to calculate Interpolate with mode=’nearest’.

class NonZeroHorizontalFusion : public ov::pass::MatcherPass #: #include <nonzero_horizontal_fusion.hpp>

NonZeroHorizontalFusion transformation makes horizontal fusion for equal NonZero layers.

class EliminateReduceReshape : public ov::pass::MatcherPass #: #include <nop_elimination.hpp>

EliminateReduceReshape eliminates Reshape from Reduce -> Reshape pattern.

class EliminatePad : public ov::pass::MatcherPass #: #include <nop_elimination.hpp>

EliminatePad eliminates pad that does nothing.

class EliminateConvert : public ov::pass::MatcherPass #: #include <nop_elimination.hpp>

EliminateConvert eliminates convert that does nothing.

class EliminateConvertNonZero : public ov::pass::MatcherPass #: #include <nop_elimination.hpp>

EliminateConvertNonZero eliminates convert before NonZero.

class EliminateConcat : public ov::pass::MatcherPass #: #include <nop_elimination.hpp>

EliminateConcat eliminates concat that does nothing.

class EliminateSplit : public ov::pass::MatcherPass #: #include <nop_elimination.hpp>

EliminateSplit eliminates split that does nothing.

class EliminateSqueeze : public ov::pass::MatcherPass #: #include <nop_elimination.hpp>

EliminateSqueeze eliminates squeeze that does nothing.

class EliminateUnsqueeze : public ov::pass::MatcherPass #: #include <nop_elimination.hpp>

EliminateUnsqueeze eliminates squeeze that does nothing.

class EliminateTranspose : public ov::pass::MatcherPass #: #include <nop_elimination.hpp>

EliminateTranspose eliminates transpose that does nothing.

class EliminateEltwise : public ov::pass::MatcherPass #: #include <nop_elimination.hpp>

EliminateEltwise eliminates eltwise ops that do nothing.

class EliminateScatterUpdate : public ov::pass::MatcherPass #: #include <nop_elimination.hpp>

EliminateScatterUpdate eliminates scatter ops that do nothing (updates/indices are empty)

class EliminateSplitConcat : public ov::pass::MatcherPass #: #include <nop_elimination.hpp>

EliminateSplit eliminates split+concat pairs which do nothing.

class EliminateNopBroadcast : public ov::pass::MatcherPass #: #include <nop_elimination.hpp>

EliminateNopBroadcast eliminates broadcast or tile with all ones on the second input.

class EliminateSliceBeforeGatherElements : public ov::pass::MatcherPass #: #include <nop_elimination.hpp>

EliminateSliceBeforeGatherElements eliminates slice before GElements if slicing from 0 It is valid since GatherElements doesn’t support negative indices and Slice won’t affect indexing of elements in the original tensor that GatherElements would like to take.

class EliminateStridedSlice : public ov::pass::MatcherPass #: #include <nop_elimination.hpp>

EliminateStridedSlice eliminates Strided Slice in case tensors were not changed.

class EliminateSlice : public ov::pass::MatcherPass #: #include <nop_elimination.hpp>

EliminateSlice eliminates Slice in case tensors were not changed.

class EliminateStridedSliceByShape : public ov::pass::MatcherPass #: #include <nop_elimination.hpp>

EliminateStridedSlice eliminates Strided Slice in case tensors were not changed.

class PrepareShapeOpsForEliminationAroundBE : public ov::pass::MatcherPass #: #include <nop_elimination.hpp>

PrepareShapeOpsForEliminationAroundBE works on the subgraph like Reshape/Squeeze/Unsqueeze -> BinaryElementwiseOperation -> Reshape/Squeeze/Unsqueeze and prepares it for the following optimizations by moving bottom op up through Binary op.

class NormalizeL2Fusion : public ov::pass::MatcherPass #: #include <normalize_l2_fusion.hpp>

NormalizeL2Fusion transformation replaces sub-graphs: x/(sqrt(max(reduce_sum(x[j0, …, jN]**2, axes), eps)) x/(sqrt(add(reduce_sum(x[j0, …, jN]**2, axes), eps)) x/(pow(max(reduce_sum(x[j0, …, jN]**2, axes), eps), 0.5) x/(pow(add(reduce_sum(x[j0, …, jN]**2, axes), eps), 0.5) x*(pow(max(reduce_sum(x[j0, …, jN]**2, axes), eps), -0.5) x*(pow(add(reduce_sum(x[j0, …, jN]**2, axes), eps), -0.5) with a NormalizeL2(x, axes, eps, eps_mode[MAX|ADD]) op.

class UselessSliceEraser : public ov::pass::ModelPass #: #include <optimize_strided_slice.hpp>

UselessSliceEraser transformation removes Slice/StridedSlice operations with equal input and output shapes.

class GroupedStridedSliceOptimizer : public ov::pass::ModelPass #: #include <optimize_strided_slice.hpp>

GroupedStridedSliceOptimizer transformation replaces group of StridedSlice operations with VariadicSplit. All StridedSlice operations must slice data with the same axis and stride = 1.

class GroupedSliceToVSplitOptimization : public ov::pass::ModelPass #: #include <optimize_strided_slice.hpp>

GroupedSliceToVSplitOptimization transformation replaces group of Slice operations with VariadicSplit. All Slice operations must slice data with the same axis and step = 1.

class SliceSequenceToSingleSlice : public ov::pass::MatcherPass #

#include <optimize_strided_slice.hpp>

SliceSequenceToSingleSlice transformation replaces group of Slice operations with single Slice. All Slice operations must slice data with the different axis.

Before: data (shape: 2, 3, 4) -> Slice (axis 0) -> Slice (axis 1) -> Slice (axis 2)

After: data (shape: 2, 3, 4) -> Slice (axes: 0, 1, 2)

class StridedSliceOptimization : public ov::pass::ModelPass #: #include <optimize_strided_slice.hpp>

StridedSliceOptimization transformation executes all transformations related to StridedSlice optimizations.

class PadFusionAvgPool : public ov::pass::MatcherPass #

#include <pad_fusion.hpp>

PadFusion transformation replaces following graph: Pad -> AvgPool to AvgPool, under following conditions.

pad mode is op::PadMode::CONSTANT
pad value is 0
exclude_pad in AvgPool is set to false or pads_begin, pads_end are set to zero

class PadFusionConvolution : public ov::pass::MatcherPass #

#include <pad_fusion.hpp>

PadFusion transformation replaces following graph: Pad -> Convolution to Convolution, under following conditions.

pad mode is op::PadMode::CONSTANT
pad value is 0

class PadFusionConvolutionBackpropData : public ov::pass::MatcherPass #

#include <pad_fusion.hpp>

PadFusion transformation replaces following graph: Pad -> ConvolutionBackpropData to ConvolutionBackpropData, under following conditions.

pad mode is op::PadMode::CONSTANT
pad value is 0
pads in ConvolutionBackpropData are greater than pads in Pad node

class PadFusionGroupConvolution : public ov::pass::MatcherPass #

#include <pad_fusion.hpp>

PadFusion transformation replaces following graph: Pad -> GroupConvolution to GroupConvolution, under following conditions.

pad mode is op::PadMode::CONSTANT
pad value is 0

class PadFusionGroupConvolutionBackpropData : public ov::pass::MatcherPass #

#include <pad_fusion.hpp>

PadFusion transformation replaces following graph: Pad -> GroupConvolutionBackpropData to GroupConvolutionBackpropData, under following conditions.

pad mode is op::PadMode::CONSTANT
pad value is 0
pads in GroupConvolutionBackpropData are greater than pads in Pad node

class PReluFusionNegativeAdd : public ov::pass::MatcherPass #: #include <prelu_fusion.hpp>

PReluFusionNegativeAdd transformation replaces a sub-graph Op / \ Relu Negative | | | Relu | | | Negative | | | Multiply \ / Add.

class PReluFusionNegativeSub : public ov::pass::MatcherPass #: #include <prelu_fusion.hpp>

PReluFusionNegativeSub transformation replaces a sub-graph Op / \ Relu Negative | | | Relu | | | Multiply \ / Sub.

class PReluFusionMultiplyAdd : public ov::pass::MatcherPass #: #include <prelu_fusion.hpp>

PReluFusionMultiplyAdd transformation replaces a sub-graph Op / \ Relu Multiply (-1) | | | Relu | | | Multiply \ / Add.

class PReluFusionMultiplySub : public ov::pass::MatcherPass #: #include <prelu_fusion.hpp>

PReluFusionMultiplySub transformation replaces a sub-graph Op / \ Relu Multiply (-1) | | | Relu | | | Multiply \ / Sub.

class PReluFusionAbsSubMulMulAdd : public ov::pass::MatcherPass #: #include <prelu_fusion.hpp>

PReluFusionAbsSubMulMulAdd transformation replaces a sub-graph Op / | \ Relu | Abs | \ | | Subtract | | | Multiply | | | Multiply (0.5) \ / Add.

class PReluFusionNegReluMulAdd : public ov::pass::MatcherPass #: #include <prelu_fusion.hpp>

PReluFusionNegReluMulAdd transformation replaces a sub-graph Op / \ Relu Negative | | | Relu | | | Multiply \ / Add.

class PReluFusion : public ov::pass::GraphRewrite #: #include <prelu_fusion.hpp>

PReluFusion transformation replaces various sub-graphs with a PRelu op.

class PullUnsqueezeThroughReduce : public ov::pass::MatcherPass #: #include <pull_through_reduce.hpp>

PullUnsqueezeThroughReduce transformation The transformation pulls Unsqueeze operator though Reduce ops if possible. In the further processing such Unsqueeze can be often skipped as nop.

class PullReshapeThroughReduce : public ov::pass::MatcherPass #: #include <pull_through_reduce.hpp>

PullReshapeThroughReduce transformation The transformation pulls Reshape operator though Reduce ops if possible. In the further processing such Reshape can be often skipped as nop.

class PullThroughReduce : public ov::pass::GraphRewrite #: #include <pull_through_reduce.hpp>

PullThroughReduce transformation The transformation pulls Reshape or Unsqueeze operators though Reduce ops if possible. In the further processing such Reshape/Unsqueeze can be often skipped as nop.

class PushConstantToSubgraph : public ov::pass::ModelPass #: #include <push_constant_to_subgraph.hpp>

PushConstantToSubgraph transformation detects MultiSubGraphOp inputs that can be constfoldable pushes that inputs to subgraphs.

class RandomUniformFusion : public ov::pass::MatcherPass #: #include <random_uniform_fusion.hpp>

RandomUniformFusion transformation replaces RandomUniform -> Add or RandomUniform -> Mul subgraph with a RandomUniform and replaces min and max const with corrected values.

class ReduceMerge : public ov::pass::MatcherPass #

#include <reduce_merge.hpp>

ReduceMerge transformation matches following graph:

+——-—+ +——-—+ | A | | B | +——-—+ +——-—+ | |

| | v v +—–

—+ +—–—+ | Reduce | | C | +—–—+ +—–—+ | | | —-— | | v v +——-—+ | Reduce | +——-—+

and replaces with:

      +----------+     +----------+
      |    B     |     |    C     |
      +----------+     +----------+
           |                |
           -------    -------
                 |    |
                 v    v

+——-—+ +—————-—+ | A | | Concat/Constant | +——-—+ +—————-—+ | | | —–— | | v v +——-—+ | Reduce | +——-—+

class ReduceReshapeFusion : public ov::pass::MatcherPass #: #include <reduce_reshape_fusion.hpp>

ReduceReshapeFusion transformation Fuse ReduceOp(keep_dims=false)+Reshape to ReduceOp(keep_dims=true)

class ReluFakeQuantizeFusion : public ov::pass::MatcherPass #

#include <relu_fake_quantize_fusion.hpp>

ReluFakeQuantizeFusion transformation replaces following graph: Relu -> FakeQuantize to FakeQuantize under following conditions:

‘input_low’ input to FakeQuantize is a Constant
’input_low’ has non negative values

class RemoveConcatZeroDimInput : public ov::pass::MatcherPass #: #include <remove_concat_zero_dim_input.hpp>

RemoveConcatZeroDimInput transformation removes input of Concat if the tensor size is equal to 0.

class ReshapePRelu : public ov::pass::MatcherPass #: #include <reshape_prelu.hpp>

ReshapePRelu reshape second input of PRelu (slope)

class ReshapeSequenceFusion : public ov::pass::MatcherPass #: #include <reshape_sequence_fusion.hpp>

ReshapeSequenceFusion fuses sequence of Reshape operation into single Reshape or eliminates full redundant sequence.

class ReverseInputChannelsFusion : public ov::pass::ModelPass #: #include <ric_fusion.hpp>

ReverseInputChannelsFusion.

class SelectWithOneValueCondition : public ov::pass::MatcherPass #: #include <select_with_one_value_condition.hpp>

SelectWithOneValueCondition transformation eliminates Select operation if the condition is constant and consists of al True or False elements.

class SequenceFusion : public ov::pass::MatcherPass #

#include <sequence_fusion.hpp>

SequenceFusion transformation replaces a chain of Cells operations with single Sequence op.

Supported cells: GRUCell, LSTMCell, RNNCell, AUGRUCell Prerequisites: the source of W,R,B inputs must be the same or it can be different Constants with the same type, shape and value.

class SharedOpOptimization : public ov::pass::ModelPass #: #include <shared_ops_optimization.hpp>

SharedOpOptimization optimizes operations which are sourcing from same Output<Node> and perform the same action on the same data.

class ShuffleChannelsFusion : public ov::pass::MatcherPass #

#include <shuffle_channels_fusion.hpp>

ShuffleChannelsFusion transformation detects Reshape-Transpose-Reshape pattern and tries to fuse it into a single ShuffleChannels layer with axis = 1.

x’ = reshape(x, [N, group, C / group, H, W]) or reshape(x, [N, group, C / group, H * W]) x’’ = transpose(x’, [0, 2, 1, 3, 4]) or transpose(x’, [0, 2, 1, 3]) y = reshape(x’’, [N, C, H, W])

Param reshape_constants_check:: the flag that defines the need for additional checks of reshapes constant Additional checks are required when ShuffleChannelsFusion using inside offline transformations and are not necessary when ShuffleChannelsFusion using inside CommonOptimizations

class GroupedGatherElimination : public ov::pass::MatcherPass #: #include <simplify_shape_of_sub_graph.hpp>

GroupedGatherElimination transformation replaces group of Gather operations with the first Gather in this group and updated indices input in case all Gathers in the group are consumed by the same Concat in incremental order.

class GatherNopElimination : public ov::pass::MatcherPass #: #include <simplify_shape_of_sub_graph.hpp>

GatherNopElimination transformation optimizes out useless Gather operations.

class AbsSinking : public ov::pass::MatcherPass #: #include <simplify_shape_of_sub_graph.hpp>

AbsSinking optimizes out the Abs which input is non negative. Has a special case for Concat -> Abs graph, it moves Abs up through Concat to its inputs, tries to constant fold new Abs ops. In case folding fails applies optimization to the leftover Abs ops.

class SkipGatherBeforeTransposeAndReshape : public ov::pass::MatcherPass #: #include <skip_gather_before_transpose_and_reshape.hpp>

SkipGatherBeforeTransposeAndReshape transformation removes Gather from the Gather->Transpose->Reshape sequence in case when input has batch=1 and gather has axis=0 and indices={0}. Also, this transformation corrects a transpose constant to save semantic.

class SoftmaxFusion : public ov::pass::MatcherPass #

#include <softmax_fusion.hpp>

SoftmaxFusion transformation replaces following graphs:

       +---------------+
       │               │
       │     input     │
       │               │
       +---------------+
           │      │
           │      v
           │ +-----------+
           │ │           │
           │ │ ReduceMax │
           │ │           │
           │ +-----------+
           │      │
           │      │
           v      v
       +---------------+
       │               │
       │      Sub      │
       │               │
       +---------------+
               |
               |
               v
       +---------------+
       │               │
       │      Exp      │
       │               │
       +---------------+
           │      │
           │      v
           │ +-----------+
           │ │           │
           │ │ ReduceSum │
           │ │           │
           │ +-----------+
           │      │
           │      │
           v      v
        +-------------+
        |             │
        |     Div     │
        │             │
        +-------------+

and +—————+ │ │ │ input │ │ │ +—————+

v +—————+ │ │ │ Exp │ │ │ +—————+ │ │ │ v │ +——–—+ │ │ │ │ │ ReduceSum │ │ │ │ │ +——–—+ │ │ │ │ v v +———-—+ | │ | Div │ │ │ +———-—+

to a single Softmax node

Restrictions:
- ReduceMax and ReduceSum axes must be scalar constants and they have to point to the same axis

class SoftPlusFusion : public ov::pass::MatcherPass #: #include <softplus_fusion.hpp>

SoftPlusFusion transformation replaces group of operations: log(exp(x) + 1) to SoftPlus op.

class SoftPlusToMishFusion : public ov::pass::MatcherPass #: #include <softplus_to_mish_fusion.hpp>

SoftPlusToMishFusion transformation replaces group of operations: x * tanh(softplus(x)) to Mish op.

class SpaceToBatchFusion : public ov::pass::MatcherPass #

#include <space_to_batch_fusion.hpp>

SpaceToBatchFusion transformation replaces following graph: Transpose (or Reshape) -> Pad -> SpaceToDepth -> Transpose (or Reshape) to SpaceToBatch Restrictions:

input rank must be 4
Transpose permutation must be [1, 0, 2, 3]
pad value is 0, PadMode is CONSTANT
SpaceToDepthMode must be BLOCKS_FIRST

class SplitConcatPairToInterpolateFusion : public ov::pass::MatcherPass #: #include <split_concat_pair_to_interpolate_fusion.hpp>

SplitConcatPairToInterpolateFusion transformation replaces group of operations: Split -> Concat to Interpolate op.

class SplitSqueezeConcatFusion : public ov::pass::MatcherPass #: #include <split_squeeze_concat_fusion.hpp>

SplitSqueezeConcatFusion transformation replaces group of operations: Split -> Squeeze (multiple) -> Concat to Transpose -> Reshape ops.

class ConvStridesPropagation : public ov::pass::MatcherPass #: #include <strides_optimization.hpp>

ConvStridesPropagation either propagates stride (greater than 1) from Convolution up through the graph or inserts pooling between current node and its consumers if the consumers have different StridesProp attributes. Strides can be propagated if Convolution kernel is {1, 1, …}.

class SupportedNodesStridesPropagation : public ov::pass::MatcherPass #: #include <strides_optimization.hpp>

SupportedNodesStridesPropagation either propagates stride (greater than 1) from current node up through the graph or inserts pooling between current node and its consumers if the consumers have different StridesProp attributes.

class UnsupportedNodesStridesPropagation : public ov::pass::MatcherPass #: #include <strides_optimization.hpp>

UnsupportedNodesStridesPropagation inserts pooling between current node and its consumers if the consumers have different StridesProp attributes.

class StridesOptimization : public ov::pass::BackwardGraphRewrite #: #include <strides_optimization.hpp>

StridesOptimization transformation works backward on function and propagates strides up through the graph if possible.

class SubtractFusion : public ov::pass::MatcherPass #: #include <subtract_fusion.hpp>

SubtractFusion transformation replaces a sub-graph Mul(y, -1) + x or x + Mul(y, -1) with Subtract(x,y)

class SwishFusionWithSigmoid : public ov::pass::MatcherPass #: #include <swish_fusion.hpp>

SwishFusionWithSigmoid replaces a sub-graphs x * Sigmoid(x) with a Swish op.

class SwishFusionWithSigmoidWithBeta : public ov::pass::MatcherPass #: #include <swish_fusion.hpp>

SwishFusionWithSigmoid replaces a sub-graphs x * Sigmoid(x * beta) with a Swish op.

class SwishFusionWithBeta : public ov::pass::MatcherPass #: #include <swish_fusion.hpp>

SwishFusionWithSigmoid replaces a sub-graphs x / (1.0 + exp(-x * beta)) with a Swish op.

class SwishFusionWithoutBeta : public ov::pass::MatcherPass #: #include <swish_fusion.hpp>

SwishFusionWithSigmoid replaces a sub-graphs x / (1.0 + exp(-x)) with a Swish op.

class SwishFusion : public ov::pass::GraphRewrite #: #include <swish_fusion.hpp>

SwishFusion transformation replaces various sub-graphs with a Swish op.

class TransposeReshapeEliminationForMatmul : public ov::pass::MatcherPass #: #include <transpose_reshape_elimination_for_matmul.hpp>

TransposeReshapeEliminationForMatmul transformation eliminates Transpose and Reshape which were created to align input and output dimension ranks before second MatMul input and after MatMul output (for example, after Einsum Decomposition inside TensorFlow 1 and OpenVINO EinsumDecomposition transformation)

class TransposeReduction : public ov::pass::MatcherPass #: #include <transpose_sinking.hpp>

TransposeReduction transformation sinks Transpose through Reduce operations.

class TransposeFQReduction : public ov::pass::MatcherPass #: #include <transpose_sinking.hpp>

TransposeFQReduction transformation sinks Transpose through FakeQuantize in case it is followed by reduction or squeeze.

class TransposeConvert : public ov::pass::MatcherPass #: #include <transpose_sinking.hpp>

TransposeConvert transformation sinks Transpose through Convert.

class TransposeEltwise : public ov::pass::MatcherPass #: #include <transpose_sinking.hpp>

TransposeEltwise transformation sinks Transpose through Eltwise.

class TransposeFuse : public ov::pass::MatcherPass #: #include <transpose_sinking.hpp>

TransposeFuse transformation eliminates 2 consequtive Transposes if they result in no changes to input or fuses them to single Transpose if input gets changed.

class TransposeSinking : public ov::pass::GraphRewrite #: #include <transpose_sinking.hpp>

TransposeSinking transformation sinks Transposes through known operations.

class TransposeToReshape : public ov::pass::MatcherPass #: #include <transpose_to_reshape.hpp>

TransposeToReshape transformation replaces suitable Transposes with Reshape operation or optimizes them out.

class WeightsDequantizeToFakeQuantize : public ov::pass::MatcherPass #: #include <weights_dequantize_to_fake_quantize.hpp>

WeightsDequantizeToFakeQuantize transformation replaces Constant (i8) -> Convert (to fp) -> Subtract (zp) -> Multiply (scale) -> with Constant (i8) -> Convert (to fp) -> FakeQuantize -> deducing levels and FakeQuantize limits according to actual values in the weights Constant.

class WrapInterpolateIntoTransposes : public ov::pass::MatcherPass #: #include <wrap_interpolate_into_transposes.hpp>

WrapInterpolateIntoTransposes transformation replaces Interpolate with Transpose -> Interpolate -> Transpose when 1) the source Interpolate has the static input rank; 2) ‘axes’ input is a Constant; 3) number of axes is equal to input rank minus 2; 4) axes contain 0 or 1. The reason of this transformation is that now CPU plugin supports interpolation only with respect to spatial dimensions, but TensorFlow frontend gives Interpolate with axes {1, 2} for 4D tensors.

class UnrollIf : public ov::pass::ModelPass #: #include <unroll_if.hpp>

The transformation replaces ‘If’ operations with one of the internal functions (bodies) if the provided condition is constant. The condition is true: ‘If’ op is replaced with then_body The condition is false ‘If’ op is replaced with else_body.

class UnrollTensorIterator : public ov::pass::ModelPass #: #include <unroll_tensor_iterator.hpp>

Unrolls the body of the TensorIterator layer. Multiple body copies, the number of which is determined by the number of iterations of the TensorIterator layer, are created and connected to each other and to the external network. If the number of TensorIterator iterations is greater than 1, then additional Concat and Split layers are added to the network.

struct EnumClassHash#

#include <convert_precision.hpp>

ConvertPrecision transformation convert precision for entire ov::Model List of supported precision conversion: FROM -> TO u8 -> i32 u16 -> i32 u32 -> i32 u64 -> i32 i64 -> i32 f16 -> f32 bool -> u8 bool -> i32.

For all operations from opset1-opset4 this conversions can be applied without adding Conversion operations. That is possible because all operations that produces “FROM” type can produce “TO” type. And for this operations we have created special fuse_type_into_<type> function (can be found in cpp file) that performs type fusion into operation. m_additional_type_to_fuse_map allows to rewrite existing type convertors.

List of operations that are supported by this transformations for i64 -> i32 conversion: opset4::Parameter opset4::Convert opset4::ShapeOf opset4::Range opset3::NonMaxSuppression opset4::NonMaxSuppression opset4::TopK opset4::NonZero opset4::Bucketize

List of operations that are supported by this transformations for bool -> u8 conversion: LogicalAnd LogicalNot LogicalOr LogicalXor ReduceLogicalAnd ReduceLogicalOr Equal NotEqual Greater GreaterEqual Less LessEqual

class AlignMixedFP32FP16Types : public ov::pass::ModelPass #: #include <align_mixed_fp32_fp16_types.hpp>

AlignMixedFP32FP16Types adds Converts to keep mixed FP16/FP32 graph type consistent.

class ConvertCompressedOnlyToLegacy : public ov::pass::ModelPass #: #include <convert_compression_only_to_legacy.hpp>

ConvertCompressedOnlyToLegacy transformation converts compression only FP16 format to legacy FP16 format.

class EnableDecompressionConvertConstantFolding : public ov::pass::MatcherPass #: #include <mark_decompression_convert_constant_folding.hpp>

Enables ConstantFolding for Convert operation in compressed function.

class DisableDecompressionConvertConstantFolding : public ov::pass::MatcherPass #: #include <mark_decompression_convert_constant_folding.hpp>

Disables ConstantFolding for Convert operation in compressed function.

class KeepConstAndDecompression : public ov::pass::MatcherPass #: #include <mark_decompression_convert_constant_folding.hpp>

Disables ConstantFolding for Convert operation and prevents conversion of f16 Consts to f32.

class KeepConstantsPrecisionAndAddConverts : public ov::pass::MatcherPass #: #include <mark_decompression_convert_constant_folding.hpp>

Prevents Consts precision conversion and adds Convert with disabled ConstantFolding.

class MarkCompressedFloatConstants : public ov::pass::MatcherPass #: #include <mark_decompression_convert_constant_folding.hpp>

Prevents ConstantFolding for f16/bf16 Const + Convert_To_FP32 to keep original FW float Constants. Original precision should be kept as long as possible, this prevents redundant conversions and saves memory. E.g. if original FW model was already compressed no need to upcast during CF, store intermediate f32 consts and then again compress them to f16 during save_model.

class MarkFloatingPointRange : public ov::pass::MatcherPass #: #include <mark_floatpoint_range.hpp>

This transformation markups the marks paths that involve Range operations with floating point output data types, as well as their users allowed for propagation. This pass is needed to prevent accuracy data loss in cases of high range generation, which could suffer due to lowered precision.

class InitNodeInfo : public ov::pass::ModelPass #

#include <init_node_info.hpp>

InitNodeInfo transformation helps to set runtime info attributes in a single place.

Every runtime info attribute that needs to be initialized should be registered in run_on_function method. Also do not forget to override init methods for registered attribute. This transformations should be called first in transformation pipeline. If attribute was already set initialization will be skipped for this node.

class MarkDequantization : public ov::pass::MatcherPass #

#include <mark_dequantization_subgraph.hpp>

MarkDequantization matches Dequantization subgraphs and marks Subtract and Multiply nodes with the dequantization attribute. Also if Convert nodes are part of the subgraph they might be marked with the disable_const_folding attribute.

If Convert -> Reshape/Unsqueeze are part of the Dequantization subraph, Convert and Reshape/Unsqueeze nodes will be swapped to eliminate Reshape/Unsqueeze in the next ConstantFolding.

Dequantization subgraph may have two forms: with and without Subtract. ZeroPoints and Scale might be present as subgraphs and include Convert ops.

Input       ZeroPoints
  │             │
  ▼             ▼
Convert   (opt) Reshape/Unsqueeze
      │      │
      ▼      ▼    Scale                        Input     Scale
      Subtract     │                            │         │
           │       ▼                            ▼         ▼
           │     (opt) Reshape/Unsqueeze       Convert  (opt) Reshape/Unsqueeze
           │      │                               │      │
           ▼      ▼                               ▼      ▼
           Multiply                               Multiply

class KeepConstsPrecision : public ov::pass::MatcherPass #

#include <mark_dequantization_subgraph.hpp>

KeepConstsPrecision matches Dequantization subgraphs and if Input/ZeroPoints/Scale are Constants they might be marked with keep_const_precision attribute.

Dequantization subgraph may have two forms: with and without Subtract.

   Input
     │
     ▼
  Convert  ZeroPoints
      │      │
      ▼      ▼                        Input
      Subtract                          │
           │                            ▼
           │     Scale               Convert   Scale
           │      │                     │      │
           ▼      ▼                     ▼      ▼
           Multiply                     Multiply

class BidirectionalLSTMSequenceDecomposition : public ov::pass::MatcherPass #: #include <bidirectional_sequences_decomposition.hpp>

Decompose LSTMSequence to forward and reverse LSTMSequence.

class BidirectionalGRUSequenceDecomposition : public ov::pass::MatcherPass #: #include <bidirectional_sequences_decomposition.hpp>

Decompose GRUSequence to forward and reverse GRUSequence.

class BidirectionalRNNSequenceDecomposition : public ov::pass::MatcherPass #: #include <bidirectional_sequences_decomposition.hpp>

Decompose RNNSequence to forward and reverse RNNSequence.

class BidirectionalSequenceDecomposition : public ov::pass::GraphRewrite #: #include <bidirectional_sequences_decomposition.hpp>

Container for all types of sequences decomposition.

class ConvertAvgPool14ToAvgPool1 : public ov::pass::MatcherPass #: #include <convert_avgpool_downgrade.hpp>

Converts AvgPool v14 to AvgPool v1.

class ConvertBatchToSpace : public ov::pass::MatcherPass #

#include <convert_batch_to_space.hpp>

ConvertBatchToSpace transformation decomposes BatchToSpace layer to Reshape->Transpose->Reshape->Crop.

Param convert_by_elements:: - reduces the maximum number of dimensions that arise during the transformation if enabled. Default value: true. false - BatchToSpace decomposes to Reshape->Transpose->Reshape->Crop. During transformation, the number of tensor dimensions can be increased by length of block_shape input of BatchToSpace layer. true - BatchToSpace decomposes to N x (Reshape->Transpose->Reshape)->Crop, where N = length of block_shape input of BatchToSpace layer. During transformation, the number of tensor dimensions can be increased by 1.

class ConvertBitwiseToLogical : public ov::pass::GraphRewrite #: #include <convert_bitwise_to_logical_bool.hpp>

Converts Bitwise operators to Logical for boolean datatype for plugins that don’t support opset13 Bitwise and to allow for constant folding for bool.

class ConvertDeformableConv8To1 : public ov::pass::MatcherPass #: #include <convert_deformable_conv_v8_to_v1.hpp>

ConvertDeformableConv8To1 converts v8::DeformableConvolution into v1::DeformableConvolution.

class ConvertEmbeddingBagOffsets15ToEmbeddingBagOffsetsSum3 : public ov::pass::MatcherPass #: #include <convert_embedding_bag_offsets15_downgrade.hpp>

Converts EmbeddingBagOffsets v15 to EmbeddingBagOffsets v3.

class ConvertEmbeddingBagPacked15ToEmbeddingBagPackedSum3 : public ov::pass::MatcherPass #: #include <convert_embedding_bag_packed15_downgrade.hpp>

Converts EmbeddingBagPacked v15 to EmbeddingBagPacked v3.

class ConvertGather0D : public ov::pass::MatcherPass #: #include <convert_gather_0d.hpp>

ConvertGather0D decomposes v1::Gather operation into v0::Unsqueeze + v1::Gather + v0::Squeeze pattern when gather indices is scalar.

class ConvertGather7ToGather1 : public ov::pass::MatcherPass #: #include <convert_gather_downgrade.hpp>

ConvertGather7ToGather1 converts v7::Gather into v1::Gather.

class ConvertGather8ToGather7 : public ov::pass::MatcherPass #: #include <convert_gather_downgrade.hpp>

ConvertGather8ToGather7 converts v8::Gather into v7::Gather.

class ConvertGather1ToGather7 : public ov::pass::MatcherPass #: #include <convert_gather_upgrade.hpp>

ConvertGather1ToGather7 converts v1::Gather into v7::Gather.

class ConvertGather7ToGather8 : public ov::pass::MatcherPass #: #include <convert_gather_upgrade.hpp>

ConvertGather7ToGather8 converts v7::Gather into v8::Gather.

class ConvertInterpolate11ToInterpolate4 : public ov::pass::MatcherPass #: #include <convert_interpolate11_downgrade.hpp>

Converts Interpolate version 11 to Interpolate version 4 if the new op uses any of the v4 allowed interpolation modes.

class ConvertInterpolate1ToInterpolate4 : public ov::pass::MatcherPass #: #include <convert_interpolate1_to_interpolate4.hpp>

ConvertInterpolate1ToInterpolate4 covert v0:interpolate into v4::Interpolate.

class ConvertMaxPool8ToMaxPool1 : public ov::pass::MatcherPass #: #include <convert_maxpool_downgrade.hpp>

ConvertMaxPool8ToMaxPool1 converts v8::MaxPool into v1::MaxPool.

class ConvertMaxPool14ToMaxPool8 : public ov::pass::MatcherPass #: #include <convert_maxpool_downgrade.hpp>

ConvertMaxPool14ToMaxPool8 converts v14::MaxPool into v8::MaxPool.

class ConvertMaxPool1ToMaxPool8 : public ov::pass::MatcherPass #: #include <convert_maxpool_upgrade.hpp>

ConvertMaxPool1ToMaxPool8 converts v1::MaxPool into v8::MaxPool.

class ConvertMVN1ToMVN6 : public ov::pass::MatcherPass #: #include <convert_mvn1_to_mvn6.hpp>

ConvertMVN1ToMVN6 covert v0:MVN into v6::MVN.

class ConvertPad12ToPad1 : public ov::pass::MatcherPass #: #include <convert_pad12_downgrade.hpp>

Converts Pad v12 to Pad v1.

class ConvertPadToGroupConvolution : public ov::pass::MatcherPass #

#include <convert_pad_to_group_conv.hpp>

ConvertPadToGroupConvolution transformation replaces Pad operation with GroupConvolution but has some restrictions on Pad parameters:

PadMode must be Constant and value is equal to 0
Padding must be applied only for spatial dimensions
Input shape rank must be static and greater than 3
Padding values must be non-negative

class ConvertPriorBox8To0 : public ov::pass::MatcherPass #: #include <convert_prior_box_v8_to_v0.hpp>

ConvertPriorBox8To1 converts v8::PriorBox into v0::PriorBox.

class ConvertROIAlign3To9 : public ov::pass::MatcherPass #: #include <convert_roi_align_v3_to_v9.hpp>

ConvertROIAlign3To9 converts v3::ROIAlign into v9::ROIAlign.

class ConvertROIAlign9To3 : public ov::pass::MatcherPass #: #include <convert_roi_align_v9_to_v3.hpp>

ConvertROIAlign9To3 converts v9::ROIAlign into v3::ROIAlign.

class ConvertScatterElementsToScatter : public ov::pass::MatcherPass #: #include <convert_scatter_elements_to_scatter.hpp>

ConvertScatterElementsToScatter convert opset3::ScatterElementsUpdate to opset3::ScatterUpdate.

class ConvertScatterElementsUpdate12ToScatterElementsUpdate3 : public ov::pass::MatcherPass #: #include <convert_scatter_elements_update12_downgrade.hpp>

Converts Pad v12 to Pad v1.

class ConvertScatterNDUpdate15ToScatterNDUpdate3 : public ov::pass::MatcherPass #: #include <convert_scatter_nd_update15_downgrade.hpp>

Converts ScatterNDUpdate version 15 to ScatterNDUpdate version 3 if ScatterNDUpdate reduction attribute is set to None.

class ConvertRNNSequenceToTensorIterator : public ov::pass::MatcherPass #: #include <convert_sequences_to_tensor_iterator.hpp>

ConvertRNNSequenceToTensorIterator transformation converts RNNSequence layer to TensorIterator *.

class ConvertGRUSequenceToTensorIterator : public ov::pass::MatcherPass #: #include <convert_sequences_to_tensor_iterator.hpp>

ConvertGRUSequenceToTensorIterator transformation converts GRUSequence layer to TensorIterator *.

class ConvertLSTMSequenceToTensorIterator : public ov::pass::MatcherPass #: #include <convert_sequences_to_tensor_iterator.hpp>

ConvertLSTMSequenceToTensorIterator transformation converts LSTMSequence layer to TensorIterator *.

class SliceToStridedSlice : public ov::pass::MatcherPass #: #include <convert_slice_to_strided_slice.hpp>

SliceToStridedSlice transformation convert v8::Slice to v1::StridedSlice.

class ConvertSoftMax8ToSoftMax1 : public ov::pass::MatcherPass #: #include <convert_softmax_downgrade.hpp>

ConvertSoftMax8ToSoftMax1 converts v8::SoftMax into v1::SoftMax.

class ConvertSoftMax1ToSoftMax8 : public ov::pass::MatcherPass #: #include <convert_softmax_upgrade.hpp>

ConvertSoftMax1ToSoftMax8 converts v1::SoftMax into v8::SoftMax.

class ConvertSpaceToBatch : public ov::pass::MatcherPass #

#include <convert_space_to_batch.hpp>

ConvertSpaceToBatch transformation decomposes SpaceToBatch layer to Pad->Reshape->Transpose->Reshape.

Param convert_by_elements:: - reduces the maximum number of dimensions that arise during the transformation if enabled. Default value: true. false - SpaceToBatch decomposes to Pad->Reshape->Transpose->Reshape. During transformation, the number of tensor dimensions can be increased by length of block_shape input of SpaceToBatch layer. true - SpaceToBatch decomposes to Pad-> N x (Reshape->Transpose->Reshape), where N = length of block_shape input of SpaceToBatch layer. During transformation, the number of tensor dimensions can be increased by 1.

class ConvertSqueeze15ToSqueeze0 : public ov::pass::MatcherPass #: #include <convert_squeeze15_downgrade.hpp>

Converts Squeeze v15 to Squeeze v0.

class ConvertTensorIteratorToLSTMSequence : public ov::pass::MatcherPass #: #include <convert_ti_to_sequences.hpp>

Finds all TensorIterator layers, detects the pattern Squeeze->LSTMCell->Unsqueeze in the TensorIterator body, converts this pattern to LSTMSequence layer and replaces them TensorIterator.

class ConvertTensorIteratorToRNNSequence : public ov::pass::MatcherPass #: #include <convert_ti_to_sequences.hpp>

Finds all TensorIterator layers, detects the pattern Squeeze->RNNCell->Unsqueeze in the TensorIterator body, converts this pattern to RNNSequence layer and replaces them TensorIterator.

class ConvertTensorIteratorToGRUSequence : public ov::pass::MatcherPass #: #include <convert_ti_to_sequences.hpp>

Finds all TensorIterator layers, detects the pattern Squeeze->GRUCell->Unsqueeze in the TensorIterator body, converts this pattern to GRUSequence layer and replaces them TensorIterator.

class ConvertLoopToLSTMSequence : public ov::pass::GraphRewrite #: #include <convert_ti_to_sequences.hpp>

Replaces Loop with LSTMCell inside to LSTMSequence.

class FuseReverseLSTMSequence : public ov::pass::MatcherPass #: #include <convert_ti_to_sequences.hpp>

Fuses ReverseSequence->LSTM->ReverseSequence to LSTMSequence with REVERSE direction flag.

class FuseLSTMSequencesToBidirectionalLSTMSequence : public ov::pass::MatcherPass #: #include <convert_ti_to_sequences.hpp>

Replaces two LSTMSequences to one bidirectional LSTMSequence.

class ConvertTopK11ToTopK3 : public ov::pass::MatcherPass #: #include <convert_topk11_downgrade.hpp>

Converts TopK version 11 to TopK version 3 if TopK 11 stable attribute is set to false.

class ConvertXorToLogicalXor : public ov::pass::MatcherPass #: #include <convert_xor_to_logical_xor.hpp>

ConvertXorToLogicalXor converts v0::Xor to v1::LogicalXor.

class ConvertDetectionOutput8ToDetectionOutput1 : public ov::pass::MatcherPass #: #include <detection_output_downgrade.hpp>

ConvertDetectionOutput8ToDetectionOutput1 converts v8::DetectionOutput into v0::DetectionOutput.

class ConvertDetectionOutput1ToDetectionOutput8 : public ov::pass::MatcherPass #: #include <detection_output_upgrade.hpp>

ConvertDetectionOutput1ToDetectionOutput8 converts v0::DetectionOutput into v8::DetectionOutput.

class EinsumDecomposition : public ov::pass::MatcherPass #: #include <einsum_decomposition.hpp>

EinsumDecomposition transformation decomposes Einsum-7 operation into a sub-graph with more simple operations: Transpose, Reshape, MatMul, ReduceSum, Unsqueeze, ShapeOf, ReduceProd, StridedSlice, and Concat.

class EyeDecomposition : public ov::pass::MatcherPass #: #include <eye_decomposition.hpp>

Do eye decomposition to sub-graph (model).

class FakeConvertDecomposition : public ov::pass::MatcherPass #: #include <fake_convert_decomposition.hpp>

FakeConvertDecomposition transformation decomposes FakeConvert layer. f8: f8e4m3, f8e5m2 downconvert: f32->f8, f16->f8, bf16->f8 upconvert: f8->f32, f8->f16, f8->bf16 output = (upconvert(downconvert(input * scale - shift)) + shift) / scale.

class FakeQuantizeDecomposition : public ov::pass::MatcherPass #

#include <fq_decomposition.hpp>

FakeQuantizeDecomposition transformation decomposes FakeQuantize layer.

Expression from specification: if x <= min(input_low, input_high): output = output_low elif x > max(input_low, input_high): output = output_high else: output = round((x - input_low) / (input_high - input_low) * (levels-1)) / (levels-1) * (output_high - output_low) + output_low

expand brackets into round: round(x * (levels-1) / (input_high - input_low) - input_low * (levels-1) / (input_high - input_low)) div on (levels-1) and mult on (output_high - output_low) => mult on (output_high - output_low) / (levels-1)

=> round(x * (levels-1) / (input_high - input_low) - input_low * (levels-1) / (input_high - input_low)) * (output_high - output_low) / (levels-1) + output_low

This transformation doesn’t support following cases:

At least one ‘range’ input is not Constant
At least one ‘input_low’ input value greater or equal than ‘input_high’ input value

class Gelu7Downgrade : public ov::pass::MatcherPass #: #include <gelu7_downgrade.hpp>

Gelu7Downgrade converts v7::Gelu operation to v2::Gelu unconditionally. This is done because only limited set of plugins support v7::Gelu which has an attribute specifying approximation mode. For other plugins the behaviour is to use v2 version of the operation which does not support the approximation mode.

class GRUCellDecomposition : public ov::pass::MatcherPass #

#include <gru_cell_decomposition.hpp>

GRUCellDecomposition transformation decomposes GRUCell layer with inputs X, H, W, R, B to Add, Split, MatMul, Multiply and Subtract ops according to the formula: (.) - Denotes element-wise multiplication.

Denotes dot product. f, g - are activation functions

zt = f(Xt*(Wz^T) + Ht-1*(Rz^T) + Wbz + Rbz) rt = f(Xt*(Wr^T) + Ht-1*(Rr^T) + Wbr + Rbr) ht = g(Xt*(Wh^T) + (rt (.) Ht-1)*(Rh^T) + Rbh + Wbh) # when linear_before_reset := false # (default) ht = g(Xt*(Wh^T) + (rt (.) (Ht-1*(Rh^T) + Rbh)) + Wbh) # when linear_before_reset:= true Ht = (1 - zt) (.) ht + zt (.) Ht-1

class HardSigmoidDecomposition : public ov::pass::MatcherPass #: #include <hard_sigmoid_decomposition.hpp>

HardSigmoidDecomposition transformation into sub-graph.

class HSigmoidDecomposition : public ov::pass::MatcherPass #: #include <hsigmoid_decomposition.hpp>

HSigmoidDecomposition transformation into sub-graph (min(Relu(x + 3), 6) * const(1/6).

class HSwishDecomposition : public ov::pass::MatcherPass #: #include <hswish_decomposition.hpp>

HSwishDecomposition transformation into sub-graph x * (min(Relu(x + 3), 6) * const(1/6).

class LogSoftmaxDecomposition : public ov::pass::MatcherPass #: #include <log_softmax_decomposition.hpp>

LogSoftmaxDecomposition transformation into sub-graph x - log(reduce_sum(exp(x), axis)).

class LSTMCellDecomposition : public ov::pass::MatcherPass #

#include <lstm_cell_decomposition.hpp>

LSTMCellDecomposition transformation decomposes LSTMCell layer with inputs X, H, C, W, R, B to Add, Split, MatMul, Multiply ops according to the formula: (.) - Denotes element-wise multiplication.

Denotes dot product. f, g, h - are activation functions.

it = f(Xt*(Wi^T) + Ht-1*(Ri^T) + Wbi + Rbi) ft = f(Xt*(Wf^T) + Ht-1*(Rf^T) + Wbf + Rbf) ct = g(Xt*(Wc^T) + Ht-1*(Rc^T) + Wbc + Rbc) ot = f(Xt*(Wo^T) + Ht-1*(Ro^T) + Wbo + Rbo) Ct = ft (.) Ct-1 + it (.) ct Ht = ot (.) h(Ct)

class MVN6Decomposition : public ov::pass::MatcherPass #: #include <mvn6_decomposition.hpp>

MVN6Decomposition transformation into sub-graph x - ReduceMean(x, axes) if normalize_variance is false and into sub-graph (x - ReduceMean(x, axes)) / Sqrt(ReduceMean((x - ReduceMean(x, axes)) ^ 2)) if normalize_variance is true.

class NormalizeL2Decomposition : public ov::pass::MatcherPass #: #include <normalize_l2_decomposition.hpp>

Decomposes NormalizeL2 into subgraph.

class ReduceL1Decomposition : public ov::pass::MatcherPass #: #include <reduce_l1_decomposition.hpp>

Decomposes ReduceL1 into ReduceSum(abs(x)).

class ReduceL2Decomposition : public ov::pass::MatcherPass #: #include <reduce_l2_decomposition.hpp>

Decomposes ReduceL2 into sqrt(ReduceSum(x * x)).

class RNNCellDecomposition : public ov::pass::MatcherPass #

#include <rnn_cell_decomposition.hpp>

RNNCellDecomposition transformation decomposes RNNCell layer with inputs X, H, W, R, B to Add, MatMul ops according to the formula:

Denotes dot product. f - is an activation functions.

Ht = f(Xt*(Wi^T) + Ht-1*(Ri^T) + Wbi + Rbi)

class SoftmaxDecomposition : public ov::pass::MatcherPass #

#include <softmax_decomposition.hpp>

SoftmaxDecomposition transformation replaces softmax with following graph:

       +---------------+
       │               │
       │     input     │
       │               │
       +---------------+
           │      │
           │      v
           │ +-----------+
           │ │           │
           │ │ ReduceMax │
           │ │           │
           │ +-----------+
           │      │
           │      │
           v      v
       +---------------+
       │               │
       │      Sub      │
       │               │
       +---------------+
               |
               |
               v
       +---------------+
       │               │
       │      Exp      │
       │               │
       +---------------+
           │      │
           │      v
           │ +-----------+
           │ │           │
           │ │ ReduceSum │
           │ │           │
           │ +-----------+
           │      │
           │      │
           v      v
        +-------------+
        |             │
        |     Div     │
        │             │
        +-------------+

class SoftPlusDecomposition : public ov::pass::MatcherPass #: #include <softplus_decomposition.hpp>

SoftPlusDecomposition transformation replaces SoftPlus op to group of operations: log(exp(x) + 1).

class SoftSignDecomposition : public ov::pass::MatcherPass #

#include <softsign_decomposition.hpp>

SoftSignDecomposition transformation replaces SoftSign with the following graph.

  Input ---> Abs
    |         |
    |         |
    |         |
    |         V
    |        Add <--- 1
    |         |
    |         |
    V         |
  Divide <----|
    |
    |
    |
    V
  Output

class ResolveNameCollisions : public ov::pass::ModelPass #

#include <resolve_names_collisions.hpp>

ResolveNameCollisions transformation helps to fix names collisions if nodes with autogenerated names have conflicts with other node names.

Every transformation call can change the graph structure and create some additional operations, autogenerated name is used if new operation doesn’t have friendly name. This transformations should be called after the transformation pipeline in order to fix names collisions.

There is also an additional mode “resolve_all_names”, the logic of which is the same, but for all “friendly_names” in the model ov, not only for autogenerated.

class BroadcastConstRangeReplacement : public ov::pass::MatcherPass #: #include <broadcast_const_range_replacement.hpp>

BroadcastConstRangeReplacement replaces Constant filled with range values starting from 0 and replaces it with Range op.

class LSTMStatesBroadcast : public ov::pass::ModelPass #: #include <lstm_states_broadcast.hpp>

In case LSTMCell has constant initial hidden and cell state with single batch size we make them broadcast-able by batch.

class ReshapeAMatMul : public ov::pass::MatcherPass #

#include <matmul_sr.hpp>

ReshapeAMatMul and ReshapeBMatMul transformations relax hard-coded Reshape followed by MatMul operation For 2D Reshape search patterns are:

MatMul(Reshape(any_input, any_input), any_input)
MatMul(any_input, Reshape(any_input, any_input))

class Proposal1Scales : public ov::pass::MatcherPass #

#include <proposal_scales_stridedslice.hpp>

ProposalScales transformation helps to silently avoid reshape issues on the scale-input of Proposal layer.

Expected sub-graph looks like: Parameter [batch, 3 or 4] -> Reshape [-1] -(in: 3)-> PriorBox

PriorBox operation accepts 3 or 4 values as scales from specification standpoint PriorBox uses first set (batch) of scale values to proceed in the plugins According to this we explicitly take first batch of scales with StridedSlice operation

Resulting sub-graph: Parameter [batch, 3 or 4] -> Reshape [-1] -> StridedSlice[0: 3 or 4] -(in: 3)-> PriorBox

class ReshapeSinkingMatMul : public ov::pass::MatcherPass #: #include <reshape_sinking.hpp>

ReshapeSinkingMatMul transformation looks for MatMul followed by optional Add surrounded with Reshape operations which are only needed to merge and unmerge dimensions into MatMuls batch. In case of success upscales MatMul to work with multidimensional batch and updates Reshape operators to make batch propagate through freely.

class ReshapeTo1D : public ov::pass::MatcherPass #: #include <reshape_to_1D.hpp>

ReshapeTo1D transformation looks for Reshape from nD to 1D tensor and replaces its pattern to [-1].

class ShapeOfConstFolding : public ov::pass::MatcherPass #: #include <shape_of_const_folding.hpp>

ShapeOfConstFolding constantfolds ShapeOf->Constant subgraph.

class StridedSliceSqueeze : public ov::pass::MatcherPass #: #include <strided_slice_squeeze.hpp>

StridedSliceSqueeze transformation looks for SS -> Squeeze and corrects SS inputs and attributes for SS output to be squeeze-able.

class SqueezeStridedSlice : public ov::pass::MatcherPass #: #include <strided_slice_squeeze.hpp>

StridedSliceSqueeze transformation looks for Squeeze -> SSe and corrects SS inputs and attributes for SS output to be squeeze-able.

class ChainedMaximumOptimization : public ov::pass::MatcherPass #: #include <chained_maximum.hpp>

Optimizes graphs based on value symbols Maximum(Maximum(A, B), B) -> Maximum(A, B) Maximum(Maximum(A, B), A) -> Maximum(A, B)

class DeReshapeMatMul : public ov::pass::MatcherPass #

#include <dereshape_matmul.hpp>

Transformation uses symbol information to optimize out Reshape operations surrounding MatMul. It checks that surrounding Reshapes are only manipulating with batch dimensions of tensor in a do-undo kind of way.

Example: Before: [A,B,C,D] -> Reshape -> [A*B,C,D] MatMul [A*B,C,E] -> Reshape -> [A,B,C,E] [A,B,D,E] -> Reshape -> [A*B,D,E]

After: [A,B,C,D] -> MatMul -> [A,B,C,E] [A,B,D,E] ->

Transformation allows slightly different variations of the pattern on inputs of MatMul.

Simplest pattern contains only Reshape operation on MatMul input: Reshape -> MatMul
The next acceptable variation is Concat of two inputs on MatMul input: Reshape -[-> Concat -]-> MatMul This variation would be transformed with realignment of the other input of Concat and the other outputs of Concat with the help of Reshape operations
The most complex variation on the MatMul input pattern is with Binary Elementwise Operation with scalar second input: Reshape -[-> Concat -]-[-> BEA (scalar) -]-> MatMul

Additionally, transformation supports variation of the pattern on output of MatMul. It allows for Binary Elementwise Arithmetic operation without second input scalar restriction. MatMul -[-> BEA -]-> Reshape this pattern variation is only applicable for the case when input reshapes are 4D -> 3D and output reshape is 3D -> 4D. Additionally, shape symbols on output of MatMul should be equal to the input shape symbols of the last Reshape, meaning that this Binary Elementwise Arithmetic doesn’t perform any broadcasting of input coming from MatMul — only other input may be broadcasted to the MatMul input of this BEA. This effect (equality of MatMul output shape symbols and output shape of BEA) is being handled by LabelResolvingThroughSelect transformation in the particular models that this variation targets.

Full pattern this transformation searches for: -> Reshape -[-> Concat -]-[-> BEA (scalar) -]-> MatMul -[-> BEA -]-> Reshape -> -> Reshape -[-> Concat -]-[-> BEA (scalar) -]->

NOTE: input branches could be (and in observed model cases are) asymmetrical, meaning that the presence of Concat on one input of MatMul doesn’t require the other input to also have Concat

class DeReshapeFullyConnected : public ov::pass::MatcherPass #

#include <dereshape_matmul.hpp>

Transformation uses symbol information to optimize out Reshape operations surrounding special cases of MatMul. It checks that surrounding Reshapes are only manipulating with batch dimensions of tensor in a do-undo kind of way. The difference with previous optimization is that this case has Reshape only on one input of MatMul and the other input is strictly 2D. Such MatMuls are also called FullyConnected.

Example: Before: [A,B,4096] -> Reshape -> [A*B,4096] MatMul [A*B,4608] -> Reshape -> [A,B,4608] [4096,4608]

After: [A,B,4096] -> MatMul -> [A,B,4608] [4096,4608] ->

class NopBroadcast : public ov::pass::MatcherPass #: #include <nop_broadcast.hpp>

Optimizes out Broadcast(data, Maximum(shape, ones)) if labels on data and shape are equal Use case with data being empty should not be considered here since original graph has Maximum with ones.

class ReshapeOptimizations : public ov::pass::MatcherPass #

#include <reshape_optimizations.hpp>

Searches for Flatten-like Reshape operations and simplifies 2nd input of such Reshape using special zero feature. The transformation works in 2 cases:

all in/out dims are static, or we can match them via the symbols.

only one out dim doesn’t have the corresponding input static dim, and we can’t match it using symbols. Besides that the output shape must not contain zero dims, because then value -1 in 2nd input to Reshape op can’t guarantee an unambiguous determination of the remaining dim value.

 for example:
       Before:
       +-------------+    +----------+
       |data         |    | Concat   |
       |shape: (0, 0)|    | shape (2)|<- the values might be determined in runtime
       +-----+-------+    +-----+----+   the empty data tensor can be reshaped to
             |                  |        any other empty shape, e.g. (0, 800)
       +-----v---------------+  |
       | Reshape             |  |
       | shape (0,-1)        <--+
       | special zero = False|
       +---------------------+

       After:
                          +---------------+
      +--------------+    | Constant      |
      | data         |    | shape (2)     |
      | shape: (0, 0)|    | values (0, -1)|<- -1 means copy the corresponding input dim
      +-----+--------+    +-------+-------+
            |                     |
       +----v----------------+    |
       | Reshape             <----+
       | shape (0,0)         |  <- it might cause inconsistency
       | special zero = True |
       +---------------------+

class ApplySymbolEquivalence : public ov::pass::ModelPass #: #include <symbol_optimization.hpp>

Resets symbols on output shapes and values according to symbol equivalence. It allows to reduce number of labels used in the model and to disambiguate symbol values.

class OptimizeSymbolsUsedAsValues : public ov::pass::ModelPass #: #include <symbol_optimization.hpp>

Collects sources where each symbol initially appeared (on shape or shape sub-graph) and attaches all value usages of this label to this initial source.

class SymbolicOptimizations : public ov::pass::ModelPass #: #include <symbolic_optimizations.hpp>

Runs optimizations which are based on symbolic shape inference.

class SymbolicPropagation : public ov::pass::ModelPass #: #include <symbolic_optimizations.hpp>

Assigns labels / symbols to all tensors on shapes and values. Uses shape inference and other special rules to propagate labels / symbols.

class LabelResolvingThroughSelect : public ov::pass::MatcherPass #

#include <symbolic_optimizations.hpp>

Transformation requires equal labels on one input of Add and output of last Reshape in the pattern: -> Add -> Reshape -[then or else input]-> Select -> Softmax -> Reshape ->

If shape labels onn mentioned tensors are equal we proved that no broadcasting of this input was done for Add and for Select. Therefore, we can put the same labels on the output of Add and Select. This transformation helps propagate labels and will not be needed if we would use information on equality of products of input and output dimensions of Reshape operations

class TSForwardBase : public ov::pass::MatcherPass #

#include <ts_base.hpp>

TSForwardBase is a base class for all forward transformations.

class TSBinaryForward : public ov::pass::transpose_sinking::TSForwardBase #: #include <ts_binary.hpp>

TSBinaryForward transformation sinks Transpose through BinaryElementwiseArithmetic, BinaryElementwiseComparison, BinaryElementwiseLogical and PRelu operations in the forward direction.

class TSBinaryBackward : public ov::pass::MatcherPass #: #include <ts_binary.hpp>

TSBinaryBackward transformation sinks Transpose through BinaryElementwiseArithmetic, BinaryElementwiseComparison, BinaryElementwiseLogical and PRelu operations in the backward direction.

class TSConcatForward : public ov::pass::transpose_sinking::TSForwardBase #: #include <ts_concat.hpp>

TSConcatForward transformation sinks Transpose through Concat operation in the forward direction.

class TSConcatBackward : public ov::pass::MatcherPass #: #include <ts_concat.hpp>

TSConcatBackward transformation sinks Transpose through Concat operation in the backward direction.

class TSCumSumForward : public ov::pass::transpose_sinking::TSForwardBase #: #include <ts_cumsum.hpp>

TSCumSumForward transformation sinks Transpose through CumSum in the forward direction.

class TSCumSumBackward : public ov::pass::MatcherPass #: #include <ts_cumsum.hpp>

TSCumSumBackward transformation sinks Transpose through CumSum in the backward direction.

class TSDataMovementForward : public ov::pass::transpose_sinking::TSForwardBase #: #include <ts_data_movement.hpp>

TSDataMovementForward transformation sinks Transpose through BatchToSpace, SpaceToBatch, ReverseSequence and Pad operations in the forward direction. These operations are categorized as “DataMovement” and are handled in a similar way in this transformation.

class TSDataMovementBackward : public ov::pass::MatcherPass #: #include <ts_data_movement.hpp>

TSDataMovementBackward transformation sinks Transpose through BatchToSpace, SpaceToBatch, ReverseSequence and Pad operations in the backward direction. These operations are categorized as “DataMovement” and are handled in a similar way in this transformation.

class TSFuse : public ov::pass::MatcherPass #: #include <ts_fuse.hpp>

TSFuse transformation eliminates 2 consecutive Transposes if they result in no changes to input or fuses them to single Transpose if input gets changed.

class TSGatherForward : public ov::pass::transpose_sinking::TSForwardBase #: #include <ts_gather.hpp>

TSGatherForward transformation sinks Transpose through Gather operations in the forward direction.

class TSGatherBackward : public ov::pass::MatcherPass #: #include <ts_gather.hpp>

TSGatherBackward transformation sinks Transpose through Gather operation in the backward direction.

class TSGeneralForward : public ov::pass::GraphRewrite #: #include <ts_general.hpp>

TSGeneralForward transformation combines all TransposeSinkingForward* transformations into single GraphRewrite pass.

class TSGeneralBackward : public ov::pass::GraphRewrite #: #include <ts_general.hpp>

TSGeneralBackward transformation combines all TransposeSinkingBackward* transformations into single GraphRewrite pass.

class TSGeneral : public ov::pass::ModelPass #: #include <ts_general.hpp>

TSGeneral transformation combines TSGeneralForward and TSGeneralBackward transformations into single ModelPass pass and inserts ConstantFolding pass after them.

class TSInterpolateForward : public ov::pass::transpose_sinking::TSForwardBase #: #include <ts_interpolate.hpp>

TSInterpolateForward transformation sinks Transpose through Interpolate operation in the forward direction.

class TSInterpolateBackward : public ov::pass::MatcherPass #: #include <ts_interpolate.hpp>

TSInterpolateBackward transformation sinks Transpose through Interpolate operation in the backward direction.

class TSReductionForward : public ov::pass::transpose_sinking::TSForwardBase #: #include <ts_reduction.hpp>

TSReductionForward transformation sinks Transpose through Reduce operations in the forward direction.

class TSReductionBackward : public ov::pass::MatcherPass #: #include <ts_reduction.hpp>

TSReductionBackward transformation sinks Transpose through Reduce operations in the backward direction.

class TSResetNoSinkingAttribute : public ov::pass::MatcherPass #: #include <ts_reset_no_sinking_attribute.hpp>

TSResetNoSinkingAttribute transformation resets all NoSinkingAttribute runtime attributes in Transpose operations.

class TSShapeOfForward : public ov::pass::transpose_sinking::TSForwardBase #

#include <ts_shape_of.hpp>

TSShapeOfForward transformation sinks Transpose through ShapeOf in the forward direction.

It replaces:

+———+ |Transpose| +———+ | v +———+ | ShapeOf | +———+

with the following:

+———+ | ShapeOf | +———+ | v +——+ |Gather| +——+

class TSSplitForward : public ov::pass::transpose_sinking::TSForwardBase #: #include <ts_split.hpp>

TSSplitForward transformation sinks Transpose through Split, VariadicSplit operations in the forward direction.

class TSSplitBackward : public ov::pass::MatcherPass #: #include <ts_split.hpp>

TSSplitBackward transformation sinks Transpose through Split, VariadicSplit operations in the backward direction.

class TSSqueezeForward : public ov::pass::transpose_sinking::TSForwardBase #: #include <ts_squeeze.hpp>

TSSqueezeForward transformation sinks Transpose through Reshape, Squeeze operations in the forward direction.

class TSSqueezeBackward : public ov::pass::MatcherPass #: #include <ts_squeeze.hpp>

TSSqueezeBackward transformation sinks Transpose through Reshape, Squeeze operations in the backward direction.

class TSTileForward : public ov::pass::transpose_sinking::TSForwardBase #: #include <ts_tile.hpp>

TSTileForward transformation sinks Transpose through Tile in the forward direction.

class TSTileBackward : public ov::pass::MatcherPass #: #include <ts_tile.hpp>

TSTileBackward transformation sinks Transpose through Tile in the backward direction.

class TSUnaryForward : public ov::pass::transpose_sinking::TSForwardBase #: #include <ts_unary.hpp>

TSUnaryForward transformation sinks Transpose through UnaryElementwiseArithmetic, Clamp, Elu, SoftPlus, LogicalNot, Convert, IsInf, IsNaN, IsFinite operations in the forward direction.

class TSUnaryBackward : public ov::pass::MatcherPass #: #include <ts_unary.hpp>

TSUnaryBackward transformation sinks Transpose through UnaryElementwiseArithmetic, Clamp, Elu, SoftPlus, LogicalNot, Convert, IsInf, IsNaN, IsFinite in the backward direction.

class TSUnsqueezeForward : public ov::pass::transpose_sinking::TSForwardBase #: #include <ts_unsqueeze.hpp>

TSUnsqueezeForward transformation sinks Transpose through Unsqueeze, Reshape operations in the forward direction.

class TSUnsqueezeBackward : public ov::pass::MatcherPass #: #include <ts_unsqueeze.hpp>

TSUnsqueezeBackward transformation sinks Transpose through Unsqueeze, Reshape operations in the backward direction.

namespace ov#

transformation aligns elementwise constant inputs ranks with its output rank

A namespace with const values for Execution Graph parameters names.

Executable Model Info is represented in ov::Model format with general ExecutionNode nodes inside including connections between the nodes. Each node describes an executable hardware-specific primitive and stores its parameters within ExecutionNode::get_rt_info map. There is a list of general keys for the parameters map.

OpenVINO C++ API.

Resolves transpose_b key from MatMul operation if corresponding input is constant or FakeQuantize by inserting Transpose.

Unnamed Group

OPENVINO_API void serialize (const std::shared_ptr< const ov::Model > &m, const std::string &xml_path, const std::string &bin_path="", ov::pass::Serialize::Version version=ov::pass::Serialize::Version::UNSPECIFIED)

Serialize given model into IR. The generated .xml and .bin files will be saved into provided paths. This method serializes model “as-is” that means no weights compression and other possible transformations are applied. It is recommended to use ov::save_model function instead of ov::serialize, because it is aligned with default model conversion flow.

Parameters:

m – Model which will be converted to IR representation.
xml_path – Path where .xml file will be saved.
bin_path – Path where .bin file will be saved (optional). The same name as for xml_path will be used by default.
version – Version of the generated IR (optional).

Functions

LP_TRANSFORMATIONS_API void mark_as_bias (const std::shared_ptr< Node > &node)

LP_TRANSFORMATIONS_API bool marked_as_bias (const std::shared_ptr< const Node > &node)

std::ostream &operator<<(std::ostream &out, const Mask &mask)#

Mask::Ptr getMask(const Output<const Node> &output)#

Mask::Ptr getMask(const Output<Node> &output)#

void setMask(Output<Node> output, const Mask::Ptr &mask)#

void setMask(Input<Node> node, const Mask::Ptr &mask)#

std::ostream &operator<<(std::ostream &s, const op::internal::GLU::GluType &reduction)#

void mark_as_decompression(const std::shared_ptr<Node> &node)#

void unmark_as_decompression(const std::shared_ptr<Node> &node)#

bool is_decompression(const std::shared_ptr<Node> &node)#

void mark_as_dequantization_node(const std::shared_ptr<Node> &node)#

bool is_dequantization_node(const std::shared_ptr<const Node> &node)#

void disable_fp16_compression(const std::shared_ptr<Node> &node)#

void enable_fp16_compression(const std::shared_ptr<Node> &node)#

bool fp16_compression_is_disabled(const std::shared_ptr<const Node> &node)#

void postpone_fp16_compression(RTMap &rt_info)#

bool is_fp16_compression_postponed(const RTMap &rt_info)#

void do_not_postpone_fp16_compression(RTMap &rt_info)#

std::string getFusedNames(const std::shared_ptr<ov::Node> &node)#

getFusedNames return string with operation names separated by coma in alphabetical order

Parameters:: node – [in] The node will be used to get FusedNames attribute

std::vector<std::string> getFusedNamesVector(const std::shared_ptr<ov::Node> &node)#

getFusedNamesVector return vector of fused names sorted in alphabetical order

Parameters:: node – [in] The node will be used to get FusedNames attribute
Returns:: vector of strings

void mark_shape_subgraph(const std::shared_ptr<Node> &node)#

void unmark_shape_subgraph(const std::shared_ptr<Node> &node)#

bool is_shape_subgraph(const std::shared_ptr<const Node> &node)#

void enable_keep_const_precision(const std::shared_ptr<Node> &node)#

void disable_keep_const_precision(const std::shared_ptr<Node> &node)#

bool is_keep_const_precision(const std::shared_ptr<const Node> &node)#

bool has_nms_selected_indices(const Node *node)#

void set_nms_selected_indices(Node *node)#

void disable_divide_conversion(const std::shared_ptr<Node> &node)#

void enable_divide_conversion(const std::shared_ptr<Node> &node)#

bool divide_is_nonconvertible(const std::shared_ptr<Node> &node)#

inline bool has_old_api_map_element_type(const std::shared_ptr<Node> &node)#

inline OldApiMapElementType get_old_api_map_element_type(const std::shared_ptr<Node> &node)#

inline void set_old_api_map_element_type(const std::shared_ptr<Node> &node, const OldApiMapElementType &old_api_map)#

inline bool has_old_api_map_order(const std::shared_ptr<Node> &node)#

inline OldApiMapOrder get_old_api_map_order(const std::shared_ptr<Node> &node)#

inline void set_old_api_map_order(std::shared_ptr<Node> &node, const OldApiMapOrder &old_api_map)#

void set_original_precision_attribute(const std::shared_ptr<Node> &node, const element::Type_t original_precision)#

void reset_original_precision_attribute(const std::shared_ptr<Node> &node)#

element::Type_t get_original_precision(const std::shared_ptr<Node> &node)#

bool is_preprocesing_node(const std::shared_ptr<Node> &node)#

void set_is_preprocessing_node(std::shared_ptr<Node> node)#

std::string getPrimitivesPriority(const std::shared_ptr<Node> &node)#

getPrimitivesPriority return string with primitive priorities value

Parameters:: node – [in] The node will be used to get PrimitivesPriority attribute

bool has_strides_prop(const Input<Node> &node)#

ov::Strides get_strides_prop(const Input<Node> &node)#

void insert_strides_prop(Input<Node> &node, const Strides &strides)#

void remove_strides_prop(Input<Node> &node)#

void mark_as_no_sinking_node(const std::shared_ptr<Node> &node)#

void reset_no_sinking_attribute(const std::shared_ptr<Node> &node)#

bool is_sinking_node(const std::shared_ptr<Node> &node)#

bool is_sinking_node(const Node *node)#

bool is_sinking_node(ov::Output<ov::Node> output)#

std::shared_ptr<ov::MappedMemory> load_mmap_object(const std::string &path)#

Returns mapped memory for a file from provided path. Instead of reading files, we can map the memory via mmap for Linux in order to avoid time-consuming reading and reduce memory consumption.

Parameters:: path – Path to a file which memory will be mmaped.
Returns:: MappedMemory shared ptr object which keep mmaped memory and control the lifetime.

template<typename A, typename B> A copy_from(B &b)#

OPENVINO_API std::ostream & operator<< (std::ostream &s, const AxisSet &axis_set)

OPENVINO_API std::ostream & operator<< (std::ostream &s, const AxisVector &axis_vector)

OPENVINO_API std::ostream & operator<< (std::ostream &s, const Coordinate &coordinate)

OPENVINO_API std::ostream & operator<< (std::ostream &s, const CoordinateDiff &coordinate_diff)

OPENVINO_API std::ostream & operator<< (std::ostream &str, const Dimension &dimension)

Insert a human-readable representation of a dimension into an output stream.

Inserts the string ? if dimension is dynamic; else inserts dimension.get_length().

Parameters:

str – The output stream targeted for insertion.
dimension – The dimension to be inserted into str.

Returns:

A reference to str after insertion.

template<typename Type, typename Value> std::enable_if<std::is_convertible<Value, std::string>::value, Type>::type as_enum(const Value &value)#: Returns the enum value matching the string.

template<typename Value> const std::string &as_string(Value value)#: Returns the string matching the enum value.

static inline std::ostream &write_all_to_stream(std::ostream &str)#

template<typename T, typename ...TS> std::ostream &write_all_to_stream(std::ostream &str, T &&arg, TS&&... args)#

template<class T, typename std::enable_if<!std::is_same<typename std::decay<T>::type, std::string>::value>::type* = nullptr> std::string stringify(T &&arg)#

template<class T, typename std::enable_if<std::is_same<typename std::decay<T>::type, std::string>::value>::type* = nullptr> T &stringify(T &&arg)#

OPENVINO_API void traverse_nodes (const std::shared_ptr< const Model > &p, const std::function< void(const std::shared_ptr< Node > &)> &f)

OPENVINO_API void traverse_nodes (const Model *p, const std::function< void(const std::shared_ptr< Node > &)> &f)

OPENVINO_API void traverse_nodes (const NodeVector &subgraph_results, const std::function< void(const std::shared_ptr< Node > &)> &f, const NodeVector &subgraph_params={})

Visit each node in a sub-graph of the entire graph.

Traverses a sub-graph starting from subgraph_results moving up towards parameter nodes. Traversal stops if it hits a node in subgraph_params.

Most useful for finding parameters of a graph directly from the result nodes and not from function parameters or extracting a subgraph relevant to the computation of certain outputs

Parameters:

subgraph_results – The output nodes of the sub-graph
f – Model to execute at each node in the traversal
subgraph_params – Input nodes of the sub-graph (optional)

OPENVINO_API void replace_node (const std::shared_ptr< Node > &target, const std::shared_ptr< Node > &replacement, const std::vector< int64_t > &output_order)

Replace the node target with the node replacement, i.e., redirect all users and control dependencies of target to replacement.

This is primarily used in graph-rewriting passes. For example, we might “fuse” two Concat operations as follows:

(Step 0: Original graph)

(Step 1: Construct replacement)

shared_ptr<Node> new_N1 = make_shared<op::Concat>({A,B,C},3);

A————————————-—. | | | B————-—)—. | | | | v v | | N0[Concat, concatenation_axis=3] C–—)—)—. | | | | | v v v v v N1[Concat, concatenation_axis=3] new_N1[Concat, concatenation_axis=3] | | v v some_user another_user

(Step 2: Replace N1 with new_N1)

replace_node(N1, new_N1);

(Step 3: N0 and N1 are now dead, nodes will be freed)

[happens automatically, once all shared_ptrs to N1 are released]

A————————————-—. | B————-—)—. | | | | C–—)—)—. | | | v v v new_N1[Concat, concatenation_axis=3] | | v v some_user another_user

NOTE 1: replace_node is not type-safe (the graph is not revalidated). For example, the following is allowed, even if node some_user requires an input of shape 2x2:

(Before) A(shape=2x2) B(shape=3x3) | v some_user(requires 2x2 input)

(After — graph is now invalid)

 replace_node(A, B);

 A(shape=2x2)  B(shape=3x3)
               |
               v
            some_user(requires 2x2 input)

NOTE 2: it is possible to insert a cycle into the graph with replace_node, resulting in an invalid graph. Care must be taken to avoid this. One common example is when you are attempting to insert a new node M “after”a nodeN`. For example, you might expect this to work:

shared_ptr<Node> M = make_shared<SomeUnaryOp>(N); replace_node(M, N);

The problem is that at replacement time, N itself is a user of M. So we end up introducing a cycle as follows:

  N
  |
  v

other users…

|||
vvv

 N------------>M
 |
 v

other users…

|||
vvv

         .----.
        |      |
        |      |
 N      `----->M
               |
               v
          other users...

To avoid the cycle, a valid way to perform the above desired insertion would be,

   auto new_N = N->clone_with_new_inputs(N->input_values());
   shared_ptr<Node> M = make_shared<SomeUnaryOp>(new_N);
   replace_node(N, M);

Parameters:

target – Node to be replaced.
replacement – Node to replace target with.
output_order – Vector determines order of replacement node’s outputs.

OPENVINO_API void replace_node (const std::shared_ptr< Node > &target, const OutputVector &replacement_values): Replace target.outputs[i] with replacement_values[i] and transfer control dependents and.

OPENVINO_API void replace_node (const std::shared_ptr< Node > &target, const std::shared_ptr< Node > &replacement)

OPENVINO_API void replace_nodes (const std::shared_ptr< Model > &f, const std::unordered_map< std::shared_ptr< op::v0::Parameter >, std::shared_ptr< op::v0::Parameter > > &parameter_replacement_map, const std::unordered_map< std::shared_ptr< Node >, std::shared_ptr< Node > > &body_replacement_map)

Replace multiple nodes in a function.

Limitations:

No check is made that the replaced nodes in parameter_replacement_map are actually among the bound parameters of f. (If a parameter appears in the map that is not bound by f, it will be silently ignored.)
If a parameter node appears as a key in both parameter_replacement_map and in body_replacement_map, behavior is unspecified.

Parameters:

f – Model where replacement is taking place.
parameter_replacement_map – A mapping from parameter shared pointers to parameter shared pointers. For each pair (k,v) in the map, parameter k is replaced by parameter v, except if k==v or k is not a parameter bound by f, in which case the pair (k,v) is ignored.
body_replacement_map – A mapping from node shared pointers to node shared pointers. For each pair (k,v) in the map, node k is replaced by node v, except if k==v, the pair (k,v) is ignored. Note that if k is a parameter, its users will be redirected to v, but k will not be replaced in the function’s parameter list.

template<typename T> std::vector<std::shared_ptr<Node>> topological_sort(T root_nodes)#: Topological sort of nodes needed to compute root_nodes.

OPENVINO_API bool compare_constants (const std::shared_ptr< Node > &n1, const std::shared_ptr< Node > &n2)

OPENVINO_API bool replace_output_update_name (Output< Node > node, const Output< Node > &node_input)

OPENVINO_API bool replace_node_update_name (const std::shared_ptr< Node > &target, const std::shared_ptr< Node > &replacement)

OPENVINO_API void save_model (const std::shared_ptr< const ov::Model > &model, const std::string &output_model, bool compress_to_fp16=true)

Save given model into IR. Floating point weights are compressed to FP16 by default. This method saves a model to IR applying all necessary transformations that usually applied in model conversion flow provided by OVC tool. Particularly, floating point weights are compressed to FP16.

Parameters:

model – Model which will be converted to IR representation.
output_model – Path to the output model file, must have extension .xml
compress_to_fp16 – Whether to compress floating point weights to FP16 (true by default)

OPENVINO_API std::ostream & operator<< (std::ostream &str, const Interval &interval)

std::shared_ptr<Model> clone_ov_model(const Model &func, std::unordered_map<Node*, std::shared_ptr<Node>> &node_map)#

OPENVINO_API std::ostream & operator<< (std::ostream &, const Model &)

OPENVINO_API ov::Dimension get_batch (const std::shared_ptr< const ov::Model > &f)

Helper method to get associated batch size for a Model.

Checks layout of each parameter in a Model and extracts value for N (B) dimension. All values are then merged and returned

Throws:

ov::AssertFailure – with details in case of error. Possible errors are:

There is no parameter with layout set. Model shall have at least one parameter with layout with ‘N’ dimension. Recommended fix is to use Parameter::set_layout API, e.g. model->get_parameters()[some_index]->set_layout("NCHW");
Several parameters have conflicting N dimension, e.g. param1 NCHW{1,3,224,224} and param2 NCHW{2,3,224,224}. This is ambiguous, most probably first dimension is incorrectly marked as ‘batch’ (N) in some layout. User shall fix it before using of ‘get_batch’ (in example above correct layout for param2 from ‘NCHW’ to ‘CHWN’)

Parameters:

f – Model where to look for a batch_size value

Returns:

Dimension representing current batch size. Can represent a number or be a dynamic

OPENVINO_API void set_batch (const std::shared_ptr< ov::Model > &model, ov::Dimension batch_size)

Helper method to set batch size to a Model.

Checks layout of each parameter in a Model and sets value for N (B) dimension. Then performs validation and type propagation

Throws:

ov::AssertFailure – with details in case of error. Possible errors are:

There is no parameter with N dimension in layout. Model shall have at least one parameter with layout with ‘N’ dimension. Recommended fix is to use Parameter::set_layout API, e.g. model->get_parameters()[some_index]->set_layout("NCHW");
Several parameters have conflicting N dimension, e.g. param1 NCHW{1,3,224,224} and param2 NCHW{3,224,224,1}. This is ambiguous (1 != 3), most probably some dimension is incorrectly marked as ‘batch’ (N) in some layout. User shall fix it before using of ‘set_batch’ (in example above correct layout for param2 from ‘NCHW’ to ‘CHWN’)
Validation fails after setting batch_size. Model becomes in inconsistent state after new batch size value is applied. Possible reason could be that layout was not set for some parameters, or batch size can’t be applied to model at all

Parameters:

model – model where to set batch_size value
batch_size – Batch size value. For dynamic batch size, Dimension::dynamic() can be passed.

OPENVINO_API std::string node_validation_failure_loc_string (const Node *node)

OPENVINO_API std::ostream & operator<< (std::ostream &, const Node &)

OPENVINO_API std::ostream & operator<< (std::ostream &, const Node *)

void OPENVINO_API check_new_args_count (const Node *const node, const OutputVector &new_args)

Check new arguments size if match node inputs count.

This check is required in cloning ov::Node.

Parameters:

node – Pointer to node.
new_args – Vector with new outputs to check.

OPENVINO_API std::ostream & operator<< (std::ostream &out, const Input< Node > &input)

OPENVINO_API std::ostream & operator<< (std::ostream &out, const Input< const Node > &input)

OPENVINO_API std::ostream & operator<< (std::ostream &out, const Output< Node > &output)

OPENVINO_API std::ostream & operator<< (std::ostream &out, const Output< const Node > &output)

OPENVINO_API OutputVector as_output_vector (const NodeVector &args)

OPENVINO_API NodeVector as_node_vector (const OutputVector &values)

OPENVINO_API ResultVector as_result_vector (const OutputVector &values): Returns a ResultVector referencing values.

OPENVINO_API PartialShape operator+ (const PartialShape &s1, const PartialShape &s2)

Elementwise addition of two PartialShape objects.

If s1 or s2 has dynamic rank, returns PartialShape::dynamic().
If s1 ands2` both have static rank, and their ranks are unequal, throws std::invalid_argument.
If s1 and s2 both have static rank, and their ranks are equal, returns a new shape whose ith dimension is s1[i] + s2[i].

Parameters:

s1 – Left operand for addition.
s2 – Right operand for addition.

Throws:

std::invalid_argument – If s1 and s2 have inconsistent ranks.

Returns:

The result of elementwise adding s1 to s2 (see description).

OPENVINO_API std::ostream & operator<< (std::ostream &str, const PartialShape &shape)

Inserts a human-readable representation of a PartialShape into an output stream.

The output to the stream is in “informal” notation. In other words:

If shape has dynamic rank, inserts the string ?.
If shape has static rank, inserts the string {, then inserts each dimension of shape into the output stream separated by commas, then inserts }.

Example:

PartialShape s1{PartialShape::dynamic())};
PartialShape s2{};
PartialShape s3{1,Dimension::dynamic(),2,3};
PartialShape s4{2,3,4};
std::cout << s1 << std::endl
          << s2 << std::endl
          << s3 << std::endl
          << s4 << std::endl;

Output:

?
{}
{1,?,2,3}
{2,3,4}

Parameters:

str – The output stream targeted for insertion.
shape – The shape to be inserted into str.

Returns:

A reference to str after insertion.

OPENVINO_API void copy_runtime_info (const std::shared_ptr< ov::Node > &from, const std::shared_ptr< ov::Node > &to)

OPENVINO_API void copy_runtime_info (const std::shared_ptr< ov::Node > &from, ov::NodeVector to)

OPENVINO_API void copy_runtime_info (const ov::NodeVector &from, const std::shared_ptr< ov::Node > &to)

OPENVINO_API void copy_runtime_info (const ov::NodeVector &from, ov::NodeVector to)

OPENVINO_API void copy_output_runtime_info (const ov::OutputVector &from, ov::OutputVector to)

OPENVINO_API std::ostream & operator<< (std::ostream &os, const RuntimeAttribute &attrubute)

template<typename ForwardIt> size_t shape_size(ForwardIt start_dim, const ForwardIt end_dim)#: Number of elements in a subset of dimensions of a shape. Returns a product of dimensions in a range [start_dim;end_dim)

template<typename SHAPE_TYPE> size_t shape_size(const SHAPE_TYPE &shape)#: Number of elements in spanned by a shape.

template<typename SHAPE_TYPE> std::vector<size_t> row_major_strides(const SHAPE_TYPE &shape)#: Row-major strides for a shape.

template<typename SHAPE_TYPE> size_t row_major_stride(const SHAPE_TYPE &shape, size_t axis)#

template<typename SHAPE_TYPE> inline bool is_scalar(const SHAPE_TYPE &shape)#

template<typename SHAPE_TYPE> inline bool is_vector(const SHAPE_TYPE &shape)#

OPENVINO_API std::ostream & operator<< (std::ostream &s, const Shape &shape)

OPENVINO_API std::ostream & operator<< (std::ostream &s, const Strides &strides)

OPENVINO_API std::ostream & operator<< (std::ostream &s, const DiscreteTypeInfo &info)

bool ::type is_type (Value value)

Type *::type as_type (Value value)

template<typename T, typename U> auto as_type_ptr(const U &value) -> decltype(::ov::util::AsTypePtr<U>::template call<T>(value))#: Casts a std::shared_ptr<Value> to a std::shared_ptr<Type> if it is of type Type, nullptr otherwise

OPENVINO_API std::ostream & operator<< (std::ostream &s, const Version &version)

OPENVINO_API std::ostream & operator<< (std::ostream &s, const std::map< std::string, Version > &versions)

OPENVINO_API_C (const Version) get_openvino_version() noexcept

Gets the current OpenVINO version.

Returns:: The current OpenVINO version

OPENVINO_API std::ostream & operator<< (std::ostream &s, const op::v1::BinaryConvolution::BinaryConvolutionMode &type)

OPENVINO_API std::ostream & operator<< (std::ostream &s, const op::v0::DepthToSpace::DepthToSpaceMode &type)

OPENVINO_API std::ostream & operator<< (std::ostream &s, const op::v9::GridSample::InterpolationMode &type)

OPENVINO_API std::ostream & operator<< (std::ostream &s, const op::v9::GridSample::PaddingMode &type)

OPENVINO_API std::ostream & operator<< (std::ostream &s, const op::v0::Interpolate::InterpolateMode &type)

OPENVINO_API std::ostream & operator<< (std::ostream &s, const op::LSTMWeightsFormat &type)

OPENVINO_API std::ostream & operator<< (std::ostream &s, const op::v8::MatrixNms::DecayFunction &type)

OPENVINO_API std::ostream & operator<< (std::ostream &s, const op::v8::MatrixNms::SortResultType &type)

OPENVINO_API std::ostream & operator<< (std::ostream &s, const op::v1::NonMaxSuppression::BoxEncodingType &type)

OPENVINO_API std::ostream & operator<< (std::ostream &s, const op::v3::NonMaxSuppression::BoxEncodingType &type)

OPENVINO_API std::ostream & operator<< (std::ostream &s, const op::v5::NonMaxSuppression::BoxEncodingType &type)

OPENVINO_API std::ostream & operator<< (std::ostream &s, const op::v9::NonMaxSuppression::BoxEncodingType &type)

OPENVINO_API std::ostream & operator<< (std::ostream &s, const op::v1::Reverse::Mode &type)

std::ostream &operator<<(std::ostream &s, const op::v3::ROIAlign::PoolingMode &mode)#

std::ostream &operator<<(std::ostream &s, const op::v9::ROIAlign::PoolingMode &mode)#

std::ostream &operator<<(std::ostream &s, const op::v9::ROIAlign::AlignedMode &mode)#

OPENVINO_API std::ostream & operator<< (std::ostream &s, const op::v5::Round::RoundMode &type)

OPENVINO_API std::ostream & operator<< (std::ostream &s, const op::v15::ScatterNDUpdate::Reduction &reduction)

OPENVINO_API std::ostream & operator<< (std::ostream &s, const op::v0::SpaceToDepth::SpaceToDepthMode &type)

OPENVINO_API std::ostream & operator<< (std::ostream &s, const op::util::EmbeddingBagOffsetsBase::Reduction &reduction)

OPENVINO_API std::ostream & operator<< (std::ostream &s, const op::util::EmbeddingBagPackedBase::Reduction &reduction)

OPENVINO_API std::ostream & operator<< (std::ostream &s, const op::util::InterpolateBase::InterpolateMode &type)

OPENVINO_API std::ostream & operator<< (std::ostream &s, const op::util::InterpolateBase::CoordinateTransformMode &type)

OPENVINO_API std::ostream & operator<< (std::ostream &s, const op::util::InterpolateBase::NearestMode &type)

OPENVINO_API std::ostream & operator<< (std::ostream &s, const op::util::InterpolateBase::ShapeCalcMode &type)

OPENVINO_API std::ostream & operator<< (std::ostream &s, const op::util::MulticlassNmsBase::SortResultType &type)

void OPENVINO_API mark_as_precision_sensitive (ov::Input< ov::Node > node_input)

void OPENVINO_API unmark_as_precision_sensitive (ov::Input< ov::Node > node_input)

bool OPENVINO_API is_precision_sensitive (const ov::Input< ov::Node > &node_input)

OPENVINO_API void skip_invalidation (const ov::Output< ov::Node > &output)

OPENVINO_API bool skip_invalidation (const ov::descriptor::Tensor &tensor)

OPENVINO_API void remove_skip_invalidation_rti (const std::shared_ptr< ov::Model > &model, bool outermost_model=true)

OPENVINO_API void populate_tensor_with_missing_symbols (ov::descriptor::Tensor &tensor)

const OPENVINO_API OpSet & get_opset1 (): Returns opset1.

const OPENVINO_API OpSet & get_opset2 (): Returns opset2.

const OPENVINO_API OpSet & get_opset3 (): Returns opset3.

const OPENVINO_API OpSet & get_opset4 (): Returns opset4.

const OPENVINO_API OpSet & get_opset5 (): Returns opset5.

const OPENVINO_API OpSet & get_opset6 (): Returns opset6.

const OPENVINO_API OpSet & get_opset7 (): Returns opset7.

const OPENVINO_API OpSet & get_opset8 (): Returns opset8.

const OPENVINO_API OpSet & get_opset9 (): Returns opset9.

const OPENVINO_API OpSet & get_opset10 (): Returns opset10.

const OPENVINO_API OpSet & get_opset11 (): Returns opset11.

const OPENVINO_API OpSet & get_opset12 (): Returns opset12.

const OPENVINO_API OpSet & get_opset13 (): Returns opset13.

const OPENVINO_API OpSet & get_opset14 (): Returns opset14.

const OPENVINO_API OpSet & get_opset15 (): Returns opset15.

const OPENVINO_API OpSet & get_opset16 (): Returns opset16.

const OPENVINO_API std::map< std::string, std::function< const ov::OpSet &()> > & get_available_opsets (): Returns map of available opsets.

OPENVINO_API pass::pattern::op::ValuePredicate operator|| (const std::function< bool(Output< Node >)> &a, const std::function< bool(Output< Node >)> &b)

OPENVINO_API pass::pattern::op::ValuePredicate operator|| (const pass::pattern::op::ValuePredicate &a, const pass::pattern::op::ValuePredicate &b)

OPENVINO_API pass::pattern::op::ValuePredicate operator|| (const pass::pattern::op::NodePredicate &a, const pass::pattern::op::NodePredicate &b)

OPENVINO_API pass::pattern::op::ValuePredicate operator|| (const pass::pattern::op::ValuePredicate &a, const pass::pattern::op::NodePredicate &b)

OPENVINO_API pass::pattern::op::ValuePredicate operator|| (const pass::pattern::op::NodePredicate &a, const pass::pattern::op::ValuePredicate &b)

OPENVINO_API pass::pattern::op::ValuePredicate operator&& (const std::function< bool(Output< Node >)> &a, const std::function< bool(Output< Node >)> &b)

OPENVINO_API pass::pattern::op::ValuePredicate operator&& (const pass::pattern::op::ValuePredicate &a, const pass::pattern::op::ValuePredicate &b)

OPENVINO_API pass::pattern::op::ValuePredicate operator&& (const pass::pattern::op::NodePredicate &a, const pass::pattern::op::NodePredicate &b)

OPENVINO_API pass::pattern::op::ValuePredicate operator&& (const pass::pattern::op::ValuePredicate &a, const pass::pattern::op::NodePredicate &b)

OPENVINO_API pass::pattern::op::ValuePredicate operator&& (const pass::pattern::op::NodePredicate &a, const pass::pattern::op::ValuePredicate &b)

template<class ElementIter> constexpr bool is_nf4_iterator()#

bool with_cpu_x86_avx2()#

Checks whether CPU supports AVX2 capability.

Returns:: True is AVX2 instructions are available, false otherwise

std::size_t coordinate_index(const Coordinate &c, const Shape &s)#

size_t coordinate_offset(const std::vector<size_t> &coordinate, const std::vector<size_t> &strides)#

Calculate offset from begin of buffer based on coordinate and strides.

If coordinates and strides have different sizes then result is undefined behaviour.

Parameters:

coordinate – Vector with multi-dimension coordinates.
strides – Vector with multi-dimension strides

Returns:

Offset of element from start of buffer.

template<class T> constexpr bool is_floating_point()#

Check if T is OpenVINO floating point precision.

Returns:: True if OpenVino floating point precision.

OV_ITT_DOMAIN(OV_PP_CAT(TYPE_LIST_, ov_eval))#

template<class TContainer> constexpr auto make_tensor_accessor(const TContainer &c) -> TensorAccessor<TContainer>#

Makes TensorAccessor for specific tensor container.

See also

ov::Core::import_model

Parameters:: model_stream – Output stream to store the model to.

void set_property(const AnyMap &properties)

Sets properties for the current compiled model.

Parameters:: properties – Map of pairs: (property name, property value).

template<typename ...Properties> inline util::EnableIfAllStringAny<void, Properties...> set_property(Properties&&... properties)

Sets properties for the current compiled model.

Template Parameters:: Properties – Should be the pack of std::pair<std::string, ov::Any> types.
Parameters:: properties – Optional pack of pairs: (property name, property value).

Any get_property(const std::string &name) const

Gets properties for current compiled model.

The method is responsible for extracting information that affects compiled model inference. The list of supported configuration values can be extracted via CompiledModel::get_property with the ov::supported_properties key, but some of these keys cannot be changed dynamically, for example, ov::device::id cannot be changed if a compiled model has already been compiled for a particular device.

Parameters:: name – Property key, can be found in openvino/runtime/properties.hpp.
Returns:: Property value.

template<typename T, PropertyMutability mutability> inline T get_property(const ov::Property<T, mutability> &property) const

Gets properties related to device behaviour.

The method extracts information that can be set via the set_property method.

Template Parameters:: T – Type of a returned value.
Parameters:: property – Property object.
Returns:: Value of property.

void release_memory()

Release intermediate memory.

This method forces the Compiled model to release memory allocated for intermediate structures, e.g. caches, tensors, temporal buffers etc., when possible

RemoteContext get_context() const

Returns pointer to device-specific shared context on a remote accelerator device that was used to create this CompiledModel.

Returns:: A context.

bool operator!() const noexcept

Checks if the current CompiledModel object is not initialized.

Returns:: true if the current CompiledModel object is not initialized; false, otherwise.

explicit operator bool() const noexcept

Checks if the current CompiledModel object is initialized.

Returns:: true if the current CompiledModel object is initialized; false, otherwise.

class Coordinate : public std::vector<size_t>#: #include <coordinate.hpp>

Coordinates for a tensor element.

class CoordinateDiff : public std::vector<std::ptrdiff_t>#: #include <coordinate_diff.hpp>

A difference (signed) of tensor element coordinates.

class CoordinateIterator#

#include <coordinate_transform.hpp>

A useful class that allows to iterate over the tensor coordinates. For example, for tensor with dimensions {2, 3} this iterator produces the following coordinates: {0,0}, {0,1}, {0,2}, {1,0}, {1,1}, {2,2}.

Public Functions

inline CoordinateIterator(const Shape &target_shape)#

Coordinates iterator constructor.

Parameters:: target_shape – The target shape for coordinates iteration

void operator++()#: The postfix operation increment the iterator by one.

CoordinateIterator operator++(int)#: The prefix operation increment the iterator by one.

void operator+=(size_t n)#

Increments iterator n times.

Parameters:: n – number of elements it should be advanced

const Coordinate &operator*() const noexcept#: Iterator dereferencing operator returns reference to current pointed coordinate.

bool operator!=(const CoordinateIterator &it) const noexcept#

Checks for iterator inequality.

Parameters:: it – second iterator to compare

bool operator==(const CoordinateIterator &it) const noexcept#

Checks for iterator equality.

Parameters:: it – second iterator to compare

size_t advance(size_t axis) noexcept#

Increments iterator using specified axis of the shape n times.

Parameters:: axis – index used for iteration

Public Static Functions

static const CoordinateIterator &end()#: Useful function to build the last iterator. Returns a singleton that points to the last iterator.

class CoordinateTransformBasic#

#include <coordinate_transform.hpp>

Class which allows to calculate item index with given coordinates in tensor and helps to iterate over all coordinates. Tensor items should be placed in memory in row-major order.

Public Functions

CoordinateIterator begin() const noexcept#: Returns an iterator to the first coordinate of the tensor.

const CoordinateIterator &end() const noexcept#: Returns an iterator to the coordinate following the last element of the tensor.

class Core

#include <core.hpp>

This class represents an OpenVINO runtime Core entity.

User applications can create several Core class instances, but in this case the underlying plugins are created multiple times and not shared between several Core instances. The recommended way is to have a single Core instance per application.

Unnamed Group

std::shared_ptr<ov::Model> read_model(const std::string &model_path, const std::string &bin_path = {}, const ov::AnyMap &properties = {}) const

Reads models from IR / ONNX / PDPD / TF / TFLite file formats.

Parameters:

model_path – Path to a model.
bin_path – Path to a data file. For IR format (*.bin):
- if bin_path is empty, will try to read a bin file with the same name as xml and
- if the bin file with the same name is not found, will load IR without weights. For the following file formats the bin_path parameter is not used:
- ONNX format (*.onnx)
- PDPD (*.pdmodel)
- TF (*.pb, *.meta, SavedModel directory)
- TFLite (*.tflite)
properties – Optional map of pairs: (property name, property value) relevant only for this read operation.

Returns:

A model.

Unnamed Group

template<typename ...Properties> inline util::EnableIfAllStringAny<CompiledModel, Properties...> read_model(const std::string &model_path, const std::string &bin_path, Properties&&... properties) const

Reads models from IR / ONNX / PDPD / TF / TFLite file formats.

Parameters:

model_path – Path to a model.
bin_path – Path to a data file. For IR format (*.bin):
- if bin_path is empty, will try to read a bin file with the same name as xml and
- if the bin file with the same name is not found, will load IR without weights. For the following file formats the bin_path parameter is not used:
- ONNX format (*.onnx)
- PDPD (*.pdmodel)
- TF (*.pb, *.meta, SavedModel directory)
- TFLite (*.tflite)
properties – Optional pack of pairs: (property name, property value) relevant only for this read operation.

Returns:

A model.

Unnamed Group

CompiledModel compile_model(const std::string &model_path, const AnyMap &properties = {})

Reads and loads a compiled model from the IR/ONNX/PDPD file to the default OpenVINO device selected by the AUTO plugin.

This can be more efficient than using the Core::read_model + Core::compile_model(model_in_memory_object) flow, especially for cases when caching is enabled and a cached model is available.

Parameters:

model_path – Path to a model.
properties – Optional map of pairs: (property name, property value) relevant only for this load operation.

Returns:

A compiled model.

Unnamed Group

template<typename ...Properties> inline util::EnableIfAllStringAny<CompiledModel, Properties...> compile_model(const std::string &model_path, Properties&&... properties)

Reads and loads a compiled model from IR / ONNX / PDPD file to the default OpenVINO device selected by AUTO plugin.

This can be more efficient than using read_model + compile_model(Model) flow especially for cases when caching is enabled and cached model is available

Template Parameters:

Properties – Should be the pack of std::pair<std::string, ov::Any> types

Parameters:

model_path – path to model with string or wstring
properties – Optional pack of pairs: (property name, property value) relevant only for this load operation

Returns:

A compiled model

Unnamed Group

CompiledModel compile_model(const std::string &model_path, const std::string &device_name, const AnyMap &properties = {})

Reads a model and creates a compiled model from the IR/ONNX/PDPD file.

This can be more efficient than using the Core::read_model + Core::compile_model(model_in_memory_object) flow, especially for cases when caching is enabled and a cached model is available.

Parameters:

model_path – Path to a model.
device_name – Name of a device to load a model to.
properties – Optional map of pairs: (property name, property value) relevant only for this load operation.

Returns:

A compiled model.

Unnamed Group

template<typename ...Properties> inline util::EnableIfAllStringAny<CompiledModel, Properties...> compile_model(const std::string &model_path, const std::string &device_name, Properties&&... properties)

Reads a model and creates a compiled model from the IR/ONNX/PDPD file.

This can be more efficient than using read_model + compile_model(Model) flow especially for cases when caching is enabled and cached model is available.

Template Parameters:

Properties – Should be a pack of std::pair<std::string, ov::Any> types.

Parameters:

model_path – Path to a model.
device_name – Name of a device to load a model to.
properties – Optional pack of pairs: (property name, property value) relevant only for this load operation.

Returns:

A compiled model.

Unnamed Group

void add_extension(const std::string &library_path)

Registers an extension to a Core object.

Parameters:: library_path – Path to the library with ov::Extension.

Public Functions

explicit Core(const std::string &xml_config_file = {})

Constructs an OpenVINO Core instance with devices and their plugins description.

There are two ways how to configure device plugins:

(default) Use XML configuration file in case of dynamic libraries build;
Use strictly defined configuration in case of static libraries build.

Parameters:

xml_config_file – Path to the .xml file with plugins to load from. If path contains only file name with extension, file will be searched in a folder with OpenVINO runtime shared library. If the XML configuration file is not specified, default OpenVINO Runtime plugins are loaded from:

(dynamic build) default plugins.xml file located in the same folder as OpenVINO runtime shared library;
(static build) statically defined configuration. In this case path to the .xml file is ignored.

std::map<std::string, Version> get_versions(const std::string &device_name) const

Returns device plugins version information. Device name can be complex and identify multiple devices at once like HETERO:CPU,GPU; in this case, std::map contains multiple entries, each per device.

Parameters:: device_name – Device name to identify a plugin.
Returns:: A vector of versions.

std::shared_ptr<ov::Model> read_model(const std::string &model, const Tensor &weights) const

Reads models from IR / ONNX / PDPD / TF / TFLite formats.

Note

Created model object shares the weights with the weights object. Thus, do not create weights on temporary data that can be freed later, since the model constant data will point to an invalid memory.

Parameters:

model – String with a model in IR / ONNX / PDPD / TF / TFLite format.
weights – Shared pointer to a constant tensor with weights. Reading ONNX / PDPD / TF / TFLite models does not support loading weights from the weights tensors.

Returns:

A model.

CompiledModel compile_model(const std::shared_ptr<const ov::Model> &model, const AnyMap &properties = {})

Creates and loads a compiled model from a source model to the default OpenVINO device selected by the AUTO plugin.

Users can create as many compiled models as they need and use them simultaneously (up to the limitation of the hardware resources).

Parameters:

model – Model object acquired from Core::read_model.
properties – Optional map of pairs: (property name, property value) relevant only for this load operation.

Returns:

A compiled model.

template<typename ...Properties> inline util::EnableIfAllStringAny<CompiledModel, Properties...> compile_model(const std::shared_ptr<const ov::Model> &model, Properties&&... properties)

Creates and loads a compiled model from a source model to the default OpenVINO device selected by AUTO plugin.

Users can create as many compiled models as they need and use them simultaneously (up to the limitation of the hardware resources)

Template Parameters:

Properties – Should be the pack of std::pair<std::string, ov::Any> types

Parameters:

model – Model object acquired from Core::read_model
properties – Optional pack of pairs: (property name, property value) relevant only for this load operation

Returns:

A compiled model

CompiledModel compile_model(const std::shared_ptr<const ov::Model> &model, const std::string &device_name, const AnyMap &properties = {})

Creates a compiled model from a source model object.

Users can create as many compiled models as they need and use them simultaneously (up to the limitation of the hardware resources).

Parameters:

model – Model object acquired from Core::read_model.
device_name – Name of a device to load a model to.
properties – Optional map of pairs: (property name, property value) relevant only for this load operation.

Returns:

A compiled model.

template<typename ...Properties> inline util::EnableIfAllStringAny<CompiledModel, Properties...> compile_model(const std::shared_ptr<const ov::Model> &model, const std::string &device_name, Properties&&... properties)

Creates a compiled model from a source model object.

Users can create as many compiled models as they need and use them simultaneously (up to the limitation of the hardware resources)

Template Parameters:

Properties – Should be the pack of std::pair<std::string, ov::Any> types

Parameters:

model – Model object acquired from Core::read_model
device_name – Name of device to load model to
properties – Optional pack of pairs: (property name, property value) relevant only for this load operation

Returns:

A compiled model

CompiledModel compile_model(const std::string &model, const ov::Tensor &weights, const std::string &device_name, const AnyMap &properties = {})

Reads a model and creates a compiled model from the IR/ONNX/PDPD memory.

Note

Parameters:

model – String with a model in IR/ONNX/PDPD format.
weights – Shared pointer to a constant tensor with weights. Reading ONNX/PDPD models does not support loading weights from the weights tensors.
device_name – Name of a device to load a model to.
properties – Optional map of pairs: (property name, property value) relevant only for this load operation.

Returns:

A compiled model.

template<typename ...Properties> inline util::EnableIfAllStringAny<CompiledModel, Properties...> compile_model(const std::string &model, const ov::Tensor &weights, const std::string &device_name, Properties&&... properties)

Reads a model and creates a compiled model from the IR/ONNX/PDPD memory.

Note

Parameters:

model – String with a model in IR/ONNX/PDPD format.
weights – Shared pointer to a constant tensor with weights. Reading ONNX/PDPD models does not support loading weights from the weights tensors.
device_name – Name of a device to load a model to.

Template Parameters:

Properties – Should be a pack of std::pair<std::string, ov::Any> types.

Returns:

A compiled model.

CompiledModel compile_model(const std::shared_ptr<const ov::Model> &model, const RemoteContext &context, const AnyMap &properties = {})

Creates a compiled model from a source model within a specified remote context.

Parameters:

model – Model object acquired from Core::read_model.
context – A reference to a RemoteContext object.
properties – Optional map of pairs: (property name, property value) relevant only for this load operation.

Returns:

A compiled model object.

template<typename ...Properties> inline util::EnableIfAllStringAny<CompiledModel, Properties...> compile_model(const std::shared_ptr<const ov::Model> &model, const RemoteContext &context, Properties&&... properties)

Creates a compiled model from a source model within a specified remote context.

Template Parameters:

Properties – Should be the pack of std::pair<std::string, ov::Any> types

Parameters:

model – Model object acquired from Core::read_model
context – Pointer to RemoteContext object
properties – Optional pack of pairs: (property name, property value) relevant only for this load operation

Returns:

A compiled model object

void add_extension(const std::shared_ptr<ov::Extension> &extension)

Registers an extension to a Core object.

Parameters:: extension – Pointer to the extension.

void add_extension(const std::vector<std::shared_ptr<ov::Extension>> &extensions)

Registers extensions to a Core object.

Parameters:: extensions – Vector of loaded extensions.

template<class T, typename std::enable_if<std::is_base_of<ov::Extension, T>::value, bool>::type = true> inline void add_extension(const T &extension)

Registers an extension to a Core object.

Parameters:: extension – Extension class that is inherited from the ov::Extension class.

template<class T, class ...Targs, typename std::enable_if<std::is_base_of<ov::Extension, T>::value, bool>::type = true> inline void add_extension(const T &extension, Targs... args)

Registers extensions to a Core object.

Parameters:

extension – Extension class that is inherited from the ov::Extension class.
args – A list of extensions.

template<class T, typename std::enable_if<std::is_base_of<ov::op::Op, T>::value, bool>::type = true> inline void add_extension(): Registers a custom operation inherited from ov::op::Op.

template<class T, class ...Targs, typename std::enable_if<std::is_base_of<ov::op::Op, T>::value && sizeof...(Targs), bool>::type = true> inline void add_extension(): Registers custom operations inherited from ov::op::Op.

CompiledModel import_model(std::istream &model_stream, const std::string &device_name, const AnyMap &properties = {})

Imports a compiled model from the previously exported one.

Parameters:

model_stream – std::istream input stream containing a model previously exported using the ov::CompiledModel::export_model method.
device_name – Name of a device to import a compiled model for. Note, if device_name device was not used to compile the original mode, an exception is thrown.
properties – Optional map of pairs: (property name, property value) relevant only for this load operation.

Returns:

A compiled model.

template<typename ...Properties> inline util::EnableIfAllStringAny<CompiledModel, Properties...> import_model(std::istream &model_stream, const std::string &device_name, Properties&&... properties)

Imports a compiled model from the previously exported one.

Template Parameters:

Properties – Should be the pack of std::pair<std::string, ov::Any> types.

Parameters:

model_stream – Model stream.
device_name – Name of a device to import a compiled model for. Note, if device_name device was not used to compile the original mode, an exception is thrown.
properties – Optional pack of pairs: (property name, property value) relevant only for this load operation.

Returns:

A compiled model.

CompiledModel import_model(std::istream &model_stream, const RemoteContext &context, const AnyMap &properties = {})

Imports a compiled model from the previously exported one with the specified remote context.

Parameters:

model_stream – std::istream input stream containing a model previously exported from ov::CompiledModel::export_model
context – A reference to a RemoteContext object. Note, if the device from context was not used to compile the original mode, an exception is thrown.
properties – Optional map of pairs: (property name, property value) relevant only for this load operation.

Returns:

A compiled model.

template<typename ...Properties> inline util::EnableIfAllStringAny<CompiledModel, Properties...> import_model(std::istream &model_stream, const RemoteContext &context, Properties&&... properties)

Imports a compiled model from the previously exported one with the specified remote context.

Template Parameters:

Properties – Should be the pack of std::pair<std::string, ov::Any> types.

Parameters:

model_stream – Model stream.
context – Pointer to a RemoteContext object.
properties – Optional pack of pairs: (property name, property value) relevant only for this load operation.

Returns:

A compiled model.

SupportedOpsMap query_model(const std::shared_ptr<const ov::Model> &model, const std::string &device_name, const AnyMap &properties = {}) const

Query device if it supports the specified model with specified properties.

Parameters:

device_name – Name of a device to query.
model – Model object to query.
properties – Optional map of pairs: (property name, property value).

Returns:

An object containing a map of pairs an operation name -> a device name supporting this operation.

template<typename ...Properties> inline util::EnableIfAllStringAny<SupportedOpsMap, Properties...> query_model(const std::shared_ptr<const ov::Model> &model, const std::string &device_name, Properties&&... properties) const

Queries a device if it supports the specified model with specified properties.

Template Parameters:

Properties – Should be the pack of std::pair<std::string, ov::Any> types.

Parameters:

device_name – Name of a device to query.
model – Model object to query.
properties – Optional pack of pairs: (property name, property value) relevant only for this query operation.

Returns:

An object containing a map of pairs an operation name -> a device name supporting this operation.

void set_property(const AnyMap &properties)

Sets properties for all the registered devices, acceptable keys can be found in openvino/runtime/properties.hpp.

Parameters:: properties – Map of pairs: (property name, property value).

template<typename ...Properties> inline util::EnableIfAllStringAny<void, Properties...> set_property(Properties&&... properties)

Sets properties for all the registered devices, acceptable keys can be found in openvino/runtime/properties.hpp.

Template Parameters:: Properties – Should be a pack of std::pair<std::string, ov::Any> types.
Parameters:: properties – Optional pack of pairs: property name, property value.

void set_property(const std::string &device_name, const AnyMap &properties)

Sets properties for a device, acceptable keys can be found in openvino/runtime/properties.hpp.

Parameters:

device_name – Name of a device.
properties – Map of pairs: (property name, property value).

template<typename ...Properties> inline util::EnableIfAllStringAny<void, Properties...> set_property(const std::string &device_name, Properties&&... properties)

Sets properties for a device, acceptable keys can be found in openvino/runtime/properties.hpp.

Template Parameters:

Properties – Should be the pack of std::pair<std::string, ov::Any> types.

Parameters:

device_name – Name of a device.
properties – Optional pack of pairs: (property name, property value).

Any get_property(const std::string &device_name, const std::string &name) const

Gets properties related to device behaviour.

The method extracts information that can be set via the set_property method.

Parameters:

device_name – Name of a device to get a property value.
name – Property name.

Returns:

Value of a property corresponding to the property name.

Any get_property(const std::string &device_name, const std::string &name, const AnyMap &arguments) const

Gets properties related to device behaviour.

The method extracts information that can be set via the set_property method.

Parameters:

device_name – Name of a device to get a property value.
name – Property name.
arguments – Additional arguments to get a property.

Returns:

Value of a property corresponding to the property name.

inline Any get_property(const std::string &name) const

Gets properties related to core behaviour.

The method extracts information that can be set via the set_property method.

Parameters:: name – Property name.
Returns:: Value of a property corresponding to the property name.

template<typename T, PropertyMutability M> inline T get_property(const std::string &device_name, const ov::Property<T, M> &property) const

Gets properties related to device behaviour.

The method is needed to request common device or system properties. It can be device name, temperature, and other devices-specific values.

Template Parameters:

T – Type of a returned value.
M – Property mutability.

Parameters:

device_name – Name of a device to get a property value.
property – Property object.

Returns:

Property value.

template<typename T, PropertyMutability M> inline T get_property(const std::string &device_name, const ov::Property<T, M> &property, const AnyMap &arguments) const

Gets properties related to device behaviour.

The method is needed to request common device or system properties. It can be device name, temperature, other devices-specific values.

Template Parameters:

T – Type of a returned value.
M – Property mutability.

Parameters:

device_name – Name of a device to get a property value.
property – Property object.
arguments – Additional arguments to get a property.

Returns:

Property value.

template<typename T, PropertyMutability M, typename ...Args> inline util::EnableIfAllStringAny<T, Args...> get_property(const std::string &device_name, const ov::Property<T, M> &property, Args&&... args) const

Gets properties related to device behaviour.

The method is needed to request common device or system properties. It can be device name, temperature, other devices-specific values.

Template Parameters:

T – Type of a returned value.
M – Property mutability.
Args – Set of additional arguments ended with property object variable.

Parameters:

device_name – Name of a device to get a property value.
property – Property object.
args – Optional pack of pairs: (argument name, argument value) ended with property object.

Returns:

Property value.

std::vector<std::string> get_available_devices() const

Returns devices available for inference. Core objects go over all registered plugins and ask about available devices.

Returns:: A vector of devices. The devices are returned as { CPU, GPU.0, GPU.1, NPU }. If there is more than one device of a specific type, they are enumerated with the .# suffix. Such enumerated device can later be used as a device name in all Core methods like Core::compile_model, Core::query_model, Core::set_property and so on.

void register_plugin(const std::string &plugin, const std::string &device_name, const ov::AnyMap &config = {})

Note

For security purposes it suggested to specify absolute path to register plugin.

Parameters:

plugin – Path (absolute or relative) or name of a plugin. Depending on platform, plugin is wrapped with shared library suffix and prefix to identify library full name. For example, on Linux platform, plugin name specified as plugin_name will be wrapped as libplugin_name.so. Plugin search algorithm:
- If plugin points to an exact library path (absolute or relative), it will be used.
- If plugin specifies file name (libplugin_name.so) or plugin name (plugin_name), it will be searched by file name (libplugin_name.so) in CWD or in paths pointed by PATH/LD_LIBRARY_PATH/DYLD_LIBRARY_PATH environment variables depending on the platform.
device_name – Device name to register a plugin for.
config – Plugin configuration options

void unload_plugin(const std::string &device_name)

Unloads the previously loaded plugin identified by device_name from OpenVINO Runtime. The method is needed to remove loaded plugin instance and free its resources. If plugin for a specified device has not been created before, the method throws an exception.

Note

This method does not remove plugin from the plugins known to OpenVINO Core object.

Parameters:: device_name – Device name identifying plugin to remove from OpenVINO Runtime.

void register_plugins(const std::string &xml_config_file)

Registers a device plugin to the OpenVINO Runtime Core instance using an XML configuration file with plugins description.

The XML file has the following structure:

<ie>
    <plugins>
        <plugin name="" location="">
            <extensions>
                <extension location=""/>
            </extensions>
            <properties>
                <property key="" value=""/>
            </properties>
        </plugin>
    </plugins>
</ie>

name identifies name of a device enabled by a plugin.
location specifies absolute path to dynamic library with a plugin. The path can also be relative to XML file directory. It allows having common config for different systems with different configurations.
properties are set to a plugin via the ov::Core::set_property method.
extensions are set to a plugin via the ov::Core::add_extension method.

Note

For security purposes it suggested to specify absolute path to register plugin.

Parameters:: xml_config_file – A path to .xml file with plugins to register.

RemoteContext create_context(const std::string &device_name, const AnyMap &remote_properties)

Creates a new remote shared context object on the specified accelerator device using specified plugin-specific low-level device API parameters (device handle, pointer, context, etc.).

Parameters:

device_name – Name of a device to create a new shared context on.
remote_properties – Map of device-specific shared context remote properties.

Returns:

Reference to a created remote context.

template<typename ...Properties> inline util::EnableIfAllStringAny<RemoteContext, Properties...> create_context(const std::string &device_name, Properties&&... remote_properties)

Creates a new shared context object on specified accelerator device using specified plugin-specific low level device API properties (device handle, pointer, etc.)

Template Parameters:

Properties – Should be the pack of std::pair<std::string, ov::Any> types

Parameters:

device_name – Name of a device to create new shared context on.
remote_properties – Pack of device-specific shared context remote properties.

Returns:

A shared pointer to a created remote context.

RemoteContext get_default_context(const std::string &device_name)

Gets a pointer to default (plugin-supplied) shared context object for the specified accelerator device.

Parameters:: device_name – Name of a device to get a default shared context from.
Returns:: Reference to a default remote context.

class Decompression : public ov::RuntimeAttribute #: #include <decompression.hpp>

Decompression class represents runtime info attribute that marks operation as used as decompression for Compressed Only format.

class DequantizationNode : public ov::RuntimeAttribute #: #include <dequantization_node.hpp>

DequantizationNode class represents runtime info attribute that marks operation that are part of dequantization subgraph.

class DeviceIDParser#: #include <device_id_parser.hpp>

Class parses device name and id.

class Dimension#

#include <dimension.hpp>

Class representing a dimension, which may be dynamic (undetermined until runtime), in a shape or shape-like object.

Static dimensions may be implicitly converted from value_type. A dynamic dimension is constructed with Dimension() or Dimension::dynamic().

Public Functions

Dimension(value_type dimension)#

Construct a static dimension.

Parameters:: dimension – Value of the dimension.

Dimension(value_type min_dimension, value_type max_dimension)#

Construct a dynamic dimension with bounded range.

Parameters:

min_dimension – The lower inclusive limit for the dimension
max_dimension – The upper inclusive limit for the dimension

Dimension(const std::string &str)#

Construct a dimension from string.

Parameters:: str – String to parse to dimension.

Dimension() = default#: Construct a dynamic dimension with range [0, …].

inline bool is_static() const#

Check whether this dimension is static.

Returns:: true if the dimension is static, else false.

inline bool is_dynamic() const#

Check whether this dimension is dynamic.

Returns:: false if the dimension is static, else true.

value_type get_length() const#

Convert this dimension to value_type. This dimension must be static and non-negative.

Throws:: std::invalid_argument – If this dimension is dynamic or negative.

inline const Interval &get_interval() const#: Return the interval of valid lengths.

bool same_scheme(const Dimension &dim) const#

Check whether this dimension represents the same scheme as the argument (both dynamic, or equal).

Parameters:: dim – The other dimension to compare this dimension to.
Returns:: true if this dimension and dim are both dynamic, or if they are both static and equal; otherwise, false.

bool compatible(const Dimension &d) const#

Check whether this dimension is capable of being merged with the argument dimension.

Two dimensions are considered compatible if it is possible to merge them. (See Dimension::merge.)

Parameters:: d – The dimension to compare this dimension with.
Returns:: true if this dimension is compatible with d, else false.

bool relaxes(const Dimension &d) const#

Check whether this dimension is a relaxation of the argument.

A dimension d1 relaxes (or is a relaxation of) d2 if d1 and d2 are static and equal, or d1 is dynamic.

d1.relaxes(d2) is equivalent to d2.refines(d1).

Parameters:: d – The dimension to compare this dimension with.
Returns:: true if this dimension relaxes d, else false.

bool refines(const Dimension &d) const#

Check whether this dimension is a refinement of the argument.

A dimension d2 refines (or is a refinement of) d1 if d1 and d2 are static and equal, or d2 is dynamic.

d1.refines(d2) is equivalent to d2.relaxes(d1).

Parameters:: d – The dimension to compare this dimension with.
Returns:: true if this dimension relaxes d, else false.

Dimension operator+(const Dimension &dim) const#

Addition operator for Dimension.

Parameters:: dim – Right operand for addition.
Returns:: Smallest interval dimension enclosing inputs

Dimension operator-(const Dimension &dim) const#

Subtraction operator for Dimension.

Parameters:: dim – Right operand for subtraction.
Returns:: Smallest interval dimension enclosing inputs

Dimension operator/(const value_type divisor) const#

Division operator for Dimension divided by a value_type parameter.

Parameters:: divisor – Right operand for division.
Returns:: Smallest interval dimension enclosing inputs

inline Dimension &operator/=(const value_type divisor)#

Divided-into operator for Dimension.

Parameters:: divisor – Right operand for multiplication.
Returns:: A reference to *this, after updating *this to the value *this * dim.

Dimension operator*(const Dimension &dim) const#

Multiplication operator for Dimension.

Parameters:: dim – Right operand for multiplicaiton.
Returns:: Smallest interval containing all “produces” which are 0 if either of this or dim has length 0, else unbounded if either is unbounded, else product of lengths.

inline Dimension &operator+=(const Dimension &dim)#

Add-into operator for Dimension.

Parameters:: dim – Right operand for addition.
Returns:: A reference to *this, after updating *this to the value *this + dim.

inline Dimension &operator*=(const Dimension &dim)#

Multiply-into operator for Dimension.

Parameters:: dim – Right operand for multiplication.
Returns:: A reference to *this, after updating *this to the value *this * dim.

Dimension operator&(const Dimension &dim) const#: Intersection of dimensions.

Dimension &operator&=(const Dimension &dim)#: Intersection of dimensions.

std::string to_string() const#: String representation of Dimension.

bool has_symbol() const#: Indicates if meaningful symbol was set to the Dimension.

std::shared_ptr<ov::Symbol> get_symbol() const#: Returns symbol of the Dimension.

void set_symbol(const std::shared_ptr<ov::Symbol> &s)#: Sets symbol of the Dimension.

Public Static Functions

static bool merge(Dimension &dst, const Dimension &d1, const Dimension &d2)#

Try to merge two Dimension objects together.

If d1 is dynamic, writes d2 to dst and returns true.
If d2 is dynamic, writes d1 to dst and returns true.
If d1 and d2 are static and equal, writes d1 to dst and returns true.
If d1 and d2 are both static and unequal, leaves dst unchanged and returns false.

Parameters:

dst – [out] Reference to write the merged Dimension into.
d1 – First dimension to merge.
d2 – Second dimension to merge.

Returns:

true if merging succeeds, else false.

static bool broadcast_merge(Dimension &dst, const Dimension &d1, const Dimension &d2)#: Try to merge two Dimension objects together with implicit broadcasting of unit-sized dimension to non unit-sized dimension.

static inline Dimension dynamic()#

Create a dynamic dimension.

Returns:: A dynamic dimension.

Friends

inline friend void swap(Dimension &a, Dimension &b)#: Swap of dimensions.

template<typename AT> class DirectValueAccessor : public ov::ValueAccessor<AT>#

#include <attribute_adapter.hpp>

Public Functions

inline virtual const AT &get() override#: Returns the value.

inline virtual void set(const AT &value) override#: Sets the value.

class DisableCleanupAttribute : public ov::RuntimeAttribute #: #include <disable_cleanup_attribute.hpp>

class DisableFP16Compression : public ov::RuntimeAttribute #: #include <disable_fp16_compression.hpp>

DisableFP16Compression class represents runtime info attribute that marks operation as prohibited to convert to lower precision (e.g. to FP16) and they should be inferred precisely in the original precision.

struct DiscreteTypeInfo#

#include <type.hpp>

Type information for a type system without inheritance; instances have exactly one type not related to any other type.

Supports three functions, ov::is_type<Type>, ov::as_type<Type>, and ov::as_type_ptr<Type> for type-safe dynamic conversions via static_cast/static_ptr_cast without using C++ RTTI. Type must have a static type_info member and a virtual get_type_info() member that returns a reference to its type_info member.

template<element::Type_t> struct element_type_traits#: #include <element_type_traits.hpp>

template<> bf16 >: #include <element_type_traits.hpp>

template<> boolean >: #include <element_type_traits.hpp>

template<> f16 >: #include <element_type_traits.hpp>

template<> f32 >: #include <element_type_traits.hpp>

template<> f4e2m1 >: #include <element_type_traits.hpp>

template<> f64 >: #include <element_type_traits.hpp>

template<> f8e4m3 >: #include <element_type_traits.hpp>

template<> f8e5m2 >: #include <element_type_traits.hpp>

template<> f8e8m0 >: #include <element_type_traits.hpp>

template<> i16 >: #include <element_type_traits.hpp>

template<> i32 >: #include <element_type_traits.hpp>

template<> i4 >: #include <element_type_traits.hpp>

template<> i64 >: #include <element_type_traits.hpp>

template<> i8 >: #include <element_type_traits.hpp>

template<> nf4 >: #include <element_type_traits.hpp>

template<> string >: #include <element_type_traits.hpp>

template<> u1 >: #include <element_type_traits.hpp>

template<> u16 >: #include <element_type_traits.hpp>

template<> u2 >: #include <element_type_traits.hpp>

template<> u3 >: #include <element_type_traits.hpp>

template<> u32 >: #include <element_type_traits.hpp>

template<> u4 >: #include <element_type_traits.hpp>

template<> u6 >: #include <element_type_traits.hpp>

template<> u64 >: #include <element_type_traits.hpp>

template<> u8 >: #include <element_type_traits.hpp>

struct EncryptionCallbacks#: #include <properties.hpp>

template<typename AT> class EnumAttributeAdapterBase : public ov::ValueAccessor<std::string>#

#include <attribute_adapter.hpp>

Access an enum via a string.

Template Parameters:: AT – The attribute type enum class

Public Functions

inline virtual const std::string &get() override#: Returns the value.

inline virtual void set(const std::string &value) override#: Sets the value.

template<typename T> class EnumMask#

#include <enum_mask.hpp>

Public Types

using value_type = typename std::underlying_type<T>::type#

Make sure the template type is an enum.

Extract the underlying type of the enum.

Public Functions

constexpr EnumMask() = default#: Some bit operations are not safe for signed values, we require enum type to use unsigned underlying type.

inline bool is_any_set(const EnumMask &p) const#: Check if any of the input parameter enum bit mask match.

inline bool is_set(const EnumMask &p) const#: Check if all of the input parameter enum bit mask match.

inline bool is_any_clear(const EnumMask &p) const#: Check if any of the input parameter enum bit mask does not match.

inline bool is_clear(const EnumMask &p) const#: Check if all of the input parameter enum bit mask do not match.

template<typename EnumType> class EnumNames#

#include <enum_names.hpp>

Uses a pairings defined by EnumTypes::get() to convert between strings and enum values.

Public Static Functions

static inline EnumType as_enum(const std::string &name)#: Converts strings to enum values.

static inline const std::string &as_string(EnumType e)#: Converts enum values to strings.

class Exception : public std::runtime_error#

#include <except.hpp>

Base error for ov runtime errors.

Subclassed by ov::AssertFailure, ov::Busy, ov::Cancelled, ov::op::util::error::UnknownActivationFunction

class Extension#

#include <extension.hpp>

The class provides the base interface for OpenVINO extensions.

Subclassed by ov::BaseOpExtension, ov::frontend::ConversionExtensionBase, ov::frontend::DecoderTransformationExtension, ov::frontend::ProgressReporterExtension, ov::frontend::TelemetryExtension

class float16#: #include <float16.hpp>

class float4_e2m1#: #include <float4_e2m1.hpp>

Class to represent the f4e2m1 type.

class float8_e4m3#: #include <float8_e4m3.hpp>

Class to represent the f8e4m3 type.

class float8_e5m2#: #include <float8_e5m2.hpp>

Class to represent the f8e5m2 type.

class float8_e8m0#: #include <float8_e8m0.hpp>

Class to represent the f8e8m0 type.

class FusedNames : public ov::RuntimeAttribute #

#include <fused_names_attribute.hpp>

FusedName class represents runtime info attribute that stores all operation names that was fully or partially fused into node.

Public Functions

FusedNames() = default#: A default constructor

inline explicit FusedNames(const std::string &name)#

Constructs a new object consisting of a single name *.

Parameters:: name – [in] The name

void fuseWith(const FusedNames &names)#

Unites current set of already fused names with another FusedNames object.

Parameters:: names – [in] Another object to fuse with

std::string getNames() const#: return string with operation names separated by coma in alphabetical order

std::vector<std::string> getVectorNames() const#

return vector of fused names sorted in alphabetical order

Returns:: vector if strings

class IAsyncInferRequest : public ov::IInferRequest #

#include <iasync_infer_request.hpp>

Base class with default implementation of asynchronous multi staged inference request. To customize pipeline stages derived class should change the content of IAsyncInferRequest::m_pipeline member container. It consists of pairs of tasks and executors which will run the task. The class is recommended to be used by plugins as a base class for asynchronous inference request implementation.

Example: Here is an example of asynchronous inference request implementation for some accelerator device. It uses 5 different executors to run different stages of a synchronous inference request.

Note

To synchronize derived context with stages derived class should call IAsyncInferRequest::stop_and_wait() function in destructor.

Public Functions

virtual void start_async()#: Start inference of specified input(s) in asynchronous mode.

Note

The method returns immediately. Inference starts also immediately.

virtual void wait()#: Waits for the result to become available.

virtual bool wait_for(const std::chrono::milliseconds &timeout)#

Waits for the result to become available. Blocks until specified timeout has elapsed or the result becomes available, whichever comes first.

Parameters:: timeout – - maximum duration in milliseconds to block for
Returns:: A true if results are ready.

virtual void cancel()#: Cancel current inference request execution.

virtual void set_callback(std::function<void(std::exception_ptr)> callback)#

Set callback function which will be called on success or failure of asynchronous request.

Parameters:: callback – - function to be called with the following description:

virtual void infer() override#: Infers specified input(s) in synchronous mode.

Note

blocks all method of InferRequest while request is ongoing (running or waiting in queue)

virtual std::vector<ov::ProfilingInfo> get_profiling_info() const override#

Queries performance measures per layer to identify the most time consuming operation.

Note

Not all plugins provide meaningful data.

Returns:: Vector of profiling information for operations in a model.

virtual ov::SoPtr<ov::ITensor> get_tensor(const ov::Output<const ov::Node> &port) const override#

Gets an input/output tensor for inference.

Note

If the tensor with the specified port is not found, an exception is thrown.

Parameters:: port – Port of the tensor to get.
Returns:: Tensor for the port port.

virtual void set_tensor(const ov::Output<const ov::Node> &port, const ov::SoPtr<ov::ITensor> &tensor) override#

Sets an input/output tensor to infer.

Parameters:

port – Port of the input or output tensor.
tensor – Reference to a tensor. The element_type and shape of a tensor must match the model’s input/output element_type and size.

virtual std::vector<ov::SoPtr<ov::ITensor>> get_tensors(const ov::Output<const ov::Node> &port) const override#

Gets a batch of tensors for input data to infer by input port. Model input must have batch dimension, and the number of tensors must match the batch size. The current version supports setting tensors to model inputs only. If port is associated with output (or any other non-input node), an exception is thrown.

Parameters:

port – Port of the input tensor.
tensors – Input tensors for batched infer request. The type of each tensor must match the model input element type and shape (except batch dimension). Total size of tensors must match the input size.

Returns:

vector of tensors

virtual void set_tensors(const ov::Output<const ov::Node> &port, const std::vector<ov::SoPtr<ov::ITensor>> &tensors) override#

Sets a batch of tensors for input data to infer by input port. Model input must have batch dimension, and the number of tensors must match the batch size. The current version supports setting tensors to model inputs only. If port is associated with output (or any other non-input node), an exception is thrown.

Parameters:

port – Port of the input tensor.
tensors – Input tensors for batched infer request. The type of each tensor must match the model input element type and shape (except batch dimension). Total size of tensors must match the input size.

virtual std::vector<ov::SoPtr<ov::IVariableState>> query_state() const override#

Gets state control interface for the given infer request.

State control essential for recurrent models.

Returns:: Vector of Variable State objects.

virtual const std::shared_ptr<const ov::ICompiledModel> &get_compiled_model() const override#

Gets pointer to compiled model (usually synchronous request holds the compiled model)

Returns:: Pointer to the compiled model

virtual const std::vector<ov::Output<const ov::Node>> &get_inputs() const override#

Gets inputs for infer request.

Returns:: vector of input ports

virtual const std::vector<ov::Output<const ov::Node>> &get_outputs() const override#

Gets outputs for infer request.

Returns:: vector of output ports

class ICompiledModel : public std::enable_shared_from_this<ICompiledModel>#

#include <icompiled_model.hpp>

OpenVINO ICompiledModel interface.

Public Functions

ICompiledModel(const std::shared_ptr<const ov::Model> &model, const std::shared_ptr<const ov::IPlugin> &plugin, const std::shared_ptr<ov::threading::ITaskExecutor> &task_executor = std::make_shared<ov::threading::CPUStreamsExecutor>(ov::threading::IStreamsExecutor::Config{"Default"}), const std::shared_ptr<ov::threading::ITaskExecutor> &callback_executor = std::make_shared<ov::threading::CPUStreamsExecutor>(ov::threading::IStreamsExecutor::Config{"Callback"}))#

Constructor for ICompiledModel interface.

Parameters:

model – OpenVINO model representation
plugin – Pointer to plugin
task_executor – Task executor (CPUStreamsExecutor by default)
callback_executor – Callback executor (CPUStreamsExecutor by default)

ICompiledModel(const std::shared_ptr<const ov::Model> &model, const std::shared_ptr<const ov::IPlugin> &plugin, const ov::SoPtr<ov::IRemoteContext> &context, const std::shared_ptr<ov::threading::ITaskExecutor> &task_executor = std::make_shared<ov::threading::CPUStreamsExecutor>(ov::threading::IStreamsExecutor::Config{"Default"}), const std::shared_ptr<ov::threading::ITaskExecutor> &callback_executor = std::make_shared<ov::threading::CPUStreamsExecutor>(ov::threading::IStreamsExecutor::Config{"Callback"}))#

Constructor for ICompiledModel interface with remote context.

Parameters:

model – OpenVINO model representation
plugin – Pointer to plugin
context – Remote context
task_executor – Task executor (CPUStreamsExecutor by default)
callback_executor – Callback executor (CPUStreamsExecutor by default)

virtual const std::vector<ov::Output<const ov::Node>> &outputs() const#

Gets all outputs from compiled model.

Returns:: model outputs

virtual const std::vector<ov::Output<const ov::Node>> &inputs() const#

Gets all inputs from compiled model.

Returns:: model inputs

virtual std::shared_ptr<ov::IAsyncInferRequest> create_infer_request() const#

Create infer request.

Returns:: Asynchronous infer request interface

virtual void export_model(std::ostream &model) const = 0#

Export compiled model to stream.

Parameters:: model – output stream

virtual std::shared_ptr<const ov::Model> get_runtime_model() const = 0#

Returns runtime model.

Returns:: OpenVINO Model which represents runtime graph

virtual void set_property(const ov::AnyMap &properties) = 0#

Allows to set property.

Parameters:: properties – new plugin properties

virtual ov::Any get_property(const std::string &name) const = 0#

Returns property.

Parameters:: name – Property name
Returns:: Property value

ov::SoPtr<ov::IRemoteContext> get_context() const#

Creates device specific remote context.

Returns:: OpenVINO RemoteContext

virtual void release_memory()#: Release intermediate memory.

interface ICore#

#include <icore.hpp>

Minimal ICore interface to allow plugin to get information from Core OpenVINO class.

Public Functions

virtual std::shared_ptr<ov::Model> read_model(const std::string &model, const ov::Tensor &weights, bool frontend_mode = false) const = 0#

Reads IR xml and bin (with the same name) files.

Parameters:

model – string with IR
weights – shared pointer to constant blob with weights
frontend_mode – read network without post-processing or other transformations

Returns:

shared pointer to ov::Model

virtual std::shared_ptr<ov::Model> read_model(const std::shared_ptr<AlignedBuffer> &model, const std::shared_ptr<AlignedBuffer> &weights) const = 0#

Reads IR xml and bin from buffer.

Parameters:

model – shared pointer to aligned buffer with IR
weights – shared pointer to aligned buffer with weights

Returns:

shared pointer to ov::Model

virtual std::shared_ptr<ov::Model> read_model(const std::string &model_path, const std::string &bin_path, const AnyMap &properties) const = 0#

Reads IR xml and bin files.

Parameters:

model_path – path to IR file
bin_path – path to bin file, if path is empty, will try to read bin file with the same name as xml and if bin file with the same name was not found, will load IR without weights.
properties – Optional map of pairs: (property name, property value) relevant only for this read operation.

Returns:

shared pointer to ov::Model

virtual ov::SoPtr<ov::ICompiledModel> compile_model(const std::shared_ptr<const ov::Model> &model, const std::string &device_name, const ov::AnyMap &config = {}) const = 0#

Creates a compiled mdel from a model object.

Users can create as many models as they need and use them simultaneously (up to the limitation of the hardware resources)

Parameters:

model – OpenVINO Model
device_name – Name of device to load model to
config – Optional map of pairs: (config parameter name, config parameter value) relevant only for this load operation

Returns:

A pointer to compiled model

virtual ov::SoPtr<ov::ICompiledModel> compile_model(const std::shared_ptr<const ov::Model> &model, const ov::SoPtr<ov::IRemoteContext> &context, const ov::AnyMap &config = {}) const = 0#

Creates a compiled model from a model object.

Users can create as many models as they need and use them simultaneously (up to the limitation of the hardware resources)

Parameters:

model – OpenVINO Model
context – “Remote” (non-CPU) accelerator device-specific execution context to use
config – Optional map of pairs: (config parameter name, config parameter value) relevant only for this load operation

Returns:

A pointer to compiled model

virtual ov::SoPtr<ov::ICompiledModel> compile_model(const std::string &model_path, const std::string &device_name, const ov::AnyMap &config) const = 0#

Creates a compiled model from a model file.

Users can create as many models as they need and use them simultaneously (up to the limitation of the hardware resources)

Parameters:

model_path – Path to model
device_name – Name of device to load model to
config – Optional map of pairs: (config parameter name, config parameter value) relevant only for this load operation

Returns:

A pointer to compiled model

virtual ov::SoPtr<ov::ICompiledModel> compile_model(const std::string &model_str, const ov::Tensor &weights, const std::string &device_name, const ov::AnyMap &config) const = 0#

Creates a compiled model from a model memory.

Users can create as many models as they need and use them simultaneously (up to the limitation of the hardware resources)

Parameters:

model_str – String data of model
weights – Model’s weights
device_name – Name of device to load model to
config – Optional map of pairs: (config parameter name, config parameter value) relevant only for this load operation

Returns:

A pointer to compiled model

virtual ov::SoPtr<ov::ICompiledModel> import_model(std::istream &model, const std::string &device_name, const ov::AnyMap &config = {}) const = 0#

Creates a compiled model from a previously exported model.

Parameters:

model – model stream
device_name – Name of device load executable model on
config – Optional map of pairs: (config parameter name, config parameter value) relevant only for this load operation*

Returns:

A pointer to compiled model

virtual ov::SoPtr<ov::ICompiledModel> import_model(std::istream &modelStream, const ov::SoPtr<ov::IRemoteContext> &context, const ov::AnyMap &config = {}) const = 0#

Creates a compiled model from a previously exported model.

Parameters:

model – model stream
context – Remote context
config – Optional map of pairs: (config parameter name, config parameter value) relevant only for this load operation*

Returns:

A pointer to compiled model

virtual ov::SupportedOpsMap query_model(const std::shared_ptr<const ov::Model> &model, const std::string &device_name, const ov::AnyMap &config) const = 0#

Query device if it supports specified network with specified configuration.

Parameters:

model – OpenVINO Model
device_name – A name of a device to query
config – Optional map of pairs: (config parameter name, config parameter value)

Returns:

An object containing a map of pairs a layer name -> a device name supporting this layer.

virtual std::vector<std::string> get_available_devices() const = 0#

Returns devices available for neural networks inference.

Returns:: A vector of devices. The devices are returned as { CPU, GPU.0, GPU.1, MYRIAD } If there more than one device of specific type, they are enumerated with .# suffix.

virtual ov::SoPtr<ov::IRemoteContext> create_context(const std::string &device_name, const AnyMap &args) const = 0#

Create a new shared context object on specified accelerator device using specified plugin-specific low level device API parameters (device handle, pointer, etc.)

Parameters:

device_name – Name of a device to create new shared context on.
params – Map of device-specific shared context parameters.

Returns:

A shared pointer to a created remote context.

virtual ov::SoPtr<ov::IRemoteContext> get_default_context(const std::string &device_name) const = 0#

Get a pointer to default shared context object for the specified device.

Parameters:: device_name – - A name of a device to get create shared context from.
Returns:: A shared pointer to a default remote context.

virtual Any get_property(const std::string &device_name, const std::string &name, const AnyMap &arguments) const = 0#

Gets properties related to device behaviour.

Parameters:

device_name – Name of a device to get a property value.
name – Property name.
arguments – Additional arguments to get a property.

Returns:

Value of a property corresponding to the property name.

template<typename T, PropertyMutability M> inline T get_property(const std::string &device_name, const Property<T, M> &property) const#

Gets properties related to device behaviour.

Template Parameters:

T – Type of a returned value.
M – Property mutability.

Parameters:

deviceName – Name of a device to get a property value.
property – Property object.

Returns:

Property value.

template<typename T, PropertyMutability M> inline T get_property(const std::string &device_name, const Property<T, M> &property, const AnyMap &arguments) const#

Gets properties related to device behaviour.

Template Parameters:

T – Type of a returned value.
M – Property mutability.

Parameters:

deviceName – Name of a device to get a property value.
property – Property object.
arguments – Additional arguments to get a property.

Returns:

Property value.

virtual AnyMap get_supported_property(const std::string &full_device_name, const AnyMap &properties, const bool keep_core_property = true) const = 0#

Get only properties that are supported by specified device.

Parameters:

full_device_name – Name of a device (can be either virtual or hardware)
properties – Properties that can contains configs that are not supported by device
keep_core_property – Whether to return core-level properties

Returns:

map of properties that are supported by device

virtual ~ICore()#: Default virtual destructor.

class IInferRequest#

#include <iinfer_request.hpp>

An internal API of inference request to be implemented by plugin.

Subclassed by ov::IAsyncInferRequest, ov::ISyncInferRequest

Public Functions

virtual void infer() = 0#: Infers specified input(s) in synchronous mode.

Note

blocks all method of InferRequest while request is ongoing (running or waiting in queue)

virtual std::vector<ov::ProfilingInfo> get_profiling_info() const = 0#

Queries performance measures per layer to identify the most time consuming operation.

Note

Not all plugins provide meaningful data.

Returns:: Vector of profiling information for operations in a model.

virtual ov::SoPtr<ov::ITensor> get_tensor(const ov::Output<const ov::Node> &port) const = 0#

Gets an input/output tensor for inference.

Note

If the tensor with the specified port is not found, an exception is thrown.

Parameters:: port – Port of the tensor to get.
Returns:: Tensor for the port port.

virtual void set_tensor(const ov::Output<const ov::Node> &port, const ov::SoPtr<ov::ITensor> &tensor) = 0#

Sets an input/output tensor to infer.

Parameters:

port – Port of the input or output tensor.
tensor – Reference to a tensor. The element_type and shape of a tensor must match the model’s input/output element_type and size.

virtual std::vector<ov::SoPtr<ov::ITensor>> get_tensors(const ov::Output<const ov::Node> &port) const = 0#

Parameters:

port – Port of the input tensor.
tensors – Input tensors for batched infer request. The type of each tensor must match the model input element type and shape (except batch dimension). Total size of tensors must match the input size.

Returns:

vector of tensors

virtual void set_tensors(const ov::Output<const ov::Node> &port, const std::vector<ov::SoPtr<ov::ITensor>> &tensors) = 0#

Parameters:

port – Port of the input tensor.
tensors – Input tensors for batched infer request. The type of each tensor must match the model input element type and shape (except batch dimension). Total size of tensors must match the input size.

virtual std::vector<ov::SoPtr<ov::IVariableState>> query_state() const = 0#

Gets state control interface for the given infer request.

State control essential for recurrent models.

Returns:: Vector of Variable State objects.

virtual const std::shared_ptr<const ov::ICompiledModel> &get_compiled_model() const = 0#

Gets pointer to compiled model (usually synchronous request holds the compiled model)

Returns:: Pointer to the compiled model

virtual const std::vector<ov::Output<const ov::Node>> &get_inputs() const = 0#

Gets inputs for infer request.

Returns:: vector of input ports

virtual const std::vector<ov::Output<const ov::Node>> &get_outputs() const = 0#

Gets outputs for infer request.

Returns:: vector of output ports

template<typename AT, typename VAT> class IndirectScalarValueAccessor : public ov::ValueAccessor<VAT>#

#include <attribute_adapter.hpp>

Public Functions

inline virtual const VAT &get() override#: Returns the value.

inline virtual void set(const VAT &value) override#: Sets the value.

template<typename AT, typename VAT> class IndirectVectorValueAccessor : public ov::ValueAccessor<VAT>#

#include <attribute_adapter.hpp>

Public Functions

inline virtual const VAT &get() override#: Returns the value.

inline virtual void set(const VAT &value) override#: Sets the value.

class InferRequest

#include <infer_request.hpp>

This is a class of infer request that can be run in asynchronous or synchronous manners.

Public Functions

InferRequest() = default: Default constructor.

InferRequest(const InferRequest &other) = default

Default copy constructor.

Parameters:: other – Another InferRequest object.

InferRequest &operator=(const InferRequest &other) = default

Default copy assignment operator.

Parameters:: other – Another InferRequest object.
Returns:: Reference to the current object.

InferRequest(InferRequest &&other) = default

Default move constructor.

Parameters:: other – Another InferRequest object.

InferRequest &operator=(InferRequest &&other) = default

Default move assignment operator.

Parameters:: other – Another InferRequest object.
Returns:: Reference to the current object.

~InferRequest(): Destructor that preserves unloading order of implementation object and reference to the library.

Note

To preserve destruction order inside the default generated assignment operator, _impl is stored before _so. Use the destructor to remove implementation object before referencing to the library explicitly.

void set_tensor(const std::string &tensor_name, const Tensor &tensor)

Sets an input/output tensor to infer on.

Parameters:

tensor_name – Name of the input or output tensor.
tensor – Reference to the tensor. The element_type and shape of the tensor must match the model’s input/output element_type and size.

void set_tensor(const ov::Output<const ov::Node> &port, const Tensor &tensor)

Sets an input/output tensor to infer.

Parameters:

port – Port of the input or output tensor. Use the following methods to get the ports:
- ov::Model::input()
- ov::Model::inputs()
- ov::Model::outputs()
- ov::Model::outputs()
- ov::CompiledModel::input()
- ov::CompiledModel::inputs()
- ov::CompiledModel::outputs()
- ov::CompiledModel::outputs()
tensor – Reference to a tensor. The element_type and shape of a tensor must match the model’s input/output element_type and size.

void set_tensor(const ov::Output<ov::Node> &port, const Tensor &tensor)

Sets an input/output tensor to infer.

Parameters:

port – Port of the input or output tensor. Use the following methods to get the ports:
- ov::Model::input()
- ov::Model::inputs()
- ov::Model::outputs()
- ov::Model::outputs()
- ov::CompiledModel::input()
- ov::CompiledModel::inputs()
- ov::CompiledModel::outputs()
- ov::CompiledModel::outputs()
tensor – Reference to a tensor. The element_type and shape of a tensor must match the model’s input/output element_type and size.

void set_tensors(const std::string &tensor_name, const std::vector<Tensor> &tensors)

Sets a batch of tensors for input data to infer by tensor name. Model input must have batch dimension, and the number of tensors must match the batch size. The current version supports setting tensors to model inputs only. If tensor_name is associated with output (or any other non-input node), an exception is thrown.

Parameters:

tensor_name – Name of the input tensor.
tensors – Input tensors for batched infer request. The type of each tensor must match the model input element type and shape (except batch dimension). Total size of tensors must match the input size.

void set_tensors(const ov::Output<const ov::Node> &port, const std::vector<Tensor> &tensors)

Parameters:

port – Port of the input tensor.
tensors – Input tensors for batched infer request. The type of each tensor must match the model input element type and shape (except batch dimension). Total size of tensors must match the input size.

void set_input_tensor(size_t idx, const Tensor &tensor)

Sets an input tensor to infer.

Parameters:

idx – Index of the input tensor. If idx is greater than the number of model inputs, an exception is thrown.
tensor – Reference to the tensor. The element_type and shape of the tensor must match the model’s input/output element_type and size.

void set_input_tensor(const Tensor &tensor)

Sets an input tensor to infer models with single input.

Note

If model has several inputs, an exception is thrown.

Parameters:: tensor – Reference to the input tensor.

void set_input_tensors(const std::vector<Tensor> &tensors)

Sets a batch of tensors for single input data. Model input must have batch dimension, and the number of tensors must match the batch size.

Parameters:: tensors – Input tensors for batched infer request. The type of each tensor must match the model input element type and shape (except batch dimension). Total size of tensors must match the input size.

void set_input_tensors(size_t idx, const std::vector<Tensor> &tensors)

Sets a batch of tensors for input data to infer by input name. Model input must have batch dimension, and number of tensors must match the batch size.

Parameters:

idx – Name of the input tensor.
tensors – Input tensors for batched infer request. The type of each tensor must match the model input element type and shape (except batch dimension). Total size of tensors must match the input size.

void set_output_tensor(size_t idx, const Tensor &tensor)

Sets an output tensor to infer.

Note

Index of the input preserved accross ov::Model, ov::CompiledModel, and ov::InferRequest.

Parameters:

idx – Index of the output tensor.
tensor – Reference to the output tensor. The type of the tensor must match the model output element type and shape.

void set_output_tensor(const Tensor &tensor)

Sets an output tensor to infer models with single output.

Note

If model has several outputs, an exception is thrown.

Parameters:: tensor – Reference to the output tensor.

Tensor get_tensor(const std::string &tensor_name)

Gets an input/output tensor for inference by tensor name.

Parameters:: tensor_name – Name of a tensor to get.
Returns:: The tensor with name tensor_name. If the tensor is not found, an exception is thrown.

Tensor get_tensor(const ov::Output<const ov::Node> &port)

Gets an input/output tensor for inference.

Note

If the tensor with the specified port is not found, an exception is thrown.

Parameters:: port – Port of the tensor to get.
Returns:: Tensor for the port port.

Tensor get_tensor(const ov::Output<ov::Node> &port)

Gets an input/output tensor for inference.

Note

If the tensor with the specified port is not found, an exception is thrown.

Parameters:: port – Port of the tensor to get.
Returns:: Tensor for the port port.

Tensor get_input_tensor(size_t idx)

Gets an input tensor for inference.

Parameters:: idx – Index of the tensor to get.
Returns:: Tensor with the input index idx. If the tensor with the specified idx is not found, an exception is thrown.

Tensor get_input_tensor()

Gets an input tensor for inference.

Returns:: The input tensor for the model. If model has several inputs, an exception is thrown.

Tensor get_output_tensor(size_t idx)

Gets an output tensor for inference.

Parameters:: idx – Index of the tensor to get.
Returns:: Tensor with the output index idx. If the tensor with the specified idx is not found, an exception is thrown.

Tensor get_output_tensor()

Gets an output tensor for inference.

Returns:: Output tensor for the model. If model has several outputs, an exception is thrown.

void infer(): Infers specified input(s) in synchronous mode.

Note

It blocks all methods of InferRequest while request is ongoing (running or waiting in a queue). Calling any method leads to throwing the ov::Busy exception.

void cancel(): Cancels inference request.

std::vector<ProfilingInfo> get_profiling_info() const

Queries performance measures per layer to identify the most time consuming operation.

Note

Not all plugins provide meaningful data.

Returns:: Vector of profiling information for operations in a model.

void start_async(): Starts inference of specified input(s) in asynchronous mode.

Note

It returns immediately. Inference starts also immediately. Calling any method while the request in a running state leads to throwing the ov::Busy exception.

void wait(): Waits for the result to become available. Blocks until the result becomes available.

bool wait_for(const std::chrono::milliseconds timeout)

Waits for the result to become available. Blocks until the specified timeout has elapsed or the result becomes available, whichever comes first.

Parameters:: timeout – Maximum duration, in milliseconds, to block for.
Returns:: True if inference request is ready and false, otherwise.

void set_callback(std::function<void(std::exception_ptr)> callback)

Sets a callback std::function that is called on success or failure of an asynchronous request.

Warning

Do not capture strong references to OpenVINO runtime objects into callback. Following objects should not be captured like:

ov::InferRequest
ov::ExecutableNetwork
ov::Core As specified objects implement shared reference concept do not capture this objects by value. It can lead to memory leaks or undefined behaviour! Try to use weak references or pointers.

Parameters:: callback – callback object which will be called on when inference finish.

std::vector<VariableState> query_state()

Gets state control interface for the given infer request.

State control essential for recurrent models.

Returns:: Vector of Variable State objects.

void reset_state(): Resets all internal variable states for relevant infer request to a value specified as default for the corresponding ReadValue node.

CompiledModel get_compiled_model()

Returns a compiled model that creates this inference request.

Returns:: Compiled model object.

bool operator!() const noexcept

Checks if the current InferRequest object is not initialized.

Returns:: True if the current InferRequest object is not initialized; false, otherwise.

explicit operator bool() const noexcept

Checks if the current InferRequest object is initialized.

Returns:: True if the current InferRequest object is initialized; false, otherwise.

bool operator!=(const InferRequest &other) const noexcept

Compares whether this request wraps the same impl underneath.

Parameters:: other – Another inference request.
Returns:: True if the current InferRequest object does not wrap the same impl as the operator’s arg.

bool operator==(const InferRequest &other) const noexcept

Compares whether this request wraps the same impl underneath.

Parameters:: other – Another inference request.
Returns:: True if the current InferRequest object wraps the same impl as the operator’s arg.

template<typename NodeType> class Input#: #include <node_input.hpp>

template<> class Input<const Node>#

#include <node_input.hpp>

A handle for one of a node’s inputs.

Public Functions

Input(const Node *node, size_t index)#

Constructs a Input.

Parameters:

node – Pointer to the node for the input handle.
index – The index of the input.

const Node *get_node() const#

Returns:: A pointer to the node referenced by this input handle.

size_t get_index() const#

Returns:: The index of the input referred to by this input handle.

OV_NO_DANGLING const element::Type & get_element_type () const

Returns:: The element type of the input referred to by this input handle.

OV_NO_DANGLING const Shape & get_shape () const

Returns:: The shape of the input referred to by this input handle.

OV_NO_DANGLING const PartialShape & get_partial_shape () const

Returns:: The partial shape of the input referred to by this input handle.

Output<Node> get_source_output() const#

Returns:: A handle to the output that is connected to this input.

OV_NO_DANGLING descriptor::Tensor & get_tensor () const

Returns:: A reference to the tensor descriptor for this input.

std::shared_ptr<descriptor::Tensor> get_tensor_ptr() const#

Returns:: A shared pointer to the tensor descriptor for this input.

bool get_is_relevant_to_shapes() const#

Returns:: true if this input is relevant to its node’s output shapes; else false.

bool get_is_relevant_to_values() const#

Returns:: true if this input is relevant to its node’s output values; else false.

OV_NO_DANGLING const RTMap & get_rt_info () const

Returns:: The constant reference to runtime info map

template<> class Input<Node>#

#include <node_input.hpp>

A handle for one of a node’s inputs.

Public Functions

Input(Node *node, size_t index)#

Constructs a Input.

Parameters:

node – Pointer to the node for the input handle.
index – The index of the input.

Node *get_node() const#

Returns:: A pointer to the node referenced by this input handle.

size_t get_index() const#

Returns:: The index of the input referred to by this input handle.

OV_NO_DANGLING const element::Type & get_element_type () const

Returns:: The element type of the input referred to by this input handle.

OV_NO_DANGLING const Shape & get_shape () const

Returns:: The shape of the input referred to by this input handle.

OV_NO_DANGLING const PartialShape & get_partial_shape () const

Returns:: The partial shape of the input referred to by this input handle.

Output<Node> get_source_output() const#

Returns:: A handle to the output that is connected to this input.

OV_NO_DANGLING descriptor::Tensor & get_tensor () const

Returns:: A reference to the tensor descriptor for this input.

std::shared_ptr<descriptor::Tensor> get_tensor_ptr() const#

Returns:: A shared pointer to the tensor descriptor for this input.

bool get_is_relevant_to_shapes() const#

Returns:: true if this input is relevant to its node’s output shapes; else false.

bool get_is_relevant_to_values() const#

Returns:: true if this input is relevant to its node’s output values; else false.

void replace_source_output(const Output<Node> &new_source_output) const#

Replaces the source output of this input.

Parameters:: new_source_output – A handle for the output that will replace this input’s source.

RTMap &get_rt_info()#

Returns:: The reference to runtime info map

OV_NO_DANGLING const RTMap & get_rt_info () const

Returns:: The constant reference to runtime info map

class Interval#

#include <interval.hpp>

Interval arithmetic.

An interval is the set of integers from m_min_val through m_max_val. The value s_max acts like infinity. The addition, subtraction, or multiplication of intervals is the smallest interval containing the sums, differences, or products of elements of the two intervals. An empty interval is canonicalized to [s_max, s_max].

Public Functions

Interval() = default#: Interval of everything.

Interval(const Interval &interval) = default#: Copy constructor.

Interval(value_type min_val, value_type max_val)#: Closed interval {x|min_val <= x <= max_val}.

Interval(value_type val)#: Single-valued interval; just contains val.

inline size_type size() const#: The number of elements in the interval. Zero if max < min.

inline bool empty() const#: Returns true if the interval has no elements.

inline value_type get_min_val() const#: the inclusive lower bound of the interval

inline void set_min_val(value_type val)#: Set the inclusive lower bound of the interval.

inline value_type get_max_val() const#: the inclusive upper bound of the interval

inline void set_max_val(value_type val)#: Set the inclusive upper bound of the interval.

inline bool has_upper_bound() const#: True if the upper bound is finite.

bool operator==(const Interval &interval) const#: True if min and max bounds match.

Interval operator+(const Interval &interval) const#: The interval whose elements are a sum of an element from each interval.

Interval &operator+=(const Interval &interval)#: Extend this interval to sums of elements in this interval and interval.

Interval operator-(const Interval &interval) const#: The interval whose elements are a difference of an element from each interval.

Interval &operator-=(const Interval &interval)#: Extend this interval to differences of elements in this interval and interval.

Interval operator*(const Interval &interval) const#: The smallest interval whose elements are a product of an element from each interval.

Interval &operator*=(const Interval &interval)#: Extend this interval to products of elements in this interval and interval.

Interval operator&(const Interval &interval) const#: The interval that is the intersection of this interval and interval.

Interval &operator&=(const Interval &interval)#: Change this interval to only include elements also in interval.

inline bool contains(value_type value) const#: True if this interval includes value.

bool contains(const Interval &interval) const#: True if this interval includes all the values in interval.

Public Static Attributes

static constexpr value_type s_max = {std::numeric_limits<value_type>::max()}#: The value used for no upper bound.

class IntervalsAlignmentAttribute : public SharedAttribute<IntervalsAlignmentSharedValue>

#include <intervals_alignment_attribute.hpp>

IntervalsAlignmentAttribute defines subgraph with the same quantization intervals alignment. FakeQuantize operations are included. The attribute is used by quantization operations.

For more details about the attribute, refer to IntervalsAlignmentAttribute page in the OpenVINO Developer Guide.

class IntervalsAlignmentSharedValue

#include <intervals_alignment_attribute.hpp>

IntervalsAlignmentSharedValue is used by IntervalsAlignmentAttribute as attribute shared value.

class Interval: #include <intervals_alignment_attribute.hpp>

class IPlugin : public std::enable_shared_from_this<IPlugin>#

#include <iplugin.hpp>

OpenVINO Plugin Interface 2.0.

Public Functions

void set_version(const Version &version)#

Sets a plugin version.

Parameters:: version – A version to set

const Version &get_version() const#

Returns a plugin version.

Returns:: A constant ov::Version object

virtual std::shared_ptr<ov::ICompiledModel> compile_model(const std::shared_ptr<const ov::Model> &model, const ov::AnyMap &properties) const = 0#

Compiles model from ov::Model object.

Parameters:

model – A model object acquired from ov::Core::read_model or source construction
properties – A ov::AnyMap of properties relevant only for this load operation

Returns:

Created Compiled Model object

virtual std::shared_ptr<ov::ICompiledModel> compile_model(const std::string &model_path, const ov::AnyMap &properties) const#

Compiles model from ov::Model object.

Parameters:

model_path – A path to model (path can be converted from unicode representation)
properties – A ov::AnyMap of properties relevant only for this load operation

Returns:

Created Compiled Model object

virtual std::shared_ptr<ov::ICompiledModel> compile_model(const std::shared_ptr<const ov::Model> &model, const ov::AnyMap &properties, const ov::SoPtr<ov::IRemoteContext> &context) const = 0#

Compiles model from ov::Model object, on specified remote context.

Parameters:

model – A model object acquired from ov::Core::read_model or source construction
properties – A ov::AnyMap of properties relevant only for this load operation
context – A pointer to plugin context derived from RemoteContext class used to execute the model

Returns:

Created Compiled Model object

virtual void set_property(const ov::AnyMap &properties) = 0#

Sets properties for plugin, acceptable keys can be found in openvino/runtime/properties.hpp.

Parameters:: properties – ov::AnyMap of properties

virtual ov::Any get_property(const std::string &name, const ov::AnyMap &arguments) const = 0#

Gets properties related to plugin behaviour.

Parameters:

name – Property name.
arguments – Additional arguments to get a property.

Returns:

Value of a property corresponding to the property name.

virtual ov::SoPtr<ov::IRemoteContext> create_context(const ov::AnyMap &remote_properties) const = 0#

Creates a remote context instance based on a map of properties.

Parameters:: remote_properties – Map of device-specific shared context remote properties.
Returns:: A remote context object

virtual ov::SoPtr<ov::IRemoteContext> get_default_context(const ov::AnyMap &remote_properties) const = 0#

Provides a default remote context instance if supported by a plugin.

Parameters:: remote_properties – Map of device-specific shared context remote properties.
Returns:: The default context.

virtual std::shared_ptr<ov::ICompiledModel> import_model(std::istream &model, const ov::AnyMap &properties) const = 0#

Creates an compiled model from an previously exported model using plugin implementation and removes OpenVINO Runtime magic and plugin name.

Parameters:

model – Reference to model output stream
properties – A ov::AnyMap of properties

Returns:

An Compiled model

virtual std::shared_ptr<ov::ICompiledModel> import_model(std::istream &model, const ov::SoPtr<ov::IRemoteContext> &context, const ov::AnyMap &properties) const = 0#

Creates an compiled model from an previously exported model using plugin implementation and removes OpenVINO Runtime magic and plugin name.

Parameters:

model – Reference to model output stream
context – A pointer to plugin context derived from RemoteContext class used to execute the network
properties – A ov::AnyMap of properties

Returns:

An Compiled model

virtual ov::SupportedOpsMap query_model(const std::shared_ptr<const ov::Model> &model, const ov::AnyMap &properties) const = 0#

Queries a plugin about supported layers in model.

Parameters:

model – Model object to query.
properties – Optional map of pairs: (property name, property value).

Returns:

An object containing a map of pairs an operation name -> a device name supporting this operation.

void set_core(const std::weak_ptr<ov::ICore> &core)#

Sets pointer to ICore interface.

Parameters:: core – Pointer to Core interface

std::shared_ptr<ov::ICore> get_core() const#

Gets reference to ICore interface.

Returns:: Reference to ICore interface

const std::shared_ptr<ov::threading::ExecutorManager> &get_executor_manager() const#

Gets reference to tasks execution manager.

Returns:: Reference to ExecutorManager interface

class IRemoteContext : public std::enable_shared_from_this<IRemoteContext>#

#include <iremote_context.hpp>

Public Functions

virtual const ov::AnyMap &get_property() const = 0#

Returns a map of device-specific parameters required for low-level operations with underlying object. Parameters include device/context handles, access flags, etc. Contents of the map returned depend on remote execution context that is currently set on the device (working scenario). Abstract method.

Returns:: A map of name/Any elements.

virtual ov::SoPtr<ov::IRemoteTensor> create_tensor(const ov::element::Type &type, const ov::Shape &shape, const ov::AnyMap &params = {}) = 0#

Allocates memory tensor in device memory or wraps user-supplied memory handle using the specified tensor description and low-level device-specific parameters. Returns a pointer to the object that implements the RemoteTensor interface.

Parameters:

type – Defines the element type of the tensor.
shape – Defines the shape of the tensor.
params – Map of the low-level tensor object parameters.

Returns:

Pointer to a plugin object that implements the RemoteTensor interface.

virtual ov::SoPtr<ov::ITensor> create_host_tensor(const ov::element::Type type, const ov::Shape &shape)#

This method is used to create a host tensor object friendly for the device in current context. For example, GPU context may allocate USM host memory (if corresponding extension is available), which could be more efficient than regular host memory.

Parameters:

type – Tensor element type.
shape – Tensor shape.

Returns:

A tensor instance with device friendly memory.

class IRemoteTensor : public ITensor#

#include <iremote_tensor.hpp>

Public Functions

virtual const AnyMap &get_properties() const = 0#

Returns additional information associated with tensor.

Returns:: Map of property names to properties

class ISyncInferRequest : public ov::IInferRequest #

#include <isync_infer_request.hpp>

Interface for syncronous infer request.

Public Functions

ISyncInferRequest(const std::shared_ptr<const ov::ICompiledModel> &compiled_model)#

Constructs syncronous inference request.

Parameters:: compiled_model – pointer to compiled model

virtual ov::SoPtr<ov::ITensor> get_tensor(const ov::Output<const ov::Node> &port) const override#

Gets an input/output tensor for inference.

Note

If the tensor with the specified port is not found, an exception is thrown.

Parameters:: port – Port of the tensor to get.
Returns:: Tensor for the port port.

virtual void set_tensor(const ov::Output<const ov::Node> &port, const ov::SoPtr<ov::ITensor> &tensor) override#

Sets an input/output tensor to infer.

Parameters:

port – Port of the input or output tensor.
tensor – Reference to a tensor. The element_type and shape of a tensor must match the model’s input/output element_type and size.

virtual std::vector<ov::SoPtr<ov::ITensor>> get_tensors(const ov::Output<const ov::Node> &port) const override#

Parameters:

port – Port of the input tensor.
tensors – Input tensors for batched infer request. The type of each tensor must match the model input element type and shape (except batch dimension). Total size of tensors must match the input size.

Returns:

vector of tensors

virtual void set_tensors(const ov::Output<const ov::Node> &port, const std::vector<ov::SoPtr<ov::ITensor>> &tensors) override#

Parameters:

port – Port of the input tensor.
tensors – Input tensors for batched infer request. The type of each tensor must match the model input element type and shape (except batch dimension). Total size of tensors must match the input size.

virtual const std::vector<ov::Output<const ov::Node>> &get_inputs() const override#

Gets inputs for infer request.

Returns:: vector of input ports

virtual const std::vector<ov::Output<const ov::Node>> &get_outputs() const override#

Gets outputs for infer request.

Returns:: vector of output ports

virtual const std::shared_ptr<const ov::ICompiledModel> &get_compiled_model() const override#

Gets pointer to compiled model (usually synchronous request holds the compiled model)

Returns:: Pointer to the compiled model

class ITensorAccessor#

#include <tensor_data_accessor.hpp>

Interface for data accessor.

Subclassed by ov::TensorAccessor< TContainer >

Public Functions

virtual Tensor operator()(size_t port) const = 0#

Get tensor at port.

Parameters:: port – Number of data port (operator input) to get tensor.
Returns:: Tensor to data at port.

interface IVariableState : public std::enable_shared_from_this<IVariableState>#

#include <ivariable_state.hpp>

Minimal interface for variable state implementation.

Public Functions

virtual const std::string &get_name() const#

Gets a variable state name.

Returns:: A string representing variable state name

virtual void reset()#: Reset internal variable state for relevant infer request, to a value specified as default for according ReadValue node.

virtual void set_state(const ov::SoPtr<ov::ITensor> &state)#

Sets the new state for the next inference.

Parameters:: newState – A new state

virtual ov::SoPtr<ov::ITensor> get_state() const#

Returns the value of the variable state.

Returns:: The value of the variable state

class KeepConstPrecision : public ov::RuntimeAttribute #: #include <keep_const_precision.hpp>

KeepConstPrecision class represents runtime info attribute that marks a Constant as prohibitted to fuse precision in ConvertPrecision.

class Layout#

#include <layout.hpp>

ov::Layout represents the text information of tensor’s dimensions/axes. E.g. layout NCHW means that 4D tensor {-1, 3, 480, 640} will have:

0: N = -1: batch dimension is dynamic
1: C = 3: number of channels is ‘3’
2: H = 480: image height is 480
3: W = 640: image width is 640

Examples: ov::Layout can be specified for:

Preprocessing purposes. E.g.
- To apply normalization (means/scales) it is usually required to set ‘C’ dimension in a layout.
- To resize the image to specified width/height it is needed to set ‘H’ and ‘W’ dimensions in a layout
- To transpose image - source and target layout can be set (see ov::preprocess::PreProcessSteps::convert_layout)
To set/get model’s batch (see ov::get_batch/ov::set_batch) it is required in general to specify ‘N’ dimension in layout for appropriate inputs

Refer also to ov::layout namespace for various additional helper functions of ov::Layout

Public Functions

Layout()#: Constructs a dynamic Layout with no layout information.

inline Layout(const char *layoutStr)#

Constructs a Layout with static or dynamic layout information based on string representation.

Parameters:

layoutStr – The string used to construct Layout from. The string representation can be in the following form:

can define order and meaning for dimensions “NCHW”
partial layout specialization:
- ”NC?” defines 3 dimensional layout, first two NC, 3rd one is not defined
- ”N…C” defines layout with dynamic rank where 1st dimension is N, last one is C
- ”NC…” defines layout with dynamic rank where first two are NC, others are not defined
only order of dimensions “adbc” (0312)
Advanced syntax can be used for multi-character names like “[N,C,H,W,…,CustomName]”

bool operator==(const Layout &rhs) const#: Comparison operator (equal)

bool operator!=(const Layout &rhs) const#: Comparison operator (not equal)

bool has_name(const std::string &dimensionName) const#

Checks if dimension with specified name is in layout.

Returns:: true if layout has information about dimension index with a given name

std::int64_t get_index_by_name(const std::string &dimensionName) const#

Gets index of dimension with a specified name.

Throws:: ov::AssertFailure – if dimension name is not found in a layout
Returns:: Index of given dimension name

std::string to_string() const#: String representation of Layout.

inline bool empty() const#: Returns ‘true’ if layout has no information, i.e. equals to Layout()

Public Static Functions

static Layout scalar()#: Constructs layout representing scalar.

class LayoutAttribute : public ov::RuntimeAttribute #: #include <layout.hpp>

class MappedMemory#: #include <mmap_object.hpp>

This class represents a mapped memory. Instead of reading files, we can map the memory via mmap for Linux or MapViewOfFile for Windows. The MappedMemory class is a abstraction to handle such memory with os-dependent details.

class Mask : public std::vector<std::set<uint64_t>>, public std::enable_shared_from_this<Mask>#: #include <mask_attribute.hpp>

each element in vector represents dimension and each element in set is an id of dimensions which contains zeros.

struct MemBandwidthPressure#: #include <performance_heuristics.hpp>

class MemorySolver#

#include <memory_solver.hpp>

Helps to solve issue of optimal memory allocation only for particular execution order.

It works with abstract data description where

Node is index in execution order
Edge is Box object with size and start-finish indexes (live time)

Example:

Mem(offset) | |____| Box {4, 5} | |_____________| Box {2, 6} | |____| Box {3, 4} | |____| Box {2, 3} | |____| Box {6, 7} |_____________________________________ 1 2 3 4 5 6 7 8 9 ExecOrder

Boxes which has an ExecOrder-axis intersection should have no Mem-axis intersections. The goal is to define a minimal required memory blob to store all boxes with such constraints and specify all corresponding position on Mem axis(through offset field).

NOTE! Exec order is predefined.

Public Functions

inline int64_t solve()#

Solve memory location with maximal reuse.

Returns:: Size of common memory blob required for storing all

inline int64_t get_offset(int id) const#: Provides calculated offset for specified box id

inline int64_t max_depth()#: Additional info. Max sum of box sizes required for any time stamp.

inline int64_t max_top_depth()#: Additional info. Max num of boxes required for any time stamp.

Public Static Functions

static inline int normalize_boxes(std::vector<Box> &boxes)#

Performes inplace normalization of the input boxes.

Returns:: lifespan of all boxes

struct Box#

#include <memory_solver.hpp>

Representation of edge (size and live time)

Public Members

int start#: Execution order index of first use. The data will be produced here.

int finish#: The execution order index of last use. After that data will be released. -1 is a reserved value for “till to end”. The data will be alive to very end of execution.

int64_t size#: Size of data. In abstract unit of measure (byte, simd, cache line, …)

int64_t id#: Box identifier, unique for each box. Will be used to querying calculated offset.

class Model : public std::enable_shared_from_this<Model>#

#include <model.hpp>

A user-defined model.

Public Functions

explicit Model(const ov::OutputVector &results, const std::string &name = "")#: Constructs a Model. Lists of parameters and variables will be generated automatically based on traversing the graph from the results.

Model(const ov::OutputVector &results, const ov::SinkVector &sinks, const std::string &name = "")#: Constructs a Model. Lists of parameters and variables will be generated automatically based on traversing the graph from the results and the sinks.

size_t get_output_size() const#: Return the number of outputs for this Model.

std::shared_ptr<ov::Node> get_output_op(size_t i) const#: Return the op that generates output i.

std::shared_ptr<ov::Model> clone() const#: Clones the original model.

std::vector<ov::Output<ov::Node>> outputs()#: Model outputs.

std::vector<ov::Output<ov::Node>> inputs()#: Model inputs.

const ov::element::Type &get_output_element_type(size_t i) const#: Return the element type of output i.

const Shape &get_output_shape(size_t i) const#: Return the shape of element i.

const PartialShape &get_output_partial_shape(size_t i) const#: Return the partial shape of element i.

std::shared_ptr<ov::Node> get_result() const#: Check that there is a single result and return it.

const std::string &get_name() const#

Get the unique name of the model.

Returns:: A const reference to the model’s unique name.

void set_friendly_name(const std::string &name)#

Sets a friendly name for a model. This does not overwrite the unique name of the model and is retrieved via get_friendly_name(). Used mainly for debugging.

Parameters:: name – is the friendly name to set

const std::string &get_friendly_name() const#

Gets the friendly name for a model. If no friendly name has been set via set_friendly_name then the model’s unique name is returned.

Returns:: A const reference to the model’s friendly name.

size_t get_graph_size() const#: Returns the sum of the size of all nodes in the graph plus the size of all constant data. This has little value beyond comparing the relative size of graphs and should not be considered the actual memory consumption of a graph.

bool is_dynamic() const#: Returns true if any of the op’s defined in the model contains partial shape.

void replace_parameter(size_t parameter_index, const std::shared_ptr<ov::op::v0::Parameter> &parameter)#

Replace the parameter_indexth parameter of the model with parameter.

All users of the parameter_indexth parameter are redirected to parameter, and the parameter_indexth entry in the model parameter list is replaced with parameter.

Parameters:

parameter_index – The index of the parameter to replace.
parameter – The parameter to substitute for the parameter_indexth parameter.

inline const ov::ParameterVector &get_parameters() const#: Return the model parameters.

inline const ov::ResultVector &get_results() const#: Return a list of model’s outputs.

int64_t get_parameter_index(const std::shared_ptr<ov::op::v0::Parameter> &parameter) const#: Index for parameter, or -1.

int64_t get_result_index(const ov::Output<ov::Node> &value) const#

Return the index of this model’s Result represented by the “value” Output object. This method returns -1 if an the passed output is not related to the Results of a model.

Parameters:: value – Output containing Node

int64_t get_result_index(const ov::Output<const ov::Node> &value) const#

Return the index of this model’s Result represented by the “value” Output object. This method returns -1 if an the passed output is not related to the Results of a model.

Parameters:: value – Output containing Node

bool evaluate(ov::TensorVector &output_tensors, const ov::TensorVector &input_tensors, ov::EvaluationContext &evaluation_context) const#

Evaluate the model on inputs, putting results in outputs.

Parameters:

output_tensors – Tensors for the outputs to compute. One for each result
input_tensors – Tensors for the inputs. One for each inputs.
evaluation_context – Storage of additional settings and attributes that can be used when evaluating the model. This additional information can be shared across nodes.

bool evaluate(ov::TensorVector &output_tensors, const ov::TensorVector &input_tensors) const#

Evaluate the model on inputs, putting results in outputs.

Parameters:

output_tensors – Tensors for the outputs to compute. One for each result
input_tensors – Tensors for the inputs. One for each inputs.

inline const ov::SinkVector &get_sinks() const#: Return a list of model’s sinks.

void add_sinks(const ov::SinkVector &sinks)#

Add new sink nodes to the list. Method doesn’t validate graph, it should be done manually after all changes.

Parameters:: sinks – new sink nodes

void remove_sink(const std::shared_ptr<ov::op::Sink> &sink)#

Delete sink node from the list of sinks. Method doesn’t delete node from graph.

Parameters:: sink – Sink to delete

void add_results(const ov::ResultVector &results)#

Add new Result nodes to the list. Method doesn’t validate graph, it should be done manually after all changes.

Parameters:: results – new Result nodes

void remove_result(const std::shared_ptr<ov::op::v0::Result> &result)#

Delete Result node from the list of results. Method will not delete node from graph.

Parameters:: result – Result node to delete

void add_parameters(const ov::ParameterVector &params)#

Add new Parameter nodes to the list.

Method doesn’t change or validate graph, it should be done manually. For example, if you want to replace ReadValue node by Parameter, you should do the following steps:

replace node ReadValue by Parameter in graph
call add_parameter() to add new input to the list
call graph validation to check correctness of changes

Parameters:: params – new Parameter nodes

void remove_parameter(const std::shared_ptr<ov::op::v0::Parameter> &param)#

Delete Parameter node from the list of parameters. Method will not delete node from graph. You need to replace Parameter with other operation manually. Attention: Indexing of parameters can be changed.

Possible use of method is to replace input by variable. For it the following steps should be done:

Parameter node should be replaced by ReadValue
call remove_parameter(param) to remove input from the list
check if any parameter indexes are saved/used somewhere, update it for all inputs because indexes can be changed
call graph validation to check all changes

Parameters:: param – Parameter node to delete

void add_variables(const ov::op::util::VariableVector &variables)#

Add new variables to the list. Method doesn’t validate graph, it should be done manually after all changes.

Parameters:: variables – new variables to add

void remove_variable(const ov::op::util::Variable::Ptr &variable)#

Delete variable from the list of variables. Method doesn’t delete nodes that used this variable from the graph.

Parameters:: variable – Variable to delete

inline const ov::op::util::VariableVector &get_variables() const#: Return a list of model’s variables.

ov::op::util::Variable::Ptr get_variable_by_id(const std::string &variable_id) const#: Return a variable by specified variable_id.

inline RTMap &get_rt_info()#

Returns a runtime info.

Returns:: reference to ov::AnyMap with runtime info

inline const RTMap &get_rt_info() const#

Returns a constant runtime info.

Returns:: reference to const ov::AnyMap with runtime info

template<class T, class ...Args, typename std::enable_if<!std::is_same<T, ov::Any>::value, bool>::type = true> inline const T &get_rt_info(Args... args) const#

Returns a runtime attribute for the path, throws an ov::Exception if path doesn’t exist.

Template Parameters:

T – the type of returned value
Args – types of variadic arguments

Parameters:

args – path to the runtime attribute

Returns:

constant reference to value from runtime info

template<class T, class ...Args, typename std::enable_if<std::is_same<T, ov::Any>::value, bool>::type = true> inline const T &get_rt_info(Args... args) const#

Returns a runtime attribute for the path, throws an ov::Exception if path doesn’t exist.

Template Parameters:

T – the type of returned value
Args – types of variadic arguments

Parameters:

args – path to the runtime attribute

Returns:

constant reference to value from runtime info

template<class T, typename std::enable_if<!std::is_same<T, ov::Any>::value, bool>::type = true> inline const T &get_rt_info(const std::vector<std::string> &args) const#

Returns a runtime attribute for the path, throws an ov::Exception if path doesn’t exist.

Template Parameters:: T – the type of returned value
Parameters:: args – vector with path to the runtime attribute
Returns:: constant reference to value from runtime info

template<class T, typename std::enable_if<std::is_same<T, ov::Any>::value, bool>::type = true> inline const T &get_rt_info(const std::vector<std::string> &args) const#

Returns a runtime attribute for the path, throws an ov::Exception if path doesn’t exist.

Template Parameters:: T – the type of returned value
Parameters:: args – vector with path to the runtime attribute
Returns:: constant reference to value from runtime info

template<class ...Args> inline bool has_rt_info(Args... args) const#

Checks if given path exists in runtime info.

Template Parameters:: Args – types of variadic arguments
Parameters:: args – path to the runtime attribute
Returns:: true if path exists, otherwise false

bool has_rt_info(const std::vector<std::string> &args) const#

Checks if given path exists in runtime info.

Parameters:: args – vector with path to the runtime attribute
Returns:: true if path exists, otherwise false

template<class T, class ...Args> inline void set_rt_info(const T &argument, Args... args)#

Add value inside the runtime info.

Template Parameters:

T – type of new value
Args – types of variadic arguments

Parameters:

argument – value for the runtime info
args – path to the runtime attribute

template<class T> inline void set_rt_info(const T &argument, const std::vector<std::string> &args)#

Add value inside the runtime info.

Template Parameters:

T – type of new value

Parameters:

argument – value for the runtime info
args – vector with path to the runtime attribute

class ModelCache#: #include <compilation_context.hpp>

class NmsSelectedIndices : public ov::RuntimeAttribute #: #include <nms_selected_indices.hpp>

class Node : public std::enable_shared_from_this<Node>#

#include <node.hpp>

Nodes are the backbone of the graph of Value dataflow. Every node has zero or more nodes as arguments and one value, which is either a tensor or a (possibly empty) tuple of values.

Subclassed by ov::op::Op, ov::pass::pattern::op::Pattern

Public Functions

virtual void validate_and_infer_types()#

Verifies that attributes and inputs are consistent and computes output shapes and element types. Must be implemented by concrete child classes so that it can be run any number of times.

Throws if the node is invalid.

virtual const ov::op::AutoBroadcastSpec &get_autob() const#

Returns:: the autobroadcasr spec

virtual bool has_evaluate() const#: Allows to get information about availability of evaluate method for the current operation.

virtual bool evaluate(ov::TensorVector &output_values, const ov::TensorVector &input_values) const#

Evaluates the op on input_values putting results in output_values.

Parameters:

output_values – Tensors for the outputs to compute. One for each result
input_values – Tensors for the inputs. One for each inputs.

Returns:

true if successful

virtual bool evaluate(ov::TensorVector &output_values, const ov::TensorVector &input_values, const ov::EvaluationContext &evaluationContext) const#

Evaluates the op on input_values putting results in output_values.

Parameters:

output_values – Tensors for the outputs to compute. One for each result
input_values – Tensors for the inputs. One for each inputs.
evaluation_context – Storage of additional settings and attributes that can be used when evaluating the op.

Returns:

true if successful

inline virtual OutputVector decompose_op() const#

Decomposes the FusedOp into a sub-graph consisting of core openvino ops.

Returns:: A vector of nodes comprising the sub-graph. The order of output tensors must match the match output tensors of the FusedOp

virtual const type_info_t &get_type_info() const = 0#: Returns the NodeTypeInfo for the node’s class. During transition to type_info, returns a dummy type_info for Node if the class has not been updated yet.

void set_arguments(const NodeVector &arguments)#: Sets/replaces the arguments with new arguments.

void set_arguments(const OutputVector &arguments)#: Sets/replaces the arguments with new arguments.

void set_argument(size_t position, const Output<Node> &argument)#: Sets/replaces the arguments with new arguments.

void set_output_size(size_t output_size)#: Sets the number of outputs.

virtual std::string description() const#

Get the string name for the type of the node, such as Add or Multiply. The class name, must not contain spaces as it is used for codegen.

Returns:: A const reference to the node’s type name

const std::string &get_name() const#

Get the unique name of the node.

Returns:: A const reference to the node’s unique name.

void set_friendly_name(const std::string &name)#

Sets a friendly name for a node. This does not overwrite the unique name of the node and is retrieved via get_friendly_name(). Used mainly for debugging. The friendly name may be set exactly once.

Parameters:: name – is the friendly name to set

const std::string &get_friendly_name() const#

Gets the friendly name for a node. If no friendly name has been set via set_friendly_name then the node’s unique name is returned.

Returns:: A const reference to the node’s friendly name.

virtual std::ostream &write_description(std::ostream &os, uint32_t depth = 0) const#

Writes a description of a node to a stream.

Parameters:

os – The stream; should be returned
depth – How many levels of inputs to describe

Returns:

The stream os

const std::vector<std::shared_ptr<Node>> &get_control_dependencies() const#: Get control dependencies registered on the node.

const std::vector<Node*> &get_control_dependents() const#: Get nodes dependent on this node.

void add_control_dependency(std::shared_ptr<Node> node)#: This node cannot execute until node executes.

void remove_control_dependency(std::shared_ptr<Node> node)#: Remove the dependency of this node on node.

void clear_control_dependencies()#: Remove all dependencies from this node.

void clear_control_dependents()#: Remove this node as a dependency from all dependent nodes.

void add_node_control_dependencies(const std::shared_ptr<const Node> &source_node)#: This node absorbs the control dependencies of source_node.

void add_node_control_dependents(const std::shared_ptr<const Node> &source_node)#: This node becomes a dependent of every node dependent on source_node.

void transfer_control_dependents(std::shared_ptr<Node> replacement)#: This node’s control dependencies are replaced by replacement.

size_t get_output_size() const#: Returns the number of outputs from the node.

const element::Type &get_output_element_type(size_t i) const#: Returns the element type for output i.

const element::Type &get_element_type() const#: Checks that there is exactly one output and returns its element type.

const Shape &get_output_shape(size_t i) const#: Returns the shape for output i.

const PartialShape &get_output_partial_shape(size_t i) const#: Returns the partial shape for output i.

Output<const Node> get_default_output() const#: Return the output to use when converting to an Output<Node> with no index specified. Throws when not supported.

virtual size_t get_default_output_index() const#: Returns the output of the default output, or throws if there is none.

size_t no_default_index() const#: Throws no default.

const Shape &get_shape() const#: Checks that there is exactly one output and returns its shape.

descriptor::Tensor &get_output_tensor(size_t i) const#: Returns the tensor for output or input i.

size_t get_input_size() const#: Returns the number of inputs for the op.

const element::Type &get_input_element_type(size_t i) const#: Returns the element type of input i.

const Shape &get_input_shape(size_t i) const#: Returns the shape of input i.

const PartialShape &get_input_partial_shape(size_t i) const#: Returns the partial shape of input i.

bool has_same_type(std::shared_ptr<const Node> node) const#: True if this and node have one output with same element type and shape.

NodeVector get_users(bool check_is_used = false) const#: Get all the nodes that uses the current node.

inline bool operator<(const Node &other) const#: Use instance ids for comparison instead of memory addresses to improve determinism.

std::vector<Input<Node>> inputs()#

Returns:: A vector containing a handle for each of this node’s inputs, in order.

std::vector<Input<const Node>> inputs() const#

Returns:: A vector containing a handle for each of this node’s inputs, in order.

std::vector<Output<Node>> input_values() const#

Returns:: A vector containing the values for each input

std::vector<Output<Node>> outputs()#

Returns:: A vector containing a handle for each of this node’s outputs, in order.

std::vector<Output<const Node>> outputs() const#

Returns:: A vector containing a handle for each of this node’s outputs, in order.

Input<Node> input(size_t input_index)#

Throws:: std::out_of_range – if the node does not have at least input_index+1 inputs.
Returns:: A handle to the input_indexth input of this node.

Input<const Node> input(size_t input_index) const#

Throws:: std::out_of_range – if the node does not have at least input_index+1 inputs.
Returns:: A handle to the input_indexth input of this node.

Output<Node> output(size_t output_index)#

Throws:: std::out_of_range – if the node does not have at least output_index+1 outputs.
Returns:: A handle to the output_indexth output of this node.

Output<const Node> output(size_t output_index) const#

Throws:: std::out_of_range – if the node does not have at least output_index+1 outputs.
Returns:: A handle to the output_indexth output of this node.

class NodeValidationFailure : public ov::AssertFailure #

#include <node.hpp>

Public Functions

template<> OPENVINO_API void create (const char *file, int line, const char *check_string, std::pair< const Node *, const std::vector< PartialShape > * > &&ctx, const std::string &explanation)

Specialization to throw the NodeValidationFailure for shape inference using PartialShape

Parameters:

check_loc_info – Exception location details to print.
ctx – NodeValidationFailure context which got pointer to node and input shapes used for shape inference.
explanation – Exception explanation string.

class NonconvertibleDivide : public ov::RuntimeAttribute #: #include <nonconvertible_divide.hpp>

NonconvertibleDivide class represents runtime info attribute that marks a Divide as prohibitted to transform it to power.

class NotImplemented : public ov::AssertFailure #: #include <except.hpp>

Exception class to be thrown on not implemented code.

class NoTransposeSinkingAttr : public ov::RuntimeAttribute #: #include <transpose_sinking_attr.hpp>

NoTransposeSinkingAttr class represents runtime info attribute that marks transpose operation should not be moved be backward sinking propagation.

class OldApiMapElementType : public ov::RuntimeAttribute #

#include <old_api_map_element_type_attribute.hpp>

OldApiMapElementType class represents runtime info attribute that stores legacy type that is required for obtaining IR in old API.

Public Functions

OldApiMapElementType() = default#: A default constructor

inline OldApiMapElementType(const ov::element::Type &value)#

Constructs a new OldApiMapElementType object.

Parameters:: value – [in] The object that stores values of OldApiMapElementType.

class OldApiMapOrder : public ov::RuntimeAttribute #

#include <old_api_map_order_attribute.hpp>

OldApiMapOrder class represents runtime info attribute that stores order of the transpose that is required for obtaining IR in old API.

OldApiMapOrder stores the following information. Parameter: Order of the transpose which should be applied to Parameter with old API layout to obtain Parameter with new API layout.

Result: Order of the transpose which should be applied to Result with new API layout to obtain Result with old API layout.

Public Functions

OldApiMapOrder() = default#: A default constructor

inline OldApiMapOrder(const std::vector<uint64_t> &value)#

Constructs a new OldApiMapOrder object.

Parameters:: value – [in] The object that stores values of OldApiMapOrder.

template<class T> class OpExtension : public ov::BaseOpExtension #

#include <op_extension.hpp>

The default implementation of OpenVINO operation extensions.

Public Functions

inline OpExtension()#: Default constructor.

inline virtual const ov::DiscreteTypeInfo &get_type_info() const override#

Returns the type info of operation.

Returns:: ov::DiscreteTypeInfo

inline virtual ov::OutputVector create(const ov::OutputVector &inputs, ov::AttributeVisitor &visitor) const override#

Method creates an OpenVINO operation.

Parameters:

inputs – vector of input ports
visitor – attribute visitor which allows to read necessaty arguments

Returns:

vector of output ports

inline virtual std::vector<ov::Extension::Ptr> get_attached_extensions() const override#

Returns extensions that should be registered together with this extension class object.

Attached extensions may include frontend extensions that OpenVINO op to framework ops or necessary transformations that should be applied to the network which consist of target op.

Returns:

class OpSet#

#include <opset.hpp>

Run-time opset information.

Public Functions

template<typename OP_TYPE> inline void insert(const std::string &name)#: Insert OP_TYPE into the opset with a special name and the default factory.

template<typename OP_TYPE> inline void insert()#: Insert OP_TYPE into the opset with the default name and factory.

ov::Node *create(const std::string &name) const#: Create the op named name using it’s factory.

ov::Node *create_insensitive(const std::string &name) const#: Create the op named name using it’s factory.

bool contains_type(const NodeTypeInfo &type_info) const#: Return true if OP_TYPE is in the opset.

template<typename OP_TYPE> inline bool contains_type() const#: Return true if OP_TYPE is in the opset.

bool contains_type(const std::string &name) const#: Return true if name is in the opset.

bool contains_type_insensitive(const std::string &name) const#: Return true if name is in the opset.

bool contains_op_type(const Node *node) const#: Return true if node’s type is in the opset.

template<class T> class optional#

#include <ov_optional.hpp>

Store optional object of type T (basic version of std::optional).

Note

If cpp17 used this class should be replaced by std::optional.

Template Parameters:: T – Type of stored object.

class OriginalPrecisionAttribute : public ov::RuntimeAttribute #: #include <original_precision_attribute.hpp>

OriginalPrecisionAttribute stores the original precision of the node to pass this information to plugins.

template<typename NodeType> class Output#: #include <node_output.hpp>

template<> class Output<const Node>#

#include <node_output.hpp>

A handle for one of a node’s outputs.

Public Functions

Output(const Node *node, size_t index)#

Constructs a Output.

Parameters:

node – A pointer to the node for the output handle.
index – The index of the output.

Output(const std::shared_ptr<const Node> &node, size_t index)#

Constructs a Output.

Parameters:

node – A shared_ptr to the node for the output handle.
index – The index of the output.

template<typename T> inline Output(const std::shared_ptr<const T> &node)#

Constructs a Output, referencing the zeroth output of the node.

Parameters:: node – A shared_ptr to the node for the output handle.

Output() = default#: A null output.

const Node *get_node() const#

Returns:: A pointer to the node referred to by this output handle.

std::shared_ptr<const Node> get_node_shared_ptr() const#

Returns:: A shared_ptr to the node referred to by this output handle.

size_t get_index() const#

Returns:: The index of the output referred to by this output handle.

OV_NO_DANGLING descriptor::Tensor & get_tensor () const

Returns:: A reference to the tensor descriptor for this output.

std::shared_ptr<descriptor::Tensor> get_tensor_ptr() const#

Returns:: A shared point to the tensor ptr for this output.

OV_NO_DANGLING const element::Type & get_element_type () const

Returns:: The element type of the output referred to by this output handle.

OV_NO_DANGLING const Shape & get_shape () const

Returns:: The shape of the output referred to by this output handle.

OV_NO_DANGLING const PartialShape & get_partial_shape () const

Returns:: The partial shape of the output referred to by this output handle.

OV_NO_DANGLING const RTMap & get_rt_info () const

Returns:: The constant reference to runtime info map

OV_NO_DANGLING const std::unordered_set< std::string > & get_names () const

Returns:: The tensor names associated with this output

std::string get_any_name() const#

Returns:: Any tensor name associated with this output

std::set<Input<Node>> get_target_inputs() const#

Returns:: A set containing handles for all inputs targeted by the output referenced by this output handle.

template<> class Output<Node>#

#include <node_output.hpp>

A handle for one of a node’s outputs.

Public Functions

Output(Node *node, size_t index)#

Constructs a Output.

Parameters:

node – A pointer to the node for the output handle.
index – The index of the output.

Output(const std::shared_ptr<Node> &node, size_t index)#

Constructs a Output.

Parameters:

node – A shared_ptr to the node for the output handle.
index – The index of the output.

template<typename T> inline Output(const std::shared_ptr<T> &node)#

Constructs a Output, referencing the default output of the node. If the node doesn’t have a default output, an exception will be thrown.

Parameters:: node – A shared_ptr to the node for the output handle.

Output() = default#: A null output.

Node *get_node() const#

Returns:: A pointer to the node referred to by this output handle.

std::shared_ptr<Node> get_node_shared_ptr() const#

Returns:: A shared_ptr to the node referred to by this output handle.

size_t get_index() const#

Returns:: The index of the output referred to by this output handle.

OV_NO_DANGLING descriptor::Tensor & get_tensor () const

Returns:: A reference to the tensor descriptor for this output.

std::shared_ptr<descriptor::Tensor> get_tensor_ptr() const#

Returns:: A shared point to the tensor ptr for this output.

void set_tensor_ptr(std::shared_ptr<descriptor::Tensor> tensor_ptr)#

Returns:: Set new tensor desc shared pointer to this output

OV_NO_DANGLING const element::Type & get_element_type () const

Returns:: The element type of the output referred to by this output handle.

OV_NO_DANGLING const Shape & get_shape () const

Returns:: The shape of the output referred to by this output handle.

OV_NO_DANGLING const PartialShape & get_partial_shape () const

Returns:: The partial shape of the output referred to by this output handle.

RTMap &get_rt_info()#

Returns:: The reference to runtime info map

OV_NO_DANGLING const RTMap & get_rt_info () const

Returns:: The constant reference to runtime info map

OV_NO_DANGLING const std::unordered_set< std::string > & get_names () const

Returns:: The tensor names associated with this output

std::string get_any_name() const#

Returns:: Any tensor names associated with this output

void set_names(const std::unordered_set<std::string> &names)#

Returns:: Set tensor names associated with this output

void add_names(const std::unordered_set<std::string> &names)#

Returns:: Add tensor names associated with this output

std::set<Input<Node>> get_target_inputs() const#

Returns:: A set containing handles for all inputs targeted by the output referenced by this output handle.

void remove_target_input(const Input<Node> &target_input) const#

Removes a target input from the output referenced by this output handle.

Parameters:: target_input – The target input to remove.

void replace(const Output<Node> &replacement)#: Replace all users of this value with replacement.

class PartialShape#

#include <partial_shape.hpp>

Class representing a shape that may be partially or totally dynamic.

A PartialShape may have:

Dynamic rank. (Informal notation: ?)
Static rank, but dynamic dimensions on some or all axes. (Informal notation examples: {1,2,?,4}, {?,?,?})
Static rank, and static dimensions on all axes. (Informal notation examples: {1,2,3,4}, {6}, {})

Public Functions

PartialShape(std::initializer_list<Dimension> init)#

Constructs a shape with static rank from an initializer list of Dimension.

Examples:

PartialShape s{2,3,4};                     // rank=3, all dimensions static
PartialShape s{};                          // rank=0
PartialShape s{2,Dimension::dynamic(),3};  // rank=3, dimension 1 dynamic

Parameters:: init – The Dimension values for the constructed shape.

PartialShape(std::vector<Dimension> dimensions)#

Constructs a PartialShape with static rank from a vector of Dimension.

Parameters:: dimensions – The Dimension values for the constructed shape.

PartialShape(const std::vector<Dimension::value_type> &dimensions)#

Constructs a PartialShape with static rank from a vector of dimensions values.

Parameters:: dimensions – The Dimension values for the constructed shape.

PartialShape()#: Constructs a static PartialShape with zero rank (the shape of a scalar).

PartialShape(const Shape &shape)#

Constructs a static PartialShape from a PartialShape.

Parameters:: shape – The PartialShape to convert into PartialShape.

PartialShape(const std::string &shape)#

Constructs a static PartialShape from a string.

Parameters:: shape – The string to parse into PartialShape.

bool is_static() const#

Check if this shape is static.

A shape is considered static if it has static rank, and all dimensions of the shape are static.

Returns:: true if this shape is static, else false.

inline bool is_dynamic() const#

Check if this shape is dynamic.

A shape is considered static if it has static rank, and all dimensions of the shape are static.

Returns:: false if this shape is static, else true.

inline Rank rank() const#

Get the rank of the shape.

Returns:: The rank of the shape. This will be Rank::dynamic() if the rank of the shape is dynamic.

bool compatible(const PartialShape &s) const#

Check whether this shape is compatible with the argument, i.e., whether it is possible to merge them.

Two shapes are compatible if

one or both of them has dynamic rank, or
both shapes have dynamic and equal rank, and their dimensions are elementwise compatible (see Dimension::compatible()).

Parameters:: s – The shape to be checked for compatibility with this shape.
Returns:: true if this shape is compatible with s, else false.

bool same_scheme(const PartialShape &s) const#

Check whether this shape represents the same scheme as the argument.

Two shapes s1 and s2 represent the same scheme if

they both have dynamic rank, or
they both have static and equal rank r, and for every i from 0 to r-1, s1[i] represents the same scheme as s2[i] (see Dimension::same_scheme()).

Parameters:: s – The shape whose scheme is being compared with this shape.
Returns:: true if this shape represents the same scheme as s, else false.

bool relaxes(const PartialShape &s) const#

Check whether this shape is a relaxation of the argument.

Intuitively, a PartialShape s1 is said to relax s2 (or is a relaxation of s2) if it is “more permissive” than s2. In other words, s1 is a relaxation of s2 if anything you can form by plugging things into the dynamic dimensions of s2 is also something you can form by plugging things into the dynamic dimensions of s1, but not necessarily the other way around.

s1.relaxes(s2) is equivalent to s2.refines(s1).

Formally, PartialShape s1 is said to relax PartialShape s2 if:

For every i from 0 to r-1, either s1[i] contains s2[i].

Parameters:: s – The shape which is being compared against this shape.
Returns:: true if this shape relaxes s, else false.

bool refines(const PartialShape &s) const#

Check whether this shape is a refinement of the argument.

Intuitively, a PartialShape s1 is said to relax s2 (or is a relaxation of s2) if it is “less permissive” than s2. In other words, s1 is a relaxation of s2 if anything you can form by plugging things into the dynamic dimensions of s1 is also something you can form by plugging things into the dynamic dimensions of s2, but not necessarily the other way around.

s1.refines(s2) is equivalent to s2.relaxes(s1).

Formally, PartialShape s1 is said to refine PartialShape s2 if:

s2 has dynamic rank, or
s1 and s2 both have static rank r, and for every i from 0 to r-1, either s2[i] is dynamic, or s1[i] == s2[i].

Parameters:: s – The shape which is being compared against this shape.
Returns:: true if this shape refines s, else false.

bool merge_rank(const Rank &r)#

Checks that this shape’s rank is compatible with r, and, if this shape’s rank is dynamic and r is static, updates this shape to have a rank of r with dimensions all dynamic.

Returns:: true if this shape’s rank is compatible with r, else false.

Shape to_shape() const#

Convert a static PartialShape to a PartialShape.

Throws:: std::invalid_argument – If this PartialShape is dynamic.
Returns:: A new PartialShape s where s[i] = size_t((*this)[i]).

bool all_non_negative() const#: Returns true if all static dimensions of the tensor are non-negative, else false.

Dimension &operator[](std::ptrdiff_t i)#

Index operator for PartialShape, with bound checking.

Parameters:: i – The index of the dimension being selected in range [-rank, rank).
Returns:: A reference to the ith Dimension of this shape.

const Dimension &operator[](std::ptrdiff_t i) const#

Index operator for PartialShape, with bound checking.

Parameters:: i – The index of the dimension being selected in range [-rank, rank).
Returns:: A reference to the ith Dimension of this shape.

inline explicit operator std::vector<Dimension>() const#: Returns a vector of the dimensions. This has no meaning if dynamic.

Shape get_max_shape() const#: Get the max bounding shape.

Shape get_min_shape() const#: Get the min bounding shape.

Shape get_shape() const#: Get the unique shape.

inline iterator begin() noexcept#: Returns a read/write iterator that points to the first element in the shape. Iteration is done in ordinary element order.

inline const_iterator begin() const noexcept#: Returns a read-only (constant) iterator that points to the first element in the shape. Iteration is done in ordinary element order.

inline iterator end() noexcept#: Returns a read/write iterator that points one past the last element in the shape. Iteration is done in ordinary element order.

inline const_iterator end() const noexcept#: Returns a read-only (constant) iterator that points one past the last element in the shape. Iteration is done in ordinary element order.

inline reverse_iterator rbegin() noexcept#: Returns a read/write reverse iterator that points to the last element in the shape. Iteration is done in reverse element order.

inline const_reverse_iterator rbegin() const noexcept#: Returns a read-only (constant) reverse iterator that points to the last element in the shape. Iteration is done in reverse element order.

inline reverse_iterator rend() noexcept#: Returns a read/write reverse iterator that points to one before the first element in the shape. Iteration is done in reverse element order.

inline const_reverse_iterator rend() const noexcept#: Returns a read-only (constant) reverse iterator that points to one before the first element in the shape. Iteration is done in reverse element order.

inline const_iterator cbegin() const noexcept#: Returns a read-only (constant) iterator that points to the first element in the shape. Iteration is done in ordinary element order.

inline const_iterator cend() const noexcept#: Returns a read-only (constant) iterator that points one past the last element in the shape. Iteration is done in ordinary element order.

inline const_reverse_iterator crbegin() const noexcept#: Returns a read-only (constant) reverse iterator that points to the last element in the shape. Iteration is done in reverse element order.

inline const_reverse_iterator crend() const noexcept#: Returns a read-only (constant) reverse iterator that points to one before the first element in the shape. Iteration is done in reverse element order.

inline void resize(size_t count)#: Resizes dimensions container to contain count elements.

inline size_t size() const#: Returns size of dimension vector. Requires rank to be static.

inline iterator insert(iterator position, const Dimension &val)#: Returns a read/write iterator that points to the inserted element in the shape.

inline void insert(iterator position, size_t n, const Dimension &val)#: Inserts count copies of the value before position.

template<class InputIterator> inline void insert(iterator position, InputIterator first, InputIterator last)#: Inserts elements from range [first, last) before position.

inline void reserve(size_t n)#: Requests that the dimensions vector capacity be enough to contain n elements.

inline void push_back(const Dimension &val)#: push element to the end of partial shape

template<class ...Args> inline void emplace_back(Args&&... args)#: emplace element to the end of partial shape

std::string to_string() const#: String representation of PartialShape.

Public Static Functions

static PartialShape dynamic(Rank r = Rank::dynamic())#

Construct a PartialShape with the given rank and all dimensions (if any) dynamic.

Returns:: A PartialShape with the given rank, and all dimensions (if any) dynamic.

static bool merge_into(PartialShape &dst, const PartialShape &src)#

Try to merge one shape into another.

Merges src into dst, returning true on success and false on failure. If false is returned, the effect on dst is unspecified.

To merge two partial shapes s1 and s2 is to find the most permissive partial shape s that is no more permissive than s1 or s2, if s exists. For example:

merge(?,?) -> ?
merge(?,{?,?}) -> {?,?}
merge({?,?},{?,?}) -> {?,?}
merge({1,2,3,4},?) -> {1,2,3,4}
merge({1,2},{1,?}) -> {1,2}
merge({1,2,?,?},{1,?,3,?}) -> {1,2,3,?}
merge({1,2,3},{1,2,3}) -> {1,2,3}

merge({1,?},{2,?}) fails [dimension 0 constraints are inconsistent]
merge({?,?},{?,?,?}) fails [ranks are inconsistent]

This function (merge_into) performs the “merge” operation described above on dst and src, but overwrites dst with the result and returns true if merging is successful; if merging is unsuccessful, the function returns false and may make unspecified changes to dst.

Parameters:

dst – [inout] The shape that src will be merged into.
src – The shape that will be merged into dst.

Returns:

true if merging succeeds, else false.

static bool broadcast_merge_into(PartialShape &dst, const PartialShape &src, const ov::op::AutoBroadcastSpec &autob)#: Try to merge one shape into another along with implicit broadcasting.

Friends

friend OPENVINO_API std::ostream & operator<< (std::ostream &str, const PartialShape &shape)

Inserts a human-readable representation of a PartialShape into an output stream.

The output to the stream is in “informal” notation. In other words:

If shape has dynamic rank, inserts the string ?.
If shape has static rank, inserts the string {, then inserts each dimension of shape into the output stream separated by commas, then inserts }.

Example:

PartialShape s1{PartialShape::dynamic())};
PartialShape s2{};
PartialShape s3{1,Dimension::dynamic(),2,3};
PartialShape s4{2,3,4};
std::cout << s1 << std::endl
          << s2 << std::endl
          << s3 << std::endl
          << s4 << std::endl;

Output:

?
{}
{1,?,2,3}
{2,3,4}

Parameters:

str – The output stream targeted for insertion.
shape – The shape to be inserted into str.

Returns:

A reference to str after insertion.

friend OPENVINO_API PartialShape operator+ (const PartialShape &s1, const PartialShape &s2)

Elementwise addition of two PartialShape objects.

If s1 or s2 has dynamic rank, returns PartialShape::dynamic().
If s1 ands2` both have static rank, and their ranks are unequal, throws std::invalid_argument.
If s1 and s2 both have static rank, and their ranks are equal, returns a new shape whose ith dimension is s1[i] + s2[i].

Parameters:

s1 – Left operand for addition.
s2 – Right operand for addition.

Throws:

std::invalid_argument – If s1 and s2 have inconsistent ranks.

Returns:

The result of elementwise adding s1 to s2 (see description).

class PrecisionPreservedAttribute : public SharedAttribute<bool>

#include <precision_preserved_attribute.hpp>

PrecisionPreservedAttribute defines the precision preserved operation. If the attribute is absent, then an operation is not precision preserved.

For more details about the attribute, refer to PrecisionPreservedAttribute page in the OpenVINO Developer Guide.

Subclassed by ov::AvgPoolPrecisionPreservedAttribute

class PrecisionsAttribute : public SharedAttribute<std::vector<ov::element::Type>>

#include <precisions_attribute.hpp>

PrecisionsAttribute defines precision which is required for input/output port or an operation.

For more details about the attribute, refer to PrecisionsAttribute page in the OpenVINO Developer Guide.

class PrecisionSensitive : public ov::RuntimeAttribute #: #include <precision_sensitive_attribute.hpp>

PrecisionSensitive class represents runtime info attribute that marks input to an operation as a precision sensitive and disables compression to FP16 of the subgraph before this input.

class PreprocessingAttribute : public ov::RuntimeAttribute #: #include <preprocessing_attribute.hpp>

class PrimitivesPriority : public ov::RuntimeAttribute #: #include <primitives_priority_attribute.hpp>

struct ProfilingInfo

#include <profiling_info.hpp>

Represents basic inference profiling information per operation.

If the operation is executed using tiling, the sum time per each tile is indicated as the total execution time. Due to parallel execution, the total execution time for all nodes might be greater than the total inference time.

Public Types

enum class Status

Defines the general status of a node.

Values:

enumerator NOT_RUN: A node is not executed.

enumerator OPTIMIZED_OUT: A node is optimized out during graph optimization phase.

enumerator EXECUTED: A node is executed.

Public Members

Status status: Defines the node status.

std::chrono::microseconds real_time: The absolute time, in microseconds, that the node ran (in total).

std::chrono::microseconds cpu_time: The net host CPU time that the node ran.

std::string node_name: Name of a node.

std::string exec_type: Execution type of a unit.

std::string node_type: Node type.

template<typename T, PropertyMutability mutability_ = PropertyMutability::RW> class Property : public util::BaseProperty<T, PropertyMutability::RW>#

#include <properties.hpp>

This class is used to bind property name with value type.

Template Parameters:: T – type of value used to set or get property

Public Functions

template<typename ...Args> inline std::pair<std::string, Any> operator()(Args&&... args) const#

Constructs property.

Template Parameters:: Args – property constructor arguments types
Parameters:: args – property constructor arguments
Returns:: Pair of name and type erased value.

template<typename T> RO > : public util::BaseProperty< T, PropertyMutability::RO >

#include <properties.hpp>

This class is used to bind read-only property name with value type.

Template Parameters:: T – type of value used to pass or get property

struct PropertyName : public std::string#

#include <properties.hpp>

This class is used to return property name and its mutability attribute.

Public Functions

inline PropertyName(const std::string &str, PropertyMutability mutability = PropertyMutability::RW)#

Constructs property name object.

Parameters:

str – property name
mutability – property mutability

inline bool is_mutable() const#

check property mutability

Returns:: true if property is mutable

class QuantizationAlignmentAttribute : public SharedAttribute<bool>

#include <quantization_alignment_attribute.hpp>

QuantizationAlignmentAttribute defines subgraph with the same quantization alignment. FakeQuantize operations are not included. The attribute is used by quantization operations.

For more details about the attribute, refer to QuantizationAlignmentAttribute page in the OpenVINO Developer Guide.

class QuantizationGranularityAttribute : public ov::RuntimeAttribute

#include <quantization_granularity_attribute.hpp>

QuantizationGranularityAttribute defines quantization granularity of operation inputs.

For more details about the attribute, refer to QuantizationGranularityAttribute page in the OpenVINO Developer Guide.

class QuantizationModeAttribute : public ov::RuntimeAttribute #: #include <quantization_mode_attribute.hpp>

struct RawNodeOutput#: #include <node.hpp>

class RemoteContext

#include <remote_context.hpp>

This class represents an abstraction

for remote (non-CPU) accelerator device-specific inference context. Such context represents a scope on the device within which compiled models and remote memory tensors can exist, function, and exchange data.

Subclassed by ov::intel_gpu::ocl::ClContext, ov::intel_npu::level_zero::ZeroContext

Public Functions

RemoteContext() = default: Default constructor.

RemoteContext(const RemoteContext &other) = default

Default copy constructor.

Parameters:: other – Another RemoteContext object.

RemoteContext &operator=(const RemoteContext &other) = default

Default copy assignment operator.

Parameters:: other – Another RemoteContext object.
Returns:: Reference to the current object.

RemoteContext(RemoteContext &&other) = default

Default move constructor.

Parameters:: other – Another RemoteContext object.

RemoteContext &operator=(RemoteContext &&other) = default

Default move assignment operator.

Parameters:: other – Another RemoteContext object.
Returns:: Reference to the current object.

operator bool() const noexcept

Checks if current RemoteContext object is initialized.

Returns:: true if current RemoteContext object is initialized, false - otherwise

~RemoteContext(): Destructor that preserves unloading order of implementation object and reference to the library.

template<typename T> inline bool is() const noexcept

Checks if the RemoteContext object can be cast to the type T.

Template Parameters:: T – Type to be checked. Must represent a class derived from RemoteContext.
Returns:: True if this object can be dynamically cast to the type T*; false, otherwise.

template<typename T> inline const T as() const

Casts this RemoteContext object to the type T.

Template Parameters:: T – Type to cast to. Must represent a class derived from RemoteContext.
Returns:: T Object.

RemoteTensor create_tensor(const element::Type &type, const Shape &shape, const AnyMap &params = {})

Parameters:

type – Defines the element type of the tensor.
shape – Defines the shape of the tensor.
params – Map of the low-level tensor object parameters.

Returns:

Pointer to a plugin object that implements the RemoteTensor interface.

AnyMap get_params() const

Returns a map of device-specific parameters required for low-level operations with the underlying object. Parameters include device/context handles, access flags, etc. Content of the returned map depends on a remote execution context that is currently set on the device (working scenario). Abstract method.

Returns:: A map of name/parameter elements.

Tensor create_host_tensor(const element::Type type, const Shape &shape)

Parameters:

type – Tensor element type.
shape – Tensor shape.

Returns:

A tensor instance with device friendly memory.

Public Static Functions

static void type_check(const RemoteContext &remote_context, const std::map<std::string, std::vector<std::string>> &type_info = {})

Internal method: checks remote type.

Parameters:

remote_context – Remote context which type is checked.
type_info – Map with remote object runtime info.

Throws:

Exception – if type check with the specified parameters failed.

class RemoteTensor : public ov::Tensor

#include <remote_tensor.hpp>

Remote memory access and interoperability API.

Subclassed by ov::intel_gpu::ocl::ClBufferTensor, ov::intel_gpu::ocl::ClImage2DTensor, ov::intel_gpu::ocl::USMTensor, ov::intel_npu::level_zero::ZeroBufferTensor

Public Functions

RemoteTensor() = default: Default constructor.

RemoteTensor(const RemoteTensor &other, const Coordinate &begin, const Coordinate &end)

Constructs region of interest (ROI) tensor from another remote tensor.

Note

Does not perform memory allocation internally

Note

A Number of dimensions in begin and end must match number of dimensions in other.get_shape()

Parameters:

other – original tensor
begin – start coordinate of ROI object inside of the original object.
end – end coordinate of ROI object inside of the original object.

void *data(const element::Type) = delete

Access to host memory is not available for RemoteTensor. To access a device-specific memory, cast to a specific RemoteTensor derived object and work with its properties or parse device memory properties via RemoteTensor::get_params.

Returns:: Nothing, throws an exception.

void copy_to(ov::Tensor &dst) const

Copies data from this RemoteTensor to the specified destination tensor.

Parameters:: dst – The destination tensor to which data will be copied.

void copy_from(const ov::Tensor &src)

Copies data from the specified source tensor to this RemoteTensor.

Parameters:: src – The source tensor from which data will be copied.

ov::AnyMap get_params() const

Returns a map of device-specific parameters required for low-level operations with underlying object. Parameters include device/context/surface/buffer handles, access flags, etc. Content of the returned map depends on remote execution context that is currently set on the device (working scenario). Abstract method.

Returns:: A map of name/parameter elements.

Public Static Functions

static void type_check(const Tensor &tensor, const std::map<std::string, std::vector<std::string>> &type_info = {})

Checks OpenVINO remote type.

Parameters:

tensor – Tensor which type is checked.
type_info – Map with remote object runtime info.

Throws:

Exception – if type check with specified parameters failed.

template<class TShape> struct result_shape#

#include <utils.hpp>

Get correct return type of input shape when call shape_infer.

The input shapes are vector like std::vector<TShape>, where TShape can be std::vector<const size_t> This will provide correct return especially for static shape which can work as reference to dimension or hold them.

Template Parameters:: TShape – Type of input shape.

template<> Shape >: #include <utils.hpp>

Get correct result shape for ov::Shape which is same type.

template<> struct result_shape<PartialShape>#: #include <utils.hpp>

Get correct result shape for PartialShape which is same type.

class RoundingGuard#

#include <rounding_guard.hpp>

Set current round direction for scoped block.

Round direction can be one of:

FE_DOWNWARD
FE_TONEAREST
FE_TOWARDZERO
FE_UPWARD see std <cfenv> header for details.

class RuntimeAttribute#: #include <runtime_attribute.hpp>

Subclassed by SharedAttribute< IntervalsAlignmentSharedValue >, SharedAttribute< bool >, SharedAttribute< std::vector< ov::element::Type > >, SharedAttribute< T >, ov::BiasAttribute, ov::Decompression, ov::DequantizationNode, ov::DisableCleanupAttribute, ov::DisableFP16Compression, ov::FusedNames, ov::KeepConstPrecision, ov::LayoutAttribute, ov::NmsSelectedIndices, ov::NoTransposeSinkingAttr, ov::NonconvertibleDivide, ov::OldApiMapElementType, ov::OldApiMapOrder, ov::OriginalPrecisionAttribute, ov::PrecisionSensitive, ov::PreprocessingAttribute, ov::PrimitivesPriority, ov::QuantizationGranularityAttribute, ov::QuantizationModeAttribute, ov::ShapeSubgraph, ov::SkipInvalidation, ov::StridesPropagation, ov::frontend::GraphIterator, ov::pass::DisableConstantFolding, ov::pass::DisableFoldSubgraphEmptyInputs, ov::pass::DisableRemoveConcatZeroDimInput, ov::preprocess::TensorInfoMemoryType

template<class T, Direction D = Direction::FORWARD> class SeqGen#

#include <sequence_generator.hpp>

Infinite generator of sequence increasing values.

Start value can be specified.

Template Parameters:: T – Type of sequence values (must support ++ or ‘—’ operators).

class Shape : public std::vector<size_t>#

#include <shape.hpp>

Shape for a tensor.

Public Functions

OPENVINO_API Shape::reference operator[] (std::ptrdiff_t i)

Gets dimension at index.

Parameters:: i – Index to shape dimension [-rank, rank).
Returns:: A reference to i-th dimension of this shape.

OPENVINO_API Shape::const_reference operator[] (std::ptrdiff_t i) const

Gets dimension at index.

Parameters:: i – Index to shape dimension [-rank, rank).
Returns:: A const reference to i-th dimension of this shape.

OPENVINO_API Shape::reference at (std::ptrdiff_t i)

Gets dimension at index, with bounds checking.

Parameters:: i – Index to shape dimension [-rank, rank).
Returns:: A reference to i-th dimension of this shape.

OPENVINO_API Shape::const_reference at (std::ptrdiff_t i) const

Gets dimension at index, with bounds checking.

Parameters:: i – Index to shape dimension [-rank, rank).
Returns:: A const reference to i-th dimension of this shape.

class ShapeSubgraph : public ov::RuntimeAttribute #: #include <is_shape_subgraph.hpp>

ShapeSubgraph class represents runtime info attribute that marks shape subgraphs. Information whether the node belongs to the shape path or to the data path is needed during evaluate and CF.

class SkipInvalidation : public ov::RuntimeAttribute #: #include <symbolic_info.hpp>

SkipInvalidation class represents runtime info attribute that instructs ov::Output objects to skip invalidation of partial values and symbols during partial value propagation.

template<class T> struct SoPtr#

#include <so_ptr.hpp>

This class instantiate object using shared library.

Template Parameters:: T – An type of object SoPtr can hold

Public Functions

SoPtr() = default#: Default constructor.

inline ~SoPtr()#: Destructor preserves unloading order of implementation object and reference to library.

inline SoPtr(const std::shared_ptr<T> &ptr, const std::shared_ptr<void> &so)#

Constructs an object with existing shared object reference and loaded pointer.

Parameters:

ptr – pointer to the loaded object
so – Existing reference to library

inline SoPtr(const std::shared_ptr<T> &ptr)#

Constructs an object with existing shared object reference.

Parameters:: ptr – pointer to the loaded object

template<class U, typename std::enable_if<std::is_base_of<T, U>::value, bool>::type = true> inline SoPtr(const std::shared_ptr<U> &ptr)#

Constructs an object with existing shared object reference.

Parameters:: ptr – pointer to the loaded object

template<typename U> inline SoPtr(const SoPtr<U> &that)#

The copy-like constructor, can create So Pointer that dereferenced into child type if T is derived of U.

Parameters:: that – copied SoPtr object

inline T *operator->() const noexcept#

Standard pointer operator.

Returns:: underlined interface with disabled Release method

Public Members

std::shared_ptr<T> _ptr#: Gets a smart pointer to the custom object.

std::shared_ptr<void> _so#: The shared object or dynamic loaded library.

class Strides : public std::vector<size_t>#: #include <strides.hpp>

Strides for a tensor.

class StridesPropagation : public ov::RuntimeAttribute #: #include <strides_property.hpp>

class Symbol#

#include <symbol.hpp>

Class representing unique symbol for the purpose of symbolic shape inference. Equality of symbols is being tracked by Disjoint-set data structure.

Public Functions

Symbol() = default#: Default constructs a unique symbol.

class Tensor

#include <tensor.hpp>

Tensor API holding host memory It can throw exceptions safely for the application, where it is properly handled.

Subclassed by ov::RemoteTensor

Public Functions

Tensor() = default: Default constructor.

Tensor(const Tensor &other, const std::shared_ptr<void> &so)

Copy constructor with adding new shared object.

Parameters:

other – Original tensor
so – Shared object

Tensor(const Tensor &other) = default

Default copy constructor.

Parameters:: other – other Tensor object

Tensor &operator=(const Tensor &other) = default

Default copy assignment operator.

Parameters:: other – other Tensor object
Returns:: reference to the current object

Tensor(Tensor &&other) = default

Default move constructor.

Parameters:: other – other Tensor object

Tensor &operator=(Tensor &&other) = default

Default move assignment operator.

Parameters:: other – other Tensor object
Returns:: reference to the current object

~Tensor(): Destructor preserves unloading order of implementation object and reference to library.

Tensor(const element::Type &type, const Shape &shape, const Allocator &allocator = {})

Constructs Tensor using element type and shape. Allocate internal host storage using default allocator.

Parameters:

type – Tensor element type
shape – Tensor shape
allocator – allocates memory for internal tensor storage

Tensor(const element::Type &type, const Shape &shape, void *host_ptr, const Strides &strides = {})

Constructs Tensor using element type and shape. Wraps allocated host memory.

Note

Does not perform memory allocation internally

Parameters:

type – Tensor element type
shape – Tensor shape
host_ptr – Pointer to pre-allocated host memory with initialized objects
strides – Optional strides parameters in bytes. Strides are supposed to be computed automatically based on shape and element size

Tensor(const ov::Output<const ov::Node> &port, const Allocator &allocator = {})

Constructs Tensor using port from node. Allocate internal host storage using default allocator.

Parameters:

port – port from node
allocator – allocates memory for internal tensor storage

Tensor(const ov::Output<const ov::Node> &port, void *host_ptr, const Strides &strides = {})

Constructs Tensor using port from node. Wraps allocated host memory.

Note

Does not perform memory allocation internally

Parameters:

port – port from node
host_ptr – Pointer to pre-allocated host memory with initialized objects
strides – Optional strides parameters in bytes. Strides are supposed to be computed automatically based on shape and element size

Tensor(const Tensor &other, const Coordinate &begin, const Coordinate &end)

Constructs region of interest (ROI) tensor form another tensor.

Note

Does not perform memory allocation internally

Note

A Number of dimensions in begin and end must match number of dimensions in other.get_shape()

Parameters:

other – original tensor
begin – start coordinate of ROI object inside of the original object.
end – end coordinate of ROI object inside of the original object.

void set_shape(const ov::Shape &shape)

Set new shape for tensor, deallocate/allocate if new total size is bigger than previous one.

Note

Memory allocation may happen

Parameters:: shape – A new shape

const element::Type &get_element_type() const

Returns:: A tensor element type

const Shape &get_shape() const

Returns:: A tensor shape

void copy_to(ov::Tensor dst) const

Copy tensor, destination tensor should have the same element type and shape.

Parameters:: dst – destination tensor

bool is_continuous() const

Reports whether the tensor is continuous or not.

Returns:: true if tensor is continuous

size_t get_size() const

Returns the total number of elements (a product of all the dims or 1 for scalar)

Returns:: The total number of elements

size_t get_byte_size() const

Returns the size of the current Tensor in bytes.

Returns:: Tensor’s size in bytes

Strides get_strides() const

Returns:: Tensor’s strides in bytes

void *data(const element::Type &type = {}) const

Provides an access to the underlaying host memory.

Note

If type parameter is specified, the method throws an exception if specified type’s fundamental type does not match with tensor element type’s fundamental type

Parameters:: type – Optional type parameter.
Returns:: A host pointer to tensor memory

template<typename T, typename datatype = typename std::decay<T>::type> inline T *data() const

Provides an access to the underlaying host memory casted to type T

Note

Throws exception if specified type does not match with tensor element type

Returns:: A host pointer to tensor memory casted to specified type T.

bool operator!() const noexcept

Checks if current Tensor object is not initialized.

Returns:: true if current Tensor object is not initialized, false - otherwise

explicit operator bool() const noexcept

Checks if current Tensor object is initialized.

Returns:: true if current Tensor object is initialized, false - otherwise

template<typename T> inline std::enable_if<std::is_base_of<Tensor, T>::value, bool>::type is() const noexcept

Checks if the Tensor object can be cast to the type T.

Template Parameters:: T – Type to be checked. Must represent a class derived from the Tensor
Returns:: true if this object can be dynamically cast to the type const T*. Otherwise, false

template<typename T> inline const std::enable_if<std::is_base_of<Tensor, T>::value, T>::type as() const

Casts this Tensor object to the type T.

Template Parameters:: T – Type to cast to. Must represent a class derived from the Tensor
Returns:: T object

Public Static Functions

static void type_check(const Tensor &tensor)

Checks openvino tensor type.

Parameters:: tensor – a tensor which type will be checked
Throws:: Exception – if type check with specified tensor is not pass

template<class TContainer> class TensorAccessor : public ov::ITensorAccessor #

#include <tensor_data_accessor.hpp>

Tensor data accessor functor.

Creates the ov::Tensor found in tensors container. This accessor does not take ownership of tensors container. Supports following containers:

ov::TensorVector
std::unordered_map<size_t, ov::Tensor>

Template Parameters:: TContainer – Type of tensor container.

Public Functions

inline constexpr TensorAccessor(const TContainer *tensors)#

Construct a new Tensor Accessor object for tensors container.

Parameters:: tensors – Pointer to container with tensors.

virtual Tensor operator()(size_t port) const override#

Get tensor for given port number.

Parameters:: port – Port number to get data.
Returns:: Tensor to data or empty tensor if data not found.

virtual Tensor operator()(size_t port) const

Get tensor at port.

Parameters:: port – Number of data port (operator input) to get tensor.
Returns:: Tensor to data at port.

virtual Tensor operator()(size_t port) const

Get tensor at port.

Parameters:: port – Number of data port (operator input) to get tensor.
Returns:: Tensor to data at port.

virtual Tensor operator()(size_t port) const

Get tensor at port.

Parameters:: port – Number of data port (operator input) to get tensor.
Returns:: Tensor to data at port.

struct TensorTransform : public ov::element::NotSupported<void>#: #include <utils.hpp>

template<typename VAT> class ValueAccessor#

#include <attribute_adapter.hpp>

Provides access to an attribute of type AT as a value accessor type VAT.

Provides access to values via get/set methods from an m_value, typically from ValueReference.

The m_buffer holds a VAT, which may be wider than the attribute AT. For example, serializers that only support int64_t integers would use a ValueAccessor<vector<int64_t>> to reference a vector<int8_t> attribute. Destruction moves the value back to the attribute if it was changed.

Template Parameters:: VAT – The adapter value type; may be wider than the value being accessed.

Public Functions

virtual const VAT &get() = 0#: Returns the value.

virtual void set(const VAT &value) = 0#: Sets the value.

template<> class ValueAccessor<void*> : public ov::ValueAccessor<void>#: #include <attribute_adapter.hpp>

template<> class ValueAccessor<void>#

#include <attribute_adapter.hpp>

ValueAccessor<void> provides an accessor for values that do not have get/set methods via AttributeVisitor.on_adapter.

All ValueAccessors must be derived from ValueAccessor<void> so that an AttributeVisitor only needs to implement a subset of the on_adapter methods.

Subclassed by ov::ValueAccessor< void * >, ov::VisitorAdapter

Public Functions

virtual const DiscreteTypeInfo &get_type_info() const = 0#: type info enables identification of the value accessor, as well as is_type and as_type.

class VariableState

#include <variable_state.hpp>

VariableState class.

Public Functions

VariableState() = default: Default constructor.

~VariableState(): Destructor that preserves unloading order of implementation object and reference to the library.

void reset(): Resets internal variable state for relevant infer request to a value specified as default for the corresponding ReadValue node.

std::string get_name() const

Gets the name of the current variable state. If length of an array is not enough, the name is truncated by len, null terminator is inserted as well. variable_id from the corresponding ReadValue is used as variable state name.

Returns:: A string representing state name.

Tensor get_state() const

Returns the value of the variable state.

Returns:: A tensor representing a state.

void set_state(const Tensor &state)

Sets the new state for the next inference.

Parameters:: state – The current state to set.

struct Version#

#include <version.hpp>

Represents version information that describes plugins and the OpemVINO library.

Public Members

const char *buildNumber#: A null terminated string with build number.

const char *description#: A null terminated description string.

class VisitorAdapter : public ov::ValueAccessor<void>#

#include <attribute_adapter.hpp>

Adapters will see visitor.

Subclassed by ov::AttributeAdapter< ParameterVector >, ov::AttributeAdapter< ResultVector >, ov::AttributeAdapter< op::AutoBroadcastSpec >, ov::AttributeAdapter< op::BroadcastModeSpec >, ov::AttributeAdapter< ov::NodeVector >, ov::AttributeAdapter< std::shared_ptr< ov::Node > >

namespace batch_util#

Functions

void mark_batch(const std::shared_ptr<ov::op::v0::Parameter> &parameter, P2Btype &map, const std::unordered_set<std::shared_ptr<Symbol>> &batches)#

void mark_no_batch(const std::shared_ptr<ov::op::v0::Parameter> &parameter, P2Btype &map)#

void mark_layout_independent_batch(const std::shared_ptr<ov::op::v0::Parameter> &parameter, const std::shared_ptr<ov::Node> &result, P2Btype &map)#

void mark_with_unique_dimension_symbols(const std::shared_ptr<Model> &m)#

void restore_original_dimensions(const std::shared_ptr<ov::Model> &model, const std::map<std::shared_ptr<ov::op::v0::Parameter>, ov::PartialShape> &parameter_to_shape, bool leave_batch_dynamic = true, bool clear_symbols = false)#

bool check_batch_tracks_through_all_the_nodes(const std::shared_ptr<ov::Model> &m)#

P2Btype find_batch(const std::shared_ptr<ov::Model> &m)#

bool detach_detection_output(const std::shared_ptr<ov::Model> &f)#

namespace cmp#

Enums

enum Bound#

Enumerate bounds to compare.

Values:

enumerator NONE#

enumerator LOWER#

enumerator UPPER#

enumerator BOTH#

Functions

template<class T, class U, typename std::enable_if<((std::is_signed<T>::value || std::is_same<T, float16>::value || std::is_same<T, bfloat16>::value) && (std::is_signed<U>::value || std::is_same<U, float16>::value || std::is_same<U, bfloat16>::value)) || (std::is_unsigned<T>::value && std::is_unsigned<U>::value)>::type* = nullptr> constexpr bool lt(T a, U b) noexcept#

Compare two values (a < b) in safe way against lossy integer conversion.

Template Parameters:

T – Type of a value.
U – Type of b value.

Parameters:

a – Value a.
b – Value b.

Returns:

true if a less b otherwise false.

template<class T, class U> constexpr bool gt(T a, U b) noexcept#

Compare two values (a > b) in safe way against lossy integer conversion.

Template Parameters:

T – Type of a value.
U – Type of b value.

Parameters:

a – Value a.
b – Value b.

Returns:

true if a > b otherwise false.

template<class T, class U> constexpr bool le(T a, U b) noexcept#

Compare two values (a <= b) in safe way against lossy integer conversion.

Template Parameters:

T – Type of a value.
U – Type of b value.

Parameters:

a – Value a.
b – Value b.

Returns:

true if a <= b otherwise false.

template<class T, class U> constexpr bool ge(T a, U b) noexcept#

Compare two values (a >= b) in safe way against lossy integer conversion.

Template Parameters:

T – Type of a value.
U – Type of b value.

Parameters:

a – Value a.
b – Value b.

Returns:

true if a >= b otherwise false.

template<class T, Bound BMode = Bound::NONE> class Between#

#include <compare.hpp>

Compare if value is between lower and upper bounds.

The Between comparator has four modes to check value:

Bound::None (lower, upper)
Bound::LOWER [lower, upper)
Bound::UPPER (lower, upper]
Bound::BOTH [lower, upper]

Template Parameters:

T – Value type to compare.
BMode – Compare bounds mode.

template<class T> class Equal#

#include <compare.hpp>

Compare if value is equal to expected.

Template Parameters:: T – Value type to compare.

template<class T> class Less#

#include <compare.hpp>

Compare if value is less to expected.

Template Parameters:: T – Value type to compare.

namespace coordinates#

namespace descriptor#

Functions

OPENVINO_API std::ostream & operator<< (std::ostream &, const ov::descriptor::Tensor &)

class Input#

#include <input.hpp>

Public Functions

Input(Node *node, size_t index, Output &output)#

Parameters:

node – The node that owns this input
index – The position of this tensor in all input tensors
output – The output that supplies a value for this input

Input(Node *node, size_t index)#

Create an Input that is not connected to an output.

Parameters:

node – The node that owns this input
index – The position of this tensor in all input tensors

std::shared_ptr<Node> get_node() const#

Returns:: the node that this is an input of

inline Node *get_raw_pointer_node() const#

Returns:: the raw pointer to the node that this is an input of

inline size_t get_index() const#

Returns:: the position within all supplied tensors of this input

inline const Output &get_output() const#

Returns:: the connected output

inline Output &get_output()#

Returns:: the connected output

inline bool has_output() const#

Returns:: true if an output is connected to the input.

const Tensor &get_tensor() const#

Returns:: the tensor of the connected output

Tensor &get_tensor()#

Returns:: the tensor of the connected output

void replace_output(const std::shared_ptr<Node> &node, size_t i)#: Replace the current output that supplies a value for this input with output i of node.

void replace_output(Output &output)#: Replace the current output that supplies a value for this input with output.

void remove_output()#: Remove the output from this input. The node will not be valid until another output is supplied.

inline bool get_is_relevant_to_shape() const#

See Node::set_input_is_relevant_to_shape for more details.

Returns:: true if the value of this input is relevant to the output shapes of the corresponding node. (Usually this is false.)

inline bool get_is_relevant_to_value() const#

See Node::set_input_is_relevant_to_value for more details.

Returns:: true if the value of this input is relevant to the output value of the corresponding node. (Usually this is true.)

const Shape &get_shape() const#

Returns:: the shape of the connected output

const PartialShape &get_partial_shape() const#

Returns:: the partial shape of the connected output

const element::Type &get_element_type() const#

Returns:: the element type of the connected output

class Output#

#include <output.hpp>

Public Functions

Output(Node *node, size_t index, const std::shared_ptr<Tensor> &tensor)#

Parameters:

node – Node that owns this output.
index – Position of the output tensor in all output tensors
tensor – The tensor where the value will be written

const Shape &get_shape() const#

Returns:: the shape of the output

const PartialShape &get_partial_shape() const#

Returns:: the partial shape of the output

const element::Type &get_element_type() const#

Returns:: the element type of the output

class Tensor#

#include <tensor.hpp>

Compile-time descriptor of a first-class value that is a tensor.

Public Functions

Tensor(const element::Type &element_type, const PartialShape &pshape, const std::unordered_set<std::string> &names = {})#

Creates Tensor descriptor.

Parameters:

element_type – Element type
pshape – Partial shape of tensor
names – Tensor names (optional default empty).

const std::string &get_any_name() const#: Gets any tensor name. Throws if tensor has no names.

const std::unordered_set<std::string> &get_names() const#: Gets tensor names.

void set_names(const std::unordered_set<std::string> &names)#

Set new names.

Parameters:: names – Names to set.

void add_names(const std::unordered_set<std::string> &names)#

Adds new names to tensor.

Parameters:: names – new names to be added.

void set_lower_value(const ov::Tensor &value)#: sets lower bound value description

void set_upper_value(const ov::Tensor &value)#: sets upper bound value description

void set_value_symbol(const TensorSymbol &value_symbol)#: sets value symbol description

void invalidate_values()#: unsets bound value descriptions

const element::Type &get_element_type() const#: Gets element type.

const Shape &get_shape() const#: Gets shape. Throw if Tensor’s shape is not static.

const PartialShape &get_partial_shape() const#: Gets partial shape.

const ov::Tensor &get_lower_value() const#: gets lower bound value description

const ov::Tensor &get_upper_value() const#: gets upper bound value description

TensorSymbol get_value_symbol() const#: gets symbol value description

bool has_and_set_bound() const#: checks if lower and upper bound are set and point to the same Tensor

size_t size() const#

Get Tensor size in bytes.

Returns:: Size in bytes.

RTMap &get_rt_info()#

Gets runtime informations.

Returns:: Runtime information map which can be modified.

const RTMap &get_rt_info() const#

Gets runtime informations.

Returns:: Read only runtime information map.

void clone_from(const Tensor &other)#

Clones Tensor from the other.

Parameters:: other – Tensor used to clone its properties.

namespace detail#

Functions

template<class T> static auto collect_attached_extensions_onnx (std::vector< ov::Extension::Ptr > &res) -> decltype(typename T::template __openvino_framework_map_helper_onnx< T >().get(), void())

template<class> static auto collect_attached_extensions_onnx(ov::Any) -> void#

template<class T> static auto collect_attached_extensions_paddle (std::vector< ov::Extension::Ptr > &res) -> decltype(typename T::template __openvino_framework_map_helper_paddle< T >().get(), void())

template<class> static auto collect_attached_extensions_paddle(ov::Any) -> void#

template<class T> static auto collect_attached_extensions_tensorflow (std::vector< ov::Extension::Ptr > &res) -> decltype(typename T::template __openvino_framework_map_helper_tensorflow< T >().get(), void())

template<class> static auto collect_attached_extensions_tensorflow(ov::Any) -> void#

template<class T> static auto collect_attached_extensions_pytorch (std::vector< ov::Extension::Ptr > &res) -> decltype(typename T::template __openvino_framework_map_helper_pytorch< T >().get(), void())

template<class> static auto collect_attached_extensions_pytorch(ov::Any) -> void#

namespace device#

Namespace with device properties.

Enums

enum class Type#

Enum to define possible device types.

Values:

enumerator INTEGRATED#: Device is integrated into host system.

enumerator DISCRETE#: Device is not integrated into host system.

Variables

static constexpr Property<std::string> id = {"DEVICE_ID"}#: the property for setting of required device to execute on values: device id starts from “0” - first device, “1” - second device, etc

static constexpr Priorities priorities = {"MULTI_DEVICE_PRIORITIES"}#: Device Priorities config option, with comma-separated devices listed in the desired priority.

static constexpr Properties properties = {"DEVICE_PROPERTIES"}#

Property to pass set of property values to specified device

Usage Example:

core.compile_model("HETERO"
    ov::device::priorities("GPU", "CPU"),
    ov::device::properties("CPU", ov::enable_profiling(true)),
    ov::device::properties("GPU", ov::enable_profiling(false)));

static constexpr Property<std::string, PropertyMutability::RO> full_name = {"FULL_DEVICE_NAME"}#: Read-only property to get a std::string value representing a full device name.

static constexpr Property<std::string, PropertyMutability::RO> architecture = {"DEVICE_ARCHITECTURE"}#: Read-only property which defines the device architecture.

static constexpr Property<UUID, PropertyMutability::RO> uuid = {"DEVICE_UUID"}#: Read-only property which defines the UUID of the device.

static constexpr Property<LUID, PropertyMutability::RO> luid = {"DEVICE_LUID"}#: Read-only property which defines the LUID of the device.

static constexpr Property<Type, PropertyMutability::RO> type = {"DEVICE_TYPE"}#: Read-only property to get a type of device. See Type enum definition for possible return values.

static constexpr Property<std::map<element::Type, float>, PropertyMutability::RO> gops = {"DEVICE_GOPS"}#: Read-only property which defines Giga OPS per second count (GFLOPS or GIOPS) for a set of precisions supported by specified device.

static constexpr Property<PCIInfo, PropertyMutability::RO> pci_info = {"DEVICE_PCI_INFO"}#: Read-only property to get PCI bus information of device. See PCIInfo struct definition for details.

static constexpr Property<float, PropertyMutability::RO> thermal = {"DEVICE_THERMAL"}#: Read-only property to get a float of device thermal.

static constexpr Property<std::vector<std::string>, PropertyMutability::RO> capabilities = {"OPTIMIZATION_CAPABILITIES"}#: Read-only property to get a std::vector<std::string> of capabilities options per device.

struct LUID

#include <properties.hpp>

Structure which defines format of LUID.

Public Members

std::array<uint8_t, MAX_LUID_SIZE> luid: Array with luid for a device.

Public Static Attributes

static const uint64_t MAX_LUID_SIZE = 8: Max size of luid array (64 bits)

struct PCIInfo

#include <properties.hpp>

Structure to store PCI bus information of device (Domain/Bus/Device/Function)

Public Members

uint32_t domain: PCI domain ID.

uint32_t bus: PCI bus ID.

uint32_t device: PCI device ID.

uint32_t function: PCI function ID.

struct Priorities : public ov::Property<std::string>

#include <properties.hpp>

Type for device Priorities config option, with comma-separated devices listed in the desired priority.

Public Functions

template<typename ...Args> inline std::pair<std::string, Any> operator()(Args&&... args) const

Constructs device priorities.

Template Parameters:: Args – property constructor arguments types
Parameters:: args – property constructor arguments
Returns:: Pair of name and type erased value.

struct Properties : public ov::Property<std::map<std::string, std::map<std::string, Any>>>

#include <properties.hpp>

Type for property to pass set of properties to specified device.

Public Functions

inline std::pair<std::string, Any> operator()(const AnyMap &config) const

Constructs property.

Parameters:: configs – set of property values with names
Returns:: Pair of string key representation and type erased property value.

inline std::pair<std::string, Any> operator()(const std::string &device_name, const AnyMap &config) const

Constructs property.

Parameters:

device_name – device plugin alias
config – set of property values with names

Returns:

Pair of string key representation and type erased property value.

template<typename ...Properties> inline util::EnableIfAllStringAny<std::pair<std::string, Any>, Properties...> operator()(const std::string &device_name, Properties&&... configs) const

Constructs property.

Template Parameters:

Properties – Should be the pack of std::pair<std::string, ov::Any> types

Parameters:

device_name – device plugin alias
configs – Optional pack of pairs: (config parameter name, config parameter value)

Returns:

Pair of string key representation and type erased property value.

struct UUID

#include <properties.hpp>

Structure which defines format of UUID.

Public Members

std::array<uint8_t, MAX_UUID_SIZE> uuid: Array with uuid for a device.

Public Static Attributes

static const uint64_t MAX_UUID_SIZE = 16: Max size of uuid array (128 bits)

namespace capability#

Namespace with possible values for ov::device::capabilities property.

Variables

static constexpr const auto FP32 = "FP32"#: Device supports fp32 inference.

static constexpr const auto BF16 = "BF16"#: Device supports bf16 inference.

static constexpr const auto FP16 = "FP16"#: Device supports fp16 inference.

static constexpr const auto INT8 = "INT8"#: Device supports int8 inference.

static constexpr const auto INT16 = "INT16"#: Device supports int16 inference.

static constexpr const auto BIN = "BIN"#: Device supports binary inference.

static constexpr const auto WINOGRAD = "WINOGRAD"#: Device supports winograd optimization.

static constexpr const auto EXPORT_IMPORT = "EXPORT_IMPORT"#: Device supports compiled model export and import.

namespace element#

Typedefs

using TypeVector = std::vector<Type>#

Enums

enum class Type_t#

Enum to define possible element types.

Values:

enumerator undefined#: Undefined element type.

enumerator dynamic#: Dynamic element type.

enumerator boolean#: boolean element type

enumerator bf16#: bf16 element type

enumerator f16#: f16 element type

enumerator f32#: f32 element type

enumerator f64#: f64 element type

enumerator i4#: i4 element type

enumerator i8#: i8 element type

enumerator i16#: i16 element type

enumerator i32#: i32 element type

enumerator i64#: i64 element type

enumerator u1#: binary element type

enumerator u2#: u2 element type

enumerator u3#: u3 element type

enumerator u4#: u4 element type

enumerator u6#: u6 element type

enumerator u8#: u8 element type

enumerator u16#: u16 element type

enumerator u32#: u32 element type

enumerator u64#: u64 element type

enumerator nf4#: nf4 element type

enumerator f8e4m3#: f8e4m3 element type

enumerator f8e5m2#: f8e5m2 element type

enumerator string#: string element type

enumerator f4e2m1#: f4e2m1 element type

enumerator f8e8m0#: f8e8m0 element type

Functions

constexpr Type undefined(Type_t::undefined)#: undefined element type

constexpr Type dynamic(Type_t::dynamic)#: dynamic element type

constexpr Type boolean(Type_t::boolean)#: boolean element type

constexpr Type bf16(Type_t::bf16)#: bf16 element type

constexpr Type f16(Type_t::f16)#: f16 element type

constexpr Type f32(Type_t::f32)#: f32 element type

constexpr Type f64(Type_t::f64)#: f64 element type

constexpr Type i4(Type_t::i4)#: i4 element type

constexpr Type i8(Type_t::i8)#: i8 element type

constexpr Type i16(Type_t::i16)#: i16 element type

constexpr Type i32(Type_t::i32)#: i32 element type

constexpr Type i64(Type_t::i64)#: i64 element type

constexpr Type u1(Type_t::u1)#: binary element type

constexpr Type u2(Type_t::u2)#: u2 element type

constexpr Type u3(Type_t::u3)#: u3 element type

constexpr Type u4(Type_t::u4)#: u4 element type

constexpr Type u6(Type_t::u6)#: u6 element type

constexpr Type u8(Type_t::u8)#: u8 element type

constexpr Type u16(Type_t::u16)#: u16 element type

constexpr Type u32(Type_t::u32)#: u32 element type

constexpr Type u64(Type_t::u64)#: u64 element type

constexpr Type nf4(Type_t::nf4)#: nf4 element type

constexpr Type f8e4m3(Type_t::f8e4m3)#: f8e4m3 element type

constexpr Type f8e5m2(Type_t::f8e5m2)#: f8e4m3 element type

constexpr Type string(Type_t::string)#: string element type

constexpr Type f4e2m1(Type_t::f4e2m1)#: f4e2m1 element type

constexpr Type f8e8m0(Type_t::f8e8m0)#: f8e8m0 element type

template<typename T> Type from()#

template<> OPENVINO_API Type from< char > ()

template<> OPENVINO_API Type from< bool > ()

template<> OPENVINO_API Type from< float > ()

template<> OPENVINO_API Type from< double > ()

template<> OPENVINO_API Type from< int8_t > ()

template<> OPENVINO_API Type from< int16_t > ()

template<> OPENVINO_API Type from< int32_t > ()

template<> OPENVINO_API Type from< int64_t > ()

template<> OPENVINO_API Type from< uint8_t > ()

template<> OPENVINO_API Type from< uint16_t > ()

template<> OPENVINO_API Type from< uint32_t > ()

template<> OPENVINO_API Type from< uint64_t > ()

template<> OPENVINO_API Type from< ov::bfloat16 > ()

template<> OPENVINO_API Type from< ov::float16 > ()

template<> OPENVINO_API Type from< ov::float8_e4m3 > ()

template<> OPENVINO_API Type from< ov::float8_e5m2 > ()

template<> OPENVINO_API Type from< std::string > ()

template<> OPENVINO_API Type from< ov::float4_e2m1 > ()

template<> OPENVINO_API Type from< ov::float8_e8m0 > ()

OPENVINO_API Type fundamental_type_for (const Type &type)

OPENVINO_API std::ostream & operator<< (std::ostream &out, const ov::element::Type &obj)

OPENVINO_API std::istream & operator>> (std::istream &out, ov::element::Type &obj)

template<class ...Args> bool is_type_list_not_empty(Args&&... args)#

template<class T> bool is_max_of(const element::Type_t &type, const T &value)#

Check if value has got maximum value of ov::element::Type_t.

Template Parameters:

T – Input value type.

Parameters:

type – ov::element type to get its maximum.
value – Input value for check.

Returns:

True if input value has got maximum number specified by ov::element type otherwise false.

template<class T> bool is_min_of(const element::Type_t type, const T &value)#

Check if value has got minimum value of ov::element::Type_t.

Template Parameters:

T – Input value type.

Parameters:

type – ov::element type to get its minimum.
value – Input value for check.

Returns:

True if input value has got minimum number specified by ov::element type otherwise false.

template<class T, class U = T> U get_value_or_limit_of(const element::Type_t &type, const T &value)#

Checks input value for element type maximum or minimum and return limit or value.

Template Parameters:

T – Type of input value.
U – Type of return value. Default same as T.

Parameters:

type – Type of ov::element::Type_t
value – Input value for check.

Returns:

If value is maximum or minimum get limit of U otherwise value as U.

template<Type_t...> struct IfTypeOf#

Primary template defines suppoted element types.

The list of element types is used to check if runtime value of element type is one in the list. Base on this check the Visitor::visit function is called for specific element type.

Template Parameters:: List – of supported ov::element types.

template<Type_t ET, Type_t... Others> struct IfTypeOf<ET, Others...>#

#include <element_visitor.hpp>

Applies visitor action for supported element type defined by template parameters.

Template Parameters:

ET – Current ov::element type used for check with input.
Others – Others supported ov::element.

Public Static Functions

template<class Visitor, class ...Args> static inline auto apply(Type_t et, Args&&... args) -> typename Visitor::result_type#

Applies visitor action if input element type is same as ET.

Uses Visitor::visit<ET> function if et == ET, otherwise check input element type against Others.

Template Parameters:

Visitor – Visitor class implementing visit function.
Args – Types of visit parameters.

Parameters:

et – Input element type.
args – Visitor arguments.

Returns:

Value of result type returned by Visitor.

template<> struct IfTypeOf<>#

#include <element_visitor.hpp>

Applies visitor action for not supported ov::element type.

Public Static Functions

template<class Visitor, class ...Args> static inline auto apply(Type_t et, Args&&... args) -> typename Visitor::result_type#

Applies visitor default action if input element type is not not supported by IfTypeOf.

Uses Visitor::visit non-template function.

Template Parameters:

Visitor – Visitor class implementing visit function.
Args – Types of visit parameters.

Parameters:

et – Input element type.
args – Visitor arguments.

Returns:

Value of result type returned by Visitor.

template<Type_t ET, class T> class Iterator#

template<class R, R... value> struct NoAction#

#include <element_visitor.hpp>

Helper visitor which defines no action for not supported type.

Template Parameters:

R – Type of return value.
value – Default value returned.

template<> struct NoAction<void>#: #include <element_visitor.hpp>

Helper visitor which defines no action for not supported type if result is void type.

template<class R> struct NotSupported#

#include <element_visitor.hpp>

Helper visitor which throws ov::Exception for not supported element type.

Template Parameters:: R – Type of return value.

class Type#

#include <element_type.hpp>

Base class to define element type.

Public Functions

bool compatible(const element::Type &t) const#

Checks whether this element type is merge-compatible with t.

Parameters:: t – The element type to compare this element type to.
Returns:: true if this element type is compatible with t, else false.

Public Static Functions

static bool merge(element::Type &dst, const element::Type &t1, const element::Type &t2)#

Merges two element types t1 and t2, writing the result into dst and returning true if successful, else returning false.

To “merge” two element types t1 and t2 is to find the least restrictive element type t that is no more restrictive than t1 and t2, if t exists. More simply:

merge(dst,element::Type::dynamic,t) writes t to dst and returns true

merge(dst,t,element::Type::dynamic) writes t to dst and returns true

merge(dst,t1,t2) where t1, t2 both static and equal writes t1 to dst and returns true

merge(dst,t1,t2) where t1, t2 both static and unequal does nothing to dst, and returns false

namespace exec_model_info#

Variables

static const char ORIGINAL_NAMES[] = "originalLayersNames"#: Used to get a string of layer names separated by a comma from the original IR, which were fused/merged to the current executable primitive.

static const char IMPL_TYPE[] = "primitiveType"#: Used to get a type of the executable primitive.

static const char OUTPUT_PRECISIONS[] = "outputPrecisions"#: Used to get output precisions of the executable primitive.

static const char PERF_COUNTER[] = "execTimeMcs"#: Used to get a value of execution time of the executable primitive, where Mcs = microseconds (1μs=0.000001s).

static const char OUTPUT_LAYOUTS[] = "outputLayouts"#: Used to get output layouts of primitive.

static const char EXECUTION_ORDER[] = "execOrder"#: Used to get an execution order of primitive.

static const char LAYER_TYPE[] = "layerType"#: Used to get a type of primitive.

static const char RUNTIME_PRECISION[] = "runtimePrecision"#: Used to get runtime precision of the executable primitive.

class ExecutionNode : public ov::op::Op #

#include <exec_model_info.hpp>

The Execution node which is used to represent node in execution graph.

It contains the following type of information in node runtime information:

ExecGraphInfoSerialization::ORIGINAL_NAMES
ExecGraphInfoSerialization::IMPL_TYPE
ExecGraphInfoSerialization::OUTPUT_PRECISIONS
ExecGraphInfoSerialization::PERF_COUNTER
ExecGraphInfoSerialization::OUTPUT_LAYOUTS
ExecGraphInfoSerialization::EXECUTION_ORDER
ExecGraphInfoSerialization::LAYER_TYPE
ExecGraphInfoSerialization::RUNTIME_PRECISION

Public Functions

ExecutionNode()#: A default constructor with no node inputs and 0 output ports.

ExecutionNode(const ov::OutputVector &arguments, size_t output_size = 1)#

Constructs a new execution node with a given parameters.

Parameters:

arguments – [in] Inputs nodes
output_size – [in] A number of output ports

std::shared_ptr<ov::Node> clone_with_new_inputs(const ov::OutputVector &inputs) const override#

Creates a new execution node with the same state, but different input nodes.

Parameters:: inputs – [in] The input nodes
Returns:: A newly created execution node

virtual bool visit_attributes(ov::AttributeVisitor&) override#

Visits attributes of the node.

Parameters:: visitor – [in] An attribute visitor
Returns:: Returns true if an operation has completed successfully

namespace frontend#

Typedefs

template<typename OVOpType = void> using OpExtension = ov::frontend::OpExtensionBase<ov::frontend::ConversionExtension, OVOpType>#

using FrontEndFactory = std::function<FrontEnd::Ptr()>#

using FrontEndVersion = uint64_t#: Each frontend plugin is responsible to export get_api_version function returning version of frontend API used for this plugin If version is not matched with OV_FRONTEND_API_VERSION - plugin will not be loaded by FrontEndManager.

using NamedOutputVector = std::vector<NamedOutput>#

using CreatorFunction = std::function<OutputVector(const NodeContext&)>#

using CreatorFunctionNamed = std::function<std::map<std::string, OutputVector>(const NodeContext&)>#

using CreatorFunctionNamedAndIndexed = std::function<NamedOutputVector(const NodeContext&)>#

Functions

inline const ov::OpSet &get_opset_by_name(const std::string &opset_name)#

The helper function to return an instance of OpSet class initialized with operations from provided opset by name.

Parameters:: opset_name – Opset name (opsetN) to initialize OpSet class.

inline std::shared_ptr<ov::Node> create_ov_node_by_name(const std::string &ov_type_name)#

The helper function to create an instance of ov::Node class initialized by provided type name. Expected formats:

opsetN::OpName
opsetN.OpName
OpName

Parameters:: ov_type_name – Type name of created ov::Node.

inline OutputVector indexed_from_named(const NamedOutputVector &outputs)#

inline NamedOutputVector named_from_indexed(const OutputVector &outputs)#

class ComplexTypeMark : public ov::op::util::FrameworkNode #

#include <complex_type_mark.hpp>

Public Functions

inline virtual void validate_and_infer_types() override#

Verifies that attributes and inputs are consistent and computes output shapes and element types. Must be implemented by concrete child classes so that it can be run any number of times.

Throws if the node is invalid.

class ConversionExtension : public ov::frontend::ConversionExtensionBase #: #include <conversion.hpp>

class ConversionExtensionBase : public ov::Extension #: #include <conversion.hpp>

Subclassed by ov::frontend::ConversionExtension

class DecoderBase : public ov::frontend::IDecoder #

#include <decoder.hpp>

Public Functions

virtual ov::Any get_attribute(const std::string &name) const = 0#

Get attribute value by name.

Parameters:: name – Attribute name
Returns:: Shared pointer to appropriate value converted to openvino data type if it exists, ‘nullptr’ otherwise

virtual size_t get_input_size() const = 0#: Get a number of inputs.

virtual void get_input_node(size_t input_port_idx, std::string &producer_name, std::string &producer_output_port_name, size_t &producer_output_port_index) const = 0#

Get a producer name and its output port index.

Parameters:

input_port_idx – Input port index by which data is consumed
producer_name – A producer name
producer_output_port_name – Output port name if exists
producer_output_port_index – Output port index from which data is generated

virtual const std::string &get_op_type() const = 0#: Get operation type.

virtual const std::string &get_op_name() const = 0#: Get node name.

virtual ~DecoderBase()#: Destructor.

class DecoderTransformationExtension : public ov::Extension #

#include <decoder_transformation.hpp>

Holds a transformation that is applied just after the original model graph is decoded. This class is a holder for transformation. The transformation can be specified as FunctionPass or MathcerPass derivatives or as a function that can be used to build corresponding FunctionPass or MatcherPass object. The type of the extension is determined in the moment of creation by calling corresponding ctor.

Public Functions

explicit DecoderTransformationExtension(const std::function<bool(std::shared_ptr<ov::Model>)> &function_pass)#: Create a custom functional pass where code of the pass is implemented as a function.

explicit DecoderTransformationExtension(const std::function<void(ov::pass::MatcherPass*)> &matcher_pass_initializer)#: Create a custom matcher pass where the code of matcher pass initialization is a given function.

template<typename Transformation, typename std::enable_if<std::is_base_of<ov::pass::PassBase, Transformation>::value, bool>::type = true> inline explicit DecoderTransformationExtension(const Transformation &transformation)#: Register existing transformation object which will be copied and kept for further registration.

void register_pass(ov::pass::Manager &manager) const#: Register pass from this object in a given pass manager object.

struct ExtensionHolder#: #include <holder.hpp>

class FrontEnd#

#include <frontend.hpp>

An interface for identifying a frontend for a particular framework. Provides an ability to load and convert of input model.

Unnamed Group

void add_extension(const std::string &library_path)#

Registers extension.

Parameters:: library_path – path to library with ov::Extension

Public Functions

FrontEnd()#: Default constructor.

template<typename ...Types> inline bool supported(const Types&... vars) const#

Validates if FrontEnd can recognize model with parameters specified. Same parameters should be used to load model.

Parameters:: vars – Any number of parameters of any type. What kind of parameters are accepted is determined by each FrontEnd individually, typically it is std::string containing path to the model file. For more information please refer to specific FrontEnd documentation.
Returns:: true if model recognized, false - otherwise.

template<typename ...Types> inline InputModel::Ptr load(const Types&... vars) const#

Loads an input model by any specified arguments. Each FrontEnd separately defines what arguments it can accept.

Parameters:: vars – Any number of parameters of any type. What kind of parameters are accepted is determined by each FrontEnd individually, typically it is std::string containing path to the model file. For more information please refer to specific FrontEnd documentation.
Returns:: Loaded input model.

virtual std::shared_ptr<ov::Model> convert(const InputModel::Ptr &model) const#

Completely convert and normalize entire Model, throws if it is not possible.

Parameters:: model – Input model
Returns:: fully converted OV Model

virtual void convert(const std::shared_ptr<ov::Model> &partially_converted) const#

Completely convert the remaining, not converted part of a Model.

Parameters:: partiallyConverted – partially converted OV Model

virtual std::shared_ptr<ov::Model> convert_partially(const InputModel::Ptr &model) const#

Convert only those parts of the model that can be converted leaving others as-is wrapped by FrameworkNode. Converted parts are normalized by additional transformations like it is done in convert method. If part of the graph cannot be converted, it is not guaranteed that the converted regions are completely normalized. Normalize should be called for each completely converted parts individually in this case.

Parameters:: model – Input model
Returns:: partially converted OV Model

virtual std::shared_ptr<ov::Model> decode(const InputModel::Ptr &model) const#

Convert operations with one-to-one mapping with decoding nodes. Each decoding node is an OV node representing a single FW operation node with all attributes represented in FW-independent way.

Parameters:: model – Input model
Returns:: OV Model after decoding

virtual void normalize(const std::shared_ptr<ov::Model> &model) const#

Runs normalization passes on Model that was loaded with partial conversion.

Parameters:: Model – partially converted OV Model

virtual std::string get_name() const#

Gets name of this FrontEnd. Can be used by clients if frontend is selected automatically by FrontEndManager::load_by_model.

Returns:: Current frontend name. Empty string if not implemented

virtual void add_extension(const std::shared_ptr<ov::Extension> &extension)#

Parameters:: extension – base extension

void add_extension(const std::vector<std::shared_ptr<ov::Extension>> &extensions)#

Parameters:: extensions – vector of extensions

template<class T, typename std::enable_if<std::is_base_of<ov::Extension, T>::value, bool>::type = true> inline void add_extension(const T &extension)#

Registers extension.

Parameters:: extension – Extension class which is inherited from ov::BaseOpExtension class

template<class T, class ...Targs, typename std::enable_if<std::is_base_of<ov::Extension, T>::value, bool>::type = true> inline void add_extension(const T &extension, Targs... args)#

Registers extensions.

Parameters:: extension – Extension class which is inherited from ov::Extension class

class FrontEndManager#

#include <manager.hpp>

Frontend management class, loads available frontend plugins on construction Allows load of frontends for particular framework, register new and list available frontends This is a main frontend entry point for client applications.

Public Functions

FrontEndManager()#: Default constructor. Searches and loads of available frontends.

FrontEndManager(FrontEndManager&&) noexcept#: Default move constructor.

FrontEndManager &operator=(FrontEndManager&&) noexcept#: Default move assignment operator.

~FrontEndManager()#: Default destructor.

FrontEnd::Ptr load_by_framework(const std::string &framework)#

Loads frontend by name of framework and capabilities.

Parameters:: framework – Framework name. Throws exception if name is not in list of available frontends
Returns:: Frontend interface for further loading of models

template<typename ...Types> inline FrontEnd::Ptr load_by_model(const Types&... vars)#

Loads frontend by model fragments described by each FrontEnd documentation. Selects and loads appropriate frontend depending on model file extension and other file info (header)

Parameters:: vars – Any number of parameters of any type. What kind of parameters are accepted is determined by each FrontEnd individually, typically it is std::string containing path to the model file. For more information please refer to specific FrontEnd documentation.
Returns:: Frontend interface for further loading of model. Returns ‘nullptr’ if no suitable frontend is found

std::vector<std::string> get_available_front_ends()#: Gets list of registered frontends. Any not loaded frontends will be loaded by this call.

void register_front_end(const std::string &name, FrontEndFactory creator)#

Parameters:

name – Name of front end
creator – Creation factory callback. Will be called when frontend is about to be created

void register_front_end(const std::string &name, const std::string &library_path)#

Parameters:

name – Name of front end
library_path – Path (absolute or relative) or name of a frontend library. If name is provided, depending on platform, it will be wrapped with shared library suffix and prefix to identify library full name

struct FrontEndPluginInfo#: #include <manager.hpp>

Each frontend plugin is responsible to export get_front_end_data function returning heap-allocated pointer to this structure. Will be used by FrontEndManager during loading of plugins.

class FWVisitor : public ov::AttributeVisitor #

#include <op.hpp>

Public Functions

inline virtual void on_adapter(const std::string &name, ValueAccessor<void> &adapter) override#

handles all specialized on_adapter methods implemented by the visitor.

The adapter implements get_type_info(), which can be used to determine the adapter directly or via is_type and as_type on any platform

class FWVisitorInputAttributes : public ov::AttributeVisitor #

#include <op.hpp>

Public Functions

inline virtual void on_adapter(const std::string &name, ValueAccessor<void> &adapter) override#

handles all specialized on_adapter methods implemented by the visitor.

The adapter implements get_type_info(), which can be used to determine the adapter directly or via is_type and as_type on any platform

class GeneralFailure : public ov::AssertFailure #: #include <exception.hpp>

class GraphIterator : private ov::RuntimeAttribute #

#include <graph_iterator.hpp>

Abstract representation for an input model graph that gives nodes in topologically sorted order.

Public Functions

virtual size_t size() const = 0#: Get a number of operation nodes in the graph.

virtual void reset() = 0#: Set iterator to the start position.

virtual void next() = 0#: Move to the next node in the graph.

virtual bool is_end() const = 0#: Returns true if iterator goes out of the range of available nodes.

virtual std::shared_ptr<DecoderBase> get_decoder() const = 0#: Return a pointer to a decoder of the current node.

virtual std::shared_ptr<GraphIterator> get_body_graph_iterator(const std::string &func_name) const = 0#: Checks if the main model graph contains a function of the requested name in the library Returns GraphIterator to this function and nullptr, if it does not exist.

virtual std::vector<std::string> get_input_names() const = 0#: Returns a vector of input names in the original order.

virtual std::vector<std::string> get_output_names() const = 0#: Returns a vector of output names in the original order.

virtual std::map<std::string, std::string> get_input_names_map() const#: Returns a map from internal tensor name to (user-defined) external name for inputs.

virtual std::map<std::string, std::string> get_output_names_map() const#: Returns a map from internal tensor name to (user-defined) external name for outputs.

class HashTable : public ov::frontend::Variable #

#include <hash_table.hpp>

HashTable is a special type of Variable that has a complex value including keys and values. Keys and values are represented with two separate graph at each time step.

Public Functions

inline virtual void validate_and_infer_types() override#

Verifies that attributes and inputs are consistent and computes output shapes and element types. Must be implemented by concrete child classes so that it can be run any number of times.

Throws if the node is invalid.

inline virtual ov::Output<ov::Node> get_value() override#: Returns a value at the current step of conversion.

inline ov::Output<ov::Node> get_keys()#: Returns a value corresponding keys of hash table.

inline ov::Output<ov::Node> get_values()#: Returns a value corresponding values of hash table.

class IDecoder#

#include <decoder.hpp>

Plays a role of node, block and module decoder.

Subclassed by ov::frontend::DecoderBase

class InitializationFailure : public ov::AssertFailure #: #include <exception.hpp>

class InputModel#

#include <input_model.hpp>

InputModel class represents an original, not yet converted model graph in a framework format given services to find places of interest in a graph or specialize/edit the model before conversion.

Editing requests may affect ability to convert the original model to OV Model. Aim to provide these editing capabilities is to unlock conversion for models that are not natively supported “as-is” because of undefined shapes, types or operations.

Specific front-end implementation is supposed to have a lazy implementation for all methods, not doing a complete load of a model without an explicit method call. For example, the list of all inputs are not pre-fetched by InputModel derived class instance creation, but only when get_inputs method is called. But it is not an obligation, the most convenient way should be chosen depending on the framework model representation.

All editing requests affect the model representation that is held behind the scene successive method calls observe a new graph structure.

Note

Class methods are divided into several groups: searching for places, naming and annotation, topology editing, setting tensor properties.

Public Functions

virtual std::vector<Place::Ptr> get_inputs() const#

Returns all inputs for a model An input is a place in a graph where data is supposed to flow inside graph from outside. It can be a tensor, port, operation; which kind of place can be an output is FW dependent. Usually framework models have a dedicated artifact to code model input, it can be a tensor without producer, that writes to it in ONNX, or a special operation like Placeholder in TensorFlow.

Returns:: A vector of input place references

virtual std::vector<Place::Ptr> get_outputs() const#

Returns all output for a model An output is a terminal place in a graph where data escapes the flow. It can be a tensor, port, operation; which kind of place can be an output is FW dependent. In comparison to a graph input, the output is less formally defined thing and determination of initial list of outputs may include some conventions defined by a frontend itself, not a framework. For example, all output ports without consumers may be considered as outputs.

Returns:: A vector of output place references

virtual Place::Ptr get_place_by_tensor_name(const std::string &tensor_name) const#

Returns a tensor place by a tensor name following framework conventions, or nullptr if a tensor with this name doesn’t exist.

Parameters:: tensor_name – Name of tensor
Returns:: Tensor place corresponding to specified tensor name or nullptr if not exists

virtual Place::Ptr get_place_by_input_index(size_t input_idx) const#

Returns a tensor place by an input index.

Parameters:: input_idx – Index of model input
Returns:: Tensor place corresponding to specified input index or nullptr

virtual Place::Ptr get_place_by_operation_name(const std::string &operation_name) const#

Returns an operation place by an operation name following framework conventions, or nullptr if an operation with this name doesn’t exist.

Parameters:: operation_name – Name of operation
Returns:: Place representing operation or nullptr if not exists

virtual Place::Ptr get_place_by_operation_name_and_input_port(const std::string &operation_name, int input_port_index)#

Returns an input port place by operation name and appropriate port index.

Parameters:

operation_name – Name of operation
input_port_index – Index of input port for this operation

Returns:

Place representing input port of operation or nullptr if not exists

virtual Place::Ptr get_place_by_operation_name_and_output_port(const std::string &operation_name, int output_port_index)#

Returns an output port place by operation name and appropriate port index.

Parameters:

operation_name – Name of operation
output_port_index – Index of output port for this operation

Returns:

Place representing output port of operation or nullptr if not exists

virtual void set_name_for_tensor(const Place::Ptr &tensor, const std::string &new_name)#

Sets name for tensor. Overwrites existing names of this place.

Parameters:

tensor – Tensor place
new_name – New name for this tensor

virtual void add_name_for_tensor(const Place::Ptr &tensor, const std::string &new_name)#

Adds new name for tensor.

Parameters:

tensor – Tensor place
new_name – New name to be added to this place

virtual void set_name_for_operation(const Place::Ptr &operation, const std::string &new_name)#

Sets name for operation. Overwrites existing names of this place.

Parameters:

operation – Operation place
new_name – New name for this operation

virtual void free_name_for_tensor(const std::string &name)#

Unassign specified name from tensor place(s)

Parameters:: name – Name of tensor

virtual void free_name_for_operation(const std::string &name)#

Unassign specified name from operation place(s)

Parameters:: name – Name of operation

virtual void set_name_for_dimension(const Place::Ptr &place, size_t shape_dim_index, const std::string &dim_name)#

Set name for a particular dimension of a place (e.g. batch dimension)

Parameters:

place – Model’s place
shape_dim_index – Dimension index
dim_name – Name to assign on this dimension

virtual void cut_and_add_new_input(const Place::Ptr &place, const std::string &new_name_optional = "")#

Cut immediately before this place and assign this place as new input; prune all nodes that don’t contribute to any output.

Parameters:

place – New place to be assigned as input
new_name_optional – Optional new name assigned to this input place

virtual void cut_and_add_new_output(const Place::Ptr &place, const std::string &new_name_optional = "")#

Cut immediately after this place and assign this place as new output; prune all nodes that don’t contribute to any output.

Parameters:

place – New place to be assigned as output
new_name_optional – Optional new name assigned to this output place

virtual Place::Ptr add_output(const Place::Ptr &place)#

Assign this place as new output or add necessary nodes to represent a new output.

Parameters:: place – Anchor point to add an output
Returns:: new output place, may be the same as a given place

virtual void remove_output(const Place::Ptr &place)#

Removes any sinks directly attached to this place with all inbound data flow if it is not required by any other output.

Parameters:: place – Model place

virtual void override_all_outputs(const std::vector<Place::Ptr> &outputs)#

Replaces all existing outputs with new ones removing all data flow that is not required for new outputs.

Parameters:

outputs – Vector with places that will become new outputs; may intersect existing outputs.
outputs – Array of new output places

virtual void override_all_inputs(const std::vector<Place::Ptr> &inputs)#

Modifies the graph to use new inputs instead of existing ones. New inputs should completely satisfy all existing outputs.

Parameters:: inputs – Array of new input places

virtual void extract_subgraph(const std::vector<Place::Ptr> &inputs, const std::vector<Place::Ptr> &outputs)#

Leaves only subgraph that are defined by new inputs and new outputs.

Parameters:

inputs – Array of new input places
outputs – Array of new output places

virtual void set_partial_shape(const Place::Ptr &place, const ov::PartialShape &shape)#

Defines all possible shape that may be used for this place; place should be uniquely refer to some data. This partial shape will be converted to corresponding shape of results OV nodes and will define shape inference when the model is converted to OV.

Parameters:

place – Model place
shape – Partial shape for this place

virtual ov::PartialShape get_partial_shape(const Place::Ptr &place) const#

Returns current partial shape used for this place.

Parameters:: place – Model place
Returns:: Partial shape for this place

virtual void set_element_type(const Place::Ptr &place, const ov::element::Type &type)#

Sets new element type for a place.

Parameters:

place – Model place
type – New element type

virtual ov::element::Type get_element_type(const Place::Ptr &place) const#

Returns current element type used for this place.

Parameters:: place – Model place
Returns:: Element type for this place

virtual void set_tensor_value(const Place::Ptr &place, const void *value)#

Freezes a tensor with statically defined value or replace existing value for already constant node or tensor.

Parameters:

place – Tensor place
value – Value for tensor place representing a memory buffer

virtual void set_tensor_partial_value(const Place::Ptr &place, const void *min_value, const void *max_value)#

Defines partial value (lower bound and upper bound) for a tensor place TODO: more details for min_value and max_value format; who defines shape?

Parameters:

place – Tensor place
min_value – Lower bound of partial value for tensor place
max_value – Upper bound of partial value for tensor place

struct NamedOutput#: #include <node_context.hpp>

class NodeContext#

#include <node_context.hpp>

Public Functions

inline virtual size_t get_input_size() const#: Returns a number of inputs.

inline virtual size_t get_input_size(const std::string &port_name) const#: Returns a number of inputs.

inline virtual Output<Node> get_input(int idx) const#: Returns exactly one input with a given idx; throws if there is no inputs or there are more than one input.

inline virtual Output<Node> get_input(const std::string &name, int idx) const#: Returns exactly one input with a given name and idx; throws if there is no inputs or there are more than one input.

inline virtual Output<Node> get_input(const std::string &name) const#: Returns exactly one input with a given name; throws if there is no inputs or there are more than one input.

inline virtual Output<Node> get_input_by_reference(int idx) const#: Returns output of Variable node (or Variable value). Variable is a special node that stores a value represented with a sub-graph. Variable has a concrete value at each conversion step. The current (consuming) operation node can change its value so consumers of this Variable will have a new value at next conversion steps. See ov::frontend::Variable class for more details.

inline virtual Any get_values_from_const_input(int idx) const#: Returns values from Constant input with the given index as ov::Any. Throws an exception if the input cannot be represented as Constant.

template<class T> inline T get_attribute(const std::string &name) const#: Returns node attribute by name.

template<class T> inline T get_attribute(const std::string &name, const T &def) const#: Returns node attribute by name. Returns ‘def’ value if attribute does not exist.

inline bool has_attribute(const std::string &name) const#: Check if an attribute of a given name exist.

virtual ov::Any get_attribute_as_any(const std::string &name) const = 0#: Returns node attribute by name as ov::Any.

inline virtual size_t get_subgraph_size() const#: Returns the number of sub-graphs that can be enumerated with get_subgraph.

inline virtual std::shared_ptr<Model> get_subgraph(int idx) const#: Returns subgraph converted on demand by the first access If there is no query for specific sub-graph it shouldn’t be converted idx should be in range 0..get_subgraph_size()-1.

class NotImplementedFailure : public ov::AssertFailure #: #include <exception.hpp>

class OpConversionFailure : public ov::AssertFailure #: #include <exception.hpp>

class OpConversionFunction#: #include <op.hpp>

class OpConversionFunctionInputAttributes#: #include <op.hpp>

class OpConversionFunctionNamed#: #include <op.hpp>

template<typename BaseConversionType, typename OVOpType = void> class OpExtensionBase : public BaseConversionType #: #include <op.hpp>

template<typename BaseConversionType> class OpExtensionBase<BaseConversionType, void> : public BaseConversionType #: #include <op.hpp>

class OpValidationFailure : public ov::AssertFailure #: #include <exception.hpp>

class Place#

#include <place.hpp>

An interface for identifying a place in a graph and iterate over it; can refer to an operation node, tensor, port etc.

Place can refer to Tensor, Input Edge, Input Port, Operation, Output Port, Output Edge

           [Tensor A]
               |
               | [Input Edge]
               |
               V
      -------------------
      [  [Input Port 0] ]
      [                 ]
      [   Operation A   ]
      [                 ]
      [ [Output Port 0] ]
      -------------------
               |
               | [Output Edge]
               |
               V
           [Tensor B]
               |
               | [Input Edge]
               |
               V
      -------------------
      [  [Input Port 0] ]
      [                 ]
      [   Operation B   ]
      [                 ]
      [ [Output Port 0] ]
      -------------------
               |
               | [Output Edge]
               |
               V
           [Tensor C]

Note

Each front end implementation provides specialization of this interface to represent a place in a model graph. Various methods in the front end classes accept and retrieve instances of Place to point to particular node part which should be modified or satisfies some criteria. For example, this class is used to report model inputs and outputs, for searching operations and tensors by name, for setting shape etc.

Public Functions

virtual std::vector<std::string> get_names() const#

All associated names (synonyms) that identify this place in the graph in a framework specific way.

Returns:: A vector of strings each representing a name that identifies this place in the graph. Can be empty if there are no names associated with this place or name cannot be attached.

virtual std::vector<Ptr> get_consuming_operations() const#

Returns references to all operation nodes that consume data from this place.

Note

It can be called for any kind of graph place searching for the first consuming operations. It is optional if place has only one output port

Returns:: A vector with all operation node references that consumes data from this place

virtual std::vector<Ptr> get_consuming_operations(int output_port_index) const#

Returns references to all operation nodes that consume data from this place for specified output port.

Note

It can be called for any kind of graph place searching for the first consuming operations.

Parameters:: output_port_index – If place is an operational node it specifies which output port should be considered.
Returns:: A vector with all operation node references that consumes data from this place

virtual std::vector<Ptr> get_consuming_operations(const std::string &outputName) const#

Returns references to all operation nodes that consume data from this place for specified output port.

Note

It can be called for any kind of graph place searching for the first consuming operations.

Parameters:: outputName – If a given place is itself an operation node, this specifies name of output port group
Returns:: A vector with all operation node references that consumes data from this place

virtual std::vector<Ptr> get_consuming_operations(const std::string &outputName, int outputPortIndex) const#

Returns references to all operation nodes that consume data from this place for specified output port.

Note

It can be called for any kind of graph place searching for the first consuming operations.

Parameters:

outputName – If a given place is itself an operation node, this specifies name of output port group, each group can have multiple ports
outputPortIndex – If place is an operational node it specifies which output port should be considered.

Returns:

A vector with all operation node references that consumes data from this place

virtual Ptr get_target_tensor() const#

Returns a tensor place that gets data from this place; applicable for operations, output ports and output edges which have only one output port.

Returns:: A tensor place which hold the resulting value for this place

virtual Ptr get_target_tensor(const std::string &outputName) const#

Returns a tensor place that gets data from this place; applicable for operations.

Parameters:: outputName – Name of output port group
Returns:: A tensor place which hold the resulting value for this place

virtual Ptr get_target_tensor(const std::string &outputName, int outputPortIndex) const#

Returns a tensor place that gets data from this place; applicable for operations.

Parameters:

outputName – Name of output port group, each group can have multiple ports
outputPortIndex – Output port index if the current place is an operation node and has multiple output ports

Returns:

A tensor place which hold the resulting value for this place

virtual Ptr get_target_tensor(int output_port_index) const#

Returns a tensor place that gets data from this place; applicable for operations.

Parameters:: output_port_index – Output port index if the current place is an operation node and has multiple output ports
Returns:: A tensor place which hold the resulting value for this place

virtual Ptr get_source_tensor() const#

Returns a tensor place that supplies data for this place; applicable for operations, input ports and input edges which have only one input port.

Returns:: A tensor place which supplies data for this place

virtual Ptr get_source_tensor(int input_port_index) const#

Returns a tensor place that supplies data for this place; applicable for operations.

Parameters:: input_port_index – Input port index for operational nodes.
Returns:: A tensor place which supplies data for this place

virtual Ptr get_source_tensor(const std::string &inputName) const#

Returns a tensor place that supplies data for this place; applicable for operations.

Parameters:: inputName – Name of input port group
Returns:: A tensor place which supplies data for this place

virtual Ptr get_source_tensor(const std::string &inputName, int inputPortIndex) const#

Returns a tensor place that supplies data for this place; applicable for operations.

Parameters:

inputName – If a given place is itself an operation node, this specifies name of output port group, each group can have multiple ports
inputPortIndex – Input port index for operational nodes.

Returns:

A tensor place which supplies data for this place

virtual Ptr get_producing_operation() const#

Get an operation node place that immediately produces data for this place; applicable if place has only one input port.

Returns:: An operation place that produces data for this place

virtual Ptr get_producing_operation(int input_port_index) const#

Get an operation node place that immediately produces data for this place.

Parameters:: input_port_index – If a given place is itself an operation node, this specifies a port index
Returns:: An operation place that produces data for this place

virtual Ptr get_producing_operation(const std::string &inputName) const#

Get an operation node place that immediately produces data for this place.

Parameters:: inputName – If a given place is itself an operation node, this specifies name of output port group
Returns:: An operation place that produces data for this place

virtual Ptr get_producing_operation(const std::string &inputName, int inputPortIndex) const#

Get an operation node place that immediately produces data for this place.

Parameters:

inputName – If a given place is itself an operation node, this specifies name of output port group, each group can have multiple ports
inputPortIndex – If a given place is itself an operation node, this specifies a port index

Returns:

An operation place that produces data for this place

virtual Ptr get_producing_port() const#: Returns a port that produces data for this place.

virtual Ptr get_input_port() const#

For operation node returns reference to an input port; applicable if operation node has only one input port.

Returns:: Input port place or nullptr if not exists

virtual Ptr get_input_port(int input_port_index) const#

For operation node returns reference to an input port with specified index.

Parameters:: input_port_index – Input port index
Returns:: Appropriate input port place or nullptr if not exists

virtual Ptr get_input_port(const std::string &input_name) const#

For operation node returns reference to an input port with specified name; applicable if port group has only one input port.

Parameters:: input_name – Name of port group
Returns:: Appropriate input port place or nullptr if not exists

virtual Ptr get_input_port(const std::string &input_name, int input_port_index) const#

For operation node returns reference to an input port with specified name and index.

Parameters:

input_name – Name of port group, each group can have multiple ports
input_port_index – Input port index in a group

Returns:

Appropriate input port place or nullptr if not exists

virtual Ptr get_output_port() const#

For operation node returns reference to an output port; applicable for operations with only one output port.

Returns:: Appropriate output port place or nullptr if not exists

virtual Ptr get_output_port(int output_port_index) const#

For operation node returns reference to an output port with specified index.

Parameters:: output_port_index – Output port index
Returns:: Appropriate output port place or nullptr if not exists

virtual Ptr get_output_port(const std::string &output_name) const#

For operation node returns reference to an output port with specified name; applicable if port group has only one output port.

Parameters:: output_name – Name of output port group
Returns:: Appropriate output port place or nullptr if not exists

virtual Ptr get_output_port(const std::string &output_name, int output_port_index) const#

For operation node returns reference to an output port with specified name and index.

Parameters:

output_name – Name of output port group, each group can have multiple ports
output_port_index – Output port index

Returns:

Appropriate output port place or nullptr if not exists

virtual std::vector<Place::Ptr> get_consuming_ports() const#: Returns all input ports that consume data flows through this place.

virtual bool is_input() const#: Returns true if this place is input for a model.

virtual bool is_output() const#: Returns true if this place is output for a model.

virtual bool is_equal(const Ptr &another) const#

Returns true if another place is the same as this place.

Parameters:: another – Another place object

virtual bool is_equal_data(const Ptr &another) const#

Returns true if another place points to the same data.

Note

The same data means all places on path: output port -> output edge -> tensor -> input edge -> input port.

Parameters:: another – Another place object

class ProgressReporterExtension : public ov::Extension #

#include <progress_reporter.hpp>

Public Types

using progress_notifier_callback = std::function<void(float, unsigned int, unsigned int)>#

A progress reporting callback signature. A FunctionObject that matches this signature should be passed to the constructor of this extension. The extension will then invoke this as a callback each time the progress needs to be reported. The callback itself is responsible for consuming the reported values.

Param progress:: A float value in the range [0.0, 1.0] indicating the total progress of an operation.
Param total_steps:: The total number of steps that a given instance of this extension is tracking
Param completed_completed:: The current number of completed steps (out of the total number of steps to take)

Public Functions

inline ProgressReporterExtension()#: The default constructor which creates a reporter that doesn’t report progress.

void report_progress(float progress, unsigned int total_steps, unsigned int completed_steps) const#

The main method of this extension used to report the progress. This method forwards its arguments to the callback stored in this class.

Parameters:

progress – A float value in the range [0.0, 1.0] indicating the total progress of an operation.
total_steps – The total number of steps that a given instance of this extension is tracking
completed_steps – The current number of completed steps (out of the total number of steps to take)

class TelemetryExtension : public ov::Extension #: #include <telemetry.hpp>

Provides callback to report telemetry information back to Python code.

class Variable : public ov::op::util::FrameworkNode #

#include <variable.hpp>

Variable is a special node used in a conversion step It can have several values (or states) during the conversion. Variable value at some time step is represented with a graph.

Subclassed by ov::frontend::HashTable

Public Functions

inline virtual void validate_and_infer_types() override#

Verifies that attributes and inputs are consistent and computes output shapes and element types. Must be implemented by concrete child classes so that it can be run any number of times.

Throws if the node is invalid.

inline bool is_initialized() const#: Checks if variable is initialized with some value.

inline virtual ov::Output<ov::Node> get_value()#: Returns a value at the current step of conversion.

inline uint64_t get_init_counter() const#: Returns a counter value (a number of values that have assigned to this variable)

namespace type#

struct List#: #include <decoder.hpp>

struct PyNone#: #include <decoder.hpp>

struct PyScalar#: #include <decoder.hpp>

struct Str#: #include <decoder.hpp>

struct Tensor#: #include <decoder.hpp>

namespace gen_pattern#

Functions

static bool matcher_verbose_enabled()#

inline Symbol operator-(const Symbol &lhs)#

inline Symbol operator+(const Symbol &lhs, const Symbol &rhs)#

inline Symbol operator-(const Symbol &lhs, const Symbol &rhs)#

inline Symbol operator*(const Symbol &lhs, const Symbol &rhs)#

inline Symbol operator/(const Symbol &lhs, const Symbol &rhs)#

inline Symbol sqrt(Symbol lhs)#

inline std::shared_ptr<Node> GenInput(values_info vt = nullptr)#

inline std::shared_ptr<Node> makePattern()#

inline std::shared_ptr<Node> makePattern(ov::Rank rank)#

inline std::shared_ptr<Node> makePattern(values_info vt)#

inline std::shared_ptr<Node> makeConst(const ov::element::Type &type, const ov::PartialShape &pshape, std::function<bool(ov::op::v0::Constant &node)> pred)#

template<typename T> std::shared_ptr<Node> makeConst(const ov::element::Type &type, const ov::Shape &shape, std::initializer_list<T> values)#

inline std::shared_ptr<Node> makeConst(const std::vector<Symbol> &v)#

template<typename T> std::shared_ptr<Node> makeConst(const ov::element::Type &type, const ov::Shape &shape, const std::vector<T> &values)#

template<class T> std::shared_ptr<Node> makePattern(const std::vector<detail::PatternNode> &inputs, detail::AttrMap attrmap = {}, const char *vt = nullptr, const char *friendly_name = nullptr, int line_no = -1, const char *file = "")#

template<class T> std::shared_ptr<Node> makeOP(const std::vector<detail::PatternNode> &inputs, detail::AttrMap attrmap = {}, const char *friendly_name = nullptr)#

template<typename T> std::shared_ptr<Node> GenConst_tril(values_info vt)#

inline std::shared_ptr<Node> operator|(const Output<Node> &lhs, const Output<Node> &rhs)#

inline std::shared_ptr<Node> operator|(const std::shared_ptr<Node> &lhs, const std::shared_ptr<Node> &rhs)#

inline std::shared_ptr<Node> GenStridedSlice(detail::PatternNode data, detail::PatternNode start, detail::PatternNode stop, detail::PatternNode step, size_t axis, int line_no = -1, const char *file = "")#

inline std::shared_ptr<Node> GenSlice(detail::PatternNode data, Symbol start, Symbol stop, Symbol step, size_t axis, int line_no = -1, const char *file = "")#

class PatternValidator#: #include <gen_pattern.hpp>

class Symbol#: #include <gen_pattern.hpp>

struct values_info#: #include <gen_pattern.hpp>

namespace detail#

Typedefs

using SymbolObservationVector = std::vector<std::pair<Symbol, double>>#

using AttrMap = std::map<std::string, AttrAny>#

Functions

inline std::vector<std::string> split_string(const std::string &s, const std::string &delimiter)#

template<typename T> std::string vec2str(const std::vector<T> &vec, int cnt_limit = 9)#

template<typename T> void add_symbol_observed(SymbolObservationVector &sov, const Symbol &sym, const T &value)#

struct AttrAny#: #include <gen_pattern.hpp>

class AttrMatcher : public ov::AttributeVisitor #

#include <gen_pattern.hpp>

Public Functions

inline virtual void on_adapter(const std::string &name, ov::ValueAccessor<void> &adapter) override#

handles all specialized on_adapter methods implemented by the visitor.

The adapter implements get_type_info(), which can be used to determine the adapter directly or via is_type and as_type on any platform

class AttrSetter : public ov::AttributeVisitor #

#include <gen_pattern.hpp>

Public Functions

inline virtual void on_adapter(const std::string &name, ov::ValueAccessor<void> &adapter) override#

handles all specialized on_adapter methods implemented by the visitor.

The adapter implements get_type_info(), which can be used to determine the adapter directly or via is_type and as_type on any platform

class GenericPattern : public ov::pass::pattern::op::Pattern #: #include <gen_pattern.hpp>

struct PatternNode#: #include <gen_pattern.hpp>

namespace helpers#

Functions

template<typename ACT, typename ...T, size_t N_ARGS = NumOfLambdaArgs<ACT>::value> std::enable_if<N_ARGS == sizeof...(T) + 2, void>::type call_with_args(const ACT &body, size_t g_id, size_t iwork, T... arg)#

template<typename ACT, typename ...T, size_t N_ARGS = NumOfLambdaArgs<ACT>::value> std::enable_if<N_ARGS == sizeof...(T) + 1, void>::type call_with_args(const ACT &body, size_t g_id, size_t iwork, T... arg)#

template<typename ACT, typename ...T, size_t N_ARGS = NumOfLambdaArgs<ACT>::value> std::enable_if<N_ARGS == sizeof...(T), void>::type call_with_args(const ACT &body, size_t g_id, size_t iwork, T... arg)#

template<typename T> struct NumOfLambdaArgs#: #include <parallel.hpp>

template<typename C, typename R, typename... Args> *)(Args...) const >: #include <parallel.hpp>

namespace hint#

Namespace with hint properties.

Enums

enum class Priority#

Enum to define possible priorities hints.

Values:

enumerator LOW#: Low priority.

enumerator MEDIUM#: Medium priority.

enumerator HIGH#: High priority.

enumerator DEFAULT#: Default priority is MEDIUM.

enum class PerformanceMode#

Enum to define possible performance mode hints.

Values:

enumerator LATENCY#: Optimize for latency.

enumerator THROUGHPUT#: Optimize for throughput.

enumerator CUMULATIVE_THROUGHPUT#: Optimize for cumulative throughput.

enum class SchedulingCoreType#

This enum contains definition of core type can be used for CPU tasks on different devices.

Values:

enumerator ANY_CORE#: Any processors can be used.

enumerator PCORE_ONLY#: Only processors of performance-cores can be used.

enumerator ECORE_ONLY#: Only processors of efficient-cores can be used.

enum class ModelDistributionPolicy#

Values:

enumerator TENSOR_PARALLEL#

enumerator PIPELINE_PARALLEL#

enum class ExecutionMode#

Enum to define possible execution mode hints.

Values:

enumerator PERFORMANCE#: Optimize for max performance, may apply properties which slightly affect accuracy.

enumerator ACCURACY#: Optimize for max accuracy.

Variables

static constexpr Property<element::Type, PropertyMutability::RW> inference_precision = {"INFERENCE_PRECISION_HINT"}#: Hint for device to use specified precision for inference.

static constexpr Property<Priority> model_priority = {"MODEL_PRIORITY"}#: High-level OpenVINO model priority hint Defines what model should be provided with more performant bounded resource first.

static constexpr Property<PerformanceMode> performance_mode = {"PERFORMANCE_HINT"}#: High-level OpenVINO Performance Hints unlike low-level properties that are individual (per-device), the hints are something that every device accepts and turns into device-specific settings.

static constexpr Property<SchedulingCoreType> scheduling_core_type = {"SCHEDULING_CORE_TYPE"}#

This property defines CPU core type which can be used during inference.

Developer can use this property to select specific CPU cores for inference. Please refer SchedulingCoreType for all definition of core type.

The following code is an example to only use efficient-cores for inference on hybrid CPU. If user sets this configuration on a platform with only performance-cores, CPU inference will still run on the performance-cores.

ie.set_property(ov::hint::scheduling_core_type(ov::hint::SchedulingCoreType::ECORE_ONLY));

static constexpr Property<std::set<ModelDistributionPolicy>> model_distribution_policy = {"MODEL_DISTRIBUTION_POLICY"}#

This property defines model distribution policy for inference with multiple sockets/devices.

This property can be used to select model distribution policy between execution units (e.g. between CPU sockets/NUMA nodes or between different GPUs). — TENSOR_PARALLEL : Distribute tensor to multiple sockets/devices during model compilation. At inference time, sockets/devices process individual tensor in parallel. — PIPELINE_PARALLEL : Distribute tensor to multiple sockets/devices during model compilation. At inference time, sockets/devices process individual tensor one by one. And each socket/device processes a portion of a different tensor in parallel.

The following code is an example how TENSOR_PARALLEL or PIPELINE_PARALLEL model distribution policy might be enabled.

ie.set_property(ov::hint::model_distribution_policy({ov::hint::ModelDistributionPolicy::TENSOR_PARALLEL}));
ie.set_property(ov::hint::model_distribution_policy({ov::hint::ModelDistributionPolicy::PIPELINE_PARALLEL}));

static constexpr Property<bool> enable_cpu_pinning = {"ENABLE_CPU_PINNING"}#

This property allows CPU pinning during inference.

Developer can use this property to enable or disable CPU pinning during inference on Windows and Linux. MacOS does not support CPU pinning, and this property is always disabled. If user does not explicitly set value for this property, OpenVINO may choose any desired value based on internal logic.

The following is an example of CPU fixed behavior on a hybrid CPU (8 performance cores and 16 efficiency cores). For stream with 4 threads on performance cores, if CPU pinning is enabled, each thread is bound to a specific performance core. If CPU pinning is disabled, OS will schedule 4 threads on performance cores only. For stream with 24 threads on all cores, if CPU pinning is enabled, each thread is bound to a specific performance core. If CPU pinning is disabled, OS will schedule 24 threads on both performance cores and efficiency cores.

The following code is example to use this property.

ie.set_property(ov::hint::enable_cpu_pinning(true));
ie.set_property(ov::hint::enable_cpu_pinning(false));

static constexpr Property<bool> enable_hyper_threading = {"ENABLE_HYPER_THREADING"}#

This property define if using hyper threading during inference.

Developer can use this property to use or not use CPU pinning during inference. If user does not explicitly set value for this property, OpenVINO may choose any desired value based on internal logic.

The following code is example to use this property.

ie.set_property(ov::hint::enable_hyper_threading(true));
ie.set_property(ov::hint::enable_hyper_threading(false));

static constexpr Property<uint32_t> num_requests = {"PERFORMANCE_HINT_NUM_REQUESTS"}#: (Optional) property that backs the (above) Performance Hints by giving additional information on how many inference requests the application will be keeping in flight usually this value comes from the actual use-case (e.g. number of video-cameras, or other sources of inputs)

static constexpr Property<std::shared_ptr<ov::Model>> model = {"MODEL_PTR"}#: This key identifies shared pointer to the ov::Model, required for some properties (ov::max_batch_size and ov::optimal_batch_size)

static constexpr Property<bool, PropertyMutability::RW> allow_auto_batching = {"ALLOW_AUTO_BATCHING"}#: Special key for auto batching feature configuration. Enabled by default.

static constexpr Property<ExecutionMode> execution_mode = {"EXECUTION_MODE_HINT"}#: High-level OpenVINO Execution hint unlike low-level properties that are individual (per-device), the hints are something that every device accepts and turns into device-specific settings Execution mode hint controls preferred optimization targets (performance or accuracy) for given model.

static constexpr Property<uint64_t, PropertyMutability::RW> dynamic_quantization_group_size{"DYNAMIC_QUANTIZATION_GROUP_SIZE"}#

This property defines group size for dynamic quantization optimization.

Dynamic quantization optimization provides an ability to get performance benefit from int8 compute. In contrast with static quantization dynamic approach assumes activations are quantized during inference. Despite the fact dynamic quantization has some runtime overheads, it might provide better accuracy metrics. This property defines granularity (aka block size) for dynamic quantization algorithms. Lower group size values might result in better accuracy, but the drawback is worse performance. Group size equal 0 means dynamic quantization optimization is disabled.

static constexpr Property<element::Type, PropertyMutability::RW> kv_cache_precision = {"KV_CACHE_PRECISION"}#: Hint for device to use specified precision for kv cache compression.

static constexpr Property<float, PropertyMutability::RW> activations_scale_factor = {"ACTIVATIONS_SCALE_FACTOR"}#: This property scales down activations to prevent overflows when inference precision is f16.

namespace intel_auto#

Namespace with Intel AUTO specific properties.

Enums

enum class SchedulePolicy#

Enum to define the policy of scheduling inference request to target device in cumulative throughput mode on AUTO.

Values:

enumerator ROUND_ROBIN#

enumerator DEVICE_PRIORITY#

enumerator DEFAULT#: Default schedule policy is DEVICE_PRIORITY.

Variables

static constexpr Property<bool> enable_startup_fallback = {"ENABLE_STARTUP_FALLBACK"}#: auto device setting that enable/disable CPU as acceleration (or helper device) at the beginning

static constexpr Property<bool> enable_runtime_fallback = {"ENABLE_RUNTIME_FALLBACK"}#: auto device setting that enable/disable runtime fallback to other devices when infer fails on current selected device

static constexpr Property<SchedulePolicy> schedule_policy = {"SCHEDULE_POLICY"}#: High-level OpenVINO model policy hint Defines what scheduling policy should be used in AUTO CUMULATIVE_THROUGHPUT or MULTI case.

namespace intel_cpu#

Namespace with Intel CPU specific properties.

Variables

static constexpr Property<bool> denormals_optimization = {"CPU_DENORMALS_OPTIMIZATION"}#

This property define whether to perform denormals optimization.

Computation with denormals is very time consuming. FTZ(Flushing denormals to zero) and DAZ(Denormals as zero) could significantly improve the performance, but it does not comply with IEEE standard. In most cases, this behavior has little impact on model accuracy. Users could enable this optimization if no or acceptable accuracy drop is seen. The following code enables denormals optimization

ie.set_property(ov::denormals_optimization(true)); // enable denormals optimization

The following code disables denormals optimization

ie.set_property(ov::denormals_optimization(false)); // disable denormals optimization

static constexpr Property<float> sparse_weights_decompression_rate = {"CPU_SPARSE_WEIGHTS_DECOMPRESSION_RATE"}#

This property defines threshold for sparse weights decompression feature activation.

Sparse weights decompression feature allows to pack weights for Matrix Multiplication operations directly in the CPU plugin at the model compilation stage and store non-zero values in a special packed format. Then, during the execution of the model, the weights are unpacked and used in the computational kernel. Since the weights are loaded from DDR/L3 cache in the packed format this significantly decreases memory consumption and as a consequence improve inference performance. The following code allows to set the sparse rate value.

core.set_property(ov::intel_cpu::sparse_weights_decompression_rate(0.8));

namespace intel_gpu#

Namespace with Intel GPU specific properties.

Typedefs

using gpu_handle_param = void*#

Enums

enum class ContextType#

Enum to define the type of the shared context.

Values:

enumerator OCL#: Pure OpenCL context.

enumerator VA_SHARED#: Context shared with a video decoding device.

enum class SharedMemType#

Enum to define the type of the shared memory buffer.

Values:

enumerator OCL_BUFFER#: Shared OpenCL buffer blob.

enumerator OCL_IMAGE2D#: Shared OpenCL 2D image blob.

enumerator USM_USER_BUFFER#: Shared USM pointer allocated by user.

enumerator USM_HOST_BUFFER#: Shared USM pointer type with host allocation type allocated by plugin.

enumerator USM_DEVICE_BUFFER#: Shared USM pointer type with device allocation type allocated by plugin.

enumerator VA_SURFACE#: Shared video decoder surface or D3D 2D texture blob.

enumerator DX_BUFFER#: Shared D3D buffer blob.

Variables

static constexpr Property<uint64_t, PropertyMutability::RO> device_total_mem_size = {"GPU_DEVICE_TOTAL_MEM_SIZE"}#: Read-only property which defines size of memory in bytes available for the device. For iGPU it returns host memory size, for dGPU - dedicated gpu memory size.

static constexpr Property<std::string, PropertyMutability::RO> uarch_version = {"GPU_UARCH_VERSION"}#: Read-only property to get microarchitecture identifier in major.minor.revision format.

static constexpr Property<int32_t, PropertyMutability::RO> execution_units_count = {"GPU_EXECUTION_UNITS_COUNT"}#: Read-only property to get count of execution units for current GPU.

static constexpr Property<std::map<std::string, uint64_t>, PropertyMutability::RO> memory_statistics{"GPU_MEMORY_STATISTICS"}#: Read-only property to get statistics of GPU memory allocated by engine for each allocation type It contains information about current memory usage.

static constexpr Property<bool> enable_loop_unrolling = {"GPU_ENABLE_LOOP_UNROLLING"}#: Turning on this key enables to unroll recurrent layers such as TensorIterator or Loop with fixed iteration count. This key is turned on by default. Turning this key on will achieve better inference performance for loops with not too many iteration counts (less than 16, as a rule of thumb). Turning this key off will achieve better performance for both graph loading time and inference time with many iteration counts (greater than 16). Note that turning this key on will increase the graph loading time in proportion to the iteration counts. Thus, this key should be turned off if graph loading time is considered to be most important target to optimize.

static constexpr Property<bool> disable_winograd_convolution = {"GPU_DISABLE_WINOGRAD_CONVOLUTION"}#: Turning on this key disables winograd convolution. Winograd convolution has different characteristics for accuracy and performance compared to other convolution implementations.

static constexpr Property<ContextType> context_type = {"CONTEXT_TYPE"}#: Shared device context type: can be either pure OpenCL (OCL) or shared video decoder (VA_SHARED) context.

static constexpr Property<gpu_handle_param> ocl_context = {"OCL_CONTEXT"}#: This key identifies OpenCL context handle in a shared context or shared memory blob parameter map.

static constexpr Property<int> ocl_context_device_id = {"OCL_CONTEXT_DEVICE_ID"}#: This key identifies ID of device in OpenCL context if multiple devices are present in the context.

static constexpr Property<int> tile_id = {"TILE_ID"}#: In case of multi-tile system, this key identifies tile within given context.

static constexpr Property<gpu_handle_param> ocl_queue = {"OCL_QUEUE"}#: This key identifies OpenCL queue handle in a shared context.

static constexpr Property<gpu_handle_param> va_device = {"VA_DEVICE"}#: This key identifies video acceleration device/display handle in a shared context or shared memory blob parameter map.

static constexpr Property<SharedMemType> shared_mem_type = {"SHARED_MEM_TYPE"}#: This key identifies type of internal shared memory in a shared memory blob parameter map.

static constexpr Property<gpu_handle_param> mem_handle = {"MEM_HANDLE"}#: This key identifies OpenCL memory handle in a shared memory blob parameter map.

static constexpr Property<gpu_handle_param> dev_object_handle = {"DEV_OBJECT_HANDLE"}#: This key identifies video decoder surface handle in a shared memory blob parameter map.

static constexpr Property<uint32_t> va_plane = {"VA_PLANE"}#: This key identifies video decoder surface plane in a shared memory blob parameter map.

namespace capability#

Possible return value for ov::device::capabilities property.

Variables

static constexpr const auto HW_MATMUL = "GPU_HW_MATMUL"#: Device has hardware block for matrix multiplication.

namespace hint#

Typedefs

using ThrottleLevel = ov::hint::Priority #

This enum represents the possible value of ov::intel_gpu::hint::queue_throttle property:

LOW is used for CL_QUEUE_THROTTLE_LOW_KHR OpenCL throttle hint
MEDIUM (DEFAULT) is used for CL_QUEUE_THROTTLE_MED_KHR OpenCL throttle hint
HIGH is used for CL_QUEUE_THROTTLE_HIGH_KHR OpenCL throttle hint

Variables

static constexpr Property<ThrottleLevel> queue_throttle = {"GPU_QUEUE_THROTTLE"}#: This key instructs the GPU plugin to use OpenCL queue throttle hints as defined in https://www.khronos.org/registry/OpenCL/specs/opencl-2.1-extensions.pdf, chapter 9.19. This option should be used with ov::intel_gpu::hint::ThrottleLevel values.

static constexpr Property<ov::hint::Priority> queue_priority = {"GPU_QUEUE_PRIORITY"}#

This key instructs the GPU plugin to use the OpenCL queue priority hint as defined in https://www.khronos.org/registry/OpenCL/specs/opencl-2.1-extensions.pdf. This option should be used with ov::hint::Priority:

LOW is used for CL_QUEUE_PRIORITY_LOW_KHR OpenCL priority hint
MEDIUM (DEFAULT) is used for CL_QUEUE_PRIORITY_MED_KHR OpenCL priority hint
HIGH is used for CL_QUEUE_PRIORITY_HIGH_KHR OpenCL priority hint

static constexpr Property<ov::hint::Priority> host_task_priority = {"GPU_HOST_TASK_PRIORITY"}#

This key instructs the GPU plugin which cpu core type of TBB affinity used in load network. This option has 3 types of levels: HIGH, LOW, and ANY. It is only affected on Hybrid CPUs.

LOW - instructs the GPU Plugin to use LITTLE cores if they are available
MEDIUM (DEFAULT) - instructs the GPU Plugin to use any available cores (BIG or LITTLE cores)
HIGH - instructs the GPU Plugin to use BIG cores if they are available

static constexpr Property<int64_t> available_device_mem = {"AVAILABLE_DEVICE_MEM_SIZE"}#: This key identifies available device memory size in bytes.

static constexpr Property<bool> enable_sdpa_optimization = {"GPU_ENABLE_SDPA_OPTIMIZATION"}#: Turning on this key disables SDPA operation decomposition and keeps SDPA operation in the graph. Enabling SDPA optimization may provide performance improvements and memory usage reduction. This key serves as a recommendation and may be ignored in known sub-optimal cases.

static constexpr Property<bool> enable_kernels_reuse = {"GPU_ENABLE_KERNELS_REUSE"}#: Turning on this property enables kernels reuse between implementations, resulting in a lower memory footprint. However, as a drawback, OpenCL set_arguments() call will be made more often, resulting in higher host pressure and slower execution in some host-bottleneck cases. This property is available only for single-stream scenarios and will be ignored in other cases.

namespace memory_type#

These keys instruct the GPU plugin to use surface/buffer memory type.

Variables

static constexpr auto surface = "GPU_SURFACE"#: Native video decoder surface.

namespace ocl#

Namespace with Intel GPU OpenCL specific remote objects.

Typedefs

using gpu_handle_param = void*#: Shortcut for defining a handle parameter.

class ClBufferTensor : public ov::RemoteTensor

#include <ocl.hpp>

This class represents an abstraction for GPU plugin remote tensor which can be shared with user-supplied OpenCL buffer. The plugin object derived from this class can be obtained with ClContext::create_tensor() call.

Note

User can obtain OpenCL buffer handle from this class.

Subclassed by ov::intel_gpu::ocl::D3DBufferTensor

Public Functions

inline cl_mem get()

Returns the underlying OpenCL memory object handle.

Returns:: underlying OpenCL memory object handle

inline operator cl_mem()

OpenCL memory handle conversion operator.

Returns:: cl_mem

inline operator cl::Buffer()

Standard Khronos cl::Buffer wrapper conversion operator.

Returns:: cl::Buffer object

Public Static Functions

static inline void type_check(const Tensor &tensor)

Checks that type defined runtime parameters are presented in remote object.

Parameters:: tensor – a tensor to check

class ClContext : public ov::RemoteContext

#include <ocl.hpp>

This class represents an abstraction for GPU plugin remote context which is shared with OpenCL context object. The plugin object derived from this class can be obtained either with CompiledModel::get_context() or Core::create_context() calls.

Subclassed by ov::intel_gpu::ocl::D3DContext, ov::intel_gpu::ocl::VAContext

Public Functions

inline ClContext(Core &core, cl_context ctx, int ctx_device_id = 0)

Constructs context object from user-supplied OpenCL context handle.

Parameters:

core – A reference to OpenVINO Runtime Core object
ctx – A OpenCL context to be used to create shared remote context
ctx_device_id – An ID of device to be used from ctx

inline ClContext(Core &core, cl_command_queue queue)

Constructs context object from user-supplied OpenCL context handle.

Note

Only latency mode is supported for such context sharing case.

Parameters:

core – A reference to OpenVINO Runtime Core object
queue – An OpenCL queue to be used to create shared remote context. Queue will be reused inside the plugin.

inline cl_context get()

Returns the underlying OpenCL context handle.

Returns:: cl_context

inline operator cl_context()

OpenCL context handle conversion operator for the ClContext object.

Returns:: cl_context

inline operator cl::Context()

Standard Khronos cl::Context wrapper conversion operator for the ClContext object.

Returns:: cl::Context object

inline std::pair<ClImage2DTensor, ClImage2DTensor> create_tensor_nv12(const cl::Image2D &nv12_image_plane_y, const cl::Image2D &nv12_image_plane_uv)

This function is used to construct a NV12 compound tensor object from two cl::Image2D wrapper objects. The resulting compound contains two remote tensors for Y and UV planes of the surface.

Parameters:

nv12_image_plane_y – cl::Image2D object containing Y plane data.
nv12_image_plane_uv – cl::Image2D object containing UV plane data.

Returns:

A pair of remote tensors for each plane

inline ClBufferTensor create_tensor(const element::Type type, const Shape &shape, const cl_mem buffer)

This function is used to obtain remote tensor object from user-supplied cl_mem object.

Parameters:

type – Tensor element type
shape – Tensor shape
buffer – A cl_mem object that should be wrapped by a remote tensor

Returns:

A remote tensor instance

inline ClBufferTensor create_tensor(const element::Type type, const Shape &shape, const cl::Buffer &buffer)

This function is used to obtain remote tensor object from user-supplied cl::Buffer object.

Parameters:

type – Tensor element type
shape – Tensor shape
buffer – A cl::Buffer object that should be wrapped by a remote tensor

Returns:

A remote tensor instance

inline ClImage2DTensor create_tensor(const element::Type type, const Shape &shape, const cl::Image2D &image)

This function is used to obtain remote tensor object from user-supplied cl::Image2D object.

Parameters:

type – Tensor element type
shape – Tensor shape
image – A cl::Image2D object that should be wrapped by a remote tensor

Returns:

A remote tensor instance

inline USMTensor create_tensor(const element::Type type, const Shape &shape, void *usm_ptr)

This function is used to obtain remote tensor object from user-supplied USM pointer.

Parameters:

type – Tensor element type
shape – Tensor shape
usm_ptr – A USM pointer that should be wrapped by a remote tensor

Returns:

A remote tensor instance

inline USMTensor create_usm_host_tensor(const element::Type type, const Shape &shape)

This function is used to allocate USM tensor with host allocation type.

Parameters:

type – Tensor element type
shape – Tensor shape

Returns:

A remote tensor instance

inline USMTensor create_usm_device_tensor(const element::Type type, const Shape &shape)

This function is used to allocate USM tensor with device allocation type.

Parameters:

type – Tensor element type
shape – Tensor shape

Returns:

A remote tensor instance

RemoteTensor create_tensor(const element::Type &type, const Shape &shape, const AnyMap &params = {})

Parameters:

type – Defines the element type of the tensor.
shape – Defines the shape of the tensor.
params – Map of the low-level tensor object parameters.

Returns:

Pointer to a plugin object that implements the RemoteTensor interface.

Public Static Functions

static inline void type_check(const RemoteContext &remote_context)

Checks that type defined runtime parameters are presented in remote object.

Parameters:: remote_context – A remote context to check

class ClImage2DTensor : public ov::RemoteTensor

#include <ocl.hpp>

This class represents an abstraction for GPU plugin remote tensor which can be shared with user-supplied OpenCL 2D Image. The plugin object derived from this class can be obtained with ClContext::create_tensor() call.

Note

User can obtain OpenCL image handle from this class.

Subclassed by ov::intel_gpu::ocl::D3DSurface2DTensor, ov::intel_gpu::ocl::VASurfaceTensor

Public Functions

inline cl_mem get()

Returns the underlying OpenCL memory object handle.

Returns:: underlying OpenCL memory object handle

inline operator cl_mem()

OpenCL memory handle conversion operator.

Returns:: cl_mem

inline operator cl::Image2D()

Standard Khronos cl::Image2D wrapper conversion operator for the ClContext object.

Returns:: cl::Image2D object

Public Static Functions

static inline void type_check(const Tensor &tensor)

Checks that type defined runtime parameters are presented in remote object.

Parameters:: tensor – a tensor to check

class D3DBufferTensor : public ov::intel_gpu::ocl::ClBufferTensor

#include <dx.hpp>

This class represents an abstraction for GPU plugin remote tensor which is shared with Direct3D 11 buffer. The plugin object derived from this class can be obtained with D3DContext::create_tensor() call.

Note

User can also obtain OpenCL buffer handle from this class.

Public Functions

inline operator ID3D11Buffer*()

ID3D11Buffer conversion operator for the D3DContext object.

Returns:: Pointer to underlying ID3D11Buffer interface

Public Static Functions

static inline void type_check(const Tensor &tensor)

Checks that type defined runtime parameters are presented in remote object.

Parameters:: tensor – a tensor to check

class D3DContext : public ov::intel_gpu::ocl::ClContext

#include <dx.hpp>

This class represents an abstraction for GPU plugin remote context which is shared with Direct3D 11 device. The plugin object derived from this class can be obtained either with CompiledModel::get_context() or Core::create_context() calls.

Note

User can also obtain OpenCL context handle from this class.

Public Functions

inline operator ID3D11Device*()

ID3D11Device conversion operator for the D3DContext object.

Returns:: Pointer to underlying ID3D11Device interface

inline D3DContext(Core &core, ID3D11Device *device, int target_tile_id = -1)

Constructs D3DContext remote context object from ID3D11Device.

Parameters:

core – OpenVINO Runtime Core object instance
device – A pointer to ID3D11Device to be used to create a remote context
target_tile_id – Desired tile id within given context for multi-tile system. Default value (-1) means that root device should be used

inline std::pair<D3DSurface2DTensor, D3DSurface2DTensor> create_tensor_nv12(const size_t height, const size_t width, ID3D11Texture2D *nv12_surf)

This function is used to obtain a NV12 tensor from NV12 DXGI video decoder output. The resulting tensor contains two remote tensors for Y and UV planes of the surface.

Parameters:

height – Height of Y plane
width – Width of Y plane
nv12_surf – A ID3D11Texture2D instance to create NV12 tensor from

Returns:

A pair of remote tensors for each plane

inline D3DBufferTensor create_tensor(const element::Type type, const Shape &shape, ID3D11Buffer *buffer)

This function is used to obtain remote tensor object from ID3D11Buffer.

Parameters:

type – Tensor element type
shape – Tensor shape
buffer – A pointer to ID3D11Buffer instance to create remote tensor based on

Returns:

A remote tensor instance

inline D3DSurface2DTensor create_tensor(const element::Type type, const Shape &shape, ID3D11Texture2D *surface, uint32_t plane = 0)

This function is used to obtain remote tensor object from ID3D11Texture2D.

Note

The underlying ID3D11Texture2D can also be a plane of output surface of DXGI video decoder

Parameters:

type – Tensor element type
shape – Tensor shape
surface – Pointer to ID3D11Texture2D interface of the objects that owns NV12 texture
plane – ID of the plane to be shared (0 or 1)

Returns:

D3DSurface2DTensor tensor

inline ClBufferTensor create_tensor(const element::Type type, const Shape &shape, const cl_mem buffer)

This function is used to obtain remote tensor object from user-supplied cl_mem object.

Parameters:

type – Tensor element type
shape – Tensor shape
buffer – A cl_mem object that should be wrapped by a remote tensor

Returns:

A remote tensor instance

inline ClBufferTensor create_tensor(const element::Type type, const Shape &shape, const cl::Buffer &buffer)

This function is used to obtain remote tensor object from user-supplied cl::Buffer object.

Parameters:

type – Tensor element type
shape – Tensor shape
buffer – A cl::Buffer object that should be wrapped by a remote tensor

Returns:

A remote tensor instance

inline ClImage2DTensor create_tensor(const element::Type type, const Shape &shape, const cl::Image2D &image)

This function is used to obtain remote tensor object from user-supplied cl::Image2D object.

Parameters:

type – Tensor element type
shape – Tensor shape
image – A cl::Image2D object that should be wrapped by a remote tensor

Returns:

A remote tensor instance

inline USMTensor create_tensor(const element::Type type, const Shape &shape, void *usm_ptr)

This function is used to obtain remote tensor object from user-supplied USM pointer.

Parameters:

type – Tensor element type
shape – Tensor shape
usm_ptr – A USM pointer that should be wrapped by a remote tensor

Returns:

A remote tensor instance

Public Static Functions

static inline void type_check(const RemoteContext &remote_context)

Checks that type defined runtime parameters are presented in remote object.

Parameters:: remote_context – A remote context to check

class D3DSurface2DTensor : public ov::intel_gpu::ocl::ClImage2DTensor

#include <dx.hpp>

This class represents an abstraction for GPU plugin remote tensor which is shared with Direct3D 11 2D texture. The plugin object derived from this class can be obtained with D3DContext::create_tensor() call.

Note

User can also obtain OpenCL 2D image handle from this class.

Public Functions

inline operator ID3D11Texture2D*()

ID3D11Texture2D conversion operator for the D3DContext object.

Returns:: Pointer to underlying ID3D11Texture2D interface

inline uint32_t plane()

Returns plane ID of underlying video decoder surface, or 0 if no video surface was shared.

Returns:: Plane ID

Public Static Functions

static inline void type_check(const Tensor &remote_tensor)

Checks that type defined runtime parameters are presented in remote object.

Parameters:: remote_tensor – remote tensor to check

class USMTensor : public ov::RemoteTensor

#include <ocl.hpp>

This class represents an abstraction for GPU plugin remote tensor which can be shared with user-supplied USM device pointer. The plugin object derived from this class can be obtained with ClContext::create_tensor() call.

Note

User can obtain USM pointer from this class.

Public Functions

inline void *get()

Returns the underlying USM pointer.

Returns:: underlying USM pointer

Public Static Functions

static inline void type_check(const Tensor &tensor)

Checks that type defined runtime parameters are presented in remote object.

Parameters:: tensor – a tensor to check

class VAContext : public ov::intel_gpu::ocl::ClContext

#include <va.hpp>

This class represents an abstraction for GPU plugin remote context which is shared with VA display object. The plugin object derived from this class can be obtained either with CompiledModel::get_context() or Core::create_context() calls.

Note

User can also obtain OpenCL context handle from this class.

Public Functions

inline operator VADisplay()

VADisplay conversion operator for the VAContext object.

Returns:: Underlying VADisplay object handle

inline VAContext(Core &core, VADisplay device, int target_tile_id = -1)

Constructs remote context object from VA display handle.

Parameters:

core – OpenVINO Runtime Core object
device – A VADisplay to create remote context from
target_tile_id – Desired tile id within given context for multi-tile system. Default value (-1) means that root device should be used

inline std::pair<VASurfaceTensor, VASurfaceTensor> create_tensor_nv12(const size_t height, const size_t width, const VASurfaceID nv12_surf)

This function is used to obtain a NV12 tensor from NV12 VA decoder output. The resulting tensor contains two remote tensors for Y and UV planes of the surface.

Parameters:

height – A height of Y plane
width – A width of Y plane
nv12_surf – NV12 VASurfaceID to create NV12 from

Returns:

A pair of remote tensors for each plane

inline VASurfaceTensor create_tensor(const element::Type type, const Shape &shape, const VASurfaceID surface, const uint32_t plane = 0)

This function is used to create remote tensor from VA surface handle.

Parameters:

type – Tensor element type
shape – Tensor shape
surface – A VASurfaceID to create remote tensor from
plane – An index of a plane inside VASurfaceID to create tensor from

Returns:

A remote tensor wrapping VASurfaceID

inline ClBufferTensor create_tensor(const element::Type type, const Shape &shape, const cl_mem buffer)

This function is used to obtain remote tensor object from user-supplied cl_mem object.

Parameters:

type – Tensor element type
shape – Tensor shape
buffer – A cl_mem object that should be wrapped by a remote tensor

Returns:

A remote tensor instance

inline ClBufferTensor create_tensor(const element::Type type, const Shape &shape, const cl::Buffer &buffer)

This function is used to obtain remote tensor object from user-supplied cl::Buffer object.

Parameters:

type – Tensor element type
shape – Tensor shape
buffer – A cl::Buffer object that should be wrapped by a remote tensor

Returns:

A remote tensor instance

inline ClImage2DTensor create_tensor(const element::Type type, const Shape &shape, const cl::Image2D &image)

This function is used to obtain remote tensor object from user-supplied cl::Image2D object.

Parameters:

type – Tensor element type
shape – Tensor shape
image – A cl::Image2D object that should be wrapped by a remote tensor

Returns:

A remote tensor instance

inline USMTensor create_tensor(const element::Type type, const Shape &shape, void *usm_ptr)

This function is used to obtain remote tensor object from user-supplied USM pointer.

Parameters:

type – Tensor element type
shape – Tensor shape
usm_ptr – A USM pointer that should be wrapped by a remote tensor

Returns:

A remote tensor instance

RemoteTensor create_tensor(const element::Type &type, const Shape &shape, const AnyMap &params = {})

Parameters:

type – Defines the element type of the tensor.
shape – Defines the shape of the tensor.
params – Map of the low-level tensor object parameters.

Returns:

Pointer to a plugin object that implements the RemoteTensor interface.

Public Static Functions

static inline void type_check(const RemoteContext &remote_context)

Checks that type defined runtime parameters are presented in remote object.

Parameters:: remote_context – A remote context to check

class VASurfaceTensor : public ov::intel_gpu::ocl::ClImage2DTensor

#include <va.hpp>

This class represents an abstraction for GPU plugin remote tensor which is shared with VA output surface. The plugin object derived from this class can be obtained with VAContext::create_tensor() call.

Note

User can also obtain OpenCL 2D image handle from this class.

Public Functions

inline operator VASurfaceID()

VASurfaceID conversion operator for the VASurfaceTensor object.

Returns:: VASurfaceID handle

inline uint32_t plane()

Returns plane ID of underlying video decoder surface.

Returns:: Plane ID

Public Static Functions

static inline void type_check(const Tensor &tensor)

Checks that type defined runtime parameters are presented in remote object.

Parameters:: tensor – a tensor to check

namespace intel_npu#

Namespace with Intel NPU specific properties.

Typedefs

using npu_handle_param = void*#

Enums

enum class MemType#

Enum to define the type of the shared memory buffer.

Values:

enumerator L0_INTERNAL_BUF#: Internal Level Zero buffer type allocated by plugin.

enumerator SHARED_BUF#: Shared buffer.

enum class TensorType#

Enum to define the type of the tensor.

Values:

enumerator INPUT#: Tensor is only used as input.

enumerator OUTPUT#: Tensor is only used as output.

enumerator BINDED#: Tensor could be used as input and output.

Variables

static constexpr ov::Property<uint64_t, ov::PropertyMutability::RO> device_alloc_mem_size = {"NPU_DEVICE_ALLOC_MEM_SIZE"}#

[Only for NPU plugin] Type: uint64_t Read-only property to get size of already allocated NPU DDR memory (both for discrete/integrated NPU devices)

Note: Queries driver both for discrete/integrated NPU devices

static constexpr ov::Property<uint64_t, ov::PropertyMutability::RO> device_total_mem_size = {"NPU_DEVICE_TOTAL_MEM_SIZE"}#

[Only for NPU plugin] Type: uint64_t Read-only property to get size of available NPU DDR memory (both for discrete/integrated NPU devices)

Note: Queries driver both for discrete/integrated NPU devices

static constexpr ov::Property<uint32_t, ov::PropertyMutability::RO> driver_version = {"NPU_DRIVER_VERSION"}#: [Only for NPU plugin] Type: uint32_t Read-only property to get NPU driver version (for both discrete/integrated NPU devices)

static constexpr ov::Property<uint32_t, ov::PropertyMutability::RO> compiler_version = {"NPU_COMPILER_VERSION"}#: [Only for NPU plugin] Type: uint32_t Read-only property to get NPU compiler version. Composite of Major (16bit MSB) and Minor (16bit LSB)

static constexpr ov::Property<std::string> compilation_mode_params = {"NPU_COMPILATION_MODE_PARAMS"}#: [Only for NPU compiler] Type: std::string Set various parameters supported by the NPU compiler.

static constexpr ov::Property<bool> compiler_dynamic_quantization = {"NPU_COMPILER_DYNAMIC_QUANTIZATION"}#: [Only for NPU compiler] Type: boolean Set or verify state of dynamic quantization in the NPU compiler

static constexpr ov::Property<bool> turbo = {"NPU_TURBO"}#: [Only for NPU plugin] Type: std::bool Set turbo on or off. The turbo mode, where available, provides a hint to the system to maintain the maximum NPU frequency and memory throughput within the platform TDP limits. Turbo mode is not recommended for sustainable workloads due to higher power consumption and potential impact on other compute resources.

static constexpr ov::Property<int64_t> tiles = {"NPU_TILES"}#: [Only for NPU Compiler] Type: integer, default is -1 Sets the number of npu tiles to compile the model for.

static constexpr ov::Property<int64_t> max_tiles = {"NPU_MAX_TILES"}#: Type: integer, default is -1 Maximum number of tiles supported by the device we compile for. Can be set for offline compilation. If not set, it will be populated by driver.

static constexpr ov::Property<bool> bypass_umd_caching = {"NPU_BYPASS_UMD_CACHING"}#: [Only for NPU plugin] Type: std::bool Bypass caching of the compiled model by UMD cache.

static constexpr ov::Property<bool> defer_weights_load = {"NPU_DEFER_WEIGHTS_LOAD"}#: [Only for NPU Plugin] Type: boolean, default is false This option allows to delay loading the weights until inference is created

static constexpr Property<MemType> mem_type = {"MEM_TYPE"}#: This key identifies type of internal shared memory in a shared memory tensor parameter map.

static constexpr Property<npu_handle_param> mem_handle = {"MEM_HANDLE"}#: This key identifies memory handle in a shared memory tensor parameter map.

static constexpr Property<npu_handle_param> l0_context = {"L0_CONTEXT"}#: This key identifies LevelZero context handle in a shared context parameter map.

static constexpr Property<TensorType> tensor_type = {"TENSOR_TYPE"}#: This key sets the type of the internal Level Zero buffer allocated by the plugin in a shared memory tensor parameter map.

namespace level_zero#

Namespace with Intel NPU LevelZero specific remote objects.

class ZeroBufferTensor : public ov::RemoteTensor

#include <level_zero.hpp>

This class represents an abstraction for NPU plugin remote tensor which can be shared with user-supplied LevelZero buffer. The plugin object derived from this class can be obtained with ZeroContext::create_tensor() call.

Note

User can obtain Level Zero buffer handle from this class.

Public Functions

inline void *get()

Returns the underlying LevelZero memory object handle.

Returns:: underlying void* memory object handle

Public Static Functions

static inline void type_check(const Tensor &tensor)

Checks that type defined runtime parameters are presented in remote object.

Parameters:: tensor – a tensor to check

class ZeroContext : public ov::RemoteContext

#include <level_zero.hpp>

This class represents an abstraction for NPU plugin remote context which is shared with LevelZero context object. The plugin object derived from this class can be obtained either with CompiledModel::get_context() or Core::create_context() calls.

Public Functions

inline ZeroContext(Core &core)

Constructs context object from user-supplied LevelZero context handle.

Parameters:: core – A reference to OpenVINO Runtime Core object

inline void *get()

Returns the underlying LevelZero context handle.

Returns:: void*

inline ZeroBufferTensor create_tensor(const element::Type type, const Shape &shape, void *buffer)

This function is used to obtain remote tensor object from user-supplied NT handle object.

Parameters:

type – Tensor element type
shape – Tensor shape
buffer – A void* object that should be wrapped by a remote tensor

Returns:

A remote tensor instance

inline ZeroBufferTensor create_tensor(const element::Type type, const Shape &shape, int fd)

This function is used to obtain remote tensor object from user-supplied DMA-BUF System Heap object.

Parameters:

type – Tensor element type
shape – Tensor shape
fd – A int object that should be wrapped by a remote tensor

Returns:

A remote tensor instance

inline ZeroBufferTensor create_l0_host_tensor(const element::Type type, const Shape &shape, const TensorType tensor_type = TensorType::BINDED)

This function is used to obtain remote tensor object.

Parameters:

type – Tensor element type
shape – Tensor shape
tensor_type – Type of the tensor to be shared, input, output or binded

Returns:

A remote tensor instance

RemoteTensor create_tensor(const element::Type &type, const Shape &shape, const AnyMap &params = {})

Parameters:

type – Defines the element type of the tensor.
shape – Defines the shape of the tensor.
params – Map of the low-level tensor object parameters.

Returns:

Pointer to a plugin object that implements the RemoteTensor interface.

namespace internal#

Variables

static constexpr Property<std::vector<PropertyName>, PropertyMutability::RO> supported_properties{"INTERNAL_SUPPORTED_PROPERTIES"}#: Read-only property to get a std::vector<PropertyName> of supported internal properties.

static constexpr Property<std::vector<PropertyName>, PropertyMutability::RO> caching_properties = {"CACHING_PROPERTIES"}#: Read-only property to get a std::vector<PropertyName> of properties which should affect the hash calculation for model cache.

static constexpr Property<bool, PropertyMutability::RO> caching_with_mmap = {"CACHING_WITH_MMAP"}#: Read-only property to get a std::vector<PropertyName> of properties which should affect the loading time from cache.

static constexpr Property<bool, PropertyMutability::RW> exclusive_async_requests = {"EXCLUSIVE_ASYNC_REQUESTS"}#: Allow to create exclusive_async_requests with one executor.

static constexpr Property<std::string, PropertyMutability::WO> config_device_id = {"CONFIG_DEVICE_ID"}#: the property for setting of required device for which config to be updated values: device id starts from “0” - first device, “1” - second device, etc note: plugin may have different devices naming convention

static constexpr Property<int32_t, PropertyMutability::RW> threads_per_stream = {"THREADS_PER_STREAM"}#: Limit #threads that are used by IStreamsExecutor to execute parallel_for calls.

static constexpr Property<std::string, PropertyMutability::RO> compiled_model_runtime_properties{"COMPILED_MODEL_RUNTIME_PROPERTIES"}#

It contains compiled_model_runtime_properties information to make plugin runtime can check whether it is compatible with the cached compiled model, the result is returned by get_property() calling.

The information details are defined by plugin itself, each plugin may require different runtime contents. For example, CPU plugin will contain OV version, while GPU plugin will contain OV and GPU driver version, etc. Core doesn’t understand its content and only read it from plugin and write it into blob header.

static constexpr Property<bool, PropertyMutability::RO> compiled_model_runtime_properties_supported{"COMPILED_MODEL_RUNTIME_PROPERTIES_SUPPORTED"}#: Check whether the attached compiled_model_runtime_properties is supported by this device runtime.

static constexpr Property<float, PropertyMutability::RW> query_model_ratio = {"QUERY_MODEL_RATIO"}#: Read-write property to set the percentage of the estimated model size which is used to determine the query model results for further processing.

namespace itt#

namespace domains#

Functions

OV_ITT_DOMAIN(ov_eval)#

namespace layout#

Functions

OPENVINO_API bool has_batch (const Layout &layout): Checks if layout has ‘batch’ dimension.

OPENVINO_API std::int64_t batch_idx (const Layout &layout)

Returns ‘batch’ dimension index.

Throws:: ov::AssertFailure – if dimension doesn’t exist.

OPENVINO_API bool has_channels (const Layout &layout)

Checks if layout has ‘channels’ dimension.

Throws:: ov::AssertFailure – if dimension doesn’t exist.

OPENVINO_API std::int64_t channels_idx (const Layout &layout)

Returns ‘channels’ dimension index.

Throws:: ov::AssertFailure – if dimension doesn’t exist.

OPENVINO_API bool has_depth (const Layout &layout): Checks if layout has ‘depth’ dimension.

OPENVINO_API std::int64_t depth_idx (const Layout &layout)

Returns ‘depth’ dimension index.

Throws:: ov::AssertFailure – if dimension doesn’t exist.

OPENVINO_API bool has_height (const Layout &layout): Checks if layout has ‘height’ dimension.

OPENVINO_API std::int64_t height_idx (const Layout &layout)

Returns ‘height’ dimension index.

Throws:: ov::AssertFailure – if dimension doesn’t exist.

OPENVINO_API bool has_width (const Layout &layout): Checks if layout has ‘width’ dimension.

OPENVINO_API std::int64_t width_idx (const Layout &layout)

Returns ‘width’ dimension index.

Throws:: ov::AssertFailure – if dimension doesn’t exist.

OPENVINO_API void set_layout (ov::Output< ov::Node > output, const ov::Layout &layout)

Sets Layout of port.

Throws:: ov::Exception – if port is not connected with Result or Parameter

OPENVINO_API ov::Layout get_layout (const ov::Output< ov::Node > &output)

Gets Layout of port.

Returns:: layout from port and empty layout in other case

OPENVINO_API ov::Layout get_layout (const ov::Output< const ov::Node > &output)

Gets Layout of port.

Returns:: layout from port and empty layout in other case

namespace log#

Namespace with log level property and its possible values.

Enums

enum class Level#

Enum to define possible log levels.

Values:

enumerator NO#: disable any logging

enumerator ERR#: error events that might still allow the application to continue running

enumerator WARNING#: potentially harmful situations which may further lead to ERROR

enumerator INFO#: informational messages that display the progress of the application at coarse-grained level

enumerator DEBUG#: fine-grained events that are most useful to debug an application.

enumerator TRACE#: finer-grained informational events than the DEBUG

Variables

static constexpr Property<Level> level = {"LOG_LEVEL"}#: the property for setting desirable log level.

namespace op#

Enums

enum class GeluApproximationMode#

Specifies the approximation to calculate Gelu.

Values:

enumerator TANH#

enumerator ERF#

enum class LSTMWeightsFormat#

Values:

enumerator FICO#

enumerator ICOF#

enumerator IFCO#

enumerator IFOC#

enumerator IOFC#

enum class MVNEpsMode#

Specifies how eps is applied in MVN.

Values:

enumerator INSIDE_SQRT#

enumerator OUTSIDE_SQRT#

enum class PadMode#

Modes for the Pad operator.

Values:

enumerator CONSTANT#

enumerator EDGE#

enumerator REFLECT#

enumerator SYMMETRIC#

enum class PadType#

Padding Type used for Convolution and Pooling

Follows ONNX padding type definitions EXPLICIT - Pad dimensions are explicity specified SAME_LOWER - Pad dimensions computed to match input shape Ceil(num_dims/2) at the beginning and Floor(num_dims/2) at the end SAME_UPPER - Pad dimensions computed to match input shape Floor(num_dims/2) at the beginning and Ceil(num_dims/2) at the end VALID - No padding AUTO - Deprecated. User should not use it in the future NOTSET - Deprecated. User should not use it in the future

Values:

enumerator EXPLICIT#

enumerator SAME_LOWER#

enumerator SAME_UPPER#

enumerator VALID#

enumerator AUTO#

enumerator NOTSET#

enum class RoundingType#

Rounding Type used for Pooling operators.

Values:

enumerator FLOOR#

enumerator CEIL#

enumerator CEIL_TORCH#

enum class AutoBroadcastType#

Specifies the algorithm to use for implicit broadcasting of a tensor to align with another tensor.

NONE - No implicit broadcasting of tensor NUMPY - Numpy-style implicit broadcasting (https://docs.scipy.org/doc/numpy/user/basics.broadcasting.html) Right-align dimensions of the two tensors, with missing dimensions treated as size 1 dimensions. After alignment, for each dimension, their sizes should either match or one of them should be of size 1. Size 1 dimension will be implicitly broadcast to match the other size.

E.g., A: Shape(2, 1, 6) B: Shape( 3, 1) Result: Shape(2, 3, 6)

 A: Shape(2, 1, 6)
 B: Shape(   3, 1)

Result: Shape(2, 3, 6) PDPD - PaddlePaddle-style implicit broadcasting (PaddlePaddle/Paddle fluid/operators/elementwise/elementwise_op.h#L126) Broadcast B to match the shape of A, where axis is the start dimension index to align B with A. If axis is -1 (default), i axis = rank(A) - rank(B). The trailing dimensions of size 1 for B will be ignored.

E.g., A: Shape(2, 3, 4, 5) B: Shape( 3, 4 ) with axis =1 Result: Shape(2, 3, 4, 5)

 A: Shape(2, 3, 4, 5)
 B: Shape(   3, 1   ) with axis = 1

Result: Shape(2, 3, 4, 5)

Values:

enumerator NONE#

enumerator EXPLICIT#

enumerator NUMPY#

enumerator PDPD#

enum class BroadcastType#

BroadcastType specifies rules used for mapping of input tensor axes to output shape axes.

EXPLICIT - Mapping of the input data shape to output shape based on axes_mapping input. NUMPY - Numpy broadcasting rules, aligned with ONNX Broadcasting. (onnx/onnx) PDPD - PaddlePaddle-style implicit broadcasting. For more informaction see AutoBroadcastType documentation. BIDIRECTIONAL - The broadcast rule is similar to numpy.array(input) * numpy.ones(target_shape). Dimensions are right alignment.

Note

Broadcasting rules are different for Broadcast op and for element-wise ops. AutoBroadcastType::NUMPY is equivalent of BroadcastType::BIDIRECTIONAL according to spec.

Values:

enumerator NONE#

enumerator EXPLICIT#

enumerator NUMPY#

enumerator PDPD#

enumerator BIDIRECTIONAL#

enum class EpsMode#

Specifies how eps is combined with L2 value.

Values:

enumerator ADD#

enumerator MAX#

enum class TopKSortType#

Values:

enumerator NONE#

enumerator SORT_INDICES#

enumerator SORT_VALUES#

enum class TopKMode#

Values:

enumerator MAX#

enumerator MIN#

enum class PhiloxAlignment#

Values:

enumerator TENSORFLOW#

enumerator PYTORCH#

enumerator MOCK#

enum class RecurrentSequenceDirection#

This class defines possible recurrent sequence directions.

Values:

enumerator FORWARD#

enumerator REVERSE#

enumerator BIDIRECTIONAL#

Functions

std::unordered_map< size_t, std::pair< ov::Tensor, ov::Tensor > > OPENVINO_API convert_input_types (OutputVector &inputs, const element::TypeVector &types)

ov::TensorVector OPENVINO_API get_output_tensors_of_original_type (const ov::TensorVector &fake_output_tensors, const element::TypeVector &types)

void OPENVINO_API reset_input_types (const std::unordered_map< size_t, std::pair< ov::Tensor, ov::Tensor > > &original_input_vals, OutputVector &inputs)

bool OPENVINO_API convert_outputs_to_fake_type (ov::TensorVector &outputs, ov::TensorVector &original_outputs, bool is_upper)

OPENVINO_API std::ostream & operator<< (std::ostream &s, const GeluApproximationMode &type)

ov::op::util::LSTMWeightsFormat convert_lstm_weights_enums(LSTMWeightsFormat format)#

OPENVINO_API std::ostream & operator<< (std::ostream &s, const MVNEpsMode &type)

OPENVINO_API std::ostream & operator<< (std::ostream &s, const v12::ScatterElementsUpdate::Reduction &reduction)

OPENVINO_API std::ostream & operator<< (std::ostream &s, const PadMode &type)

OPENVINO_API std::ostream & operator<< (std::ostream &s, const PadType &type)

OPENVINO_API std::ostream & operator<< (std::ostream &s, const RoundingType &type)

OPENVINO_API std::ostream & operator<< (std::ostream &s, const AutoBroadcastType &type)

OPENVINO_API std::ostream & operator<< (std::ostream &s, const BroadcastType &type)

OPENVINO_API std::ostream & operator<< (std::ostream &s, const EpsMode &type)

OPENVINO_API std::ostream & operator<< (std::ostream &s, const TopKSortType &type)

OPENVINO_API std::ostream & operator<< (std::ostream &s, const TopKMode &type)

OPENVINO_API std::ostream & operator<< (std::ostream &s, const PhiloxAlignment &alignment)

OPENVINO_API std::ostream & operator<< (std::ostream &s, const RecurrentSequenceDirection &direction)

template<class TShape, class TRShape = result_shape_t<TShape>> std::vector<TRShape> copy_shape_infer(const Node *op, const std::vector<TShape> &input_shapes)#

template<class OpType, class T, class TRShape = result_shape_t<T>> std::vector<TRShape> eltwise_shape_infer(const OpType *op, const std::vector<T> &input_shapes)#

template<class T, class TRShape = result_shape_t<T>> std::vector<TRShape> shape_infer(const util::FFTBase *op, const std::vector<T> &input_shapes, const ITensorAccessor &ta = make_tensor_accessor())#

template<class TShape, class TRShape = result_shape_t<TShape>> std::vector<TRShape> shape_infer(const util::GatherBase *op, const std::vector<TShape> &input_shapes, const ITensorAccessor &tensor_accessor = make_tensor_accessor())#

template<class TShape, class TRShape = result_shape_t<TShape>> std::vector<TRShape> shape_infer(const util::ConvertColorI420Base *op, const std::vector<TShape> &input_shapes)#

template<class TShape, class TRShape = result_shape_t<TShape>> std::vector<TRShape> shape_infer(const util::MulticlassNmsBase *op, const std::vector<TShape> &input_shapes, const bool static_output = !std::is_same<PartialShape, TShape>::value, const bool ignore_bg_class = false)#

template<class TShape, class TRShape = result_shape_t<TShape>> std::vector<TRShape> shape_infer(const util::ConvertColorNV12Base *op, const std::vector<TShape> &input_shapes)#

template<class TShape, class TRShape = result_shape_t<TShape>> std::vector<TRShape> shape_infer(const util::PadBase *op, const std::vector<TShape> &input_shapes, const ITensorAccessor &tensor_accessor = make_tensor_accessor())#

template<class TShape, class TRShape = result_shape_t<TShape>> std::vector<TRShape> reduce_shape_infer(const util::ReductionBase *op, bool keep_dims, const std::vector<TShape> &input_shapes, const ITensorAccessor &tensor_accessor = make_tensor_accessor())#

template<class TShape, class TRShape = result_shape_t<TShape>> std::vector<TRShape> shape_infer(const util::ArithmeticReductionKeepDims *op, const std::vector<TShape> &input_shapes, const ITensorAccessor &tensor_accessor = make_tensor_accessor())#

template<class TShape, class TRShape = result_shape_t<TShape>> std::vector<TRShape> shape_infer(const util::LogicalReductionKeepDims *op, const std::vector<TShape> &input_shapes, const ITensorAccessor &tensor_accessor = make_tensor_accessor())#

template<class TShape, class TRShape = result_shape_t<TShape>> std::vector<TRShape> shape_infer(const util::ScatterElementsUpdateBase *op, const std::vector<TShape> &input_shapes, const ITensorAccessor &ta = make_tensor_accessor())#

template<class TShape, class TRShape = result_shape_t<TShape>> std::vector<TRShape> shape_infer(const util::ScatterNDBase *op, const std::vector<TShape> &input_shapes)#

template<class TShape, class TRShape = result_shape_t<TShape>> std::vector<TRShape> shape_infer(const util::TopKBase *op, const std::vector<TShape> &input_shapes, const ITensorAccessor &tensor_accessor = make_tensor_accessor())#

TopK shape inference.

Template Parameters:

TShape – Type of shape.

Parameters:

op – Pointer to TopK operator.
input_shapes – Input shapes of TopK.
constant_data – Map of constant data. DEfault empty.

Returns:

Vector of output shapes for

template<class TShape, class TData, class TRes = std::vector<TData>, class UnaryOperation = ov::util::Cast<TData>, typename std::enable_if<!std::is_same<TShape, ov::PartialShape>::value>::type* = nullptr> ov::optional<TRes> get_input_const_data_as(const ov::Node *op, size_t idx, const ITensorAccessor &tensor_accessor, UnaryOperation &&func = ov::util::Cast<TData>())#

Get the operator’s input const as pointer to vector of specified type.

The behaviour depends on shape type. The default output type is std::vector<TData> can be replace by other type which if is possible to construct it from constant data vector.

Template Parameters:

TShape – Shape type which enabled this version (not ov::PartialShape)
TData – Type use to cast input’s data.
TRes – Result type which has got default type as std::vector<TData>.
UnaryOperation – Unary function object applied on data with signature (Ret f(const TData &a)).
TShape – Shape type which enabled this version (ov::PartialShape)
TData – Type use to cast input’s data.
TRes – Result type which has got default type as std::vector<TData>.
UnaryOperation – Unary function object applied on data with signature (Ret f(const TData &a)).

Parameters:

op – Pointer to operator.
idx – Operator’s input number.
tensor_accessor – Tensor accessor object.
func – Unary operation function object.
op – Pointer to operator.
idx – Operator’s input number.
tensor_accessor – Tensor accessor object.
func – Unary operation function object.

Returns:

Pointer to constant data or nullptr if input has no constant data.

Returns:

Pointer to constant data or nullptr if input has no constant data.

template<class TShape, class TDimValue = typename TShape::value_type::value_type, class UnaryOperation = ov::util::InTypeRange<TDimValue>, typename std::enable_if<!std::is_same<TShape, ov::PartialShape>::value>::type* = nullptr> ov::optional<TShape> get_input_const_data_as_shape(const ov::Node *op, size_t port, const ITensorAccessor &tensor_accessor, UnaryOperation &&func = ov::util::InTypeRange<TDimValue>())#

Get the input const data as shape object.

The input data can be processed by unary operation. By default is validated and casted to shape’s dimension type.

Template Parameters:

TShape – Shape type.
TDimValue – Dimension value type.
UnaryOperation – Unary function object applied on data with signature (Ret f(const TDimValue &a)).

Parameters:

op – Pointer to operator.
port – Input port number.
tensor_accessor – Tensor accessor object.
func – Unary operation function object to apply in input data. Default ov::utils::InTypeRange<TDimValue>.

Returns:

Unique pointer to shape created from input data.

inline element::Type get_input_const_element_type(const ov::Node *const op, size_t port, const ITensorAccessor &ta)#

template<class TShape, class TData, class TResult = std::vector<std::pair<TData, TData>>> ov::optional<TResult> get_input_bounds(const ov::Node *op, size_t port, const ITensorAccessor &ta)#

Get the input bounds from constant input or try evaluate bunds and return them as vector of pairs (lower, upper).

Template Parameters:

TShape – Shape type.
TData – Bound value type.

Parameters:

op – Operator pointer.
port – Input port number.
ta – Tensor accessor to constant data.

Returns:

Return optional vector of bounds as pair lower, upper when evaluated successful.

ov::Shape infer_broadcast_shape(const ov::Node *const op, const ov::Shape &first, const ov::Shape &second)#

Inference broadcast shape for element wise operator according to broadcast specification stored in operator.

Parameters:

op – Pointer to operator.
first – First input shape.
second – Second input shape.

Returns:

Result shape from inputs with applied broadcast specification.

ov::Shape infer_broadcast_shape(const ov::Node *const op, const ov::TensorVector &inputs)#

Inference broadcast shape from input tensor shapes for element wise operator according to broadcast specification stored in operator.

Parameters:

op – Pointer to operator.
inputs – Tensors vector to get theirs shapes.

Returns:

Result shape from input tensors shape with applied broadcast specification.

struct AutoBroadcastSpec#: #include <attr_types.hpp>

Implicit broadcast specification.

struct BroadcastModeSpec#: #include <attr_types.hpp>

Implicit broadcast specification.

class Op : public ov::Node #

#include <op.hpp>

Root of all actual ops.

Subclassed by ov::exec_model_info::ExecutionNode, ov::op::Sink, ov::op::internal::DynamicQuantize, ov::op::internal::FullyConnected, ov::op::internal::GLU, ov::op::internal::NonMaxSuppressionIEInternal, ov::op::internal::RMS, ov::op::internal::RoPE, ov::op::util::AvgPoolBase, ov::op::util::BinaryElementwiseArithmetic, ov::op::util::BinaryElementwiseBitwise, ov::op::util::BinaryElementwiseComparison, ov::op::util::BinaryElementwiseLogical, ov::op::util::BroadcastBase, ov::op::util::ConvertColorI420Base, ov::op::util::ConvertColorNV12Base, ov::op::util::ConvolutionBase, ov::op::util::DetectionOutputBase, ov::op::util::EmbeddingBagOffsetsBase, ov::op::util::EmbeddingBagPackedBase, ov::op::util::FFTBase, ov::op::util::GatherBase, ov::op::util::GatherNDBase, ov::op::util::IndexReduction, ov::op::util::InterpolateBase, ov::op::util::MaxPoolBase, ov::op::util::MulticlassNmsBase, ov::op::util::PadBase, ov::op::util::RNNCellBase, ov::op::util::ROIAlignBase, ov::op::util::ReadValueBase, ov::op::util::ReductionBase, ov::op::util::ScatterBase, ov::op::util::ScatterElementsUpdateBase, ov::op::util::ScatterNDBase, ov::op::util::ShapeOfBase, ov::op::util::SqueezeBase, ov::op::util::TopKBase, ov::op::util::UnaryElementwiseArithmetic, ov::op::v0::BatchNormInference, ov::op::v0::CTCGreedyDecoder, ov::op::v0::Concat, ov::op::v0::Constant, ov::op::v0::Convert, ov::op::v0::CumSum, ov::op::v0::DepthToSpace, ov::op::v0::FakeQuantize, ov::op::v0::HardSigmoid, ov::op::v0::Interpolate, ov::op::v0::LRN, ov::op::v0::MVN, ov::op::v0::MatMul, ov::op::v0::NormalizeL2, ov::op::v0::PRelu, ov::op::v0::PSROIPooling, ov::op::v0::Parameter, ov::op::v0::PriorBox, ov::op::v0::PriorBoxClustered, ov::op::v0::Proposal, ov::op::v0::ROIPooling, ov::op::v0::Range, ov::op::v0::RegionYolo, ov::op::v0::ReorgYolo, ov::op::v0::Result, ov::op::v0::ReverseSequence, ov::op::v0::Selu, ov::op::v0::ShuffleChannels, ov::op::v0::SpaceToDepth, ov::op::v0::Tile, ov::op::v0::Unsqueeze, ov::op::v10::IsFinite, ov::op::v10::IsInf, ov::op::v10::IsNaN, ov::op::v10::Unique, ov::op::v12::GroupNormalization, ov::op::v13::BitwiseNot, ov::op::v13::FakeConvert, ov::op::v13::Multinomial, ov::op::v13::NMSRotated, ov::op::v13::ScaledDotProductAttention, ov::op::v14::ConvertPromoteTypes, ov::op::v14::Inverse, ov::op::v15::Col2Im, ov::op::v15::STFT, ov::op::v15::SearchSorted, ov::op::v15::SliceScatter, ov::op::v15::StringTensorPack, ov::op::v15::StringTensorUnpack, ov::op::v16::Identity, ov::op::v1::BatchToSpace, ov::op::v1::ConvertLike, ov::op::v1::DeformablePSROIPooling, ov::op::v1::GatherTree, ov::op::v1::LogicalNot, ov::op::v1::NonMaxSuppression, ov::op::v1::OneHot, ov::op::v1::Reshape, ov::op::v1::Reverse, ov::op::v1::Select, ov::op::v1::Softmax, ov::op::v1::SpaceToBatch, ov::op::v1::Split, ov::op::v1::StridedSlice, ov::op::v1::Transpose, ov::op::v1::VariadicSplit, ov::op::v3::Bucketize, ov::op::v3::EmbeddingSegmentsSum, ov::op::v3::ExtractImagePatches, ov::op::v3::NonMaxSuppression, ov::op::v3::NonZero, ov::op::v4::CTCLoss, ov::op::v4::Range, ov::op::v4::Swish, ov::op::v5::BatchNormInference, ov::op::v5::LogSoftmax, ov::op::v5::NonMaxSuppression, ov::op::v6::CTCGreedyDecoderSeqLen, ov::op::v6::ExperimentalDetectronDetectionOutput, ov::op::v6::ExperimentalDetectronGenerateProposalsSingleImage, ov::op::v6::ExperimentalDetectronPriorGridGenerator, ov::op::v6::ExperimentalDetectronROIFeatureExtractor, ov::op::v6::ExperimentalDetectronTopKROIs, ov::op::v6::GatherElements, ov::op::v6::MVN, ov::op::v7::Einsum, ov::op::v7::Roll, ov::op::v8::AdaptiveAvgPool, ov::op::v8::AdaptiveMaxPool, ov::op::v8::MatrixNms, ov::op::v8::PriorBox, ov::op::v8::RandomUniform, ov::op::v8::Slice, ov::op::v8::Softmax, ov::op::v9::Eye, ov::op::v9::GenerateProposals, ov::op::v9::GridSample, ov::op::v9::NonMaxSuppression

Public Functions

inline virtual const ::ov::Node::type_info_t &get_type_info() const override#: Returns the NodeTypeInfo for the node’s class. During transition to type_info, returns a dummy type_info for Node if the class has not been updated yet.

class Sink : public ov::op::Op #

#include <sink.hpp>

Root of nodes that can be sink nodes.

Subclassed by ov::op::util::AssignBase, ov::op::util::MultiSubGraphOp

class TemporaryReplaceOutputType#

#include <type_relaxed.hpp>

Set another type for a specified output for the period of time when an instance of the class exists. When the execution leaves the scope where an onject of TemporaryReplaceOutputType is defined, the type of the output is set to its original value. Used when initialized TypeRelaxed<BaseOp> operation in case when inputs have types that are not compatible with BaseOp infer function. In this case before TypeRelaxed is constructed the BaseOp contructor requires modified data types. So it should be

Public Functions

TemporaryReplaceOutputType(Output<Node> output, element::Type tmp_type)#: Replace element type for a given output port by tmp_type.

Output<Node> get() const#: Return the output port that was used in the constructor.

~TemporaryReplaceOutputType()#: Restores the original element type for the output.

template<typename BaseOp> class TypeRelaxed : public BaseOp, public ov::op::TypeRelaxedBase #

#include <type_relaxed.hpp>

Relaxes tensor element type requirements for BaseOp inputs and outputs This class template should be used with Node descendant class. Defines a new operation by extending the original BaseOp operation with ability to accept inputs and provide outputs with element type that is unusual for BaseOp. For example, TypeRelaxed<opset1::Add> can accept mixed-precision inputs and provide another type of output. New types are provided as inputs attributes for TypeRelaxed template and fixed. There is no any deduction logic for types are provided as a part of this class and it should be implemented outside if required.

Public Functions

template<typename ...Args> inline TypeRelaxed(const element::TypeVector &_input_data_types, const element::TypeVector &_output_data_types, Args&&... args)#: Creating a new TypeRelaxed operation by calling one of the original op ctors forwarding arguments directly.

class TypeRelaxedBase#

#include <type_relaxed.hpp>

A base class for templated TypeRelaxed that maintains overridden input types and output types for an operation.

Subclassed by ov::op::TypeRelaxed< BaseOp >

Public Functions

inline const element::Type &get_overridden_output_type(size_t outputIndex = 0) const#

This method may look similar to Node::get_output_element_type, but it is not the same thing, because get_output_element_type returns the result of type inference, so it is completely deduced from an operation inputs and attributes, and get_overridden_output_type returns value of the attribute that is used to deduce output type. In some cases they don’t match: get_overridden_output_type may return element::undefined for some index i, and get_output_element_type will return some real type for the same index i.

Returns:: Data type that will be set for output with a given index outputIndex. If output with a specified index outputIndex hasn’t been set before, element::undefined will returned. Undefined means no type override happens for a given outputIndex and it will deduced as original operation defineds in its infer function.

inline void set_overridden_output_type(const element::Type &element_type, size_t outputIndex = 0)#: Set data type that overrides the original data type for output port with outputIndex index In case if outputIndex is out of range of known outputs (and this class cannot detect the real number of outputs for original operation), the number of overridden outputs is changed according to a given outputIndex value.

inline const element::Type &get_origin_input_type(size_t inputIndex = 0) const#

Returns:: Data type that will be set for input when original shape/type inference function is called. If index inputIndex hasn’t been set before, element::undefined will returned. Undefined means that the type from input tensor descriptor is used for a given index.

inline void set_origin_input_type(const element::Type &element_type, size_t inputIndex = 0)#: Set data type that overrides the original data type for input port with inputIndex index. In case if inputIndex is out of range of known inputs (and this class cannot detect the real number of inputs for original operation), the number of overridden inputs is changed according to a given inputIndex value. All new entries except one added at inputIndex position are undefined.

namespace batch_norm#

Functions

template<class TShape, class TRShape = result_shape_t<TShape>, class TDimension = typename TShape::value_type> std::vector<TRShape> infer_shape(const Node *node, const std::vector<TShape> &inputs_shapes)#

namespace convolution#

Functions

template<class TOp, class TShape, typename std::enable_if<std::is_base_of<util::ConvolutionBackPropBase, TOp>::value>::type* = nullptr> size_t calculate_num_spatial(const TOp *op, const std::vector<TShape> &input_shapes, const result_shape_t<TShape> &out_spatial_shape)#

template<class TOp, class TShape, class TIter> void apply_auto_pad(const TOp *op, const TShape &data_shape, const TShape &filters_shape, const result_shape_t<TShape> &out_spatial_shape, TIter pads_begin, TIter pads_end)#

Apply auto padding for backward convolution.

The auto padding can be applied only if inputs and attributes of operator are validated. The input shapes must have got static ranks.

Parameters:

op – Pointer to convolution operator.
data_shape – Input data shape (must be static rank).
filters_shape – Input filter shape (must be static rank).
out_spatial_shape – Reference to input with out spatial shape.
pads_begin – Iterator to begin of pads begin.
pads_end – Iterator to begin of pads end.

template<class TShape> void apply_padding(const util::ConvolutionBackPropBase *op, const std::vector<TShape> &input_shapes, const result_shape_t<TShape> &out_spatial_shape, CoordinateDiff &pads_begin, CoordinateDiff &pads_end)#

Apply auto padding for back propagation convolutions.

Template Parameters:

TShape – Shape type.

Parameters:

op – Pointer to back propagation convolution operator.
data_shape – Input data shape.
filters_shape – Input filter shape.
out_spatial_shape – Input output spatial shape.

template<class TOp, class TShape, class TContainer, typename std::enable_if<std::is_base_of<ov::op::util::ConvolutionBackPropBase, TOp>::value>::type* = nullptr> void append_spatial_shape(const TOp *op, const TShape &data_shape, const TShape &filters_shape, const TContainer &pads_begin, const TContainer &pads_end, result_shape_t<TShape> &out_shape)#

Append spatial dimension at end of output shape of back propagation convolution.

Template Parameters:

TOp – Back propagation convolution operator type.
TShape – Type of shape.

Parameters:

op – Pointer to operator.
data_shape – Input data shape.
filters_shape – Input filter shape.
out_shape – Output shape to append spatial dimensions.

template<class TConv> constexpr size_t filter_non_spatial_dims_count()#

Provides convolution filter non spatial dimension count.

Note

If specific convolution operator requires different value provide specialization for this operator.

Template Parameters:: TConv – Type of convolution operator.
Returns:: Default value for convolution operators (2).

template<class TOp> bool is_auto_pad(const TOp *op)#

Checks if Op property auto_pad is set to same lower or upper.

Template Parameters:: TOp – Type of operator (must have get_auto_pad member function).
Parameters:: op – Pointer to operator.
Returns:: True if auto pad enabled.

inline void resize_empty_padding(const size_t num_spatial, CoordinateDiff &pads_begin, CoordinateDiff &pads_end)#

Resize paddings if empty to number of spatial dimensions.

Parameters:

num_spatial – Number of spatial dimensions.
pads_begin – Begin padding to resize.
pads_end – End padding to resize.

inline size_t num_spatial_from_attr(const util::ConvolutionBase *op)#

template<class TOp, class TShape, typename std::enable_if<std::is_base_of<util::ConvolutionFwdPropBase, TOp>::value>::type* = nullptr> size_t calculate_num_spatial(const TOp *op, const std::vector<TShape> &input_shapes)#

template<class TOp, class TShape, class TIter, typename std::enable_if<std::is_base_of<util::ConvolutionFwdPropBase, TOp>::value || std::is_base_of<util::DeformableConvolutionBase, TOp>::value>::type* = nullptr> void apply_auto_pad(const TOp *op, const TShape &data_shape, const TShape &filters_shape, TIter pads_begin, TIter pads_end)#

Apply auto padding for forward convolution.

The auto padding can be applied only if inputs and attributes of operator are validated. The input shapes must have got static ranks.

Parameters:

op – Pointer to convolution operator.
data_shape – Input data shape (must be static rank).
filters_shape – Input filter shape (must be static rank).
pads_begin – Iterator to begin of pads begin.
pads_end – Iterator to begin of pads end.

template<class TOp, class TShape, typename std::enable_if<std::is_base_of<util::ConvolutionFwdPropBase, TOp>::value || std::is_base_of<util::DeformableConvolutionBase, TOp>::value>::type* = nullptr> void apply_padding(const TOp *op, const TShape &data_shape, const TShape &filters_shape, CoordinateDiff &pads_begin, CoordinateDiff &pads_end)#

Apply padding to forward propagation convolution besed on padding.

Template Parameters:

TShape –

Parameters:

op – Pointer to coevolution operator.
data_shape – Input data shapes for shape inference.
filters_shape – Input filters shape for shape inference.
pads_begin – Begin padding to updated.
pads_end – End padding to update.

template<class TOp, class TShape, class TRShape = result_shape_t<TShape>, typename std::enable_if<std::is_base_of<util::ConvolutionFwdPropBase, TOp>::value || std::is_base_of<util::DeformableConvolutionBase, TOp>::value>::type* = nullptr> void append_spatial_shape(const TOp *op, const TShape &data_shape, const TShape &filters_shape, CoordinateDiff &pads_begin, CoordinateDiff &pads_end, TRShape &out_shape)#

Append spatial dimension at end of output shape of forward propagation convolution.

Template Parameters:

TOp – Forward propagation convolution operator type.
TShape – Type of shape.

Parameters:

op – Pointer to operator.
data_shape – Input data shape.
filters_shape – Input filter shape.
out_shape – Output shape to append spatial dimensions.

template<> constexpr size_t filter_non_spatial_dims_count<v1::GroupConvolutionBackpropData>()#

Defines non-spatial dimension for filters for group convolution back propagation operator.

Returns:: Value of non-spatial filter dimensions (3).

template<> constexpr size_t filter_non_spatial_dims_count<v1::GroupConvolution>()#

Defines non-spatial dimension for filters for group convolution operator.

Returns:: Value of non-spatial filter dimensions (3).

Variables

constexpr auto num_spatial_undefined = util::num_spatial_undefined #

constexpr size_t spatial_dim_offset = 2#

namespace validate#

Functions

template<class TShape> void data_shape(const v1::BinaryConvolution *op, const TShape &data_shape)#

Specific check of data shape for binary convolution data shape must be rank 4.

The shape_infer is same as for Convolution operator except this check.