openvino_genai.ContinuousBatchingPipeline#

class openvino_genai.ContinuousBatchingPipeline#

Bases: pybind11_object

This class is used for generation with LLMs with continuous batchig

__init__(*args, **kwargs)#

Overloaded function.

  1. __init__(self: openvino_genai.py_openvino_genai.ContinuousBatchingPipeline, models_path: os.PathLike | str | bytes, scheduler_config: openvino_genai.py_openvino_genai.SchedulerConfig, device: str, properties: collections.abc.Mapping[str, object] = {}, tokenizer_properties: collections.abc.Mapping[str, object] = {}, vision_encoder_properties: collections.abc.Mapping[str, object] = {}) -> None

  2. __init__(self: openvino_genai.py_openvino_genai.ContinuousBatchingPipeline, models_path: os.PathLike | str | bytes, tokenizer: openvino_genai.py_openvino_genai.Tokenizer, scheduler_config: openvino_genai.py_openvino_genai.SchedulerConfig, device: str, **kwargs) -> None

Methods

__delattr__(name, /)

Implement delattr(self, name).

__dir__()

Default dir() implementation.

__eq__(value, /)

Return self==value.

__format__(format_spec, /)

Default object formatter.

__ge__(value, /)

Return self>=value.

__getattribute__(name, /)

Return getattr(self, name).

__getstate__()

Helper for pickle.

__gt__(value, /)

Return self>value.

__hash__()

Return hash(self).

__init__(*args, **kwargs)

Overloaded function.

__init_subclass__

This method is called when a class is subclassed.

__le__(value, /)

Return self<=value.

__lt__(value, /)

Return self<value.

__ne__(value, /)

Return self!=value.

__new__(**kwargs)

__reduce__()

Helper for pickle.

__reduce_ex__(protocol, /)

Helper for pickle.

__repr__()

Return repr(self).

__setattr__(name, value, /)

Implement setattr(self, name, value).

__sizeof__()

Size of object in memory, in bytes.

__str__()

Return str(self).

__subclasshook__

Abstract classes can override this to customize issubclass().

_pybind11_conduit_v1_

add_request(*args, **kwargs)

Overloaded function.

finish_chat(self)

generate(*args, **kwargs)

Overloaded function.

get_config(self)

get_metrics(self)

get_tokenizer(self)

has_non_finished_requests(self)

start_chat(self[, system_message])

step(self)

Attributes

__annotations__

__annotations__ = {}#
__class__#

alias of pybind11_type

__delattr__(name, /)#

Implement delattr(self, name).

__dir__()#

Default dir() implementation.

__eq__(value, /)#

Return self==value.

__format__(format_spec, /)#

Default object formatter.

Return str(self) if format_spec is empty. Raise TypeError otherwise.

__ge__(value, /)#

Return self>=value.

__getattribute__(name, /)#

Return getattr(self, name).

__getstate__()#

Helper for pickle.

__gt__(value, /)#

Return self>value.

__hash__()#

Return hash(self).

__init__(*args, **kwargs)#

Overloaded function.

  1. __init__(self: openvino_genai.py_openvino_genai.ContinuousBatchingPipeline, models_path: os.PathLike | str | bytes, scheduler_config: openvino_genai.py_openvino_genai.SchedulerConfig, device: str, properties: collections.abc.Mapping[str, object] = {}, tokenizer_properties: collections.abc.Mapping[str, object] = {}, vision_encoder_properties: collections.abc.Mapping[str, object] = {}) -> None

  2. __init__(self: openvino_genai.py_openvino_genai.ContinuousBatchingPipeline, models_path: os.PathLike | str | bytes, tokenizer: openvino_genai.py_openvino_genai.Tokenizer, scheduler_config: openvino_genai.py_openvino_genai.SchedulerConfig, device: str, **kwargs) -> None

__init_subclass__()#

This method is called when a class is subclassed.

The default implementation does nothing. It may be overridden to extend subclasses.

__le__(value, /)#

Return self<=value.

__lt__(value, /)#

Return self<value.

__ne__(value, /)#

Return self!=value.

__new__(**kwargs)#
__reduce__()#

Helper for pickle.

__reduce_ex__(protocol, /)#

Helper for pickle.

__repr__()#

Return repr(self).

__setattr__(name, value, /)#

Implement setattr(self, name, value).

__sizeof__()#

Size of object in memory, in bytes.

__str__()#

Return str(self).

__subclasshook__()#

Abstract classes can override this to customize issubclass().

This is invoked early on by abc.ABCMeta.__subclasscheck__(). It should return True, False or NotImplemented. If it returns NotImplemented, the normal algorithm is used. Otherwise, it overrides the normal algorithm (and the outcome is cached).

_pybind11_conduit_v1_()#
add_request(*args, **kwargs)#

Overloaded function.

  1. add_request(self: openvino_genai.py_openvino_genai.ContinuousBatchingPipeline, request_id: typing.SupportsInt, input_ids: openvino._pyopenvino.Tensor, generation_config: openvino_genai.py_openvino_genai.GenerationConfig) -> openvino_genai.py_openvino_genai.GenerationHandle

  2. add_request(self: openvino_genai.py_openvino_genai.ContinuousBatchingPipeline, request_id: typing.SupportsInt, prompt: str, generation_config: openvino_genai.py_openvino_genai.GenerationConfig) -> openvino_genai.py_openvino_genai.GenerationHandle

  3. add_request(self: openvino_genai.py_openvino_genai.ContinuousBatchingPipeline, request_id: typing.SupportsInt, prompt: str, images: collections.abc.Sequence[openvino._pyopenvino.Tensor], generation_config: openvino_genai.py_openvino_genai.GenerationConfig) -> openvino_genai.py_openvino_genai.GenerationHandle

finish_chat(self: openvino_genai.py_openvino_genai.ContinuousBatchingPipeline) → None#
generate(*args, **kwargs)#

Overloaded function.

  1. generate(self: openvino_genai.py_openvino_genai.ContinuousBatchingPipeline, input_ids: collections.abc.Sequence[openvino._pyopenvino.Tensor], generation_config: collections.abc.Sequence[openvino_genai.py_openvino_genai.GenerationConfig], streamer: collections.abc.Callable[[str], int | None] | openvino_genai.py_openvino_genai.StreamerBase | None = None) -> list[openvino_genai.py_openvino_genai.EncodedGenerationResult]

  2. generate(self: openvino_genai.py_openvino_genai.ContinuousBatchingPipeline, prompts: collections.abc.Sequence[str], generation_config: collections.abc.Sequence[openvino_genai.py_openvino_genai.GenerationConfig], streamer: collections.abc.Callable[[str], int | None] | openvino_genai.py_openvino_genai.StreamerBase | None = None) -> list[openvino_genai.py_openvino_genai.GenerationResult]

  3. generate(self: openvino_genai.py_openvino_genai.ContinuousBatchingPipeline, prompt: str, generation_config: openvino_genai.py_openvino_genai.GenerationConfig, streamer: collections.abc.Callable[[str], int | None] | openvino_genai.py_openvino_genai.StreamerBase | None = None) -> list[openvino_genai.py_openvino_genai.GenerationResult]

  4. generate(self: openvino_genai.py_openvino_genai.ContinuousBatchingPipeline, prompts: collections.abc.Sequence[str], images: collections.abc.Sequence[collections.abc.Sequence[openvino._pyopenvino.Tensor]], generation_config: collections.abc.Sequence[openvino_genai.py_openvino_genai.GenerationConfig], streamer: collections.abc.Callable[[str], int | None] | openvino_genai.py_openvino_genai.StreamerBase | None = None) -> list[openvino_genai.py_openvino_genai.GenerationResult]

get_config(self: openvino_genai.py_openvino_genai.ContinuousBatchingPipeline) → openvino_genai.py_openvino_genai.GenerationConfig#
get_metrics(self: openvino_genai.py_openvino_genai.ContinuousBatchingPipeline) → openvino_genai.py_openvino_genai.PipelineMetrics#
get_tokenizer(self: openvino_genai.py_openvino_genai.ContinuousBatchingPipeline) → openvino_genai.py_openvino_genai.Tokenizer#
has_non_finished_requests(self: openvino_genai.py_openvino_genai.ContinuousBatchingPipeline) → bool#
start_chat(self: openvino_genai.py_openvino_genai.ContinuousBatchingPipeline, system_message: str = '') → None#
step(self: openvino_genai.py_openvino_genai.ContinuousBatchingPipeline) → None#