diff --git a/chatsky/__init__.py b/chatsky/__init__.py index e1d7365f4..2aa4673b1 100644 --- a/chatsky/__init__.py +++ b/chatsky/__init__.py @@ -43,4 +43,5 @@ import chatsky.responses as rsp import chatsky.processing as proc + import chatsky.__rebuild_pydantic_models__ diff --git a/chatsky/core/__init__.py b/chatsky/core/__init__.py index 7f18e72a7..474b04c1a 100644 --- a/chatsky/core/__init__.py +++ b/chatsky/core/__init__.py @@ -3,7 +3,27 @@ """ from chatsky.core.context import Context -from chatsky.core.message import Message, MessageInitTypes +from chatsky.core.message import ( + Message, + MessageInitTypes, + Attachment, + CallbackQuery, + Location, + Contact, + Invoice, + PollOption, + Poll, + DataAttachment, + Audio, + Video, + Animation, + Image, + Sticker, + Document, + VoiceMessage, + VideoMessage, + MediaGroup, +) from chatsky.core.pipeline import Pipeline from chatsky.core.script import Node, Flow, Script from chatsky.core.script_function import BaseCondition, BaseResponse, BaseDestination, BaseProcessing, BasePriority diff --git a/chatsky/core/message.py b/chatsky/core/message.py index 006939137..24a0c7e73 100644 --- a/chatsky/core/message.py +++ b/chatsky/core/message.py @@ -6,7 +6,8 @@ It only contains types and properties that are compatible with most messaging services. """ -from typing import Literal, Optional, List, Union, Dict, Any +from __future__ import annotations +from typing import Literal, Optional, List, Union, Dict, Any, TYPE_CHECKING from typing_extensions import TypeAlias, Annotated from pathlib import Path from urllib.request import urlopen @@ -16,7 +17,6 @@ from pydantic import Field, FilePath, HttpUrl, model_validator, field_validator, field_serializer from pydantic_core import Url -from chatsky.messengers.common.interface import MessengerInterfaceWithAttachments from chatsky.utils.devel import ( json_pickle_validator, json_pickle_serializer, @@ -25,6 +25,9 @@ JSONSerializableExtras, ) +if TYPE_CHECKING: + from chatsky.messengers.common.interface import MessengerInterfaceWithAttachments + class DataModel(JSONSerializableExtras): """ @@ -283,6 +286,7 @@ class level variables to store message information. VoiceMessage, VideoMessage, MediaGroup, + DataModel, ] ] ] = None diff --git a/chatsky/core/node_label.py b/chatsky/core/node_label.py index 1e032dcb1..622cfbc21 100644 --- a/chatsky/core/node_label.py +++ b/chatsky/core/node_label.py @@ -6,7 +6,7 @@ from __future__ import annotations -from typing import Optional, Union, Tuple, TYPE_CHECKING +from typing import Optional, Union, Tuple, List, TYPE_CHECKING from typing_extensions import TypeAlias, Annotated from pydantic import BaseModel, model_validator, ValidationInfo @@ -47,7 +47,7 @@ def validate_from_str_or_tuple(cls, data, info: ValidationInfo): Allow instantiating of this class from: - A single string (node name). Also attempt to get the current flow name from context. - - A tuple of two strings (flow and node name). + - A tuple or list of two strings (flow and node name). """ if isinstance(data, str): flow_name = None @@ -55,11 +55,13 @@ def validate_from_str_or_tuple(cls, data, info: ValidationInfo): if isinstance(context, dict): flow_name = _get_current_flow_name(context.get("ctx")) return {"flow_name": flow_name, "node_name": data} - elif isinstance(data, tuple): + elif isinstance(data, (tuple, list)): if len(data) == 2 and isinstance(data[0], str) and isinstance(data[1], str): return {"flow_name": data[0], "node_name": data[1]} else: - raise ValueError(f"Cannot validate NodeLabel from {data!r}: tuple should contain 2 strings.") + raise ValueError( + f"Cannot validate NodeLabel from {data!r}: {type(data).__name__} should contain 2 strings." + ) return data @@ -67,6 +69,7 @@ def validate_from_str_or_tuple(cls, data, info: ValidationInfo): NodeLabel, Annotated[str, "node_name, flow name equal to current flow's name"], Tuple[Annotated[str, "flow_name"], Annotated[str, "node_name"]], + Annotated[List[str], "list of two strings (flow_name and node_name)"], Annotated[dict, "dict following the NodeLabel data model"], ] """Types that :py:class:`~.NodeLabel` can be validated from.""" @@ -124,6 +127,7 @@ def check_node_exists(self, info: ValidationInfo): AbsoluteNodeLabel, NodeLabel, Tuple[Annotated[str, "flow_name"], Annotated[str, "node_name"]], + Annotated[List[str], "list of two strings (flow_name and node_name)"], Annotated[dict, "dict following the AbsoluteNodeLabel data model"], ] """Types that :py:class:`~.AbsoluteNodeLabel` can be validated from.""" diff --git a/chatsky/core/pipeline.py b/chatsky/core/pipeline.py index 8c51692bf..2da7bf1dc 100644 --- a/chatsky/core/pipeline.py +++ b/chatsky/core/pipeline.py @@ -32,6 +32,7 @@ from .utils import finalize_service_group from chatsky.core.service.actor import Actor from chatsky.core.node_label import AbsoluteNodeLabel, AbsoluteNodeLabelInitTypes +from chatsky.core.script_parsing import JSONImporter, Path logger = logging.getLogger(__name__) @@ -167,6 +168,31 @@ def __init__( super().__init__(**init_dict) self.services_pipeline # cache services + @classmethod + def from_file( + cls, + file: Union[str, Path], + custom_dir: Union[str, Path] = "custom", + **overrides, + ) -> "Pipeline": + """ + Create Pipeline by importing it from a file. + A file (json or yaml) should contain a dictionary with keys being a subset of pipeline init parameters. + + See :py:meth:`.JSONImporter.import_pipeline_file` for more information. + + :param file: Path to a file containing pipeline init parameters. + :param custom_dir: Path to a directory containing custom code. + Defaults to "./custom". + If ``file`` does not use custom code, this parameter will not have any effect. + :param overrides: You can pass init parameters to override those imported from the ``file``. + """ + pipeline = JSONImporter(custom_dir=custom_dir).import_pipeline_file(file) + + pipeline.update(overrides) + + return cls(**pipeline) + @computed_field @cached_property def actor(self) -> Actor: diff --git a/chatsky/core/script.py b/chatsky/core/script.py index f70db49fe..35ca32d33 100644 --- a/chatsky/core/script.py +++ b/chatsky/core/script.py @@ -12,7 +12,7 @@ import logging from typing import List, Optional, Dict -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, AliasChoices from chatsky.core.script_function import AnyResponse, BaseProcessing from chatsky.core.node_label import AbsoluteNodeLabel @@ -21,28 +21,34 @@ logger = logging.getLogger(__name__) -class Node(BaseModel): +class Node(BaseModel, extra="forbid"): """ Node is a basic element of the dialog graph. Usually used to represent a specific state of a conversation. """ - transitions: List[Transition] = Field(default_factory=list) + transitions: List[Transition] = Field( + validation_alias=AliasChoices("transitions", "TRANSITIONS"), default_factory=list + ) """List of transitions possible from this node.""" - response: Optional[AnyResponse] = Field(default=None) + response: Optional[AnyResponse] = Field(validation_alias=AliasChoices("response", "RESPONSE"), default=None) """Response produced when this node is entered.""" - pre_transition: Dict[str, BaseProcessing] = Field(default_factory=dict) + pre_transition: Dict[str, BaseProcessing] = Field( + validation_alias=AliasChoices("pre_transition", "PRE_TRANSITION"), default_factory=dict + ) """ A dictionary of :py:class:`.BaseProcessing` functions that are executed before transitions are processed. Keys of the dictionary act as names for the processing functions. """ - pre_response: Dict[str, BaseProcessing] = Field(default_factory=dict) + pre_response: Dict[str, BaseProcessing] = Field( + validation_alias=AliasChoices("pre_response", "PRE_RESPONSE"), default_factory=dict + ) """ A dictionary of :py:class:`.BaseProcessing` functions that are executed before response is processed. Keys of the dictionary act as names for the processing functions. """ - misc: dict = Field(default_factory=dict) + misc: dict = Field(validation_alias=AliasChoices("misc", "MISC"), default_factory=dict) """ A dictionary that is used to store metadata about the node. @@ -72,7 +78,9 @@ class Flow(BaseModel, extra="allow"): This is used to group them by a specific purpose. """ - local_node: Node = Field(alias="local", default_factory=Node) + local_node: Node = Field( + validation_alias=AliasChoices("local", "LOCAL", "local_node", "LOCAL_NODE"), default_factory=Node + ) """Node from which all other nodes in this Flow inherit properties according to :py:meth:`Node.merge`.""" __pydantic_extra__: Dict[str, Node] @@ -100,7 +108,9 @@ class Script(BaseModel, extra="allow"): It represents an entire dialog graph. """ - global_node: Node = Field(alias="global", default_factory=Node) + global_node: Node = Field( + validation_alias=AliasChoices("global", "GLOBAL", "global_node", "GLOBAL_NODE"), default_factory=Node + ) """Node from which all other nodes in this Script inherit properties according to :py:meth:`Node.merge`.""" __pydantic_extra__: Dict[str, Flow] @@ -157,17 +167,17 @@ def get_inherited_node(self, label: AbsoluteNodeLabel) -> Optional[Node]: return inheritant_node.merge(self.global_node).merge(flow.local_node).merge(node) -GLOBAL = "global" +GLOBAL = "GLOBAL" """Key for :py:attr:`~chatsky.core.script.Script.global_node`.""" -LOCAL = "local" +LOCAL = "LOCAL" """Key for :py:attr:`~chatsky.core.script.Flow.local_node`.""" -TRANSITIONS = "transitions" +TRANSITIONS = "TRANSITIONS" """Key for :py:attr:`~chatsky.core.script.Node.transitions`.""" -RESPONSE = "response" +RESPONSE = "RESPONSE" """Key for :py:attr:`~chatsky.core.script.Node.response`.""" -MISC = "misc" +MISC = "MISC" """Key for :py:attr:`~chatsky.core.script.Node.misc`.""" -PRE_RESPONSE = "pre_response" +PRE_RESPONSE = "PRE_RESPONSE" """Key for :py:attr:`~chatsky.core.script.Node.pre_response`.""" -PRE_TRANSITION = "pre_transition" +PRE_TRANSITION = "PRE_TRANSITION" """Key for :py:attr:`~chatsky.core.script.Node.pre_transition`.""" diff --git a/chatsky/core/script_parsing.py b/chatsky/core/script_parsing.py new file mode 100644 index 000000000..861dce741 --- /dev/null +++ b/chatsky/core/script_parsing.py @@ -0,0 +1,311 @@ +""" +Pipeline File Import +-------------------- +This module introduces tools that allow importing Pipeline objects from +json/yaml files. + +- :py:class:`JSONImporter` is a class that imports pipeline from files +- :py:func:`get_chatsky_objects` is a function that provides an index of objects commonly used in a Pipeline definition. +""" + +from typing import Union, Optional, Any, List, Tuple +import importlib +import importlib.util +import importlib.machinery +import sys +import logging +from pathlib import Path +import json +from inspect import ismodule +from functools import reduce +from contextlib import contextmanager + +from pydantic import JsonValue + +try: + import yaml + + yaml_available = True +except ImportError: + yaml_available = False + + +logger = logging.getLogger(__name__) + + +class JSONImportError(Exception): + """An exception for incorrect usage of :py:class:`JSONImporter`.""" + + __notes__ = [ + "Read the guide on Pipeline import from file: " + "https://deeppavlov.github.io/chatsky/user_guides/pipeline_import.html" + ] + + +class JSONImporter: + """ + Enables pipeline import from file. + + Since Pipeline and all its components are already pydantic ``BaseModel``, + the only purpose of this class is to allow importing and instantiating arbitrary objects. + + Import is done by replacing strings of certain patterns with corresponding objects. + This process is implemented in :py:meth:`resolve_string_reference`. + + Instantiating is done by replacing dictionaries where a single key is an imported object + with an initialized object where arguments are specified by the dictionary values. + This process is implemented in :py:meth:`replace_resolvable_objects` and + :py:meth:`parse_args`. + + :param custom_dir: Path to the directory containing custom code available for import under the + :py:attr:`CUSTOM_DIR_NAMESPACE_PREFIX`. + """ + + CHATSKY_NAMESPACE_PREFIX: str = "chatsky." + """ + Prefix that indicates an import from the `chatsky` library. + + This class variable can be changed to allow using a different prefix. + """ + CUSTOM_DIR_NAMESPACE_PREFIX: str = "custom." + """ + Prefix that indicates an import from the custom directory. + + This class variable can be changed to allow using a different prefix. + """ + EXTERNAL_LIB_NAMESPACE_PREFIX: str = "external:" + """ + Prefix that indicates an import from any library. + + This class variable can be changed to allow using a different prefix. + """ + + def __init__(self, custom_dir: Union[str, Path]): + self.custom_dir: Path = Path(custom_dir).absolute() + self.custom_dir_location: str = str(self.custom_dir.parent) + self.custom_dir_stem: str = str(self.custom_dir.stem) + + @staticmethod + def is_resolvable(value: str) -> bool: + """ + Check if ``value`` starts with any of the namespace prefixes: + + - :py:attr:`CHATSKY_NAMESPACE_PREFIX`; + - :py:attr:`CUSTOM_DIR_NAMESPACE_PREFIX`; + - :py:attr:`EXTERNAL_LIB_NAMESPACE_PREFIX`. + + :return: Whether the value should be resolved (starts with a namespace prefix). + """ + return ( + value.startswith(JSONImporter.CHATSKY_NAMESPACE_PREFIX) + or value.startswith(JSONImporter.CUSTOM_DIR_NAMESPACE_PREFIX) + or value.startswith(JSONImporter.EXTERNAL_LIB_NAMESPACE_PREFIX) + ) + + @staticmethod + @contextmanager + def sys_path_append(path): + """ + Append ``path`` to ``sys.path`` before yielding and + restore ``sys.path`` to initial state after returning. + """ + sys_path = sys.path.copy() + sys.path.append(path) + yield + sys.path = sys_path + + @staticmethod + def replace_prefix(string, old_prefix, new_prefix) -> str: + """ + Replace ``old_prefix`` in ``string`` with ``new_prefix``. + + :raises ValueError: If the ``string`` does not begin with ``old_prefix``. + :return: A new string with a new prefix. + """ + if not string.startswith(old_prefix): + raise ValueError(f"String {string!r} does not start with {old_prefix!r}") + return new_prefix + string[len(old_prefix) :] # noqa: E203 + + def resolve_string_reference(self, obj: str) -> Any: + """ + Import an object indicated by ``obj``. + + First, ``obj`` is pre-processed -- prefixes are replaced to allow import: + + - :py:attr:`CUSTOM_DIR_NAMESPACE_PREFIX` is replaced ``{stem}.`` where `stem` is the stem of the custom dir; + - :py:attr:`CHATSKY_NAMESPACE_PREFIX` is replaced with ``chatsky.``; + - :py:attr:`EXTERNAL_LIB_NAMESPACE_PREFIX` is removed. + + Next the resulting string is imported: + If the string is ``a.b.c.d``, the following is tried in order: + + 1. ``from a import b; return b.c.d`` + 2. ``from a.b import c; return c.d`` + 3. ``from a.b.c import d; return d`` + + For custom dir imports; parent of the custom dir is appended to ``sys.path`` via :py:meth:`sys_path_append`. + + :return: An imported object. + :raises ValueError: If ``obj`` does not begin with any of the prefixes (is not :py:meth:`is_resolvable`). + :raises JSONImportError: If a string could not be imported. Includes exceptions raised on every import attempt. + """ + # prepare obj string + if obj.startswith(self.CUSTOM_DIR_NAMESPACE_PREFIX): + if not self.custom_dir.exists(): + raise JSONImportError(f"Could not find directory {self.custom_dir}") + obj = self.replace_prefix(obj, self.CUSTOM_DIR_NAMESPACE_PREFIX, self.custom_dir_stem + ".") + + elif obj.startswith(self.CHATSKY_NAMESPACE_PREFIX): + obj = self.replace_prefix(obj, self.CHATSKY_NAMESPACE_PREFIX, "chatsky.") + + elif obj.startswith(self.EXTERNAL_LIB_NAMESPACE_PREFIX): + obj = self.replace_prefix(obj, self.EXTERNAL_LIB_NAMESPACE_PREFIX, "") + + else: + raise ValueError(f"Could not find a namespace prefix: {obj}") + + # import obj + split = obj.split(".") + exceptions: List[Exception] = [] + + for module_split in range(1, len(split)): + module_name = ".".join(split[:module_split]) + object_name = split[module_split:] + try: + with self.sys_path_append(self.custom_dir_location): + module = importlib.import_module(module_name) + return reduce(getattr, [module, *object_name]) + except Exception as exc: + exceptions.append(exc) + logger.debug(f"Exception attempting to import {object_name} from {module_name!r}", exc_info=exc) + raise JSONImportError(f"Could not import {obj}") from Exception(exceptions) + + def parse_args(self, value: JsonValue) -> Tuple[list, dict]: + """ + Parse ``value`` into args and kwargs: + + - If ``value`` is a dictionary, it is returned as kwargs; + - If ``value`` is a list, it is returned as args; + - If ``value`` is ``None``, both args and kwargs are empty; + - If ``value`` is anything else, it is returned as the only arg. + + :return: A tuple of args and kwargs. + """ + args = [] + kwargs = {} + value = self.replace_resolvable_objects(value) + if isinstance(value, dict): + kwargs = value + elif isinstance(value, list): + args = value + elif value is not None: # none is used when no argument is passed: e.g. `dst.Previous:` does not accept args + args = [value] + + return args, kwargs + + def replace_resolvable_objects(self, obj: JsonValue) -> Any: + """ + Replace any resolvable objects inside ``obj`` with their resolved versions and + initialize any that are the only key of a dictionary. + + This method iterates over every value inside ``obj`` (which is ``JsonValue``). + Any string that :py:meth:`is_resolvable` is replaced with an object return from + :py:meth:`resolve_string_reference`. + This is done only once (i.e. if a string is resolved to another resolvable string, + that string is not resolved). + + Any dictionaries that contain only one resolvable key are replaced with a result of + ``resolve_string_reference(key)(*args, **kwargs)`` (the object is initialized) + where ``args`` and ``kwargs`` is the result of :py:meth:`parse_args` + on the value of the dictionary. + + :return: A new object with replaced resolvable strings and dictionaries. + """ + if isinstance(obj, dict): + keys = obj.keys() + if len(keys) == 1: + key = keys.__iter__().__next__() + if self.is_resolvable(key): + args, kwargs = self.parse_args(obj[key]) + return self.resolve_string_reference(key)(*args, **kwargs) + + return {k: (self.replace_resolvable_objects(v)) for k, v in obj.items()} + elif isinstance(obj, list): + return [self.replace_resolvable_objects(item) for item in obj] + elif isinstance(obj, str): + if self.is_resolvable(obj): + return self.resolve_string_reference(obj) + return obj + + def import_pipeline_file(self, file: Union[str, Path]) -> dict: + """ + Import a dictionary from a json/yaml file and replace resolvable objects in it. + + :return: A result of :py:meth:`replace_resolvable_objects` on the dictionary. + :raises JSONImportError: If a file does not have a correct file extension. + :raises JSONImportError: If an imported object from file is not a dictionary. + """ + file = Path(file).absolute() + + with open(file, "r", encoding="utf-8") as fd: + if file.suffix == ".json": + pipeline = json.load(fd) + elif file.suffix in (".yaml", ".yml"): + if not yaml_available: + raise ImportError("`pyyaml` package is missing.\nRun `pip install chatsky[yaml]`.") + pipeline = yaml.safe_load(fd) + else: + raise JSONImportError("File should have a `.json`, `.yaml` or `.yml` extension") + if not isinstance(pipeline, dict): + raise JSONImportError("File should contain a dict") + + logger.info(f"Loaded file {file}") + return self.replace_resolvable_objects(pipeline) + + +def get_chatsky_objects(): + """ + Return an index of most commonly used ``chatsky`` objects (in the context of pipeline initialization). + + :return: A dictionary where keys are names of the objects (e.g. ``chatsky.core.Message``) and values + are the objects. + The items in the dictionary are all the objects from the ``__init__`` files of the following modules: + + - "chatsky.cnd"; + - "chatsky.rsp"; + - "chatsky.dst"; + - "chatsky.proc"; + - "chatsky.core"; + - "chatsky.core.service"; + - "chatsky.slots"; + - "chatsky.context_storages"; + - "chatsky.messengers". + """ + json_importer = JSONImporter(custom_dir="none") + + def get_objects_from_submodule(submodule_name: str, alias: Optional[str] = None): + module = json_importer.resolve_string_reference(submodule_name) + + return { + ".".join([alias or submodule_name, name]): obj + for name, obj in module.__dict__.items() + if not name.startswith("_") and not ismodule(obj) + } + + return { + k: v + for module in ( + "chatsky.cnd", + "chatsky.rsp", + "chatsky.dst", + "chatsky.proc", + "chatsky.core", + "chatsky.core.service", + "chatsky.slots", + "chatsky.context_storages", + "chatsky.messengers", + # "chatsky.stats", + # "chatsky.utils", + ) + for k, v in get_objects_from_submodule(module).items() + } diff --git a/chatsky/messengers/__init__.py b/chatsky/messengers/__init__.py index 40a96afc6..cc979894f 100644 --- a/chatsky/messengers/__init__.py +++ b/chatsky/messengers/__init__.py @@ -1 +1,8 @@ -# -*- coding: utf-8 -*- +from chatsky.messengers.common import ( + MessengerInterface, + MessengerInterfaceWithAttachments, + PollingMessengerInterface, + CallbackMessengerInterface, +) +from chatsky.messengers.telegram import LongpollingInterface as TelegramInterface +from chatsky.messengers.console import CLIMessengerInterface diff --git a/chatsky/processing/__init__.py b/chatsky/processing/__init__.py index 4bc71cf5d..fcd984a05 100644 --- a/chatsky/processing/__init__.py +++ b/chatsky/processing/__init__.py @@ -1,2 +1,2 @@ from .standard import ModifyResponse -from .slots import Extract, ExtractAll, Unset, UnsetAll, FillTemplate +from .slots import Extract, Unset, UnsetAll, FillTemplate diff --git a/chatsky/processing/slots.py b/chatsky/processing/slots.py index 6bf017782..898195e94 100644 --- a/chatsky/processing/slots.py +++ b/chatsky/processing/slots.py @@ -24,14 +24,16 @@ class Extract(BaseProcessing): slots: List[SlotName] """A list of slot names to extract.""" + success_only: bool = True + """If set, only successfully extracted values will be stored in the slot storage.""" - def __init__(self, *slots: SlotName): - super().__init__(slots=slots) + def __init__(self, *slots: SlotName, success_only: bool = True): + super().__init__(slots=slots, success_only=success_only) async def call(self, ctx: Context): manager = ctx.framework_data.slot_manager results = await asyncio.gather( - *(manager.extract_slot(slot, ctx) for slot in self.slots), return_exceptions=True + *(manager.extract_slot(slot, ctx, self.success_only) for slot in self.slots), return_exceptions=True ) for result in results: @@ -39,16 +41,6 @@ async def call(self, ctx: Context): logger.exception("An exception occurred during slot extraction.", exc_info=result) -class ExtractAll(BaseProcessing): - """ - Extract all slots defined in the pipeline. - """ - - async def call(self, ctx: Context): - manager = ctx.framework_data.slot_manager - await manager.extract_all(ctx) - - class Unset(BaseProcessing): """ Mark specified slots as not extracted and clear extracted values. diff --git a/chatsky/slots/slots.py b/chatsky/slots/slots.py index a6981fabc..276a28f56 100644 --- a/chatsky/slots/slots.py +++ b/chatsky/slots/slots.py @@ -239,6 +239,8 @@ async def get_value(self, ctx: Context) -> ExtractedValueSlot: logger.exception(f"Exception occurred during {self.__class__.__name__!r} extraction.", exc_info=error) extracted_value = error finally: + if not is_slot_extracted: + logger.debug(f"Slot {self.__class__.__name__!r} was not extracted: {extracted_value}") return ExtractedValueSlot.model_construct( is_slot_extracted=is_slot_extracted, extracted_value=extracted_value, @@ -362,16 +364,21 @@ def get_slot(self, slot_name: SlotName) -> BaseSlot: return slot raise KeyError(f"Could not find slot {slot_name!r}.") - async def extract_slot(self, slot_name: SlotName, ctx: Context) -> None: + async def extract_slot(self, slot_name: SlotName, ctx: Context, success_only: bool) -> None: """ Extract slot `slot_name` and store extracted value in `slot_storage`. :raises KeyError: If the slot with the specified name does not exist. + + :param slot_name: Name of the slot to extract. + :param ctx: Context. + :param success_only: Whether to store the value only if it is successfully extracted. """ slot = self.get_slot(slot_name) value = await slot.get_value(ctx) - recursive_setattr(self.slot_storage, slot_name, value) + if value.__slot_extracted__ or success_only is False: + recursive_setattr(self.slot_storage, slot_name, value) async def extract_all(self, ctx: Context): """ diff --git a/chatsky/utils/testing/__init__.py b/chatsky/utils/testing/__init__.py index bfbe04fef..334d4d077 100644 --- a/chatsky/utils/testing/__init__.py +++ b/chatsky/utils/testing/__init__.py @@ -1,10 +1,3 @@ # -*- coding: utf-8 -*- from .common import is_interactive_mode, check_happy_path from .toy_script import TOY_SCRIPT, TOY_SCRIPT_KWARGS, HAPPY_PATH - -try: - import pytest - - pytest.register_assert_rewrite("chatsky.utils.testing.telegram") -except ImportError: - ... diff --git a/docs/source/user_guides.rst b/docs/source/user_guides.rst index 6802a4b34..b8dbc376d 100644 --- a/docs/source/user_guides.rst +++ b/docs/source/user_guides.rst @@ -4,7 +4,7 @@ User guides :doc:`Basic concepts <./user_guides/basic_conceptions>` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -In the ``basic concepts`` tutorial the basics of Chatsky are described, +In the ``basic concepts`` guide the basics of Chatsky are described, those include but are not limited to: dialog graph creation, specifying start and fallback nodes, setting transitions and conditions, using ``Context`` object in order to receive information about current script execution. @@ -26,7 +26,7 @@ The ``context guide`` walks you through the details of working with the :doc:`Superset guide <./user_guides/superset_guide>` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The ``superset guide`` tutorial highlights the usage of Superset visualization tool +The ``superset guide`` highlights the usage of Superset visualization tool for exploring the telemetry data collected from your conversational services. We show how to plug in the telemetry collection and configure the pre-built Superset dashboard shipped with Chatsky. @@ -38,6 +38,12 @@ The ``optimization guide`` demonstrates various tools provided by the library that you can use to profile your conversational service, and to locate and remove performance bottlenecks. +:doc:`YAML import guide <./user_guides/pipeline_import>` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The ``yaml import guide`` shows another option for initializing ``Pipeline`` +objects -- from yaml or json files. + .. toctree:: :hidden: @@ -46,3 +52,4 @@ and to locate and remove performance bottlenecks. user_guides/context_guide user_guides/superset_guide user_guides/optimization_guide + user_guides/pipeline_import diff --git a/docs/source/user_guides/pipeline_import.rst b/docs/source/user_guides/pipeline_import.rst new file mode 100644 index 000000000..c50b56259 --- /dev/null +++ b/docs/source/user_guides/pipeline_import.rst @@ -0,0 +1,179 @@ +Pipeline YAML import guide +-------------------------- + +Introduction +~~~~~~~~~~~~ + +Instead of passing all the arguments to pipeline from a python environment, +you can initialize pipeline by getting the arguments from a file. + +The details of this process are described in this guide. + +Basics +~~~~~~ + +To initialize ``Pipeline`` from a file, call its `from_file <../apiref/chatsky.core.pipeline.html#chatsky.core.pipeline.Pipeline.from_file>`_ +method. It accepts a path to a file, a path to a custom code directory and overrides. + +File +==== + +The file should be a json or yaml file that contains a dictionary. +They keys in the dictionary are the names of pipeline init parameters and the values are the values of the parameters. + +Below is a minimalistic example of such a file: + +.. code-block:: yaml + + script: + flow: + node: + RESPONSE: Hi + TRANSITIONS: + - dst: node + cnd: true + priority: 2 + start_label: + - flow + - node + +.. note:: + + If you're using yaml files, you need to install pyyaml: + + .. code-block:: sh + + pip install chatsky[yaml] + + +Custom dir +========== + +Custom directory allows using any objects inside the yaml file. + +More on that in the :ref:`object-import` section. + +Overrides +========= + +Any pipeline init parameters can be passed to ``from_file``. +They will override parameters defined in the file (or add them if they are not defined in the file). + +.. _object-import: + +Object Import +~~~~~~~~~~~~~ + +JSON values are often not enough to build any serious script. + +For this reason, the init parameters in the pipeline file are preprocessed in two ways: + +String reference replacement +============================ + +Any string that begins with either ``chatsky.``, ``custom.`` or ``external:`` is replaced with a corresponding object. + +The ``chatsky.`` prefix indicates that an object should be found inside the ``chatsky`` library. +For example, string ``chatsky.cnd.ExactMatch`` will be replaced with the ``chatsky.cnd.ExactMatch`` object (which is a class). + +The ``custom.`` prefix allows importing object from the custom directory passed to ``Pipeline.from_file``. +For example, string ``custom.my_response`` will be replaced with the ``my_response`` object defined in ``custom/__init__.py`` +(or will throw an exception if there's no such object). + +The ``external:`` prefix can be used to import any objects (primarily, from external libraries). +For example, string ``external:os.getenv`` will be replaced with the function ``os.getenv``. + +.. note:: + + It is highly recommended to read about the import process for these strings + `here <../apiref/chatsky.core.script_parsing.html#chatsky.core.script_parsing.JSONImporter.resolve_string_reference>`_. + +.. note:: + + If you want to use different prefixes, you can edit the corresponding class variables of the + `JSONImporter <../apiref/chatsky.core.script_parsing.html#chatsky.core.script_parsing.JSONImporter>`_ class: + + .. code-block:: python + + from chatsky.core.script_parsing import JSONImporter + from chatsky import Pipeline + + JSONImporter.CHATSKY_NAMESPACE_PREFIX = "_chatsky:" + + pipeline = Pipeline.from_file(...) + + After changing the prefix variable, ``from_file`` will no longer replace strings that start with ``chatsky.``. + (and will replace strings that start with ``_chatsky:``) + +Single-key dict replacement +=========================== + +Any dictionary containing a **single** key that **begins with any of the prefixes** described in the previous section +will be replaced with a call result of the object referenced by the key. + +Call is made with the arguments passed as a value of the dictionary: + +- If the value is a dictionary; it is passed as kwargs; +- If the value is a list; it is passed as args; +- If the value is ``None``; no arguments are passed; +- Otherwise, the value is passed as the only arg. + +.. list-table:: Examples + :widths: auto + :header-rows: 1 + + * - YAML string + - Resulting object + - Note + * - .. code-block:: yaml + + external:os.getenv: TOKEN + - .. code-block:: python + + os.getenv("TOKEN") + - This falls into the 4th condition (value is not a dict, list or None) so it is passed as the only argument. + * - .. code-block:: yaml + + chatsky.dst.Previous: + - .. code-block:: python + + chatsky.dst.Previous() + - The value is ``None``, so there are no arguments. + * - .. code-block:: yaml + + chatsky.dst.Previous + - .. code-block:: python + + chatsky.dst.Previous + - This is not a dictionary, the resulting object is a class! + * - .. code-block:: yaml + + chatsky.cnd.Regexp: + pattern: "yes" + flags: external:re.I + - .. code-block:: python + + chatsky.cnd.Regexp( + pattern="yes", + flags=re.I + ) + - The value is a dictionary; it is passed as kwargs. + This also showcases that replacement is recursive ``external:re.I`` is replaced as well. + * - .. code-block:: yaml + + chatsky.proc.Extract: + - person.name + - person.age + - .. code-block:: python + + chatsky.proc.Extract( + "person.name", + "person.age" + ) + - The value is a list; it is passed as args. + +Further reading +~~~~~~~~~~~~~~~ + +* `API ref <../apiref/chatsky.core.script_parsing.html>`_ +* `Comprehensive example `_ diff --git a/poetry.lock b/poetry.lock index 2f12d0b3e..ad192e6a2 100644 --- a/poetry.lock +++ b/poetry.lock @@ -4952,7 +4952,6 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, @@ -6936,9 +6935,10 @@ redis = ["redis"] sqlite = ["aiosqlite", "sqlalchemy"] stats = ["omegaconf", "opentelemetry-exporter-otlp", "opentelemetry-instrumentation", "requests", "tqdm"] telegram = ["python-telegram-bot"] +yaml = ["pyyaml"] ydb = ["six", "ydb"] [metadata] lock-version = "2.0" python-versions = "^3.8.1,!=3.9.7" -content-hash = "a4e53a8b58504d6e4f877ac5e7901d5aa8451003bf9edf55ebfb4df7af8424ab" +content-hash = "511348f67731d8a26e0a269d3f8f032368a85289cdd4772df378335c57812201" diff --git a/pyproject.toml b/pyproject.toml index cbde2143f..a6ff0a967 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -74,6 +74,7 @@ python-telegram-bot = { version = "~=21.3", extras = ["all"], optional = true } opentelemetry-instrumentation = { version = "*", optional = true } sqlalchemy = { version = "*", extras = ["asyncio"], optional = true } opentelemetry-exporter-otlp = { version = ">=1.20.0", optional = true } # log body serialization is required +pyyaml = { version = "*", optional = true } [tool.poetry.extras] json = ["aiofiles"] @@ -87,6 +88,7 @@ ydb = ["ydb", "six"] telegram = ["python-telegram-bot"] stats = ["opentelemetry-exporter-otlp", "opentelemetry-instrumentation", "requests", "tqdm", "omegaconf"] benchmark = ["pympler", "humanize", "pandas", "altair", "tqdm"] +yaml = ["pyyaml"] [tool.poetry.group.lint] @@ -224,5 +226,4 @@ concurrency = [ exclude_also = [ "if TYPE_CHECKING:", "raise NotImplementedError", - "raise RuntimeError", ] diff --git a/tests/core/script_parsing/__init__.py b/tests/core/script_parsing/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/core/script_parsing/custom/__init__.py b/tests/core/script_parsing/custom/__init__.py new file mode 100644 index 000000000..daa9575c6 --- /dev/null +++ b/tests/core/script_parsing/custom/__init__.py @@ -0,0 +1,4 @@ +from . import submodule as sub +from .submodule import VAR as V + +recurse = "custom.V" diff --git a/tests/core/script_parsing/custom/submodule/__init__.py b/tests/core/script_parsing/custom/submodule/__init__.py new file mode 100644 index 000000000..1e4d22038 --- /dev/null +++ b/tests/core/script_parsing/custom/submodule/__init__.py @@ -0,0 +1,2 @@ +from . import submodule as sub +from .submodule import V as VAR diff --git a/tests/core/script_parsing/custom/submodule/submodule/__init__.py b/tests/core/script_parsing/custom/submodule/submodule/__init__.py new file mode 100644 index 000000000..f59f611ce --- /dev/null +++ b/tests/core/script_parsing/custom/submodule/submodule/__init__.py @@ -0,0 +1,2 @@ +from . import file as f +from .file import VAR as V diff --git a/tests/core/script_parsing/custom/submodule/submodule/file.py b/tests/core/script_parsing/custom/submodule/submodule/file.py new file mode 100644 index 000000000..7a4283691 --- /dev/null +++ b/tests/core/script_parsing/custom/submodule/submodule/file.py @@ -0,0 +1 @@ +VAR = 1 diff --git a/tests/core/script_parsing/custom_dir/__init__.py b/tests/core/script_parsing/custom_dir/__init__.py new file mode 100644 index 000000000..f46b7b8d2 --- /dev/null +++ b/tests/core/script_parsing/custom_dir/__init__.py @@ -0,0 +1 @@ +VAR = 2 diff --git a/tests/core/script_parsing/custom_dir/module.py b/tests/core/script_parsing/custom_dir/module.py new file mode 100644 index 000000000..ec8099c45 --- /dev/null +++ b/tests/core/script_parsing/custom_dir/module.py @@ -0,0 +1 @@ +VAR = 3 diff --git a/tests/core/script_parsing/pipeline.json b/tests/core/script_parsing/pipeline.json new file mode 100644 index 000000000..359643f43 --- /dev/null +++ b/tests/core/script_parsing/pipeline.json @@ -0,0 +1,15 @@ +{ + "script": { + "flow": { + "node": { + "misc": { + "key": "custom.V" + } + } + } + }, + "start_label": [ + "flow", + "node" + ] +} \ No newline at end of file diff --git a/tests/core/script_parsing/pipeline.yaml b/tests/core/script_parsing/pipeline.yaml new file mode 100644 index 000000000..5b26e179e --- /dev/null +++ b/tests/core/script_parsing/pipeline.yaml @@ -0,0 +1,28 @@ +script: + flow: + node: + response: + text: hi + misc: + key: custom.V + transitions: + - dst: + chatsky.dst.Previous: + cnd: + chatsky.cnd.HasText: t +start_label: + - flow + - node +fallback_label: + - other_flow + - other_node +slots: + person: + likes: + chatsky.slots.RegexpSlot: + regexp: "I like (.+)" + match_group_idx: 1 + age: + chatsky.slots.RegexpSlot: + regexp: "I'm ([0-9]+) years old" + match_group_idx: 1 diff --git a/tests/core/script_parsing/test_script_parsing.py b/tests/core/script_parsing/test_script_parsing.py new file mode 100644 index 000000000..9307f4f15 --- /dev/null +++ b/tests/core/script_parsing/test_script_parsing.py @@ -0,0 +1,184 @@ +from pathlib import Path + +import pytest + +import chatsky +from chatsky.core.script_parsing import JSONImporter, JSONImportError, get_chatsky_objects, yaml_available + + +current_dir = Path(__file__).parent.absolute() + + +class TestResolveStringReference: + @pytest.mark.parametrize( + "string", + [ + "custom.V", + "custom.sub.VAR", + "custom.sub.sub.V", + "custom.sub.sub.f.VAR", + "custom.submodule.VAR", + "custom.submodule.sub.V", + "custom.submodule.sub.f.VAR", + "custom.submodule.submodule.V", + "custom.submodule.submodule.f.VAR", + "custom.submodule.submodule.file.VAR", + "custom.sub.submodule.V", + "custom.sub.submodule.f.VAR", + "custom.sub.submodule.file.VAR", + ], + ) + def test_resolve_custom_object(self, string): + json_importer = JSONImporter(custom_dir=current_dir / "custom") + + assert json_importer.resolve_string_reference(string) == 1 + + def test_different_custom_location(self): + json_importer = JSONImporter(custom_dir=current_dir / "custom_dir") + + assert json_importer.resolve_string_reference("custom.VAR") == 2 + assert json_importer.resolve_string_reference("custom.module.VAR") == 3 + + @pytest.mark.parametrize( + "obj,val", + [ + ("chatsky.cnd.ExactMatch", chatsky.conditions.ExactMatch), + ("chatsky.conditions.standard.ExactMatch", chatsky.conditions.ExactMatch), + ("chatsky.core.message.Image", chatsky.core.message.Image), + ("chatsky.Message", chatsky.Message), + ("chatsky.context_storages.sql.SQLContextStorage", chatsky.context_storages.sql.SQLContextStorage), + ("chatsky.messengers.telegram.LongpollingInterface", chatsky.messengers.telegram.LongpollingInterface), + ("chatsky.LOCAL", "LOCAL"), + ], + ) + def test_resolve_chatsky_objects(self, obj, val): + json_importer = JSONImporter(custom_dir=current_dir / "none") + + assert json_importer.resolve_string_reference(obj) == val + + def test_resolve_external_objects(self): + json_importer = JSONImporter(custom_dir=current_dir / "none") + + assert json_importer.resolve_string_reference("external:logging.DEBUG") == 10 + + def test_alternative_domain_names(self, monkeypatch): + monkeypatch.setattr(JSONImporter, "CHATSKY_NAMESPACE_PREFIX", "_chatsky:") + monkeypatch.setattr(JSONImporter, "CUSTOM_DIR_NAMESPACE_PREFIX", "_custom:") + + json_importer = JSONImporter(custom_dir=current_dir / "custom") + + assert json_importer.resolve_string_reference("_chatsky:Message") == chatsky.Message + assert json_importer.resolve_string_reference("_custom:V") == 1 + + def test_non_existent_custom_dir(self): + json_importer = JSONImporter(custom_dir=current_dir / "none") + with pytest.raises(JSONImportError, match="Could not find directory"): + json_importer.resolve_string_reference("custom.VAR") + + def test_wrong_prefix(self): + json_importer = JSONImporter(custom_dir=current_dir / "none") + with pytest.raises(ValueError, match="prefix"): + json_importer.resolve_string_reference("wrong_domain.VAR") + + def test_non_existent_object(self): + json_importer = JSONImporter(custom_dir=current_dir / "custom_dir") + with pytest.raises(JSONImportError, match="Could not import"): + json_importer.resolve_string_reference("chatsky.none.VAR") + with pytest.raises(JSONImportError, match="Could not import"): + json_importer.resolve_string_reference("custom.none.VAR") + + +@pytest.mark.parametrize( + "obj,replaced", + [ + (5, 5), + (True, True), + ("string", "string"), + ("custom.V", 1), + ("chatsky.LOCAL", "LOCAL"), + ({"text": "custom.V"}, {"text": 1}), + ({"1": {"2": "custom.V"}}, {"1": {"2": 1}}), + ({"1": "custom.V", "2": "custom.V"}, {"1": 1, "2": 1}), + (["custom.V", 4], [1, 4]), + ({"chatsky.Message": None}, chatsky.Message()), + ({"chatsky.Message": ["text"]}, chatsky.Message("text")), + ({"chatsky.Message": {"text": "text", "misc": {}}}, chatsky.Message("text", misc={})), + ({"chatsky.Message": ["chatsky.LOCAL"]}, chatsky.Message("LOCAL")), + ({"chatsky.Message": {"text": "LOCAL"}}, chatsky.Message("LOCAL")), + ], +) +def test_replace_resolvable_objects(obj, replaced): + json_importer = JSONImporter(custom_dir=current_dir / "custom") + + assert json_importer.replace_resolvable_objects(obj) == replaced + + +def test_nested_replacement(): + json_importer = JSONImporter(custom_dir=current_dir / "none") + + obj = json_importer.replace_resolvable_objects({"chatsky.cnd.Negation": {"chatsky.cnd.HasText": {"text": "text"}}}) + + assert isinstance(obj, chatsky.cnd.Negation) + assert isinstance(obj.condition, chatsky.cnd.HasText) + assert obj.condition.text == "text" + + +def test_no_recursion(): + json_importer = JSONImporter(custom_dir=current_dir / "custom") + + obj = json_importer.replace_resolvable_objects( + {"chatsky.cnd.Negation": {"chatsky.cnd.HasText": {"text": "custom.recurse"}}} + ) + + assert obj.condition.text == "custom.V" + + +class TestImportPipelineFile: + @pytest.mark.skipif(not yaml_available, reason="YAML dependencies missing") + def test_normal_import(self): + pipeline = chatsky.Pipeline.from_file( + current_dir / "pipeline.yaml", + custom_dir=current_dir / "custom", + fallback_label=("flow", "node"), # override the parameter + ) + + assert pipeline.start_label.node_name == "node" + assert pipeline.fallback_label.node_name == "node" + start_node = pipeline.script.get_node(pipeline.start_label) + assert start_node.response.root == chatsky.Message("hi", misc={"key": 1}) + assert start_node.transitions[0].dst == chatsky.dst.Previous() + assert start_node.transitions[0].cnd == chatsky.cnd.HasText("t") + + assert pipeline.slots.person.likes == chatsky.slots.RegexpSlot(regexp="I like (.+)", match_group_idx=1) + assert pipeline.slots.person.age == chatsky.slots.RegexpSlot(regexp="I'm ([0-9]+) years old", match_group_idx=1) + + def test_import_json(self): + pipeline = chatsky.Pipeline.from_file(current_dir / "pipeline.json", custom_dir=current_dir / "custom") + + assert pipeline.script.get_node(pipeline.start_label).misc == {"key": 1} + + def test_wrong_file_ext(self): + with pytest.raises(JSONImportError, match="extension"): + chatsky.Pipeline.from_file(current_dir / "__init__.py") + + def test_wrong_object_type(self): + with pytest.raises(JSONImportError, match="dict"): + chatsky.Pipeline.from_file(current_dir / "wrong_type.json") + + +@pytest.mark.parametrize( + "key,value", + [ + ("chatsky.cnd.ExactMatch", chatsky.conditions.ExactMatch), + ("chatsky.core.Image", chatsky.core.message.Image), + ("chatsky.core.Message", chatsky.Message), + ("chatsky.context_storages.SQLContextStorage", chatsky.context_storages.sql.SQLContextStorage), + ("chatsky.messengers.TelegramInterface", chatsky.messengers.telegram.LongpollingInterface), + ("chatsky.slots.RegexpSlot", chatsky.slots.RegexpSlot), + ], +) +def test_get_chatsky_objects(key, value): + json_importer = JSONImporter(custom_dir=current_dir / "none") + + assert json_importer.resolve_string_reference(key) == value + assert get_chatsky_objects()[key] == value diff --git a/tests/core/script_parsing/wrong_type.json b/tests/core/script_parsing/wrong_type.json new file mode 100644 index 000000000..63964002b --- /dev/null +++ b/tests/core/script_parsing/wrong_type.json @@ -0,0 +1,4 @@ +[ + 1, + 2 +] \ No newline at end of file diff --git a/tests/core/test_node_label.py b/tests/core/test_node_label.py new file mode 100644 index 000000000..8580f5f5f --- /dev/null +++ b/tests/core/test_node_label.py @@ -0,0 +1,51 @@ +import pytest +from pydantic import ValidationError + +from chatsky.core import NodeLabel, Context, AbsoluteNodeLabel, Pipeline + + +def test_init_from_single_string(): + ctx = Context.init(("flow", "node1")) + ctx.framework_data.pipeline = Pipeline({"flow": {"node2": {}}}, ("flow", "node2")) + + node = AbsoluteNodeLabel.model_validate("node2", context={"ctx": ctx}) + + assert node == AbsoluteNodeLabel(flow_name="flow", node_name="node2") + + +@pytest.mark.parametrize("data", [("flow", "node"), ["flow", "node"]]) +def test_init_from_iterable(data): + node = AbsoluteNodeLabel.model_validate(data) + assert node == AbsoluteNodeLabel(flow_name="flow", node_name="node") + + +@pytest.mark.parametrize( + "data,msg", + [ + (["flow", "node", 3], "list should contain 2 strings"), + ((1, 2), "tuple should contain 2 strings"), + ], +) +def test_init_from_incorrect_iterables(data, msg): + with pytest.raises(ValidationError, match=msg): + AbsoluteNodeLabel.model_validate(data) + + +def test_init_from_node_label(): + with pytest.raises(ValidationError): + AbsoluteNodeLabel.model_validate(NodeLabel(node_name="node")) + + ctx = Context.init(("flow", "node1")) + ctx.framework_data.pipeline = Pipeline({"flow": {"node2": {}}}, ("flow", "node2")) + + node = AbsoluteNodeLabel.model_validate(NodeLabel(node_name="node2"), context={"ctx": ctx}) + + assert node == AbsoluteNodeLabel(flow_name="flow", node_name="node2") + + +def test_check_node_exists(): + ctx = Context.init(("flow", "node1")) + ctx.framework_data.pipeline = Pipeline({"flow": {"node2": {}}}, ("flow", "node2")) + + with pytest.raises(ValidationError, match="Cannot find node"): + AbsoluteNodeLabel.model_validate(("flow", "node3"), context={"ctx": ctx}) diff --git a/tests/slots/test_slot_functions.py b/tests/slots/test_slot_functions.py index b0c80a651..83c6b0171 100644 --- a/tests/slots/test_slot_functions.py +++ b/tests/slots/test_slot_functions.py @@ -56,10 +56,16 @@ def func(*args, **kwargs): async def test_basic_functions(context, manager, log_event_catcher): + await proc.Extract("0", "2", "err").wrapped_call(context) + + assert manager.get_extracted_slot("0").value == 4 + assert manager.is_slot_extracted("1") is False + assert isinstance(manager.get_extracted_slot("err").extracted_value, SlotNotExtracted) + proc_logs = log_event_catcher(proc_logger, level=logging.ERROR) slot_logs = log_event_catcher(slot_logger, level=logging.ERROR) - await proc.Extract("0", "2", "err").wrapped_call(context) + await proc.Extract("0", "2", "err", success_only=False).wrapped_call(context) assert manager.get_extracted_slot("0").value == 4 assert manager.is_slot_extracted("1") is False @@ -82,16 +88,6 @@ async def test_basic_functions(context, manager, log_event_catcher): assert await cnd.SlotsExtracted("0", "1", mode="any").wrapped_call(context) is False -async def test_extract_all(context, manager, monkeypatch, call_logger_factory): - logs, func = call_logger_factory() - - monkeypatch.setattr(SlotManager, "extract_all", func) - - await proc.ExtractAll().wrapped_call(context) - - assert logs == [{"args": (manager, context), "kwargs": {}}] - - async def test_unset_all(context, manager, monkeypatch, call_logger_factory): logs, func = call_logger_factory() diff --git a/tests/slots/test_slot_manager.py b/tests/slots/test_slot_manager.py index b6b3f6db1..33885b360 100644 --- a/tests/slots/test_slot_manager.py +++ b/tests/slots/test_slot_manager.py @@ -174,7 +174,39 @@ def test_get_slot_by_name(empty_slot_manager): ], ) async def test_slot_extraction(slot_name, expected_slot_storage, empty_slot_manager, context_with_request): - await empty_slot_manager.extract_slot(slot_name, context_with_request) + await empty_slot_manager.extract_slot(slot_name, context_with_request, success_only=False) + assert empty_slot_manager.slot_storage == expected_slot_storage + + +@pytest.mark.parametrize( + "slot_name,expected_slot_storage", + [ + ( + "person.name", + ExtractedGroupSlot( + person=ExtractedGroupSlot( + name=extracted_slot_values["person.name"], + surname=init_value_slot, + email=init_value_slot, + ), + msg_len=init_value_slot, + ), + ), + ( + "person.surname", + ExtractedGroupSlot( + person=ExtractedGroupSlot( + name=init_value_slot, + surname=init_value_slot, + email=init_value_slot, + ), + msg_len=init_value_slot, + ), + ), + ], +) +async def test_successful_extraction(slot_name, expected_slot_storage, empty_slot_manager, context_with_request): + await empty_slot_manager.extract_slot(slot_name, context_with_request, success_only=True) assert empty_slot_manager.slot_storage == expected_slot_storage diff --git a/tutorials/messengers/telegram/1_basic.py b/tutorials/messengers/telegram/1_basic.py index dec0ea10e..66dd055f1 100644 --- a/tutorials/messengers/telegram/1_basic.py +++ b/tutorials/messengers/telegram/1_basic.py @@ -48,6 +48,20 @@ class and [python-telegram-bot](https://docs.python-telegram-bot.org/) Either of the two interfaces connect the bot to Telegram. They can be passed directly to a Chatsky `Pipeline` instance. + + +
+ +Note + +You can also import `LongpollingInterface` +under the alias of `TelegramInterface` from `chatsky.messengers`: + +```python +from chatsky.messengers import TelegramInterface +``` + +
""" diff --git a/tutorials/slots/1_basic_example.py b/tutorials/slots/1_basic_example.py index 66e8cedcb..dcddfbade 100644 --- a/tutorials/slots/1_basic_example.py +++ b/tutorials/slots/1_basic_example.py @@ -78,8 +78,6 @@ Condition for checking if specified slots are extracted. - %mddoclink(api,processing.slots,Extract): A processing function that extracts specified slots. -- %mddoclink(api,processing.slots,ExtractAll): - A processing function that extracts all slots. - %mddoclink(api,processing.slots,Unset): A processing function that marks specified slots as not extracted, effectively resetting their state. diff --git a/utils/pipeline_yaml_import_example/custom_dir/__init__.py b/utils/pipeline_yaml_import_example/custom_dir/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/utils/pipeline_yaml_import_example/custom_dir/rsp.py b/utils/pipeline_yaml_import_example/custom_dir/rsp.py new file mode 100644 index 000000000..b937639e8 --- /dev/null +++ b/utils/pipeline_yaml_import_example/custom_dir/rsp.py @@ -0,0 +1,8 @@ +from chatsky import BaseResponse, Context, MessageInitTypes, cnd + + +class ListNotExtractedSlots(BaseResponse): + async def call(self, ctx: Context) -> MessageInitTypes: + not_extracted_slots = [key for key in ("name", "age") if not await cnd.SlotsExtracted(f"person.{key}")(ctx)] + + return f"You did not provide {not_extracted_slots} yet." diff --git a/utils/pipeline_yaml_import_example/pipeline.py b/utils/pipeline_yaml_import_example/pipeline.py new file mode 100644 index 000000000..970ab60db --- /dev/null +++ b/utils/pipeline_yaml_import_example/pipeline.py @@ -0,0 +1,19 @@ +from pathlib import Path +import logging + +from chatsky import Pipeline + + +logging.basicConfig(level=logging.INFO) + +current_dir = Path(__file__).parent + +pipeline = Pipeline.from_file( + file=current_dir / "pipeline.yaml", + custom_dir=current_dir / "custom_dir", + # these paths can also be relative (e.g. file="pipeline.yaml") + # but that would only work if executing pipeline in this directory +) + +if __name__ == "__main__": + pipeline.run() diff --git a/utils/pipeline_yaml_import_example/pipeline.yaml b/utils/pipeline_yaml_import_example/pipeline.yaml new file mode 100644 index 000000000..b7c638c45 --- /dev/null +++ b/utils/pipeline_yaml_import_example/pipeline.yaml @@ -0,0 +1,112 @@ +script: + GLOBAL: + TRANSITIONS: + - dst: [tech_flow, start_node] + cnd: + chatsky.cnd.ExactMatch: /start + priority: 2 + tech_flow: + start_node: + RESPONSE: + text: + "Hello. + We'd like to collect some data about you. + Do you agree? (yes/no)" + PRE_TRANSITION: + unset_all_slots: + chatsky.proc.UnsetAll: + TRANSITIONS: + - dst: [data_collection, start] + cnd: + chatsky.cnd.Regexp: + pattern: "yes" + flags: external:re.IGNORECASE + fallback_node: + RESPONSE: + "Dialog finished. + You can restart by typing /start." + data_collection: + LOCAL: + PRE_TRANSITION: + extract_slots: + chatsky.proc.Extract: + - person.name + - person.age + TRANSITIONS: + - dst: not_provided_slots + cnd: + chatsky.cnd.Negation: + chatsky.cnd.SlotsExtracted: + - person.name + - person.age + priority: 0.5 + - dst: name_extracted + cnd: + chatsky.cnd.All: + - chatsky.cnd.HasText: My name + - chatsky.cnd.SlotsExtracted: person.name + - dst: age_extracted + cnd: + chatsky.cnd.All: + - chatsky.cnd.HasText: years old + - chatsky.cnd.SlotsExtracted: person.age + - dst: [final_flow, all_slots_extracted] + cnd: + chatsky.cnd.SlotsExtracted: + - person.name + - person.age + priority: 1.5 + start: + RESPONSE: + text: + "Please provide us with the following data: + + Your *name* by sending message \"My name is X\" + + Your *age* by sending message \"I'm X years old\"" + parse_mode: external:telegram.constants.ParseMode.MARKDOWN_V2 + not_provided_slots: + RESPONSE: + custom.rsp.ListNotExtractedSlots: + name_extracted: + RESPONSE: + Got your name. Now provide your age. + age_extracted: + RESPONSE: + Got your age. Now provide your name. + final_flow: + all_slots_extracted: + RESPONSE: + chatsky.rsp.FilledTemplate: + chatsky.Message: + text: + "Thank you for providing us your data. + + Your name: {person.name}; + Your age: {person.age}. + + Here's a cute sticker as a reward:" + attachments: + - chatsky.core.Sticker: + id: CAACAgIAAxkBAAErBZ1mKAbZvEOmhscojaIL5q0u8vgp1wACRygAAiSjCUtLa7RHZy76ezQE +start_label: + - tech_flow + - start_node +fallback_label: + - tech_flow + - fallback_node +slots: + person: + name: + chatsky.slots.RegexpSlot: + regexp: "My name is (.+)" + match_group_idx: 1 + age: + chatsky.slots.RegexpSlot: + regexp: "I'm ([0-9]+) years old" + match_group_idx: 1 +messenger_interface: + chatsky.messengers.TelegramInterface: + token: + external:os.getenv: + TG_BOT_TOKEN