From 1a7b7a589b90899377c46c18819b5bde7c12658b Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 7 Jun 2023 16:53:25 +0530 Subject: [PATCH 1/9] build(deps): Bump Pydantic from v1 to v2 --- frappe/utils/typing_validations.py | 36 +++++------------------------- pyproject.toml | 2 +- 2 files changed, 7 insertions(+), 31 deletions(-) diff --git a/frappe/utils/typing_validations.py b/frappe/utils/typing_validations.py index e7ebcfbdffcd..4c43064af6b8 100644 --- a/frappe/utils/typing_validations.py +++ b/frappe/utils/typing_validations.py @@ -1,11 +1,10 @@ from functools import lru_cache, wraps from inspect import _empty, isclass, signature from types import EllipsisType -from typing import Any, Callable, ForwardRef, TypeVar, Union +from typing import Callable, ForwardRef, TypeVar, Union -from pydantic.config import BaseConfig -from pydantic.error_wrappers import ValidationError as PyValidationError -from pydantic.tools import NameFactory, _generate_parsing_type_name +from pydantic import TypeAdapter as PyTypeAdapter +from pydantic import ValidationError as PyValidationError from frappe.exceptions import FrappeTypeError @@ -69,29 +68,8 @@ def raise_type_error( @lru_cache(maxsize=2048) -def _get_parsing_type( - type_: Any, *, type_name: NameFactory | None = None, config: type[BaseConfig] = None -) -> Any: - # Note: this is a copy of pydantic.tools._get_parsing_type with the addition of allowing a config argument - from pydantic.main import create_model - - if type_name is None: - type_name = _generate_parsing_type_name - if not isinstance(type_name, str): - type_name = type_name(type_) - return create_model(type_name, __root__=(type_, ...), __config__=config) - - -def parse_obj_as( - type_: type[T], - obj: Any, - *, - type_name: NameFactory | None = None, - config: type[BaseConfig] | None = None, -) -> T: - # Note: This is a copy of pydantic.tools.parse_obj_as with the addition of allowing a config argument - model_type = _get_parsing_type(type_, type_name=type_name, config=config) # type: ignore[arg-type] - return model_type(__root__=obj).__root__ +def TypeAdapter(type_): + return PyTypeAdapter(type_) def transform_parameter_types(func: Callable, args: tuple, kwargs: dict): @@ -157,9 +135,7 @@ def transform_parameter_types(func: Callable, args: tuple, kwargs: dict): # validate the type set using pydantic - raise a TypeError if Validation is raised or Ellipsis is returned try: - current_arg_value_after = parse_obj_as( - current_arg_type, current_arg_value, type_name=current_arg, config=FrappePydanticConfig - ) + current_arg_value_after = TypeAdapter(current_arg_type).validate_python(current_arg_value) except (TypeError, PyValidationError) as e: raise_type_error(current_arg, current_arg_type, current_arg_value, current_exception=e) diff --git a/pyproject.toml b/pyproject.toml index aa89eed928e2..dde346e7aa7a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,7 +53,7 @@ dependencies = [ "psycopg2-binary~=2.9.1", "pyOpenSSL~=23.0.0", "pycryptodome~=3.10.1", - "pydantic~=1.10.2", + "pydantic~=2.0b2", "pyotp~=2.6.0", "python-dateutil~=2.8.1", "pytz==2022.1", From 5c2fe7c292262a030c1ad6894f8fec65b7388f1b Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 7 Jun 2023 17:17:13 +0530 Subject: [PATCH 2/9] test: Add tests for validate_argument_types decorator --- frappe/tests/test_utils.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/frappe/tests/test_utils.py b/frappe/tests/test_utils.py index 4b362e7b4787..e46e37b51888 100644 --- a/frappe/tests/test_utils.py +++ b/frappe/tests/test_utils.py @@ -1155,3 +1155,35 @@ def test_bankers_rounding_property(self, number, precision): def test_default_rounding(self): self.assertEqual(frappe.get_system_settings("rounding_method"), "Banker's Rounding") + + +class TestTypingValidations(FrappeTestCase): + def test_validate_argument_types(self): + from frappe.utils.typing_validations import FrappeTypeError, validate_argument_types + + @validate_argument_types + def test_simple_types(a: int, b: float, c: bool): + return a, b, c + + @validate_argument_types + def test_sequence(a: str, b: list[dict] | None = None, c: dict[str, int] | None = None): + return a, b, c + + self.assertEqual(test_simple_types(True, 2.0, True), (1, 2.0, True)) + self.assertEqual(test_simple_types(1, 2, 1), (1, 2.0, True)) + self.assertEqual(test_simple_types(1.0, 2, 1), (1, 2.0, True)) + self.assertEqual(test_simple_types(1, 2, "1"), (1, 2.0, True)) + + with self.assertRaises(FrappeTypeError): + test_simple_types(1, 2, "a") + with self.assertRaises(FrappeTypeError): + test_simple_types(1, 2, None) + + self.assertEqual(test_sequence("a", [{"a": 1}], {"a": 1}), ("a", [{"a": 1}], {"a": 1})) + self.assertEqual(test_sequence("a", None, None), ("a", None, None)) + self.assertEqual(test_sequence("a", [{"a": 1}], None), ("a", [{"a": 1}], None)) + self.assertEqual(test_sequence("a", None, {"a": 1}), ("a", None, {"a": 1})) + self.assertEqual(test_sequence("a", [{"a": 1}], {"a": "1.0"}), ("a", [{"a": 1}], {"a": 1})) + + with self.assertRaises(FrappeTypeError): + test_sequence("a", [{"a": 1}], True) From ae3a61b3fc75957d2e4c710179338d5784991049 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 7 Jun 2023 19:26:32 +0530 Subject: [PATCH 3/9] test: Add tests for Document arg validations --- frappe/tests/test_utils.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/frappe/tests/test_utils.py b/frappe/tests/test_utils.py index e46e37b51888..2e83f8b8b68e 100644 --- a/frappe/tests/test_utils.py +++ b/frappe/tests/test_utils.py @@ -1159,6 +1159,7 @@ def test_default_rounding(self): class TestTypingValidations(FrappeTestCase): def test_validate_argument_types(self): + from frappe.core.doctype.doctype.doctype import DocType from frappe.utils.typing_validations import FrappeTypeError, validate_argument_types @validate_argument_types @@ -1169,11 +1170,14 @@ def test_simple_types(a: int, b: float, c: bool): def test_sequence(a: str, b: list[dict] | None = None, c: dict[str, int] | None = None): return a, b, c + @validate_argument_types + def test_doctypes(a: DocType | dict): + return a + self.assertEqual(test_simple_types(True, 2.0, True), (1, 2.0, True)) self.assertEqual(test_simple_types(1, 2, 1), (1, 2.0, True)) self.assertEqual(test_simple_types(1.0, 2, 1), (1, 2.0, True)) self.assertEqual(test_simple_types(1, 2, "1"), (1, 2.0, True)) - with self.assertRaises(FrappeTypeError): test_simple_types(1, 2, "a") with self.assertRaises(FrappeTypeError): @@ -1184,6 +1188,11 @@ def test_sequence(a: str, b: list[dict] | None = None, c: dict[str, int] | None self.assertEqual(test_sequence("a", [{"a": 1}], None), ("a", [{"a": 1}], None)) self.assertEqual(test_sequence("a", None, {"a": 1}), ("a", None, {"a": 1})) self.assertEqual(test_sequence("a", [{"a": 1}], {"a": "1.0"}), ("a", [{"a": 1}], {"a": 1})) - with self.assertRaises(FrappeTypeError): test_sequence("a", [{"a": 1}], True) + + doctype = frappe.get_last_doc("DocType") + self.assertEqual(test_doctypes(doctype), doctype) + self.assertEqual(test_doctypes(doctype.as_dict()), doctype.as_dict()) + with self.assertRaises(FrappeTypeError): + test_doctypes("a") From ee3d4da734dc0fd646076181fc43e0e4d57a8ce0 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Wed, 7 Jun 2023 19:26:57 +0530 Subject: [PATCH 4/9] fix: Allow arbitrary objects validation in TypeAdapter --- frappe/utils/typing_validations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/utils/typing_validations.py b/frappe/utils/typing_validations.py index 4c43064af6b8..83f59fb2ece1 100644 --- a/frappe/utils/typing_validations.py +++ b/frappe/utils/typing_validations.py @@ -69,7 +69,7 @@ def raise_type_error( @lru_cache(maxsize=2048) def TypeAdapter(type_): - return PyTypeAdapter(type_) + return PyTypeAdapter(type_, config=FrappePydanticConfig) def transform_parameter_types(func: Callable, args: tuple, kwargs: dict): From 58fc6a2b940200d75ad3278cd9d5baaa6e8a97d7 Mon Sep 17 00:00:00 2001 From: gavin Date: Fri, 30 Jun 2023 22:18:45 +0530 Subject: [PATCH 5/9] build: Bump to stable Pydantic v2 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index a4d61d684533..17a75f3c0b83 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,7 +53,7 @@ dependencies = [ "psycopg2-binary~=2.9.1", "pyOpenSSL~=23.2.0", "pycryptodome~=3.18.0", - "pydantic~=2.0b3", + "pydantic==2.0", "pyotp~=2.8.0", "python-dateutil~=2.8.2", "pytz==2023.3", From 883445aefa77dd475a0d2ac27295723f17292621 Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Sun, 2 Jul 2023 16:00:50 +0530 Subject: [PATCH 6/9] perf: Defer pydantic imports until function call Pydantic adds an additional 1-2MB in memory usage. We can defer it in case an environment doesn't use it at all. --- frappe/utils/typing_validations.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/frappe/utils/typing_validations.py b/frappe/utils/typing_validations.py index 83f59fb2ece1..91a318eae4d1 100644 --- a/frappe/utils/typing_validations.py +++ b/frappe/utils/typing_validations.py @@ -3,9 +3,6 @@ from types import EllipsisType from typing import Callable, ForwardRef, TypeVar, Union -from pydantic import TypeAdapter as PyTypeAdapter -from pydantic import ValidationError as PyValidationError - from frappe.exceptions import FrappeTypeError SLACK_DICT = { @@ -69,6 +66,8 @@ def raise_type_error( @lru_cache(maxsize=2048) def TypeAdapter(type_): + from pydantic import TypeAdapter as PyTypeAdapter + return PyTypeAdapter(type_, config=FrappePydanticConfig) @@ -81,6 +80,8 @@ def transform_parameter_types(func: Callable, args: tuple, kwargs: dict): if not (args or kwargs) or not func.__annotations__: return args, kwargs + from pydantic import ValidationError as PyValidationError + annotations = func.__annotations__ new_args, new_kwargs = list(args), kwargs From af7032893087db31f5eff6c31c9473561fdd4d0b Mon Sep 17 00:00:00 2001 From: Gavin D'souza Date: Sun, 2 Jul 2023 16:19:25 +0530 Subject: [PATCH 7/9] test: Bump memory usage treshold post pydantic v2 upgrade --- frappe/core/doctype/rq_job/test_rq_job.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/core/doctype/rq_job/test_rq_job.py b/frappe/core/doctype/rq_job/test_rq_job.py index f5d5f89ed400..cb7faca18781 100644 --- a/frappe/core/doctype/rq_job/test_rq_job.py +++ b/frappe/core/doctype/rq_job/test_rq_job.py @@ -160,7 +160,7 @@ def test_memory_usage(self): # If this starts failing analyze memory usage using memray or some equivalent tool to find # offending imports/function calls. # Refer this PR: https://github.com/frappe/frappe/pull/21467 - LAST_MEASURED_USAGE = 40 + LAST_MEASURED_USAGE = 42 self.assertLessEqual(rss, LAST_MEASURED_USAGE * 1.05, msg) From 265a28e151c3e7301b7221d44b1d79b3ba52cd4c Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Sun, 2 Jul 2023 16:32:40 +0530 Subject: [PATCH 8/9] perf: preload pydantic --- frappe/app.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frappe/app.py b/frappe/app.py index 1cbdca1361a0..ee4ba7d1700d 100644 --- a/frappe/app.py +++ b/frappe/app.py @@ -33,6 +33,8 @@ # If gc.freeze is done then importing modules before forking allows us to share the memory if frappe._tune_gc: + import pydantic + import frappe.boot import frappe.client import frappe.core.doctype.user.user From 35039ca382a68ca705583c66d667d43759d22501 Mon Sep 17 00:00:00 2001 From: Ankush Menat Date: Sun, 2 Jul 2023 18:46:24 +0530 Subject: [PATCH 9/9] Revert "test: Bump memory usage treshold post pydantic v2 upgrade" This reverts commit af7032893087db31f5eff6c31c9473561fdd4d0b. --- frappe/core/doctype/rq_job/test_rq_job.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frappe/core/doctype/rq_job/test_rq_job.py b/frappe/core/doctype/rq_job/test_rq_job.py index cb7faca18781..f5d5f89ed400 100644 --- a/frappe/core/doctype/rq_job/test_rq_job.py +++ b/frappe/core/doctype/rq_job/test_rq_job.py @@ -160,7 +160,7 @@ def test_memory_usage(self): # If this starts failing analyze memory usage using memray or some equivalent tool to find # offending imports/function calls. # Refer this PR: https://github.com/frappe/frappe/pull/21467 - LAST_MEASURED_USAGE = 42 + LAST_MEASURED_USAGE = 40 self.assertLessEqual(rss, LAST_MEASURED_USAGE * 1.05, msg)