diff --git a/poetry.lock b/poetry.lock index f32658c39d1..da3fd16d7fa 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.9.0.dev0 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. [[package]] name = "build" @@ -1027,6 +1027,17 @@ files = [ {file = "ptyprocess-0.7.0.tar.gz", hash = "sha256:5c5d0a3b48ceee0b48485e0c26037c0acd7d29765ca3fbb5cb3831d347423220"}, ] +[[package]] +name = "py-cpuinfo" +version = "9.0.0" +description = "Get CPU info with pure Python" +optional = false +python-versions = "*" +files = [ + {file = "py-cpuinfo-9.0.0.tar.gz", hash = "sha256:3cdbbf3fac90dc6f118bfd64384f309edeadd902d7c8fb17f02ffa1fc3f49690"}, + {file = "py_cpuinfo-9.0.0-py3-none-any.whl", hash = "sha256:859625bc251f64e21f077d099d4162689c762b5d6a4c3c97553d56241c9674d5"}, +] + [[package]] name = "pycparser" version = "2.21" @@ -1074,6 +1085,26 @@ tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} [package.extras] testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] +[[package]] +name = "pytest-benchmark" +version = "4.0.0" +description = "A ``pytest`` fixture for benchmarking code. It will group the tests into rounds that are calibrated to the chosen timer." +optional = false +python-versions = ">=3.7" +files = [ + {file = "pytest-benchmark-4.0.0.tar.gz", hash = "sha256:fb0785b83efe599a6a956361c0691ae1dbb5318018561af10f3e915caa0048d1"}, + {file = "pytest_benchmark-4.0.0-py3-none-any.whl", hash = "sha256:fdb7db64e31c8b277dff9850d2a2556d8b60bcb0ea6524e36e28ffd7c87f71d6"}, +] + +[package.dependencies] +py-cpuinfo = "*" +pytest = ">=3.8" + +[package.extras] +aspect = ["aspectlib"] +elasticsearch = ["elasticsearch"] +histogram = ["pygal", "pygaljs"] + [[package]] name = "pytest-cov" version = "4.1.0" @@ -1195,7 +1226,6 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, @@ -1595,4 +1625,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "4dcace57c7bc2c7ff2cccb8647530d2e86862d2e21993a79198ff1c0546a880b" +content-hash = "6b87066e98c493626fdd659870e50668ba7dc0a4321891f70ee1a53479b94ae3" diff --git a/pyproject.toml b/pyproject.toml index 10f1bf89a3f..40c2e6be673 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,6 +59,7 @@ xattr = { version = "^1.0.0", markers = "sys_platform == 'darwin'" } [tool.poetry.group.dev.dependencies] pre-commit = ">=2.10" +pytest-benchmark = "^4.0.0" [tool.poetry.group.test.dependencies] coverage = ">=7.2.0" diff --git a/src/poetry/installation/wheel_installer.py b/src/poetry/installation/wheel_installer.py index 27a867f85bc..8ca39104721 100644 --- a/src/poetry/installation/wheel_installer.py +++ b/src/poetry/installation/wheel_installer.py @@ -1,8 +1,10 @@ from __future__ import annotations import logging +import os import platform import sys +import tempfile from pathlib import Path from typing import TYPE_CHECKING @@ -65,9 +67,49 @@ def write_to_fs( return RecordEntry(path, Hash(self.hash_algorithm, hash_), size) +class WheelDestinationCopy(SchemeDictionaryDestination): + """SchemeDictionaryDestination that copies the files with os.replace to avoid race condition""" + + def write_to_fs( + self, + scheme: Scheme, + path: str, + stream: BinaryIO, + is_executable: bool, + ) -> RecordEntry: + from installer.records import Hash + from installer.records import RecordEntry + from installer.utils import copyfileobj_with_hashing + from installer.utils import make_file_executable + + target_path = Path(self.scheme_dict[scheme]) / path + + parent_folder = target_path.parent + if not parent_folder.exists(): + # Due to the parallel installation it can happen + # that two threads try to create the directory. + parent_folder.mkdir(parents=True, exist_ok=True) + + with tempfile.NamedTemporaryFile(delete=False) as fp: + with open(fp.name, "wb") as f: + hash_, size = copyfileobj_with_hashing(stream, f, self.hash_algorithm) + if target_path.exists(): + # Contrary to the base library we don't raise an error here since it can + # break pkgutil-style and pkg_resource-style namespace packages. + # We instead log a warning and ignore it. See issue #9158. + logger.warning(f"{target_path} already exists. Overwritting.") + os.replace(fp.name, target_path) + + if is_executable: + make_file_executable(target_path) + + return RecordEntry(path, Hash(self.hash_algorithm, hash_), size) + + class WheelInstaller: - def __init__(self, env: Env) -> None: + def __init__(self, env: Env, wheel_destination_class=WheelDestination) -> None: self._env = env + self.wheel_destination_class = wheel_destination_class script_kind: LauncherKind if not WINDOWS: @@ -99,7 +141,7 @@ def install(self, wheel: Path) -> None: scheme_dict["headers"] = str( Path(scheme_dict["include"]) / source.distribution ) - destination = WheelDestination( + destination = self.wheel_destination_class( scheme_dict, interpreter=str(self._env.python), script_kind=self._script_kind, diff --git a/src/poetry/masonry/builders/editable.py b/src/poetry/masonry/builders/editable.py index a2bb9a514b7..86e94e579c6 100644 --- a/src/poetry/masonry/builders/editable.py +++ b/src/poetry/masonry/builders/editable.py @@ -1,12 +1,10 @@ from __future__ import annotations import csv -import hashlib import json import locale import os -from base64 import urlsafe_b64encode from pathlib import Path from typing import TYPE_CHECKING @@ -17,6 +15,7 @@ from poetry.utils._compat import WINDOWS from poetry.utils._compat import decode from poetry.utils.env import build_environment +from poetry.utils.helpers import get_file_hash_urlsafe from poetry.utils.helpers import is_dir_writable from poetry.utils.pip import pip_install @@ -261,26 +260,12 @@ def _add_dist_info(self, added_files: list[Path]) -> None: with record.open("w", encoding="utf-8", newline="") as f: csv_writer = csv.writer(f) for path in added_files: - hash = self._get_file_hash(path) - size = path.stat().st_size - csv_writer.writerow((path, f"sha256={hash}", size)) + hash_, size = get_file_hash_urlsafe(path) + csv_writer.writerow((path, f"sha256={hash_}", size)) # RECORD itself is recorded with no hash or size csv_writer.writerow((record, "", "")) - def _get_file_hash(self, filepath: Path) -> str: - hashsum = hashlib.sha256() - with filepath.open("rb") as src: - while True: - buf = src.read(1024 * 8) - if not buf: - break - hashsum.update(buf) - - src.seek(0) - - return urlsafe_b64encode(hashsum.digest()).decode("ascii").rstrip("=") - def _debug(self, msg: str) -> None: if self._io.is_debug(): self._io.write_line(msg) diff --git a/src/poetry/utils/helpers.py b/src/poetry/utils/helpers.py index fbc29e5b4dd..a8b49ee4ac3 100644 --- a/src/poetry/utils/helpers.py +++ b/src/poetry/utils/helpers.py @@ -1,5 +1,6 @@ from __future__ import annotations +import base64 import hashlib import io import logging @@ -18,6 +19,7 @@ from pathlib import Path from typing import TYPE_CHECKING from typing import Any +from typing import BinaryIO from typing import overload import requests @@ -332,6 +334,24 @@ def get_file_hash(path: Path, hash_name: str = "sha256") -> str: return h.hexdigest() +def get_file_hash_urlsafe( + file: Path | BinaryIO, hash_algorithm: str = "sha256" +) -> tuple[str, int]: + hashsum = hashlib.new(hash_algorithm) + with file.open("rb") if isinstance(file, Path) else file as src: + size = 0 + while True: + buf = src.read(1024 * 8) + if not buf: + break + hashsum.update(buf) + size += len(buf) + + src.seek(0) + + return base64.urlsafe_b64encode(hashsum.digest()).decode("ascii").rstrip("="), size + + def get_highest_priority_hash_type( hash_types: set[str], archive_name: str ) -> str | None: diff --git a/tests/fixtures/distributions/Django-5.0.3-py3-none-any.whl b/tests/fixtures/distributions/Django-5.0.3-py3-none-any.whl new file mode 100644 index 00000000000..c2bf3461e1b Binary files /dev/null and b/tests/fixtures/distributions/Django-5.0.3-py3-none-any.whl differ diff --git a/tests/installation/test_wheel_installer.py b/tests/installation/test_wheel_installer.py index b7b3d7c7c93..cfe7814d604 100644 --- a/tests/installation/test_wheel_installer.py +++ b/tests/installation/test_wheel_installer.py @@ -1,6 +1,7 @@ from __future__ import annotations import re +import shutil from pathlib import Path from typing import TYPE_CHECKING @@ -9,6 +10,7 @@ from poetry.core.constraints.version import parse_constraint +from poetry.installation.wheel_installer import WheelDestinationCopy from poetry.installation.wheel_installer import WheelInstaller from poetry.utils.env import MockEnv @@ -29,6 +31,11 @@ def demo_wheel(fixture_dir: FixtureDirGetter) -> Path: return fixture_dir("distributions/demo-0.1.0-py2.py3-none-any.whl") +@pytest.fixture(scope="module") +def django_wheel(fixture_dir: FixtureDirGetter) -> Path: + return fixture_dir("distributions/Django-5.0.3-py3-none-any.whl") + + @pytest.fixture(scope="module") def default_installation(tmp_path_factory: TempPathFactory, demo_wheel: Path) -> Path: env = MockEnv(path=tmp_path_factory.mktemp("default_install")) @@ -81,3 +88,71 @@ def test_enable_bytecode_compilation( assert not list(cache_dir.glob("*.opt-2.pyc")) else: assert not cache_dir.exists() + + +def delete_tmp_dir_func_factory(tmp_path: Path): + def delete_tmp_dir(): + shutil.rmtree(tmp_path) + tmp_path.mkdir() + + return delete_tmp_dir + + +@pytest.mark.benchmark( + group="demo_wheel", +) +def test_bench_installer_copy_demo( + env: MockEnv, demo_wheel: Path, benchmark, tmp_path: Path +) -> None: + installer = WheelInstaller(env, wheel_destination_class=WheelDestinationCopy) + benchmark.pedantic( + installer.install, + args=(demo_wheel,), + setup=delete_tmp_dir_func_factory(tmp_path), + rounds=1000, + ) + + +@pytest.mark.benchmark( + group="demo_wheel", +) +def test_bench_installer_demo( + env: MockEnv, demo_wheel: Path, benchmark, tmp_path: Path +) -> None: + installer = WheelInstaller(env) + benchmark.pedantic( + installer.install, + args=(demo_wheel,), + setup=delete_tmp_dir_func_factory(tmp_path), + rounds=1000, + ) + + +@pytest.mark.benchmark( + group="django_wheel", +) +def test_bench_installer_copy_django( + env: MockEnv, django_wheel: Path, benchmark, tmp_path: Path +) -> None: + installer = WheelInstaller(env, wheel_destination_class=WheelDestinationCopy) + benchmark.pedantic( + installer.install, + args=(django_wheel,), + setup=delete_tmp_dir_func_factory(tmp_path), + rounds=30, + ) + + +@pytest.mark.benchmark( + group="django_wheel", +) +def test_bench_installer_django( + env: MockEnv, django_wheel: Path, benchmark, tmp_path: Path +) -> None: + installer = WheelInstaller(env) + benchmark.pedantic( + installer.install, + args=(django_wheel,), + setup=delete_tmp_dir_func_factory(tmp_path), + rounds=30, + )