diff --git a/src/poetry/installation/chooser.py b/src/poetry/installation/chooser.py index 424aaefe7bb..cdf2b87884d 100644 --- a/src/poetry/installation/chooser.py +++ b/src/poetry/installation/chooser.py @@ -6,6 +6,7 @@ from packaging.tags import Tag +from poetry.repositories.pypi_repository import PyPiRepository from poetry.utils.patterns import wheel_file_re @@ -103,6 +104,15 @@ def _get_links(self, package: Package) -> list[Link]: selected_links.append(link) continue + if ( + link.hash_name + and not link.hash_name.startswith("sha") + and isinstance(repository, PyPiRepository) + and repository.get_sha_hash_from_link(link) in hashes + ): + selected_links.append(link) + continue + h = link.hash_name + ":" + link.hash if h not in hashes: continue diff --git a/src/poetry/installation/executor.py b/src/poetry/installation/executor.py index 43ebfe51f69..00d9deddb48 100644 --- a/src/poetry/installation/executor.py +++ b/src/poetry/installation/executor.py @@ -655,16 +655,28 @@ def _validate_archive_hash(archive: Path | Link, package: Package) -> str: package.name, archive_path, ) - archive_hash = "sha256:" + file_dep.hash() known_hashes = {f["hash"] for f in package.files} + known_types = (h.split(":")[0] for h in known_hashes) - if archive_hash not in known_hashes: + archive_hashes = set() + for hash_type in known_types: + archive_hashes.add(f"{hash_type}:{file_dep.hash(hash_type)}") + + if archive_hashes.isdisjoint(known_hashes): raise RuntimeError( - f"Hash for {package} from archive {archive_path.name} not found in" - f" known hashes (was: {archive_hash})" + f"Invalid hashes ({', '.join(sorted(archive_hashes))}) for" + f" {package} using archive {archive.name}. " + f"Expected one of {', '.join(sorted(known_hashes))}." ) - return archive_hash + try: + _, sha256 = next( + filter(lambda h: h.startswith("sha256:"), archive_hashes) + ).split(":") + except StopIteration: + sha256 = file_dep.hash("sha256") + + return f"sha256={sha256}" def _download_archive(self, operation: Install | Update, link: Link) -> Path: response = self._authenticator.request( diff --git a/src/poetry/repositories/legacy_repository.py b/src/poetry/repositories/legacy_repository.py index 5a48fa4b585..12923b8501a 100644 --- a/src/poetry/repositories/legacy_repository.py +++ b/src/poetry/repositories/legacy_repository.py @@ -1,14 +1,12 @@ from __future__ import annotations import cgi -import hashlib import re import urllib.parse import warnings from collections import defaultdict from html import unescape -from pathlib import Path from typing import TYPE_CHECKING from typing import Any from typing import Iterator @@ -35,12 +33,12 @@ from poetry.repositories.pypi_repository import PyPiRepository from poetry.utils.authenticator import Authenticator from poetry.utils.helpers import canonicalize_name -from poetry.utils.helpers import download_file -from poetry.utils.helpers import temporary_directory from poetry.utils.patterns import wheel_file_re if TYPE_CHECKING: + from pathlib import Path + from poetry.core.packages.dependency import Dependency with warnings.catch_warnings(): @@ -354,35 +352,9 @@ def _get_release_info(self, name: str, version: str) -> dict: ): urls["sdist"].append(link.url) - file_hash = f"{link.hash_name}:{link.hash}" if link.hash else None - - if not link.hash or ( - link.hash_name not in ("sha256", "sha384", "sha512") - and hasattr(hashlib, link.hash_name) - ): - with temporary_directory() as temp_dir: - filepath = Path(temp_dir) / link.filename - self._download(link.url, str(filepath)) - - known_hash = ( - getattr(hashlib, link.hash_name)() if link.hash_name else None - ) - required_hash = hashlib.sha256() - - chunksize = 4096 - with filepath.open("rb") as f: - while True: - chunk = f.read(chunksize) - if not chunk: - break - if known_hash: - known_hash.update(chunk) - required_hash.update(chunk) - - if not known_hash or known_hash.hexdigest() == link.hash: - file_hash = f"{required_hash.name}:{required_hash.hexdigest()}" - - files.append({"file": link.filename, "hash": file_hash}) + files.append( + {"file": link.filename, "hash": self.get_sha_hash_from_link(link)} + ) data.files = files @@ -417,6 +389,3 @@ def _get_page(self, endpoint: str) -> Page | None: ) return Page(response.url, response.content, response.headers) - - def _download(self, url: str, dest: str) -> None: - return download_file(url, dest, session=self.session) diff --git a/src/poetry/repositories/pypi_repository.py b/src/poetry/repositories/pypi_repository.py index 467a5a9c8cb..a60e129c6bb 100644 --- a/src/poetry/repositories/pypi_repository.py +++ b/src/poetry/repositories/pypi_repository.py @@ -1,5 +1,6 @@ from __future__ import annotations +import hashlib import logging import os import urllib.parse @@ -453,3 +454,40 @@ def _download(self, url: str, dest: str) -> None: def _log(self, msg: str, level: str = "info") -> None: getattr(logger, level)(f"{self._name}: {msg}") + + def get_sha_hash_from_link(self, link: Link) -> str | None: + """Get sha256|384|512 hash for a file from the provided link. + + If the hash type included in the link is not sha256|384|512, + convert it to sha256. + + """ + file_hash = f"{link.hash_name}:{link.hash}" if link.hash else None + + if not link.hash or ( + link.hash_name not in ("sha256", "sha384", "sha512") + and hasattr(hashlib, link.hash_name) + ): + with temporary_directory() as temp_dir: + filepath = Path(temp_dir) / link.filename + self._download(link.url, str(filepath)) + + known_hash = ( + getattr(hashlib, link.hash_name)() if link.hash_name else None + ) + required_hash = hashlib.sha256() + + chunksize = 4096 + with filepath.open("rb") as f: + while True: + chunk = f.read(chunksize) + if not chunk: + break + if known_hash: + known_hash.update(chunk) + required_hash.update(chunk) + + if not known_hash or known_hash.hexdigest() == link.hash: + file_hash = f"{required_hash.name}:{required_hash.hexdigest()}" + + return file_hash diff --git a/tests/installation/test_chooser.py b/tests/installation/test_chooser.py index 973e4842ff6..e7a00d4ff0c 100644 --- a/tests/installation/test_chooser.py +++ b/tests/installation/test_chooser.py @@ -22,6 +22,7 @@ import httpretty from httpretty.core import HTTPrettyRequest + from pytest_mock import MockerFixture JSON_FIXTURES = ( @@ -255,3 +256,29 @@ def test_chooser_throws_an_error_if_package_hashes_do_not_match( with pytest.raises(RuntimeError) as e: chooser.choose_for(package) assert files[0]["hash"] in str(e) + + +@pytest.mark.parametrize("known_hash", ["sha256:1234", "md5:1234"]) +def test_chooser_can_choose_distribution_with_non_sha_hash( + env: MockEnv, known_hash: str, mock_legacy: None, mocker: MockerFixture, pool: Pool +): + chooser = Chooser(pool, env) + + package = Package( + "md5-package", + "1.2.3", + source_type="legacy", + source_reference="foo", + source_url="https://foo.bar/simple/", + ) + files = [{"hash": known_hash, "filename": "md5-package-1.2.3.tar.gz"}] + package.files = files + + mock_get_sha_hash_from_link = mocker.patch.object( + pool.repository(package.source_reference), + "get_sha_hash_from_link", + return_value="sha256:1234", + ) + + assert chooser.choose_for(package).filename == files[0]["filename"] + mock_get_sha_hash_from_link.assert_called_once() diff --git a/tests/installation/test_executor.py b/tests/installation/test_executor.py index 8060c29cf10..5d7308424c5 100644 --- a/tests/installation/test_executor.py +++ b/tests/installation/test_executor.py @@ -15,6 +15,7 @@ from poetry.core.packages.package import Package from poetry.core.packages.utils.link import Link +from poetry.installation.chef import Chef from poetry.installation.executor import Executor from poetry.installation.operations import Install from poetry.installation.operations import Uninstall @@ -582,3 +583,91 @@ def test_executor_should_be_initialized_with_correct_workers( executor = Executor(tmp_venv, pool, config, io) assert executor._max_workers == expected_workers + + +def test_executor_should_check_every_possible_hash_types( + config: Config, + io: BufferedIO, + pool: Pool, + mocker: MockerFixture, + fixture_dir: FixtureDirGetter, + tmp_dir: str, +): + mocker.patch.object( + Chef, + "get_cached_archive_for_link", + side_effect=lambda link: link, + ) + mocker.patch.object( + Executor, + "_download_archive", + return_value=fixture_dir("distributions").joinpath( + "demo-0.1.0-py2.py3-none-any.whl" + ), + ) + + env = MockEnv(path=Path(tmp_dir)) + executor = Executor(env, pool, config, io) + + package = Package("demo", "0.1.0") + package.files = [ + { + "file": "demo-0.1.0-py2.py3-none-any.whl", + "hash": "md5:15507846fd4299596661d0197bfb4f90", + } + ] + + archive = executor._download_link( + Install(package), Link("https://example.com/demo-0.1.0-py2.py3-none-any.whl") + ) + + assert archive == fixture_dir("distributions").joinpath( + "demo-0.1.0-py2.py3-none-any.whl" + ) + + +def test_executor_should_check_every_possible_hash_types_before_failing( + config: Config, + io: BufferedIO, + pool: Pool, + mocker: MockerFixture, + fixture_dir: FixtureDirGetter, + tmp_dir: str, +): + mocker.patch.object( + Chef, + "get_cached_archive_for_link", + side_effect=lambda link: link, + ) + mocker.patch.object( + Executor, + "_download_archive", + return_value=fixture_dir("distributions").joinpath( + "demo-0.1.0-py2.py3-none-any.whl" + ), + ) + + env = MockEnv(path=Path(tmp_dir)) + executor = Executor(env, pool, config, io) + + package = Package("demo", "0.1.0") + package.files = [ + {"file": "demo-0.1.0-py2.py3-none-any.whl", "hash": "md5:123456"}, + {"file": "demo-0.1.0-py2.py3-none-any.whl", "hash": "sha256:123456"}, + ] + + expected_message = ( + "Invalid hashes " + "(" + "md5:15507846fd4299596661d0197bfb4f90, " + "sha256:70e704135718fffbcbf61ed1fc45933cfd86951a744b681000eaaa75da31f17a" + ") " + "for demo (0.1.0) using archive demo-0.1.0-py2.py3-none-any.whl. " + "Expected one of md5:123456, sha256:123456." + ) + + with pytest.raises(RuntimeError, match=re.escape(expected_message)): + executor._download_link( + Install(package), + Link("https://example.com/demo-0.1.0-py2.py3-none-any.whl"), + ) diff --git a/tests/repositories/fixtures/legacy/md5-package.html b/tests/repositories/fixtures/legacy/md5-package.html new file mode 100644 index 00000000000..0523cbe3211 --- /dev/null +++ b/tests/repositories/fixtures/legacy/md5-package.html @@ -0,0 +1,15 @@ + + + + + Links for md5-package + + +

Links for md5-package

+ md5-package-1.2.3.tar.gz
+ +