Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

repository/legacy: calculate sha256 if unavailable #2958

Merged
merged 1 commit into from
Mar 22, 2021
Merged

repository/legacy: calculate sha256 if unavailable #2958

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 37 additions & 4 deletions poetry/repositories/legacy_repository.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import cgi
import hashlib
import re
import urllib.parse
import warnings
Expand Down Expand Up @@ -27,6 +28,8 @@
from poetry.core.semver.version_range import VersionRange
from poetry.locations import REPOSITORY_CACHE_DIR
from poetry.utils.helpers import canonicalize_name
from poetry.utils.helpers import download_file
from poetry.utils.helpers import temporary_directory
from poetry.utils.patterns import wheel_file_re

from ..config.config import Config
Expand Down Expand Up @@ -374,10 +377,37 @@ def _get_release_info(self, name: str, version: str) -> dict:
):
urls["sdist"].append(link.url)

h = link.hash
if h:
h = link.hash_name + ":" + link.hash
files.append({"file": link.filename, "hash": h})
file_hash = "{}:{}".format(link.hash_name, link.hash) if link.hash else None

if not link.hash or (
link.hash_name not in ("sha256", "sha384", "sha512")
and hasattr(hashlib, link.hash_name)
):
with temporary_directory() as temp_dir:
filepath = Path(temp_dir) / link.filename
self._download(link.url, str(filepath))

known_hash = (
getattr(hashlib, link.hash_name)() if link.hash_name else None
)
required_hash = hashlib.sha256()

chunksize = 4096
with filepath.open("rb") as f:
while True:
chunk = f.read(chunksize)
if not chunk:
break
if known_hash:
known_hash.update(chunk)
required_hash.update(chunk)

if not known_hash or known_hash.hexdigest() == link.hash:
file_hash = "{}:{}".format(
required_hash.name, required_hash.hexdigest()
)

files.append({"file": link.filename, "hash": file_hash})

data.files = files

Expand Down Expand Up @@ -415,3 +445,6 @@ def _get(self, endpoint: str) -> Optional[Page]:
)

return Page(response.url, response.content, response.headers)

def _download(self, url, dest): # type: (str, str) -> None
return download_file(url, dest, session=self.session)
4 changes: 2 additions & 2 deletions tests/repositories/fixtures/legacy/ipython.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,10 @@
</head>
<body>
<h1>Links for ipython</h1>
<a href="https://files.pythonhosted.org/packages/52/19/aadde98d6bde1667d0bf431fb2d22451f880aaa373e0a241c7e7cb5815a0/ipython-5.7.0-py2-none-any.whl#sha256=707d1bbfc81e41e39ead1012af931bec6f80357b87e520af352e539cf5961dc0">ipython-5.7.0-py2-none-any.whl</a><br/>
<a href="https://files.pythonhosted.org/packages/52/19/aadde98d6bde1667d0bf431fb2d22451f880aaa373e0a241c7e7cb5815a0/ipython-5.7.0-py2-none-any.whl#md5=a10a802ef98da741cd6f4f6289d47ba7">ipython-5.7.0-py2-none-any.whl</a><br/>
<a href="https://files.pythonhosted.org/packages/c7/b6/03e0b5b0972e6161d16c4cec8d41a20372bd0634f8cb4cc0c984b8a91db6/ipython-5.7.0-py3-none-any.whl#sha256=fc0464e68f9c65cd8c453474b4175432cc29ecb6c83775baedf6dbfcee9275ab">ipython-5.7.0-py3-none-any.whl</a><br/>
<a href="https://files.pythonhosted.org/packages/3c/fd/559fead731a29eaa55cc235c8029807b2520976a937c30e9ee603f3bb566/ipython-5.7.0.tar.gz#sha256=8db43a7fb7619037c98626613ff08d03dda9d5d12c84814a4504c78c0da8323c">ipython-5.7.0.tar.gz</a><br/>
<a href="https://files.pythonhosted.org/packages/a9/2e/41dce4ed129057e05a555a7f9629aa2d5f81fdcd4d16568bc24b75a1d2c9/ipython-7.5.0-py3-none-any.whl#md5=dbdc53e3918f28fa335a173432402a00" data-requires-python="&gt;=3.5">ipython-7.5.0-py3-none-any.whl</a><br/>
<a href="https://files.pythonhosted.org/packages/a9/2e/41dce4ed129057e05a555a7f9629aa2d5f81fdcd4d16568bc24b75a1d2c9/ipython-7.5.0-py3-none-any.whl#md5=a10a802ef98da741cd6f4f6289d47ba8" data-requires-python="&gt;=3.5">ipython-7.5.0-py3-none-any.whl</a><br/>
<a href="https://files.pythonhosted.org/packages/75/74/9b0ef91c8e356c907bb12297000951acb804583b54eeaddc342c5bad4d96/ipython-7.5.0.tar.gz#sha256=e840810029224b56cd0d9e7719dc3b39cf84d577f8ac686547c8ba7a06eeab26" data-requires-python="&gt;=3.5">ipython-7.5.0.tar.gz</a><br/>
</body>
</html>
Expand Down
32 changes: 30 additions & 2 deletions tests/repositories/test_legacy_repository.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -273,7 +273,7 @@ def test_get_package_retrieves_non_sha256_hashes():
expected = [
{
"file": "ipython-7.5.0-py3-none-any.whl",
"hash": "md5:dbdc53e3918f28fa335a173432402a00",
"hash": "sha256:78aea20b7991823f6a32d55f4e963a61590820e43f666ad95ad07c7f0c704efa",
},
{
"file": "ipython-7.5.0.tar.gz",
Expand All @@ -284,12 +284,40 @@ def test_get_package_retrieves_non_sha256_hashes():
assert expected == package.files


def test_get_package_retrieves_non_sha256_hashes_mismatching_known_hash():
repo = MockRepository()

package = repo.package("ipython", "5.7.0")

expected = [
{
"file": "ipython-5.7.0-py2-none-any.whl",
"hash": "md5:a10a802ef98da741cd6f4f6289d47ba7",
},
{
"file": "ipython-5.7.0-py3-none-any.whl",
"hash": "sha256:fc0464e68f9c65cd8c453474b4175432cc29ecb6c83775baedf6dbfcee9275ab",
},
{
"file": "ipython-5.7.0.tar.gz",
"hash": "sha256:8db43a7fb7619037c98626613ff08d03dda9d5d12c84814a4504c78c0da8323c",
},
]

assert expected == package.files


def test_get_package_retrieves_packages_with_no_hashes():
repo = MockRepository()

package = repo.package("jupyter", "1.0.0")

assert [] == package.files
assert [
{
"file": "jupyter-1.0.0.tar.gz",
"hash": "sha256:d9dc4b3318f310e34c82951ea5d6683f67bed7def4b259fafbfe4f1beb1d8e5f",
}
] == package.files


class MockHttpRepository(LegacyRepository):
Expand Down