From 6be8b8633a4b9a97703bb71a27242f98ce4980bd Mon Sep 17 00:00:00 2001 From: Lukas Puehringer Date: Thu, 18 Apr 2024 14:11:48 +0200 Subject: [PATCH 01/17] Return file size from render_simple_detail The simple detail file size is to be included in TUF metadata, along with its path and hash. Signed-off-by: Lukas Puehringer --- tests/unit/packaging/test_utils.py | 8 +++++--- warehouse/packaging/utils.py | 12 ++++++------ 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/tests/unit/packaging/test_utils.py b/tests/unit/packaging/test_utils.py index e256ca0300e1..8a9eebdeccc0 100644 --- a/tests/unit/packaging/test_utils.py +++ b/tests/unit/packaging/test_utils.py @@ -58,7 +58,7 @@ def test_render_simple_detail(db_request, monkeypatch, jinja): context = _valid_simple_detail_context(context) expected_content = template.render(**context, request=db_request).encode("utf-8") - content_hash, path = render_simple_detail(project, db_request) + content_hash, path, size = render_simple_detail(project, db_request) assert fakeblake2b.calls == [pretend.call(digest_size=32)] assert fake_hasher.update.calls == [pretend.call(expected_content)] @@ -69,6 +69,7 @@ def test_render_simple_detail(db_request, monkeypatch, jinja): f"{project.normalized_name}/deadbeefdeadbeefdeadbeefdeadbeef" + f".{project.normalized_name}.html" ) + assert size == len(expected_content) def test_render_simple_detail_with_store(db_request, monkeypatch, jinja): @@ -94,7 +95,7 @@ def test_render_simple_detail_with_store(db_request, monkeypatch, jinja): fake_named_temporary_file = pretend.stub( name="/tmp/wutang", - write=pretend.call_recorder(lambda data: None), + write=pretend.call_recorder(lambda data: 42), flush=pretend.call_recorder(lambda: None), ) @@ -115,7 +116,7 @@ def __exit__(self, type, value, traceback): context = _valid_simple_detail_context(context) expected_content = template.render(**context, request=db_request).encode("utf-8") - content_hash, path = render_simple_detail(project, db_request, store=True) + content_hash, path, size = render_simple_detail(project, db_request, store=True) assert fake_named_temporary_file.write.calls == [pretend.call(expected_content)] assert fake_named_temporary_file.flush.calls == [pretend.call()] @@ -153,3 +154,4 @@ def __exit__(self, type, value, traceback): f"{project.normalized_name}/deadbeefdeadbeefdeadbeefdeadbeef" + f".{project.normalized_name}.html" ) + assert size == 42 diff --git a/warehouse/packaging/utils.py b/warehouse/packaging/utils.py index 282c45747f75..b05a2dad519c 100644 --- a/warehouse/packaging/utils.py +++ b/warehouse/packaging/utils.py @@ -123,12 +123,12 @@ def render_simple_detail(project, request, store=False): f"{project.normalized_name}/{content_hash}.{project.normalized_name}.html" ) - if store: - storage = request.find_service(ISimpleStorage) - with tempfile.NamedTemporaryFile() as f: - f.write(content.encode("utf-8")) - f.flush() + with tempfile.NamedTemporaryFile() as f: + simple_detail_size = f.write(content.encode("utf-8")) + f.flush() + if store: + storage = request.find_service(ISimpleStorage) storage.store( simple_detail_path, f.name, @@ -148,7 +148,7 @@ def render_simple_detail(project, request, store=False): }, ) - return (content_hash, simple_detail_path) + return (content_hash, simple_detail_path, simple_detail_size) def _valid_simple_detail_context(context: dict) -> dict: From a4844cccd9c02cda152551a5ecd9e0ceaf7f9f35 Mon Sep 17 00:00:00 2001 From: Lukas Puehringer Date: Thu, 18 Apr 2024 14:14:20 +0200 Subject: [PATCH 02/17] Add function to update TUF repository metadata Add new TUF subpackage with functions to talk to the RSTUF API, in order to update TUF repository metadata when a project changes. The function requires RSTUF to be bootstrapped (see `make inittuf`), and can additionally be toggled via setting: - disabled by default - enabled in dev environment Signed-off-by: Lukas Puehringer --- dev/environment | 3 ++ tests/unit/test_config.py | 1 + warehouse/config.py | 8 +++++ warehouse/tuf/__init__.py | 61 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 73 insertions(+) diff --git a/dev/environment b/dev/environment index 40aad6ce7f60..073b0b4a9092 100644 --- a/dev/environment +++ b/dev/environment @@ -64,6 +64,9 @@ TWOFACTORMANDATE_AVAILABLE=true TWOFACTORMANDATE_ENABLED=true OIDC_AUDIENCE=pypi +TUF_RSTUF_API_URL="http://rstuf-api" +TUF_ENABLED=true + # Default to the reCAPTCHA testing keys from https://developers.google.com/recaptcha/docs/faq RECAPTCHA_SITE_KEY=6LeIxAcTAAAAAJcZVRqyHh71UMIEGNQ_MXjiZKhI RECAPTCHA_SECRET_KEY=6LeIxAcTAAAAAGG-vFI1TnRWxMZNFuojJ4WifJWe diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index 330d85ff6c43..f854a5f38ecd 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -336,6 +336,7 @@ def __init__(self): "warehouse.search.ratelimit_string": "5 per second", "oidc.backend": "warehouse.oidc.services.OIDCPublisherService", "integrity.backend": "warehouse.attestations.services.IntegrityService", + "tuf.enabled": False, "warehouse.organizations.max_undecided_organization_applications": 3, "reconcile_file_storages.batch_size": 100, "metadata_backfill.batch_size": 500, diff --git a/warehouse/config.py b/warehouse/config.py index 3aa1659d4eac..470c5f0b19bb 100644 --- a/warehouse/config.py +++ b/warehouse/config.py @@ -470,6 +470,14 @@ def configure(settings=None): "INTEGRITY_BACKEND", default="warehouse.attestations.services.IntegrityService", ) + maybe_set( + settings, + "tuf.enabled", + "TUF_ENABLED", + coercer=distutils.util.strtobool, + default=False, + ) + maybe_set(settings, "tuf.rstuf_api_url", "TUF_RSTUF_API_URL") # Pythondotorg integration settings maybe_set( diff --git a/warehouse/tuf/__init__.py b/warehouse/tuf/__init__.py index d03c032ccaa8..53106f19cf28 100644 --- a/warehouse/tuf/__init__.py +++ b/warehouse/tuf/__init__.py @@ -16,10 +16,16 @@ import time +from http import HTTPStatus from typing import Any import requests +from pyramid.request import Request + +from warehouse.packaging.models import Project +from warehouse.packaging.utils import render_simple_detail + class RSTUFError(Exception): pass @@ -31,6 +37,15 @@ def get_task_state(server: str, task_id: str) -> str: return resp.json()["data"]["state"] +def is_bootstrapped(server: str) -> bool: + """Call RSTUF bootstrap API to check, if RSTUF is ready to be used.""" + response = requests.get(f"{server}/api/v1/bootstrap") + if response.status_code != HTTPStatus.OK: + return False + + return response.json()["data"]["bootstrap"] + + def post_bootstrap(server: str, payload: Any) -> str: resp = requests.post(f"{server}/api/v1/bootstrap", json=payload) resp.raise_for_status() @@ -44,6 +59,16 @@ def post_bootstrap(server: str, payload: Any) -> str: return resp_data["task_id"] +def post_artifacts(server: str, payload: Any) -> str: + """Call RSTUF artifacts API to update the relevant TUF metadata. + + Returns task id of the async update task in RSTUF. + """ + resp = requests.post(f"{server}/api/v1/artifacts", json=payload) + resp.raise_for_status() + return resp.json()["data"]["task_id"] + + def wait_for_success(server: str, task_id: str): """Poll RSTUF task state API until success or error.""" @@ -72,3 +97,39 @@ def wait_for_success(server: str, task_id: str): else: raise RSTUFError("RSTUF job failed, please check payload and retry") + + +def update_metadata(request: Request, project: Project): + """Update TUF metadata to capture project changes (PEP 458). + + NOTE: PEP 458 says, TUF targets metadata must include path, hash and size of + distributions files and simple detail files. In reality, simple detail files + are enough, as they already include all relevant distribution file infos. + """ + if not request.registry.settings["tuf.enabled"]: + return + + server = request.registry.settings["tuf.rstuf_api_url"] + + if not is_bootstrapped(server): + return + + # NOTE: We ignore the returned simple detail path with the content hash as + # infix. In TUF metadata the project name and hash are listed separately, so + # that there is only one entry per target file, even if the content changes. + digest, _, size = render_simple_detail(project, request, store=True) + payload = { + "targets": [ + { + "path": project.normalized_name, + "info": { + "length": size, + "hashes": {"blake2b-256": digest}, + }, + } + ] + } + + # TODO: Handle errors: pass, retry or notify + task_id = post_artifacts(server, payload) + wait_for_success(server, task_id) From e381cfff2d2e8ba62283e261e4b1207ab703f950 Mon Sep 17 00:00:00 2001 From: Lukas Puehringer Date: Thu, 18 Apr 2024 14:20:21 +0200 Subject: [PATCH 03/17] Call TUF metadata update function in relevant views Relevant views are "file upload", "remove" (file, release, project), "yank". Signed-off-by: Lukas Puehringer --- tests/unit/forklift/test_legacy.py | 1 + tests/unit/manage/test_views.py | 2 ++ warehouse/forklift/legacy.py | 3 +++ warehouse/manage/views/__init__.py | 9 +++++++-- 4 files changed, 13 insertions(+), 2 deletions(-) diff --git a/tests/unit/forklift/test_legacy.py b/tests/unit/forklift/test_legacy.py index 6b9475f980ff..f0353dd84be6 100644 --- a/tests/unit/forklift/test_legacy.py +++ b/tests/unit/forklift/test_legacy.py @@ -455,6 +455,7 @@ def test_is_duplicate_false(self, pyramid_config, db_request): assert legacy._is_duplicate_file(db_request.db, filename, wrong_hashes) is False +@pytest.mark.usefixtures("disable_tuf") class TestFileUpload: def test_fails_disallow_new_upload(self, pyramid_config, pyramid_request): pyramid_request.flags = pretend.stub( diff --git a/tests/unit/manage/test_views.py b/tests/unit/manage/test_views.py index fb16a5a325ec..8b1ef1cd79d8 100644 --- a/tests/unit/manage/test_views.py +++ b/tests/unit/manage/test_views.py @@ -2599,6 +2599,7 @@ def test_manage_projects(self, db_request): } +@pytest.mark.usefixtures("disable_tuf") class TestManageProjectSettings: @pytest.mark.parametrize("enabled", [False, True]) def test_manage_project_settings(self, enabled, monkeypatch): @@ -4021,6 +4022,7 @@ def test_manage_project_releases(self, db_request): } +@pytest.mark.usefixtures("disable_tuf") class TestManageProjectRelease: def test_manage_project_release(self): files = pretend.stub() diff --git a/warehouse/forklift/legacy.py b/warehouse/forklift/legacy.py index 979826c5764a..4d351bdfe2b9 100644 --- a/warehouse/forklift/legacy.py +++ b/warehouse/forklift/legacy.py @@ -44,6 +44,7 @@ from sqlalchemy import and_, exists, func, orm from sqlalchemy.exc import MultipleResultsFound, NoResultFound +from warehouse import tuf from warehouse.admin.flags import AdminFlagValue from warehouse.attestations.errors import AttestationUploadError from warehouse.attestations.interfaces import IIntegrityService @@ -1306,6 +1307,8 @@ def file_upload(request): request.db.flush() # flush db now so server default values are populated for celery + tuf.update_metadata(request, release.project) + # Push updates to BigQuery dist_metadata = { "metadata_version": meta.metadata_version, diff --git a/warehouse/manage/views/__init__.py b/warehouse/manage/views/__init__.py index a5b176807cb8..a32388d17496 100644 --- a/warehouse/manage/views/__init__.py +++ b/warehouse/manage/views/__init__.py @@ -34,6 +34,7 @@ import warehouse.utils.otp as otp +from warehouse import tuf from warehouse.accounts.forms import RecoveryCodeAuthenticationForm from warehouse.accounts.interfaces import ( IPasswordBreachedService, @@ -2051,6 +2052,8 @@ def delete_project(project, request): remove_project(project, request) + tuf.update_metadata(request, project) + return HTTPSeeOther(request.route_path("manage.projects")) @@ -2221,6 +2224,7 @@ def yank_project_release(self): recipient_role=contributor_role, ) + tuf.update_metadata(self.request, self.release.project) return HTTPSeeOther( self.request.route_path( "manage.project.releases", project_name=self.release.project.name @@ -2305,7 +2309,7 @@ def unyank_project_release(self): submitter_role=submitter_role, recipient_role=contributor_role, ) - + tuf.update_metadata(self.request, self.release.project) return HTTPSeeOther( self.request.route_path( "manage.project.releases", project_name=self.release.project.name @@ -2405,6 +2409,7 @@ def delete_project_release(self): recipient_role=contributor_role, ) + tuf.update_metadata(self.request, self.release.project) return HTTPSeeOther( self.request.route_path( "manage.project.releases", project_name=self.release.project.name @@ -2503,7 +2508,7 @@ def _error(message): self.request.session.flash( f"Deleted file {release_file.filename!r}", queue="success" ) - + tuf.update_metadata(self.request, self.release.project) return HTTPSeeOther( self.request.route_path( "manage.project.release", From c982fe405bcac195a98eb8460983c9c3b9a3c0b0 Mon Sep 17 00:00:00 2001 From: Lukas Puehringer Date: Mon, 22 Apr 2024 15:53:12 +0200 Subject: [PATCH 04/17] Update TUF section in developer documentation Signed-off-by: Lukas Puehringer --- docs/dev/development/getting-started.rst | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/docs/dev/development/getting-started.rst b/docs/dev/development/getting-started.rst index 61b474f80e49..ba38e68a35c0 100644 --- a/docs/dev/development/getting-started.rst +++ b/docs/dev/development/getting-started.rst @@ -265,19 +265,16 @@ includes the TUF trust root for development and other configuration. By calling this API, RSTUF creates the TUF metadata repository, installs the TUF trust root for development, and creates the initial set of TUF metadata. -.. note:: +Once this process is finished, TUF metadata will update automatically whenever +distribution files are uploaded, yanked or removed. To disable this behavior, +you can set the ``TUF_ENABLED`` environment variable to false. TUF metadata +is hosted at: http://localhost:9001/tuf-metadata/ - The RSTUF API is exposed only for development purposes and will not be - available in production. Currently, no upload hooks or automatic metadata - update tasks are configured to interact with RSTUF. +.. note:: - Take a look at the `RSTUF API documentation - `_ - to see how you can simulate artifact upload or removal, and how they affect - the TUF metadata repository: + RSTUF and automated TUF metadata updates are currently only available in the + Warehouse development environment. - * RSTUF API: http://localhost:8001 - * TUF Metadata Repository: http://localhost:9001/tuf-metadata/ Resetting the development database From 4b7feb3333ec4fccec3eb317113f8449ba07e4e9 Mon Sep 17 00:00:00 2001 From: Lukas Puehringer Date: Wed, 19 Jun 2024 11:53:28 +0200 Subject: [PATCH 05/17] Make TUF repository metadata update async Make `tuf.update_metadata` an async task and call with `delay`. The `project` argument becomes `project_id`, because it must be serializable. This change allows us to remove the `disable_tuf` test fixture, because `delay` is mocked in the relevant tests anyway, so that `update_metadata` isn't actually called in any existing tests. However, call_recorder asserts need to be and are adapted. Signed-off-by: Lukas Puehringer --- tests/unit/forklift/test_legacy.py | 4 +++- tests/unit/manage/test_views.py | 2 -- warehouse/forklift/legacy.py | 2 +- warehouse/manage/views/__init__.py | 10 +++++----- warehouse/tuf/__init__.py | 7 ++++++- 5 files changed, 15 insertions(+), 10 deletions(-) diff --git a/tests/unit/forklift/test_legacy.py b/tests/unit/forklift/test_legacy.py index f0353dd84be6..124a2ae64d2f 100644 --- a/tests/unit/forklift/test_legacy.py +++ b/tests/unit/forklift/test_legacy.py @@ -57,6 +57,7 @@ Role, ) from warehouse.packaging.tasks import sync_file_to_cache, update_bigquery_release_files +from warehouse.tuf import update_metadata from ...common.db.accounts import EmailFactory, UserFactory from ...common.db.classifiers import ClassifierFactory @@ -455,7 +456,6 @@ def test_is_duplicate_false(self, pyramid_config, db_request): assert legacy._is_duplicate_file(db_request.db, filename, wrong_hashes) is False -@pytest.mark.usefixtures("disable_tuf") class TestFileUpload: def test_fails_disallow_new_upload(self, pyramid_config, pyramid_request): pyramid_request.flags = pretend.stub( @@ -1164,10 +1164,12 @@ def storage_service_store(path, file_path, *, meta): ] assert db_request.task.calls == [ + pretend.call(update_metadata), pretend.call(update_bigquery_release_files), pretend.call(sync_file_to_cache), ] assert delay.calls == [ + pretend.call(release.project.id), pretend.call( { "metadata_version": "1.2", diff --git a/tests/unit/manage/test_views.py b/tests/unit/manage/test_views.py index 8b1ef1cd79d8..fb16a5a325ec 100644 --- a/tests/unit/manage/test_views.py +++ b/tests/unit/manage/test_views.py @@ -2599,7 +2599,6 @@ def test_manage_projects(self, db_request): } -@pytest.mark.usefixtures("disable_tuf") class TestManageProjectSettings: @pytest.mark.parametrize("enabled", [False, True]) def test_manage_project_settings(self, enabled, monkeypatch): @@ -4022,7 +4021,6 @@ def test_manage_project_releases(self, db_request): } -@pytest.mark.usefixtures("disable_tuf") class TestManageProjectRelease: def test_manage_project_release(self): files = pretend.stub() diff --git a/warehouse/forklift/legacy.py b/warehouse/forklift/legacy.py index 4d351bdfe2b9..e4cd8cc41930 100644 --- a/warehouse/forklift/legacy.py +++ b/warehouse/forklift/legacy.py @@ -1307,7 +1307,7 @@ def file_upload(request): request.db.flush() # flush db now so server default values are populated for celery - tuf.update_metadata(request, release.project) + request.task(tuf.update_metadata).delay(release.project.id) # Push updates to BigQuery dist_metadata = { diff --git a/warehouse/manage/views/__init__.py b/warehouse/manage/views/__init__.py index a32388d17496..3e22b2bb1197 100644 --- a/warehouse/manage/views/__init__.py +++ b/warehouse/manage/views/__init__.py @@ -2052,7 +2052,7 @@ def delete_project(project, request): remove_project(project, request) - tuf.update_metadata(request, project) + request.task(tuf.update_metadata).delay(project.id) return HTTPSeeOther(request.route_path("manage.projects")) @@ -2224,7 +2224,7 @@ def yank_project_release(self): recipient_role=contributor_role, ) - tuf.update_metadata(self.request, self.release.project) + self.request.task(tuf.update_metadata).delay(self.release.project.id) return HTTPSeeOther( self.request.route_path( "manage.project.releases", project_name=self.release.project.name @@ -2309,7 +2309,7 @@ def unyank_project_release(self): submitter_role=submitter_role, recipient_role=contributor_role, ) - tuf.update_metadata(self.request, self.release.project) + self.request.task(tuf.update_metadata).delay(self.release.project.id) return HTTPSeeOther( self.request.route_path( "manage.project.releases", project_name=self.release.project.name @@ -2409,7 +2409,7 @@ def delete_project_release(self): recipient_role=contributor_role, ) - tuf.update_metadata(self.request, self.release.project) + self.request.task(tuf.update_metadata).delay(self.release.project.id) return HTTPSeeOther( self.request.route_path( "manage.project.releases", project_name=self.release.project.name @@ -2508,7 +2508,7 @@ def _error(message): self.request.session.flash( f"Deleted file {release_file.filename!r}", queue="success" ) - tuf.update_metadata(self.request, self.release.project) + self.request.task(tuf.update_metadata).delay(self.release.project.id) return HTTPSeeOther( self.request.route_path( "manage.project.release", diff --git a/warehouse/tuf/__init__.py b/warehouse/tuf/__init__.py index 53106f19cf28..d183d7eb03c3 100644 --- a/warehouse/tuf/__init__.py +++ b/warehouse/tuf/__init__.py @@ -18,11 +18,13 @@ from http import HTTPStatus from typing import Any +from uuid import UUID import requests from pyramid.request import Request +from warehouse import tasks from warehouse.packaging.models import Project from warehouse.packaging.utils import render_simple_detail @@ -99,7 +101,8 @@ def wait_for_success(server: str, task_id: str): raise RSTUFError("RSTUF job failed, please check payload and retry") -def update_metadata(request: Request, project: Project): +@tasks.task(ignore_result=True, acks_late=True) +def update_metadata(request: Request, project_id: UUID): """Update TUF metadata to capture project changes (PEP 458). NOTE: PEP 458 says, TUF targets metadata must include path, hash and size of @@ -114,6 +117,8 @@ def update_metadata(request: Request, project: Project): if not is_bootstrapped(server): return + project = request.db.query(Project).filter(Project.id == project_id).one() + # NOTE: We ignore the returned simple detail path with the content hash as # infix. In TUF metadata the project name and hash are listed separately, so # that there is only one entry per target file, even if the content changes. From b95f1a6196c8fd56cfe83e9bb2c733fc6672797d Mon Sep 17 00:00:00 2001 From: Lukas Puehringer Date: Wed, 19 Jun 2024 10:32:16 +0200 Subject: [PATCH 06/17] Do not check if RSTUF is bootstrapped Instead, allow handling a custom RSTUFNoBootstrapError, which is raised, if tuf.update_metadata is called before RSTUF is bootstrapped. Signed-off-by: Lukas Puehringer --- warehouse/tuf/__init__.py | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/warehouse/tuf/__init__.py b/warehouse/tuf/__init__.py index d183d7eb03c3..123a09f88643 100644 --- a/warehouse/tuf/__init__.py +++ b/warehouse/tuf/__init__.py @@ -16,7 +16,6 @@ import time -from http import HTTPStatus from typing import Any from uuid import UUID @@ -33,21 +32,16 @@ class RSTUFError(Exception): pass +class RSTUFNoBootstrapError(Exception): + pass + + def get_task_state(server: str, task_id: str) -> str: resp = requests.get(f"{server}/api/v1/task?task_id={task_id}") resp.raise_for_status() return resp.json()["data"]["state"] -def is_bootstrapped(server: str) -> bool: - """Call RSTUF bootstrap API to check, if RSTUF is ready to be used.""" - response = requests.get(f"{server}/api/v1/bootstrap") - if response.status_code != HTTPStatus.OK: - return False - - return response.json()["data"]["bootstrap"] - - def post_bootstrap(server: str, payload: Any) -> str: resp = requests.post(f"{server}/api/v1/bootstrap", json=payload) resp.raise_for_status() @@ -68,7 +62,15 @@ def post_artifacts(server: str, payload: Any) -> str: """ resp = requests.post(f"{server}/api/v1/artifacts", json=payload) resp.raise_for_status() - return resp.json()["data"]["task_id"] + + # 200 but no "data" means that RSTUF isn't bootstrapped yet + # TODO: Ask upstream to not return 200 on error + resp_json = resp.json() + resp_data = resp_json.get("data") + if not resp_data: + raise RSTUFNoBootstrapError(resp_json) + + return resp_data["task_id"] def wait_for_success(server: str, task_id: str): @@ -114,9 +116,6 @@ def update_metadata(request: Request, project_id: UUID): server = request.registry.settings["tuf.rstuf_api_url"] - if not is_bootstrapped(server): - return - project = request.db.query(Project).filter(Project.id == project_id).one() # NOTE: We ignore the returned simple detail path with the content hash as From db8e2c0d942641b43730f99450c1f00faeb7b75e Mon Sep 17 00:00:00 2001 From: Lukas Puehringer Date: Wed, 19 Jun 2024 10:53:41 +0200 Subject: [PATCH 07/17] Drop TUF_ENABLED setting Using TUF_RSTUF_API_URL as toggle instead should be good enough. This is also what rubygems does. h/t @simi Signed-off-by: Lukas Puehringer --- dev/environment | 1 - docs/dev/development/getting-started.rst | 2 +- tests/unit/test_config.py | 1 - warehouse/config.py | 7 ------- warehouse/tuf/__init__.py | 5 ++--- 5 files changed, 3 insertions(+), 13 deletions(-) diff --git a/dev/environment b/dev/environment index 073b0b4a9092..d3f23a1cf45c 100644 --- a/dev/environment +++ b/dev/environment @@ -65,7 +65,6 @@ TWOFACTORMANDATE_ENABLED=true OIDC_AUDIENCE=pypi TUF_RSTUF_API_URL="http://rstuf-api" -TUF_ENABLED=true # Default to the reCAPTCHA testing keys from https://developers.google.com/recaptcha/docs/faq RECAPTCHA_SITE_KEY=6LeIxAcTAAAAAJcZVRqyHh71UMIEGNQ_MXjiZKhI diff --git a/docs/dev/development/getting-started.rst b/docs/dev/development/getting-started.rst index ba38e68a35c0..c619be9b5268 100644 --- a/docs/dev/development/getting-started.rst +++ b/docs/dev/development/getting-started.rst @@ -267,7 +267,7 @@ TUF trust root for development, and creates the initial set of TUF metadata. Once this process is finished, TUF metadata will update automatically whenever distribution files are uploaded, yanked or removed. To disable this behavior, -you can set the ``TUF_ENABLED`` environment variable to false. TUF metadata +you can unset the ``TUF_RSTUF_API_URL`` environment variable. TUF metadata is hosted at: http://localhost:9001/tuf-metadata/ .. note:: diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index f854a5f38ecd..330d85ff6c43 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -336,7 +336,6 @@ def __init__(self): "warehouse.search.ratelimit_string": "5 per second", "oidc.backend": "warehouse.oidc.services.OIDCPublisherService", "integrity.backend": "warehouse.attestations.services.IntegrityService", - "tuf.enabled": False, "warehouse.organizations.max_undecided_organization_applications": 3, "reconcile_file_storages.batch_size": 100, "metadata_backfill.batch_size": 500, diff --git a/warehouse/config.py b/warehouse/config.py index 470c5f0b19bb..40345d4ea93d 100644 --- a/warehouse/config.py +++ b/warehouse/config.py @@ -470,13 +470,6 @@ def configure(settings=None): "INTEGRITY_BACKEND", default="warehouse.attestations.services.IntegrityService", ) - maybe_set( - settings, - "tuf.enabled", - "TUF_ENABLED", - coercer=distutils.util.strtobool, - default=False, - ) maybe_set(settings, "tuf.rstuf_api_url", "TUF_RSTUF_API_URL") # Pythondotorg integration settings diff --git a/warehouse/tuf/__init__.py b/warehouse/tuf/__init__.py index 123a09f88643..ab4c7cb06d83 100644 --- a/warehouse/tuf/__init__.py +++ b/warehouse/tuf/__init__.py @@ -111,10 +111,9 @@ def update_metadata(request: Request, project_id: UUID): distributions files and simple detail files. In reality, simple detail files are enough, as they already include all relevant distribution file infos. """ - if not request.registry.settings["tuf.enabled"]: - return - server = request.registry.settings["tuf.rstuf_api_url"] + if not server: + return project = request.db.query(Project).filter(Project.id == project_id).one() From bdbcf09041a23793a75c826aa573403fb5510408 Mon Sep 17 00:00:00 2001 From: Lukas Puehringer Date: Thu, 20 Jun 2024 12:17:22 +0200 Subject: [PATCH 08/17] Add tuf tests Signed-off-by: Lukas Puehringer --- tests/unit/tuf/test_tuf.py | 83 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/tests/unit/tuf/test_tuf.py b/tests/unit/tuf/test_tuf.py index 421a2e234ab9..9e7cc99c215b 100644 --- a/tests/unit/tuf/test_tuf.py +++ b/tests/unit/tuf/test_tuf.py @@ -57,6 +57,37 @@ def test_post_bootstrap(self, monkeypatch): with pytest.raises(tuf.RSTUFError): tuf.post_bootstrap(self.server, payload) + def test_post_artifacts(self, monkeypatch): + payload = { + "targets": [ + { + "path": "foo", + "info": { + "length": 42, + "hashes": {"blake2b-256": "deadbeef"}, + }, + } + ] + } + + resp_json = {"data": {"task_id": self.task_id}} + resp = stub( + raise_for_status=(lambda *a: None), json=(lambda *a, **kw: resp_json) + ) + post = call_recorder(lambda *a, **kw: resp) + monkeypatch.setattr(tuf.requests, "post", post) + + # Test success + result = tuf.post_artifacts(self.server, payload) + + assert result == self.task_id + assert post.calls == [call(f"{self.server}/api/v1/artifacts", json=payload)] + + # Test fail with incomplete response json (i.e. no bootstrap error) + del resp_json["data"] + with pytest.raises(tuf.RSTUFNoBootstrapError): + tuf.post_artifacts(self.server, payload) + def test_wait_for_success(self, monkeypatch): get_task_state = call_recorder(lambda *a: "SUCCESS") monkeypatch.setattr(tuf, "get_task_state", get_task_state) @@ -88,3 +119,55 @@ def test_wait_for_success_error(self, state, iterations, monkeypatch): tuf.wait_for_success(self.server, self.task_id) assert get_task_state.calls == [call(self.server, self.task_id)] * iterations + + def test_update_metadata(self, db_request, monkeypatch): + project_id = "id" + project_name = "name" + + project = stub(normalized_name=project_name) + + one = call_recorder(lambda: project) + db_request.db.query = lambda a: stub(filter=lambda a: stub(one=one)) + + # Test early return, if no RSTUF API URL configured + db_request.registry.settings = {"tuf.rstuf_api_url": None} + tuf.update_metadata(db_request, project_id) + + assert not one.calls + + # Test regular run + rstuf_url = "url" + index_digest = "digest" + index_size = 42 + + db_request.registry.settings = {"tuf.rstuf_api_url": rstuf_url} + + render = call_recorder(lambda *a, **kw: (index_digest, None, index_size)) + monkeypatch.setattr(tuf, "render_simple_detail", render) + + post = call_recorder(lambda *a: self.task_id) + monkeypatch.setattr(tuf, "post_artifacts", post) + + wait = call_recorder(lambda *a: None) + monkeypatch.setattr(tuf, "wait_for_success", wait) + + tuf.update_metadata(db_request, project_id) + assert one.calls == [call()] + assert render.calls == [call(project, db_request, store=True)] + assert post.calls == [ + call( + rstuf_url, + { + "targets": [ + { + "path": project_name, + "info": { + "length": index_size, + "hashes": {"blake2b-256": index_digest}, + }, + } + ] + }, + ) + ] + assert wait.calls == [call(rstuf_url, self.task_id)] From 08d5bd11150fa6fe2c3895ede1b80d3ea3678112 Mon Sep 17 00:00:00 2001 From: Lukas Puehringer Date: Thu, 20 Jun 2024 19:22:38 +0200 Subject: [PATCH 09/17] make translations Signed-off-by: Lukas Puehringer --- warehouse/locale/messages.pot | 98 +++++++++++++++++------------------ 1 file changed, 49 insertions(+), 49 deletions(-) diff --git a/warehouse/locale/messages.pot b/warehouse/locale/messages.pot index 76f6785fa667..026e8878eced 100644 --- a/warehouse/locale/messages.pot +++ b/warehouse/locale/messages.pot @@ -152,7 +152,7 @@ msgstr "" msgid "Successful WebAuthn assertion" msgstr "" -#: warehouse/accounts/views.py:569 warehouse/manage/views/__init__.py:873 +#: warehouse/accounts/views.py:569 warehouse/manage/views/__init__.py:874 msgid "Recovery code accepted. The supplied code cannot be used again." msgstr "" @@ -286,7 +286,7 @@ msgid "You are now ${role} of the '${project_name}' project." msgstr "" #: warehouse/accounts/views.py:1548 warehouse/accounts/views.py:1791 -#: warehouse/manage/views/__init__.py:1417 +#: warehouse/manage/views/__init__.py:1418 msgid "" "Trusted publishing is temporarily disabled. See https://pypi.org/help" "#admin-intervention for details." @@ -306,19 +306,19 @@ msgstr "" msgid "You can't register more than 3 pending trusted publishers at once." msgstr "" -#: warehouse/accounts/views.py:1614 warehouse/manage/views/__init__.py:1472 -#: warehouse/manage/views/__init__.py:1585 -#: warehouse/manage/views/__init__.py:1697 -#: warehouse/manage/views/__init__.py:1807 +#: warehouse/accounts/views.py:1614 warehouse/manage/views/__init__.py:1473 +#: warehouse/manage/views/__init__.py:1586 +#: warehouse/manage/views/__init__.py:1698 +#: warehouse/manage/views/__init__.py:1808 msgid "" "There have been too many attempted trusted publisher registrations. Try " "again later." msgstr "" -#: warehouse/accounts/views.py:1625 warehouse/manage/views/__init__.py:1486 -#: warehouse/manage/views/__init__.py:1599 -#: warehouse/manage/views/__init__.py:1711 -#: warehouse/manage/views/__init__.py:1821 +#: warehouse/accounts/views.py:1625 warehouse/manage/views/__init__.py:1487 +#: warehouse/manage/views/__init__.py:1600 +#: warehouse/manage/views/__init__.py:1712 +#: warehouse/manage/views/__init__.py:1822 msgid "The trusted publisher could not be registered" msgstr "" @@ -446,157 +446,157 @@ msgid "" "less." msgstr "" -#: warehouse/manage/views/__init__.py:285 +#: warehouse/manage/views/__init__.py:286 msgid "Account details updated" msgstr "" -#: warehouse/manage/views/__init__.py:315 +#: warehouse/manage/views/__init__.py:316 msgid "Email ${email_address} added - check your email for a verification link" msgstr "" -#: warehouse/manage/views/__init__.py:821 +#: warehouse/manage/views/__init__.py:822 msgid "Recovery codes already generated" msgstr "" -#: warehouse/manage/views/__init__.py:822 +#: warehouse/manage/views/__init__.py:823 msgid "Generating new recovery codes will invalidate your existing codes." msgstr "" -#: warehouse/manage/views/__init__.py:931 +#: warehouse/manage/views/__init__.py:932 msgid "Verify your email to create an API token." msgstr "" -#: warehouse/manage/views/__init__.py:1031 +#: warehouse/manage/views/__init__.py:1032 msgid "API Token does not exist." msgstr "" -#: warehouse/manage/views/__init__.py:1063 +#: warehouse/manage/views/__init__.py:1064 msgid "Invalid credentials. Try again" msgstr "" -#: warehouse/manage/views/__init__.py:1182 +#: warehouse/manage/views/__init__.py:1183 msgid "Invalid alternate repository location details" msgstr "" -#: warehouse/manage/views/__init__.py:1219 +#: warehouse/manage/views/__init__.py:1220 msgid "Added alternate repository '${name}'" msgstr "" -#: warehouse/manage/views/__init__.py:1253 -#: warehouse/manage/views/__init__.py:2154 -#: warehouse/manage/views/__init__.py:2239 -#: warehouse/manage/views/__init__.py:2340 -#: warehouse/manage/views/__init__.py:2440 +#: warehouse/manage/views/__init__.py:1254 +#: warehouse/manage/views/__init__.py:2157 +#: warehouse/manage/views/__init__.py:2243 +#: warehouse/manage/views/__init__.py:2344 +#: warehouse/manage/views/__init__.py:2445 msgid "Confirm the request" msgstr "" -#: warehouse/manage/views/__init__.py:1265 +#: warehouse/manage/views/__init__.py:1266 msgid "Invalid alternate repository id" msgstr "" -#: warehouse/manage/views/__init__.py:1276 +#: warehouse/manage/views/__init__.py:1277 msgid "Invalid alternate repository for project" msgstr "" -#: warehouse/manage/views/__init__.py:1284 +#: warehouse/manage/views/__init__.py:1285 msgid "" "Could not delete alternate repository - ${confirm} is not the same as " "${alt_repo_name}" msgstr "" -#: warehouse/manage/views/__init__.py:1322 +#: warehouse/manage/views/__init__.py:1323 msgid "Deleted alternate repository '${name}'" msgstr "" -#: warehouse/manage/views/__init__.py:1453 +#: warehouse/manage/views/__init__.py:1454 msgid "" "GitHub-based trusted publishing is temporarily disabled. See " "https://pypi.org/help#admin-intervention for details." msgstr "" -#: warehouse/manage/views/__init__.py:1566 +#: warehouse/manage/views/__init__.py:1567 msgid "" "GitLab-based trusted publishing is temporarily disabled. See " "https://pypi.org/help#admin-intervention for details." msgstr "" -#: warehouse/manage/views/__init__.py:1678 +#: warehouse/manage/views/__init__.py:1679 msgid "" "Google-based trusted publishing is temporarily disabled. See " "https://pypi.org/help#admin-intervention for details." msgstr "" -#: warehouse/manage/views/__init__.py:1787 +#: warehouse/manage/views/__init__.py:1788 msgid "" "ActiveState-based trusted publishing is temporarily disabled. See " "https://pypi.org/help#admin-intervention for details." msgstr "" -#: warehouse/manage/views/__init__.py:2022 -#: warehouse/manage/views/__init__.py:2323 -#: warehouse/manage/views/__init__.py:2431 +#: warehouse/manage/views/__init__.py:2023 +#: warehouse/manage/views/__init__.py:2327 +#: warehouse/manage/views/__init__.py:2436 msgid "" "Project deletion temporarily disabled. See https://pypi.org/help#admin-" "intervention for details." msgstr "" -#: warehouse/manage/views/__init__.py:2166 +#: warehouse/manage/views/__init__.py:2169 msgid "Could not yank release - " msgstr "" -#: warehouse/manage/views/__init__.py:2251 +#: warehouse/manage/views/__init__.py:2255 msgid "Could not un-yank release - " msgstr "" -#: warehouse/manage/views/__init__.py:2352 +#: warehouse/manage/views/__init__.py:2356 msgid "Could not delete release - " msgstr "" -#: warehouse/manage/views/__init__.py:2452 +#: warehouse/manage/views/__init__.py:2457 msgid "Could not find file" msgstr "" -#: warehouse/manage/views/__init__.py:2456 +#: warehouse/manage/views/__init__.py:2461 msgid "Could not delete file - " msgstr "" -#: warehouse/manage/views/__init__.py:2606 +#: warehouse/manage/views/__init__.py:2611 msgid "Team '${team_name}' already has ${role_name} role for project" msgstr "" -#: warehouse/manage/views/__init__.py:2713 +#: warehouse/manage/views/__init__.py:2718 msgid "User '${username}' already has ${role_name} role for project" msgstr "" -#: warehouse/manage/views/__init__.py:2780 +#: warehouse/manage/views/__init__.py:2785 msgid "${username} is now ${role} of the '${project_name}' project." msgstr "" -#: warehouse/manage/views/__init__.py:2812 +#: warehouse/manage/views/__init__.py:2817 msgid "" "User '${username}' does not have a verified primary email address and " "cannot be added as a ${role_name} for project" msgstr "" -#: warehouse/manage/views/__init__.py:2825 +#: warehouse/manage/views/__init__.py:2830 #: warehouse/manage/views/organizations.py:878 msgid "User '${username}' already has an active invite. Please try again later." msgstr "" -#: warehouse/manage/views/__init__.py:2890 +#: warehouse/manage/views/__init__.py:2895 #: warehouse/manage/views/organizations.py:943 msgid "Invitation sent to '${username}'" msgstr "" -#: warehouse/manage/views/__init__.py:2923 +#: warehouse/manage/views/__init__.py:2928 msgid "Could not find role invitation." msgstr "" -#: warehouse/manage/views/__init__.py:2934 +#: warehouse/manage/views/__init__.py:2939 msgid "Invitation already expired." msgstr "" -#: warehouse/manage/views/__init__.py:2966 +#: warehouse/manage/views/__init__.py:2971 #: warehouse/manage/views/organizations.py:1130 msgid "Invitation revoked from '${username}'." msgstr "" From 31e68a9d92b37be3152f6772db59b00a4f6471c8 Mon Sep 17 00:00:00 2001 From: Lukas Puehringer Date: Tue, 30 Jul 2024 10:02:32 +0200 Subject: [PATCH 10/17] Use variable for test value in a TUF test requested by @trishankatdatadog Signed-off-by: Lukas Puehringer --- tests/unit/packaging/test_utils.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/unit/packaging/test_utils.py b/tests/unit/packaging/test_utils.py index 8a9eebdeccc0..1f80e8213ca2 100644 --- a/tests/unit/packaging/test_utils.py +++ b/tests/unit/packaging/test_utils.py @@ -93,9 +93,10 @@ def test_render_simple_detail_with_store(db_request, monkeypatch, jinja): fakeblake2b = pretend.call_recorder(lambda *a, **kw: fake_hasher) monkeypatch.setattr(hashlib, "blake2b", fakeblake2b) + expected_size = 42 fake_named_temporary_file = pretend.stub( name="/tmp/wutang", - write=pretend.call_recorder(lambda data: 42), + write=pretend.call_recorder(lambda data: expected_size), flush=pretend.call_recorder(lambda: None), ) @@ -154,4 +155,4 @@ def __exit__(self, type, value, traceback): f"{project.normalized_name}/deadbeefdeadbeefdeadbeefdeadbeef" + f".{project.normalized_name}.html" ) - assert size == 42 + assert size == expected_size From 1b928700cec872adf82d9d67f7f8500a5b5c443c Mon Sep 17 00:00:00 2001 From: Kairo de Araujo Date: Thu, 3 Oct 2024 13:29:55 +0200 Subject: [PATCH 11/17] refactor: remove prefixes tuf/TUF from config - `tuf.rstuf_*` to `rstuf.*` - environment variable from `TUF_RSTUF*` to `RSTUF*` Signed-off-by: Kairo de Araujo --- dev/environment | 2 +- docs/dev/development/getting-started.rst | 2 +- tests/unit/tuf/test_tuf.py | 4 ++-- warehouse/config.py | 2 +- warehouse/tuf/__init__.py | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/dev/environment b/dev/environment index d3f23a1cf45c..6c6dff0dfdc8 100644 --- a/dev/environment +++ b/dev/environment @@ -64,7 +64,7 @@ TWOFACTORMANDATE_AVAILABLE=true TWOFACTORMANDATE_ENABLED=true OIDC_AUDIENCE=pypi -TUF_RSTUF_API_URL="http://rstuf-api" +RSTUF_API_URL="http://rstuf-api" # Default to the reCAPTCHA testing keys from https://developers.google.com/recaptcha/docs/faq RECAPTCHA_SITE_KEY=6LeIxAcTAAAAAJcZVRqyHh71UMIEGNQ_MXjiZKhI diff --git a/docs/dev/development/getting-started.rst b/docs/dev/development/getting-started.rst index c619be9b5268..d428c9dc99ac 100644 --- a/docs/dev/development/getting-started.rst +++ b/docs/dev/development/getting-started.rst @@ -267,7 +267,7 @@ TUF trust root for development, and creates the initial set of TUF metadata. Once this process is finished, TUF metadata will update automatically whenever distribution files are uploaded, yanked or removed. To disable this behavior, -you can unset the ``TUF_RSTUF_API_URL`` environment variable. TUF metadata +you can unset the ``RSTUF_API_URL`` environment variable. TUF metadata is hosted at: http://localhost:9001/tuf-metadata/ .. note:: diff --git a/tests/unit/tuf/test_tuf.py b/tests/unit/tuf/test_tuf.py index 9e7cc99c215b..ea24de92deff 100644 --- a/tests/unit/tuf/test_tuf.py +++ b/tests/unit/tuf/test_tuf.py @@ -130,7 +130,7 @@ def test_update_metadata(self, db_request, monkeypatch): db_request.db.query = lambda a: stub(filter=lambda a: stub(one=one)) # Test early return, if no RSTUF API URL configured - db_request.registry.settings = {"tuf.rstuf_api_url": None} + db_request.registry.settings = {"rstuf.api_url": None} tuf.update_metadata(db_request, project_id) assert not one.calls @@ -140,7 +140,7 @@ def test_update_metadata(self, db_request, monkeypatch): index_digest = "digest" index_size = 42 - db_request.registry.settings = {"tuf.rstuf_api_url": rstuf_url} + db_request.registry.settings = {"rstuf.api_url": rstuf_url} render = call_recorder(lambda *a, **kw: (index_digest, None, index_size)) monkeypatch.setattr(tuf, "render_simple_detail", render) diff --git a/warehouse/config.py b/warehouse/config.py index 40345d4ea93d..84fcfcf372f0 100644 --- a/warehouse/config.py +++ b/warehouse/config.py @@ -470,7 +470,7 @@ def configure(settings=None): "INTEGRITY_BACKEND", default="warehouse.attestations.services.IntegrityService", ) - maybe_set(settings, "tuf.rstuf_api_url", "TUF_RSTUF_API_URL") + maybe_set(settings, "rstuf.api_url", "RSTUF_API_URL") # Pythondotorg integration settings maybe_set( diff --git a/warehouse/tuf/__init__.py b/warehouse/tuf/__init__.py index ab4c7cb06d83..03ed196321f3 100644 --- a/warehouse/tuf/__init__.py +++ b/warehouse/tuf/__init__.py @@ -111,7 +111,7 @@ def update_metadata(request: Request, project_id: UUID): distributions files and simple detail files. In reality, simple detail files are enough, as they already include all relevant distribution file infos. """ - server = request.registry.settings["tuf.rstuf_api_url"] + server = request.registry.settings["rstuf.api_url"] if not server: return From 825cb5304009f79667bb358495fdfaa875025188 Mon Sep 17 00:00:00 2001 From: Kairo de Araujo Date: Thu, 3 Oct 2024 15:31:37 +0200 Subject: [PATCH 12/17] refactor: split tuf.__init__.tasks to tuf.tasks this refactor split the tasks from the tuf.__init__.py to tasks.py Signed-off-by: Kairo de Araujo --- tests/unit/forklift/test_legacy.py | 2 +- tests/unit/tuf/test_tasks.py | 78 ++++++++++++++++++++++++++++++ tests/unit/tuf/test_tuf.py | 52 -------------------- warehouse/forklift/legacy.py | 2 +- warehouse/manage/views/__init__.py | 10 ++-- warehouse/tuf/__init__.py | 42 ---------------- warehouse/tuf/tasks.py | 54 +++++++++++++++++++++ 7 files changed, 139 insertions(+), 101 deletions(-) create mode 100644 tests/unit/tuf/test_tasks.py create mode 100644 warehouse/tuf/tasks.py diff --git a/tests/unit/forklift/test_legacy.py b/tests/unit/forklift/test_legacy.py index 124a2ae64d2f..c05d13ca0b17 100644 --- a/tests/unit/forklift/test_legacy.py +++ b/tests/unit/forklift/test_legacy.py @@ -57,7 +57,7 @@ Role, ) from warehouse.packaging.tasks import sync_file_to_cache, update_bigquery_release_files -from warehouse.tuf import update_metadata +from warehouse.tuf.tasks import update_metadata from ...common.db.accounts import EmailFactory, UserFactory from ...common.db.classifiers import ClassifierFactory diff --git a/tests/unit/tuf/test_tasks.py b/tests/unit/tuf/test_tasks.py new file mode 100644 index 000000000000..40bddef4a8db --- /dev/null +++ b/tests/unit/tuf/test_tasks.py @@ -0,0 +1,78 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from pretend import call, call_recorder, stub + +from warehouse import tuf + + +class TestTUFTasks: + task_id = "123456" + + def test_update_metadata(self, db_request, monkeypatch): + project_id = "id" + project_name = "name" + + project = stub(normalized_name=project_name) + + one = call_recorder(lambda: project) + db_request.db.query = lambda a: stub(filter=lambda a: stub(one=one)) + + rstuf_url = "url" + index_digest = "digest" + index_size = 42 + + db_request.registry.settings = {"rstuf.api_url": rstuf_url} + + render = call_recorder(lambda *a, **kw: (index_digest, None, index_size)) + tuf.tasks.render_simple_detail = render + + post = call_recorder(lambda *a: self.task_id) + monkeypatch.setattr(tuf.tasks, "post_artifacts", post) + + wait = call_recorder(lambda *a: None) + monkeypatch.setattr(tuf.tasks, "wait_for_success", wait) + + tuf.tasks.update_metadata(db_request, project_id) + assert one.calls == [call()] + assert render.calls == [call(project, db_request, store=True)] + assert post.calls == [ + call( + rstuf_url, + { + "targets": [ + { + "path": project_name, + "info": { + "length": index_size, + "hashes": {"blake2b-256": index_digest}, + }, + } + ] + }, + ) + ] + assert wait.calls == [call(rstuf_url, self.task_id)] + + def test_update_metadata_no_rstuf_api_url(self, db_request): + project_id = "id" + project_name = "name" + + project = stub(normalized_name=project_name) + + one = call_recorder(lambda: project) + db_request.db.query = lambda a: stub(filter=lambda a: stub(one=one)) + + # Test early return, if no RSTUF API URL configured + db_request.registry.settings = {"rstuf.api_url": None} + tuf.tasks.update_metadata(db_request, project_id) + + assert not one.calls diff --git a/tests/unit/tuf/test_tuf.py b/tests/unit/tuf/test_tuf.py index ea24de92deff..adbcc5862ff2 100644 --- a/tests/unit/tuf/test_tuf.py +++ b/tests/unit/tuf/test_tuf.py @@ -119,55 +119,3 @@ def test_wait_for_success_error(self, state, iterations, monkeypatch): tuf.wait_for_success(self.server, self.task_id) assert get_task_state.calls == [call(self.server, self.task_id)] * iterations - - def test_update_metadata(self, db_request, monkeypatch): - project_id = "id" - project_name = "name" - - project = stub(normalized_name=project_name) - - one = call_recorder(lambda: project) - db_request.db.query = lambda a: stub(filter=lambda a: stub(one=one)) - - # Test early return, if no RSTUF API URL configured - db_request.registry.settings = {"rstuf.api_url": None} - tuf.update_metadata(db_request, project_id) - - assert not one.calls - - # Test regular run - rstuf_url = "url" - index_digest = "digest" - index_size = 42 - - db_request.registry.settings = {"rstuf.api_url": rstuf_url} - - render = call_recorder(lambda *a, **kw: (index_digest, None, index_size)) - monkeypatch.setattr(tuf, "render_simple_detail", render) - - post = call_recorder(lambda *a: self.task_id) - monkeypatch.setattr(tuf, "post_artifacts", post) - - wait = call_recorder(lambda *a: None) - monkeypatch.setattr(tuf, "wait_for_success", wait) - - tuf.update_metadata(db_request, project_id) - assert one.calls == [call()] - assert render.calls == [call(project, db_request, store=True)] - assert post.calls == [ - call( - rstuf_url, - { - "targets": [ - { - "path": project_name, - "info": { - "length": index_size, - "hashes": {"blake2b-256": index_digest}, - }, - } - ] - }, - ) - ] - assert wait.calls == [call(rstuf_url, self.task_id)] diff --git a/warehouse/forklift/legacy.py b/warehouse/forklift/legacy.py index e4cd8cc41930..4fc964c36f9f 100644 --- a/warehouse/forklift/legacy.py +++ b/warehouse/forklift/legacy.py @@ -1307,7 +1307,7 @@ def file_upload(request): request.db.flush() # flush db now so server default values are populated for celery - request.task(tuf.update_metadata).delay(release.project.id) + request.task(tuf.tasks.update_metadata).delay(release.project.id) # Push updates to BigQuery dist_metadata = { diff --git a/warehouse/manage/views/__init__.py b/warehouse/manage/views/__init__.py index 3e22b2bb1197..51cb86f52de5 100644 --- a/warehouse/manage/views/__init__.py +++ b/warehouse/manage/views/__init__.py @@ -2052,7 +2052,7 @@ def delete_project(project, request): remove_project(project, request) - request.task(tuf.update_metadata).delay(project.id) + request.task(tuf.tasks.update_metadata).delay(project.id) return HTTPSeeOther(request.route_path("manage.projects")) @@ -2224,7 +2224,7 @@ def yank_project_release(self): recipient_role=contributor_role, ) - self.request.task(tuf.update_metadata).delay(self.release.project.id) + self.request.task(tuf.tasks.update_metadata).delay(self.release.project.id) return HTTPSeeOther( self.request.route_path( "manage.project.releases", project_name=self.release.project.name @@ -2309,7 +2309,7 @@ def unyank_project_release(self): submitter_role=submitter_role, recipient_role=contributor_role, ) - self.request.task(tuf.update_metadata).delay(self.release.project.id) + self.request.task(tuf.tasks.update_metadata).delay(self.release.project.id) return HTTPSeeOther( self.request.route_path( "manage.project.releases", project_name=self.release.project.name @@ -2409,7 +2409,7 @@ def delete_project_release(self): recipient_role=contributor_role, ) - self.request.task(tuf.update_metadata).delay(self.release.project.id) + self.request.task(tuf.tasks.update_metadata).delay(self.release.project.id) return HTTPSeeOther( self.request.route_path( "manage.project.releases", project_name=self.release.project.name @@ -2508,7 +2508,7 @@ def _error(message): self.request.session.flash( f"Deleted file {release_file.filename!r}", queue="success" ) - self.request.task(tuf.update_metadata).delay(self.release.project.id) + self.request.task(tuf.tasks.update_metadata).delay(self.release.project.id) return HTTPSeeOther( self.request.route_path( "manage.project.release", diff --git a/warehouse/tuf/__init__.py b/warehouse/tuf/__init__.py index 03ed196321f3..5991c918aa4c 100644 --- a/warehouse/tuf/__init__.py +++ b/warehouse/tuf/__init__.py @@ -17,16 +17,9 @@ import time from typing import Any -from uuid import UUID import requests -from pyramid.request import Request - -from warehouse import tasks -from warehouse.packaging.models import Project -from warehouse.packaging.utils import render_simple_detail - class RSTUFError(Exception): pass @@ -101,38 +94,3 @@ def wait_for_success(server: str, task_id: str): else: raise RSTUFError("RSTUF job failed, please check payload and retry") - - -@tasks.task(ignore_result=True, acks_late=True) -def update_metadata(request: Request, project_id: UUID): - """Update TUF metadata to capture project changes (PEP 458). - - NOTE: PEP 458 says, TUF targets metadata must include path, hash and size of - distributions files and simple detail files. In reality, simple detail files - are enough, as they already include all relevant distribution file infos. - """ - server = request.registry.settings["rstuf.api_url"] - if not server: - return - - project = request.db.query(Project).filter(Project.id == project_id).one() - - # NOTE: We ignore the returned simple detail path with the content hash as - # infix. In TUF metadata the project name and hash are listed separately, so - # that there is only one entry per target file, even if the content changes. - digest, _, size = render_simple_detail(project, request, store=True) - payload = { - "targets": [ - { - "path": project.normalized_name, - "info": { - "length": size, - "hashes": {"blake2b-256": digest}, - }, - } - ] - } - - # TODO: Handle errors: pass, retry or notify - task_id = post_artifacts(server, payload) - wait_for_success(server, task_id) diff --git a/warehouse/tuf/tasks.py b/warehouse/tuf/tasks.py new file mode 100644 index 000000000000..5e12b304b6fc --- /dev/null +++ b/warehouse/tuf/tasks.py @@ -0,0 +1,54 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from uuid import UUID + +from pyramid.request import Request + +from warehouse import tasks +from warehouse.packaging.models import Project +from warehouse.packaging.utils import render_simple_detail +from warehouse.tuf import post_artifacts, wait_for_success + + +@tasks.task(ignore_result=True, acks_late=True) +def update_metadata(request: Request, project_id: UUID): + """Update TUF metadata to capture project changes (PEP 458). + + NOTE: PEP 458 says, TUF targets metadata must include path, hash and size of + distributions files and simple detail files. In reality, simple detail files + are enough, as they already include all relevant distribution file infos. + """ + server = request.registry.settings["rstuf.api_url"] + if not server: + return + + project = request.db.query(Project).filter(Project.id == project_id).one() + + # NOTE: We ignore the returned simple detail path with the content hash as + # infix. In TUF metadata the project name and hash are listed separately, so + # that there is only one entry per target file, even if the content changes. + digest, _, size = render_simple_detail(project, request, store=True) + payload = { + "targets": [ + { + "path": project.normalized_name, + "info": { + "length": size, + "hashes": {"blake2b-256": digest}, + }, + } + ] + } + + # TODO: Handle errors: pass, retry or notify + task_id = post_artifacts(server, payload) + wait_for_success(server, task_id) From e9183e589a9541c6f4ad7026de7a16b08160c1c0 Mon Sep 17 00:00:00 2001 From: Kairo de Araujo Date: Sat, 5 Oct 2024 17:31:40 +0200 Subject: [PATCH 13/17] cli: remove rstuf custom command add upstream use rstuf upstream cli update `make inittuf` to use upstream command Signed-off-by: Kairo de Araujo --- Makefile | 2 +- docs/dev/development/getting-started.rst | 2 +- requirements/dev.txt | 1 + tests/unit/cli/test_tuf.py | 38 ------------------------ warehouse/cli/tuf.py | 33 -------------------- 5 files changed, 3 insertions(+), 73 deletions(-) delete mode 100644 tests/unit/cli/test_tuf.py delete mode 100644 warehouse/cli/tuf.py diff --git a/Makefile b/Makefile index ab240a1d1404..7879aa8f5981 100644 --- a/Makefile +++ b/Makefile @@ -150,7 +150,7 @@ initdb: .state/docker-build-base .state/db-populated inittuf: .state/db-migrated docker compose up -d rstuf-api docker compose up -d rstuf-worker - docker compose run --rm web python -m warehouse tuf bootstrap dev/rstuf/bootstrap.json --api-server http://rstuf-api + docker compose run --rm web rstuf admin --api-server http://rstuf-api send bootstrap dev/rstuf/bootstrap.json runmigrations: .state/docker-build-base docker compose run --rm web python -m warehouse db upgrade head diff --git a/docs/dev/development/getting-started.rst b/docs/dev/development/getting-started.rst index d428c9dc99ac..33d3ab872199 100644 --- a/docs/dev/development/getting-started.rst +++ b/docs/dev/development/getting-started.rst @@ -256,7 +256,7 @@ You should see the following line at the bottom of the output: .. code-block:: console - Bootstrap completed using `dev/rstuf/bootstrap.json`. 🔐 🎉 + Bootstrap completed. 🔐 🎉 This command sends a static *bootstrap payload* to the RSTUF API. The payload diff --git a/requirements/dev.txt b/requirements/dev.txt index 5e4f4656dc83..521c2c265438 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -4,3 +4,4 @@ pip-tools>=1.0 pyramid_debugtoolbar>=2.5 pip-api watchdog +repository-service-tuf diff --git a/tests/unit/cli/test_tuf.py b/tests/unit/cli/test_tuf.py deleted file mode 100644 index 5486455f7ceb..000000000000 --- a/tests/unit/cli/test_tuf.py +++ /dev/null @@ -1,38 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json - -from pretend import call, call_recorder - -from warehouse.cli import tuf - - -class TestTUF: - def test_bootstrap(self, cli, monkeypatch): - task_id = "123456" - server = "rstuf.api" - payload = ["foo"] - - post = call_recorder(lambda *a: task_id) - wait = call_recorder(lambda *a: None) - monkeypatch.setattr(tuf, "post_bootstrap", post) - monkeypatch.setattr(tuf, "wait_for_success", wait) - - result = cli.invoke( - tuf.bootstrap, args=["--api-server", server, "-"], input=json.dumps(payload) - ) - - assert result.exit_code == 0 - - assert post.calls == [call(server, payload)] - assert wait.calls == [call(server, task_id)] diff --git a/warehouse/cli/tuf.py b/warehouse/cli/tuf.py deleted file mode 100644 index 1f9763ba1209..000000000000 --- a/warehouse/cli/tuf.py +++ /dev/null @@ -1,33 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json - -import click - -from warehouse.cli import warehouse -from warehouse.tuf import post_bootstrap, wait_for_success - - -@warehouse.group() -def tuf(): - """Manage TUF.""" - - -@tuf.command() -@click.argument("payload", type=click.File("rb", lazy=True), required=True) -@click.option("--api-server", required=True) -def bootstrap(payload, api_server): - """Use payload file to bootstrap RSTUF server.""" - task_id = post_bootstrap(api_server, json.load(payload)) - wait_for_success(api_server, task_id) - print(f"Bootstrap completed using `{payload.name}`. 🔐 🎉") From e5c22e58eab88d195dfb191d33be3cb56c055495 Mon Sep 17 00:00:00 2001 From: Kairo de Araujo Date: Mon, 7 Oct 2024 10:04:11 +0200 Subject: [PATCH 14/17] chore: Implement TUF Interface and RSTUF Service - Implements the tuf/interfaces.py with ITUFService - Implements the tuf/services.py with RSTUFService - Refactor the tuf/tasks.py metadata_update to use the Service - RSTUF tasks are triggered by changes in the Project db objects - Add unit tests Signed-off-by: Kairo de Araujo --- tests/conftest.py | 9 ++ tests/unit/forklift/test_legacy.py | 3 - tests/unit/test_config.py | 1 + tests/unit/tuf/test_init.py | 69 +++++++++ tests/unit/tuf/test_services.py | 239 +++++++++++++++++++++++++++++ tests/unit/tuf/test_tasks.py | 29 +--- tests/unit/tuf/test_tuf.py | 121 --------------- warehouse/config.py | 3 + warehouse/forklift/legacy.py | 3 - warehouse/manage/views/__init__.py | 9 +- warehouse/tuf/__init__.py | 95 ++---------- warehouse/tuf/interfaces.py | 35 +++++ warehouse/tuf/services.py | 89 +++++++++++ warehouse/tuf/tasks.py | 10 +- 14 files changed, 473 insertions(+), 242 deletions(-) create mode 100644 tests/unit/tuf/test_init.py create mode 100644 tests/unit/tuf/test_services.py delete mode 100644 tests/unit/tuf/test_tuf.py create mode 100644 warehouse/tuf/interfaces.py create mode 100644 warehouse/tuf/services.py diff --git a/tests/conftest.py b/tests/conftest.py index 3e9fd971a949..d7e4dc81468c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -63,6 +63,8 @@ from warehouse.packaging.interfaces import IProjectService from warehouse.subscriptions import services as subscription_services from warehouse.subscriptions.interfaces import IBillingService, ISubscriptionService +from warehouse.tuf import services as tuf_services +from warehouse.tuf.interfaces import ITUFService from .common.db import Session from .common.db.accounts import EmailFactory, UserFactory @@ -179,6 +181,7 @@ def pyramid_services( integrity_service, macaroon_service, helpdesk_service, + tuf_service, ): services = _Services() @@ -201,6 +204,7 @@ def pyramid_services( services.register_service(integrity_service, IIntegrityService, None) services.register_service(macaroon_service, IMacaroonService, None, name="") services.register_service(helpdesk_service, IHelpDeskService, None) + services.register_service(tuf_service, ITUFService, None) return services @@ -611,6 +615,11 @@ def helpdesk_service(): return helpdesk_services.ConsoleHelpDeskService() +@pytest.fixture +def tuf_service(db_session): + return tuf_services.RSTUFService(db_session) + + class QueryRecorder: def __init__(self): self.queries = [] diff --git a/tests/unit/forklift/test_legacy.py b/tests/unit/forklift/test_legacy.py index c05d13ca0b17..6b9475f980ff 100644 --- a/tests/unit/forklift/test_legacy.py +++ b/tests/unit/forklift/test_legacy.py @@ -57,7 +57,6 @@ Role, ) from warehouse.packaging.tasks import sync_file_to_cache, update_bigquery_release_files -from warehouse.tuf.tasks import update_metadata from ...common.db.accounts import EmailFactory, UserFactory from ...common.db.classifiers import ClassifierFactory @@ -1164,12 +1163,10 @@ def storage_service_store(path, file_path, *, meta): ] assert db_request.task.calls == [ - pretend.call(update_metadata), pretend.call(update_bigquery_release_files), pretend.call(sync_file_to_cache), ] assert delay.calls == [ - pretend.call(release.project.id), pretend.call( { "metadata_version": "1.2", diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index 330d85ff6c43..8719f1bbb38c 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -456,6 +456,7 @@ def __init__(self): pretend.call(".helpdesk"), pretend.call(".http"), pretend.call(".utils.row_counter"), + pretend.call(".tuf"), ] + [pretend.call(x) for x in [configurator_settings.get("warehouse.theme")] if x] + [pretend.call(".sanity")] diff --git a/tests/unit/tuf/test_init.py b/tests/unit/tuf/test_init.py new file mode 100644 index 000000000000..071a9aca6c9b --- /dev/null +++ b/tests/unit/tuf/test_init.py @@ -0,0 +1,69 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pretend + +from tests.common.db.packaging import ProjectFactory, UserFactory +from warehouse import tuf +from warehouse.tuf.interfaces import ITUFService +from warehouse.tuf.services import rstuf_factory + + +def test_update_metadata_for_project(db_request, monkeypatch): + delay = pretend.call_recorder(lambda *a: None) + config = pretend.stub( + registry=pretend.stub(settings={"rstuf.api_url": "http://rstuf"}), + task=pretend.call_recorder(lambda *a: pretend.stub(delay=delay)), + ) + + project0 = ProjectFactory.create() + user0 = UserFactory.create() + + session = pretend.stub(info={}, new={project0, user0}, dirty=set()) + + tuf.update_metadata_for_project(config, session, pretend.stub()) + + # calls only for Projects + assert config.task.calls == [pretend.call(tuf.update_metadata)] + assert delay.calls == [pretend.call(project0.id)] + + +def test_update_metadata_for_project_rstuf_disabled(db_request, monkeypatch): + delay = pretend.call_recorder(lambda *a: None) + config = pretend.stub( + registry=pretend.stub(settings={}), + task=pretend.call_recorder(lambda *a: pretend.stub(delay=delay)), + ) + + project0 = ProjectFactory.create() + + session = pretend.stub(info={}, new={project0}, dirty=set()) + + tuf.update_metadata_for_project(config, session, pretend.stub()) + + assert config.task.calls == [] + assert delay.calls == [] + + +def test_includeme(): + config = pretend.stub( + register_service_factory=pretend.call_recorder( + lambda factory, iface, name=None: None + ), + maybe_dotted=pretend.call_recorder(lambda *a: "http://rstuf"), + ) + + tuf.includeme(config) + + assert config.register_service_factory.calls == [ + pretend.call(rstuf_factory, ITUFService), + ] diff --git a/tests/unit/tuf/test_services.py b/tests/unit/tuf/test_services.py new file mode 100644 index 000000000000..8bf6f329cd04 --- /dev/null +++ b/tests/unit/tuf/test_services.py @@ -0,0 +1,239 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pretend +import pytest + +from requests import Session +from zope.interface.verify import verifyClass + +from warehouse.tuf import services +from warehouse.tuf.interfaces import ITUFService +from warehouse.tuf.services import RSTUFService + + +class TestRSTUFService: + + def test_verify_service(self): + assert verifyClass(ITUFService, RSTUFService) + + def basic_init(self, db_request): + db_request.registry.settings = {"rstuf.api_url": "http://rstuf"} + + rstuf = RSTUFService(db_request) + + assert rstuf is not None + assert rstuf.api_url == "http://rstuf" + assert isinstance(rstuf.requests, Session) + + def test_create_service(self, db_request): + db_request.registry.settings = {"rstuf.api_url": "http://rstuf"} + + rstuf = RSTUFService.create_service(db_request) + + assert rstuf is not None + assert rstuf.api_url == "http://rstuf" + assert isinstance(rstuf.requests, Session) + + def test_get_task_state(self, monkeypatch, db_request): + db_request.registry.settings = {"rstuf.api_url": "http://rstuf"} + + response = pretend.stub( + raise_for_status=pretend.call_recorder(lambda: None), + json=pretend.call_recorder(lambda: {"data": {"state": "SUCCESS"}}), + ) + test_session = pretend.stub( + get=pretend.call_recorder(lambda *a, **kw: response) + ) + fake_session = pretend.call_recorder(lambda: test_session) + monkeypatch.setattr(services, "Session", fake_session) + + rstuf = RSTUFService.create_service(db_request) + + state = rstuf.get_task_state("123456") + + assert state == "SUCCESS" + + assert test_session.get.calls == [ + pretend.call("http://rstuf/api/v1/task?task_id=123456"), + ] + assert response.raise_for_status.calls == [pretend.call()] + assert response.json.calls == [pretend.call()] + + def test_post_artifacts(self, monkeypatch, db_request): + db_request.registry.settings = {"rstuf.api_url": "http://rstuf"} + + response = pretend.stub( + raise_for_status=pretend.call_recorder(lambda: None), + json=pretend.call_recorder(lambda: {"data": {"task_id": "123456"}}), + ) + test_session = pretend.stub( + post=pretend.call_recorder(lambda *a, **kw: response) + ) + fake_session = pretend.call_recorder(lambda: test_session) + monkeypatch.setattr(services, "Session", fake_session) + + rstuf = RSTUFService.create_service(db_request) + + task_id = rstuf.post_artifacts({"targets": [{"path": "name"}]}) + + assert task_id == "123456" + + assert test_session.post.calls == [ + pretend.call( + "http://rstuf/api/v1/artifacts", json={"targets": [{"path": "name"}]} + ), + ] + assert response.raise_for_status.calls == [pretend.call()] + assert response.json.calls == [pretend.call()] + + @pytest.mark.parametrize( + ("states", "exception", "message"), + [ + ( + [ + {"data": {"state": "PENDING"}}, + {"data": {"state": "STARTED"}}, + {"data": {"state": "RECEIVED"}}, + {"data": {"state": "STARTED"}}, + {"data": {"state": "SUCCESS"}}, + ], + None, + "", + ), + ( + [ + {"data": {"state": "PENDING"}}, + {"data": {"state": "STARTED"}}, + {"data": {"state": "RECEIVED"}}, + {"data": {"state": "STARTED"}}, + {"data": {"state": "FAILURE"}}, + ], + services.RSTUFError, + "RSTUF job failed, please check payload and retry", + ), + ( + [ + {"data": {"state": "PENDING"}}, + {"data": {"state": "STARTED"}}, + {"data": {"state": "RECEIVED"}}, + {"data": {"state": "STARTED"}}, + {"data": {"state": "ERRORED"}}, + ], + services.RSTUFError, + "RSTUF internal problem, please check RSTUF health", + ), + ( + [ + {"data": {"state": "PENDING"}}, + {"data": {"state": "STARTED"}}, + {"data": {"state": "RECEIVED"}}, + {"data": {"state": "STARTED"}}, + {"data": {"state": "REVOKED"}}, + ], + services.RSTUFError, + "RSTUF internal problem, please check RSTUF health", + ), + ( + [ + {"data": {"state": "PENDING"}}, + {"data": {"state": "STARTED"}}, + {"data": {"state": "RECEIVED"}}, + {"data": {"state": "STARTED"}}, + {"data": {"state": "REJECTED"}}, + ], + services.RSTUFError, + "RSTUF internal problem, please check RSTUF health", + ), + ( + [ + {"data": {"state": "PENDING"}}, + {"data": {"state": "STARTED"}}, + {"data": {"state": "RECEIVED"}}, + {"data": {"state": "STARTED"}}, + {"data": {"state": "INVALID_STATE"}}, + ], + services.RSTUFError, + "RSTUF job returned unexpected state: INVALID_STATE", + ), + ( + [ + {"data": {"state": "PENDING"}}, + {"data": {"state": "PENDING"}}, + {"data": {"state": "PENDING"}}, + {"data": {"state": "PENDING"}}, + {"data": {"state": "PENDING"}}, + ], + services.RSTUFError, + "RSTUF job failed, please check payload and retry", + ), + ], + ) + def test_wait_for_pending_than_success( + self, monkeypatch, db_request, states, exception, message + ): + db_request.registry.settings = {"rstuf.api_url": "http://rstuf"} + + # generate iter of responses + responses = iter(states) + response = pretend.stub( + raise_for_status=pretend.call_recorder(lambda: None), + json=pretend.call_recorder(lambda: next(responses)), + ) + test_session = pretend.stub( + get=pretend.call_recorder(lambda *a, **kw: response) + ) + fake_session = pretend.call_recorder(lambda: test_session) + monkeypatch.setattr(services, "Session", fake_session) + + rstuf = RSTUFService.create_service(db_request) + rstuf.delay = 0.1 # speed up the test + if message == "RSTUF job failed, please check payload and retry": + rstuf.retries = 5 # simulate failure by limiting retries + + result = None + if exception is not None: + with pytest.raises(exception) as e: + rstuf.wait_for_success("123456") + + assert message in str(e) + else: + result = rstuf.wait_for_success("123456") + + assert result is None + + assert test_session.get.calls == [ + pretend.call("http://rstuf/api/v1/task?task_id=123456"), + pretend.call("http://rstuf/api/v1/task?task_id=123456"), + pretend.call("http://rstuf/api/v1/task?task_id=123456"), + pretend.call("http://rstuf/api/v1/task?task_id=123456"), + pretend.call("http://rstuf/api/v1/task?task_id=123456"), + ] + assert response.raise_for_status.calls == [ + pretend.call(), + pretend.call(), + pretend.call(), + pretend.call(), + pretend.call(), + ] + assert response.json.calls == [ + pretend.call(), + pretend.call(), + pretend.call(), + pretend.call(), + pretend.call(), + ] + + def test_rstuf_factory(self, db_request): + db_request.registry.settings = {"rstuf.api_url": "http://rstuf"} + rstuf = services.rstuf_factory(pretend.stub(), db_request) + + assert isinstance(rstuf, RSTUFService) diff --git a/tests/unit/tuf/test_tasks.py b/tests/unit/tuf/test_tasks.py index 40bddef4a8db..61fd5ccaa162 100644 --- a/tests/unit/tuf/test_tasks.py +++ b/tests/unit/tuf/test_tasks.py @@ -35,18 +35,16 @@ def test_update_metadata(self, db_request, monkeypatch): render = call_recorder(lambda *a, **kw: (index_digest, None, index_size)) tuf.tasks.render_simple_detail = render - post = call_recorder(lambda *a: self.task_id) - monkeypatch.setattr(tuf.tasks, "post_artifacts", post) - - wait = call_recorder(lambda *a: None) - monkeypatch.setattr(tuf.tasks, "wait_for_success", wait) + rstuf = tuf.services.RSTUFService.create_service(db_request) + rstuf.post_artifacts = call_recorder(lambda *a: self.task_id) + rstuf.wait_for_success = call_recorder(lambda *a: None) + db_request.find_service = call_recorder(lambda *a, **kw: rstuf) tuf.tasks.update_metadata(db_request, project_id) assert one.calls == [call()] assert render.calls == [call(project, db_request, store=True)] - assert post.calls == [ + assert rstuf.post_artifacts.calls == [ call( - rstuf_url, { "targets": [ { @@ -60,19 +58,4 @@ def test_update_metadata(self, db_request, monkeypatch): }, ) ] - assert wait.calls == [call(rstuf_url, self.task_id)] - - def test_update_metadata_no_rstuf_api_url(self, db_request): - project_id = "id" - project_name = "name" - - project = stub(normalized_name=project_name) - - one = call_recorder(lambda: project) - db_request.db.query = lambda a: stub(filter=lambda a: stub(one=one)) - - # Test early return, if no RSTUF API URL configured - db_request.registry.settings = {"rstuf.api_url": None} - tuf.tasks.update_metadata(db_request, project_id) - - assert not one.calls + assert rstuf.wait_for_success.calls == [call(self.task_id)] diff --git a/tests/unit/tuf/test_tuf.py b/tests/unit/tuf/test_tuf.py deleted file mode 100644 index adbcc5862ff2..000000000000 --- a/tests/unit/tuf/test_tuf.py +++ /dev/null @@ -1,121 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pytest - -from pretend import call, call_recorder, stub - -from warehouse import tuf - - -class TestTUF: - server = "rstuf.api" - task_id = "123456" - - def test_get_task_state(self, monkeypatch): - state = "SUCCESS" - - resp_json = {"data": {"state": state}} - resp = stub( - raise_for_status=(lambda *a: None), json=(lambda *a, **kw: resp_json) - ) - get = call_recorder(lambda *a: resp) - monkeypatch.setattr(tuf.requests, "get", get) - - result = tuf.get_task_state(self.server, self.task_id) - - assert result == state - assert get.calls == [call(f"{self.server}/api/v1/task?task_id={self.task_id}")] - - def test_post_bootstrap(self, monkeypatch): - payload = ["foo"] - - resp_json = {"data": {"task_id": self.task_id}} - resp = stub( - raise_for_status=(lambda *a: None), json=(lambda *a, **kw: resp_json) - ) - post = call_recorder(lambda *a, **kw: resp) - monkeypatch.setattr(tuf.requests, "post", post) - - # Test success - result = tuf.post_bootstrap(self.server, payload) - - assert result == self.task_id - assert post.calls == [call(f"{self.server}/api/v1/bootstrap", json=payload)] - - # Test fail with incomplete response json - del resp_json["data"] - with pytest.raises(tuf.RSTUFError): - tuf.post_bootstrap(self.server, payload) - - def test_post_artifacts(self, monkeypatch): - payload = { - "targets": [ - { - "path": "foo", - "info": { - "length": 42, - "hashes": {"blake2b-256": "deadbeef"}, - }, - } - ] - } - - resp_json = {"data": {"task_id": self.task_id}} - resp = stub( - raise_for_status=(lambda *a: None), json=(lambda *a, **kw: resp_json) - ) - post = call_recorder(lambda *a, **kw: resp) - monkeypatch.setattr(tuf.requests, "post", post) - - # Test success - result = tuf.post_artifacts(self.server, payload) - - assert result == self.task_id - assert post.calls == [call(f"{self.server}/api/v1/artifacts", json=payload)] - - # Test fail with incomplete response json (i.e. no bootstrap error) - del resp_json["data"] - with pytest.raises(tuf.RSTUFNoBootstrapError): - tuf.post_artifacts(self.server, payload) - - def test_wait_for_success(self, monkeypatch): - get_task_state = call_recorder(lambda *a: "SUCCESS") - monkeypatch.setattr(tuf, "get_task_state", get_task_state) - tuf.wait_for_success(self.server, self.task_id) - - assert get_task_state.calls == [call(self.server, self.task_id)] - - @pytest.mark.parametrize( - ("state", "iterations"), - [ - ("PENDING", 20), - ("RUNNING", 20), - ("RECEIVED", 20), - ("STARTED", 20), - ("FAILURE", 1), - ("ERRORED", 1), - ("REVOKED", 1), - ("REJECTED", 1), - ("bogus", 1), - ], - ) - def test_wait_for_success_error(self, state, iterations, monkeypatch): - monkeypatch.setattr(tuf.time, "sleep", lambda *a: None) - - get_task_state = call_recorder(lambda *a: state) - monkeypatch.setattr(tuf, "get_task_state", get_task_state) - - with pytest.raises(tuf.RSTUFError): - tuf.wait_for_success(self.server, self.task_id) - - assert get_task_state.calls == [call(self.server, self.task_id)] * iterations diff --git a/warehouse/config.py b/warehouse/config.py index 84fcfcf372f0..31b982fda364 100644 --- a/warehouse/config.py +++ b/warehouse/config.py @@ -942,6 +942,9 @@ def configure(settings=None): ignore=["warehouse.migrations.env", "warehouse.celery", "warehouse.wsgi"], ) + # RSTUF configuration to provide TUF metadata + config.include(".tuf") + # Sanity check our request and responses. # Note: It is very important that this go last. We need everything else # that might have added a tween to be registered prior to this. diff --git a/warehouse/forklift/legacy.py b/warehouse/forklift/legacy.py index 4fc964c36f9f..979826c5764a 100644 --- a/warehouse/forklift/legacy.py +++ b/warehouse/forklift/legacy.py @@ -44,7 +44,6 @@ from sqlalchemy import and_, exists, func, orm from sqlalchemy.exc import MultipleResultsFound, NoResultFound -from warehouse import tuf from warehouse.admin.flags import AdminFlagValue from warehouse.attestations.errors import AttestationUploadError from warehouse.attestations.interfaces import IIntegrityService @@ -1307,8 +1306,6 @@ def file_upload(request): request.db.flush() # flush db now so server default values are populated for celery - request.task(tuf.tasks.update_metadata).delay(release.project.id) - # Push updates to BigQuery dist_metadata = { "metadata_version": meta.metadata_version, diff --git a/warehouse/manage/views/__init__.py b/warehouse/manage/views/__init__.py index 51cb86f52de5..a5b176807cb8 100644 --- a/warehouse/manage/views/__init__.py +++ b/warehouse/manage/views/__init__.py @@ -34,7 +34,6 @@ import warehouse.utils.otp as otp -from warehouse import tuf from warehouse.accounts.forms import RecoveryCodeAuthenticationForm from warehouse.accounts.interfaces import ( IPasswordBreachedService, @@ -2052,8 +2051,6 @@ def delete_project(project, request): remove_project(project, request) - request.task(tuf.tasks.update_metadata).delay(project.id) - return HTTPSeeOther(request.route_path("manage.projects")) @@ -2224,7 +2221,6 @@ def yank_project_release(self): recipient_role=contributor_role, ) - self.request.task(tuf.tasks.update_metadata).delay(self.release.project.id) return HTTPSeeOther( self.request.route_path( "manage.project.releases", project_name=self.release.project.name @@ -2309,7 +2305,7 @@ def unyank_project_release(self): submitter_role=submitter_role, recipient_role=contributor_role, ) - self.request.task(tuf.tasks.update_metadata).delay(self.release.project.id) + return HTTPSeeOther( self.request.route_path( "manage.project.releases", project_name=self.release.project.name @@ -2409,7 +2405,6 @@ def delete_project_release(self): recipient_role=contributor_role, ) - self.request.task(tuf.tasks.update_metadata).delay(self.release.project.id) return HTTPSeeOther( self.request.route_path( "manage.project.releases", project_name=self.release.project.name @@ -2508,7 +2503,7 @@ def _error(message): self.request.session.flash( f"Deleted file {release_file.filename!r}", queue="success" ) - self.request.task(tuf.tasks.update_metadata).delay(self.release.project.id) + return HTTPSeeOther( self.request.route_path( "manage.project.release", diff --git a/warehouse/tuf/__init__.py b/warehouse/tuf/__init__.py index 5991c918aa4c..843ef03e1e83 100644 --- a/warehouse/tuf/__init__.py +++ b/warehouse/tuf/__init__.py @@ -10,87 +10,24 @@ # See the License for the specific language governing permissions and # limitations under the License. -""" -RSTUF API client library -""" +from warehouse import db +from warehouse.packaging.models import Project +from warehouse.tuf.interfaces import ITUFService +from warehouse.tuf.services import rstuf_factory +from warehouse.tuf.tasks import update_metadata -import time -from typing import Any +@db.listens_for(db.Session, "after_flush") +def update_metadata_for_project(config, session, flush_context): + # We will start a task to update the metadata for each project that has been + # deleted, yanked, unyanked, release deleted or release file deleted. + if config.registry.settings.get("rstuf.api_url") is None: + return -import requests + for obj in session.new | session.dirty: + if isinstance(obj, Project): + config.task(update_metadata).delay(obj.id) -class RSTUFError(Exception): - pass - - -class RSTUFNoBootstrapError(Exception): - pass - - -def get_task_state(server: str, task_id: str) -> str: - resp = requests.get(f"{server}/api/v1/task?task_id={task_id}") - resp.raise_for_status() - return resp.json()["data"]["state"] - - -def post_bootstrap(server: str, payload: Any) -> str: - resp = requests.post(f"{server}/api/v1/bootstrap", json=payload) - resp.raise_for_status() - - # TODO: Ask upstream to not return 200 on error - resp_json = resp.json() - resp_data = resp_json.get("data") - if not resp_data: - raise RSTUFError(f"Error in RSTUF job: {resp_json}") - - return resp_data["task_id"] - - -def post_artifacts(server: str, payload: Any) -> str: - """Call RSTUF artifacts API to update the relevant TUF metadata. - - Returns task id of the async update task in RSTUF. - """ - resp = requests.post(f"{server}/api/v1/artifacts", json=payload) - resp.raise_for_status() - - # 200 but no "data" means that RSTUF isn't bootstrapped yet - # TODO: Ask upstream to not return 200 on error - resp_json = resp.json() - resp_data = resp_json.get("data") - if not resp_data: - raise RSTUFNoBootstrapError(resp_json) - - return resp_data["task_id"] - - -def wait_for_success(server: str, task_id: str): - """Poll RSTUF task state API until success or error.""" - - retries = 20 - delay = 1 - - for _ in range(retries): - state = get_task_state(server, task_id) - - match state: - case "SUCCESS": - break - - case "PENDING" | "RUNNING" | "RECEIVED" | "STARTED": - time.sleep(delay) - continue - - case "FAILURE": - raise RSTUFError("RSTUF job failed, please check payload and retry") - - case "ERRORED" | "REVOKED" | "REJECTED": - raise RSTUFError("RSTUF internal problem, please check RSTUF health") - - case _: - raise RSTUFError(f"RSTUF job returned unexpected state: {state}") - - else: - raise RSTUFError("RSTUF job failed, please check payload and retry") +def includeme(config): + config.register_service_factory(rstuf_factory, ITUFService) diff --git a/warehouse/tuf/interfaces.py b/warehouse/tuf/interfaces.py new file mode 100644 index 000000000000..308837c280af --- /dev/null +++ b/warehouse/tuf/interfaces.py @@ -0,0 +1,35 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from zope.interface import Interface + + +class ITUFService(Interface): + def create_service(db_session): + """ + Create appropriate RSTUF service based on environment + """ + + def get_task_state(task_id): + """ + Fetch the RSTUF task state to based on the task id + """ + + def post_artifacts(payload): + """ + Send the Artifacts payload to RSTUF API + """ + + def wait_for_success(task_id): + """ + Wait for the RSTUF task to complete successfully + """ diff --git a/warehouse/tuf/services.py b/warehouse/tuf/services.py new file mode 100644 index 000000000000..ae8dda0f092e --- /dev/null +++ b/warehouse/tuf/services.py @@ -0,0 +1,89 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import time + +from requests import Session +from zope.interface import implementer + +from warehouse.tuf.interfaces import ITUFService + + +class RSTUFError(Exception): + pass + + +class RSTUFNoBootstrapError(Exception): + pass + + +@implementer(ITUFService) +class RSTUFService: + def __init__(self, api_url, retries=20, delay=1): + self.requests = Session() + self.api_url = api_url + # TODO make retries and delay configurable + self.retries = retries + self.delay = delay + + @classmethod + def create_service(cls, db_session): + return cls(db_session.registry.settings["rstuf.api_url"]) + + def get_task_state(self, task_id): + """Get the RSTUF task state based on the task id.""" + response = self.requests.get(f"{self.api_url}/api/v1/task?task_id={task_id}") + response.raise_for_status() + return response.json()["data"]["state"] + + def post_artifacts(self, payload): + """Call RSTUF artifacts API to update the relevant TUF metadata. + + Returns task id of the async update task in RSTUF. + """ + response = self.requests.post(f"{self.api_url}/api/v1/artifacts", json=payload) + response.raise_for_status() + + response_json = response.json() + response_data = response_json.get("data") + + return response_data["task_id"] + + def wait_for_success(self, task_id): + """Poll RSTUF task state API until success or error.""" + for _ in range(self.retries): + state = self.get_task_state(task_id) + + match state: + case "SUCCESS": + break + + case "PENDING" | "RUNNING" | "RECEIVED" | "STARTED": + time.sleep(self.delay) + continue + + case "FAILURE": + raise RSTUFError("RSTUF job failed, please check payload and retry") + + case "ERRORED" | "REVOKED" | "REJECTED": + raise RSTUFError( + "RSTUF internal problem, please check RSTUF health" + ) + + case _: + raise RSTUFError(f"RSTUF job returned unexpected state: {state}") + + else: + raise RSTUFError("RSTUF job failed, please check payload and retry") + + +def rstuf_factory(context, db_session): + return RSTUFService(db_session.registry.settings["rstuf.api_url"]) diff --git a/warehouse/tuf/tasks.py b/warehouse/tuf/tasks.py index 5e12b304b6fc..f08003880285 100644 --- a/warehouse/tuf/tasks.py +++ b/warehouse/tuf/tasks.py @@ -16,7 +16,7 @@ from warehouse import tasks from warehouse.packaging.models import Project from warehouse.packaging.utils import render_simple_detail -from warehouse.tuf import post_artifacts, wait_for_success +from warehouse.tuf.interfaces import ITUFService @tasks.task(ignore_result=True, acks_late=True) @@ -27,9 +27,7 @@ def update_metadata(request: Request, project_id: UUID): distributions files and simple detail files. In reality, simple detail files are enough, as they already include all relevant distribution file infos. """ - server = request.registry.settings["rstuf.api_url"] - if not server: - return + rstuf_service = request.find_service(ITUFService, context=None) project = request.db.query(Project).filter(Project.id == project_id).one() @@ -50,5 +48,5 @@ def update_metadata(request: Request, project_id: UUID): } # TODO: Handle errors: pass, retry or notify - task_id = post_artifacts(server, payload) - wait_for_success(server, task_id) + task_id = rstuf_service.post_artifacts(payload) + rstuf_service.wait_for_success(task_id) From f982336d4e01660a3356f9daa21d3f084c01c2d7 Mon Sep 17 00:00:00 2001 From: Kairo de Araujo Date: Mon, 7 Oct 2024 11:43:54 +0200 Subject: [PATCH 15/17] refactor: utils.render_simple_detail - remove the simple_detail_path as it is not used by any other caller - calculate size without writting the file, only write when store is true Signed-off-by: Kairo de Araujo --- tests/unit/packaging/test_utils.py | 14 +++----------- tests/unit/tuf/test_tasks.py | 4 ++-- warehouse/packaging/utils.py | 14 ++++++++------ warehouse/tuf/tasks.py | 2 +- 4 files changed, 14 insertions(+), 20 deletions(-) diff --git a/tests/unit/packaging/test_utils.py b/tests/unit/packaging/test_utils.py index 1f80e8213ca2..6318418197d8 100644 --- a/tests/unit/packaging/test_utils.py +++ b/tests/unit/packaging/test_utils.py @@ -58,17 +58,13 @@ def test_render_simple_detail(db_request, monkeypatch, jinja): context = _valid_simple_detail_context(context) expected_content = template.render(**context, request=db_request).encode("utf-8") - content_hash, path, size = render_simple_detail(project, db_request) + content_hash, size = render_simple_detail(project, db_request) assert fakeblake2b.calls == [pretend.call(digest_size=32)] assert fake_hasher.update.calls == [pretend.call(expected_content)] assert fake_hasher.hexdigest.calls == [pretend.call()] assert content_hash == "deadbeefdeadbeefdeadbeefdeadbeef" - assert path == ( - f"{project.normalized_name}/deadbeefdeadbeefdeadbeefdeadbeef" - + f".{project.normalized_name}.html" - ) assert size == len(expected_content) @@ -93,7 +89,7 @@ def test_render_simple_detail_with_store(db_request, monkeypatch, jinja): fakeblake2b = pretend.call_recorder(lambda *a, **kw: fake_hasher) monkeypatch.setattr(hashlib, "blake2b", fakeblake2b) - expected_size = 42 + expected_size = 225 fake_named_temporary_file = pretend.stub( name="/tmp/wutang", write=pretend.call_recorder(lambda data: expected_size), @@ -117,7 +113,7 @@ def __exit__(self, type, value, traceback): context = _valid_simple_detail_context(context) expected_content = template.render(**context, request=db_request).encode("utf-8") - content_hash, path, size = render_simple_detail(project, db_request, store=True) + content_hash, size = render_simple_detail(project, db_request, store=True) assert fake_named_temporary_file.write.calls == [pretend.call(expected_content)] assert fake_named_temporary_file.flush.calls == [pretend.call()] @@ -151,8 +147,4 @@ def __exit__(self, type, value, traceback): ] assert content_hash == "deadbeefdeadbeefdeadbeefdeadbeef" - assert path == ( - f"{project.normalized_name}/deadbeefdeadbeefdeadbeefdeadbeef" - + f".{project.normalized_name}.html" - ) assert size == expected_size diff --git a/tests/unit/tuf/test_tasks.py b/tests/unit/tuf/test_tasks.py index 61fd5ccaa162..0a3d8973cf62 100644 --- a/tests/unit/tuf/test_tasks.py +++ b/tests/unit/tuf/test_tasks.py @@ -28,11 +28,11 @@ def test_update_metadata(self, db_request, monkeypatch): rstuf_url = "url" index_digest = "digest" - index_size = 42 + index_size = 255 db_request.registry.settings = {"rstuf.api_url": rstuf_url} - render = call_recorder(lambda *a, **kw: (index_digest, None, index_size)) + render = call_recorder(lambda *a, **kw: (index_digest, index_size)) tuf.tasks.render_simple_detail = render rstuf = tuf.services.RSTUFService.create_service(db_request) diff --git a/warehouse/packaging/utils.py b/warehouse/packaging/utils.py index b05a2dad519c..f52a19aa696d 100644 --- a/warehouse/packaging/utils.py +++ b/warehouse/packaging/utils.py @@ -123,12 +123,14 @@ def render_simple_detail(project, request, store=False): f"{project.normalized_name}/{content_hash}.{project.normalized_name}.html" ) - with tempfile.NamedTemporaryFile() as f: - simple_detail_size = f.write(content.encode("utf-8")) - f.flush() + simple_detail_size = len(content.encode("utf-8")) + + if store: + storage = request.find_service(ISimpleStorage) + with tempfile.NamedTemporaryFile() as f: + f.write(content.encode("utf-8")) + f.flush() - if store: - storage = request.find_service(ISimpleStorage) storage.store( simple_detail_path, f.name, @@ -148,7 +150,7 @@ def render_simple_detail(project, request, store=False): }, ) - return (content_hash, simple_detail_path, simple_detail_size) + return (content_hash, simple_detail_size) def _valid_simple_detail_context(context: dict) -> dict: diff --git a/warehouse/tuf/tasks.py b/warehouse/tuf/tasks.py index f08003880285..5562b575d4cc 100644 --- a/warehouse/tuf/tasks.py +++ b/warehouse/tuf/tasks.py @@ -34,7 +34,7 @@ def update_metadata(request: Request, project_id: UUID): # NOTE: We ignore the returned simple detail path with the content hash as # infix. In TUF metadata the project name and hash are listed separately, so # that there is only one entry per target file, even if the content changes. - digest, _, size = render_simple_detail(project, request, store=True) + digest, size = render_simple_detail(project, request, store=True) payload = { "targets": [ { From 43edb0ad53b51c2dcf3f2590ae6eaf89f46aaa9a Mon Sep 17 00:00:00 2001 From: Kairo de Araujo Date: Wed, 9 Oct 2024 10:53:35 +0200 Subject: [PATCH 16/17] chore: bump RSTUF API and Worker to 1.0.0b1 - Update RSTUF services to 1.0.0b1 - Update payload to use `artifacts` instead `targets` - Add RSTUF `PRE_RUN` state Signed-off-by: Kairo de Araujo --- docker-compose.yml | 6 +++--- tests/unit/tuf/test_services.py | 10 +++++----- tests/unit/tuf/test_tasks.py | 2 +- warehouse/tuf/services.py | 2 +- warehouse/tuf/tasks.py | 2 +- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 66954eba67ac..94a5079ad315 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -173,7 +173,7 @@ services: SIMPLE_BACKEND: "warehouse.packaging.services.LocalSimpleStorage path=/var/opt/warehouse/simple/ url=http://files:9001/simple/{path}" rstuf-api: - image: ghcr.io/repository-service-tuf/repository-service-tuf-api:v0.12.0b1 + image: ghcr.io/repository-service-tuf/repository-service-tuf-api:v1.0.0b1 ports: - 8001:80 stop_signal: SIGKILL @@ -187,7 +187,7 @@ services: condition: service_started rstuf-worker: - image: ghcr.io/repository-service-tuf/repository-service-tuf-worker:v0.14.0b1 + image: ghcr.io/repository-service-tuf/repository-service-tuf-worker:v1.0.0b1 volumes: - rstuf-metadata:/var/opt/repository-service-tuf/storage - ./dev/rstuf/keys/online:/keyvault @@ -200,7 +200,7 @@ services: - RSTUF_REDIS_SERVER=redis://redis - RSTUF_REDIS_SERVER_DB_RESULT=1 - RSTUF_REDIS_SERVER_DB_REPO_SETTINGS=2 - - RSTUF_SQL_SERVER=postgresql://postgres@db:5432/rstuf + - RSTUF_DB_SERVER=postgresql://postgres@db:5432/rstuf depends_on: db: condition: service_healthy diff --git a/tests/unit/tuf/test_services.py b/tests/unit/tuf/test_services.py index 8bf6f329cd04..755db2cfa2ac 100644 --- a/tests/unit/tuf/test_services.py +++ b/tests/unit/tuf/test_services.py @@ -101,7 +101,7 @@ def test_post_artifacts(self, monkeypatch, db_request): ( [ {"data": {"state": "PENDING"}}, - {"data": {"state": "STARTED"}}, + {"data": {"state": "PRE_RUN"}}, {"data": {"state": "RECEIVED"}}, {"data": {"state": "STARTED"}}, {"data": {"state": "SUCCESS"}}, @@ -123,7 +123,7 @@ def test_post_artifacts(self, monkeypatch, db_request): ( [ {"data": {"state": "PENDING"}}, - {"data": {"state": "STARTED"}}, + {"data": {"state": "PRE_RUN"}}, {"data": {"state": "RECEIVED"}}, {"data": {"state": "STARTED"}}, {"data": {"state": "ERRORED"}}, @@ -134,7 +134,7 @@ def test_post_artifacts(self, monkeypatch, db_request): ( [ {"data": {"state": "PENDING"}}, - {"data": {"state": "STARTED"}}, + {"data": {"state": "PRE_RUN"}}, {"data": {"state": "RECEIVED"}}, {"data": {"state": "STARTED"}}, {"data": {"state": "REVOKED"}}, @@ -145,7 +145,7 @@ def test_post_artifacts(self, monkeypatch, db_request): ( [ {"data": {"state": "PENDING"}}, - {"data": {"state": "STARTED"}}, + {"data": {"state": "PRE_RUN"}}, {"data": {"state": "RECEIVED"}}, {"data": {"state": "STARTED"}}, {"data": {"state": "REJECTED"}}, @@ -156,7 +156,7 @@ def test_post_artifacts(self, monkeypatch, db_request): ( [ {"data": {"state": "PENDING"}}, - {"data": {"state": "STARTED"}}, + {"data": {"state": "PRE_RUN"}}, {"data": {"state": "RECEIVED"}}, {"data": {"state": "STARTED"}}, {"data": {"state": "INVALID_STATE"}}, diff --git a/tests/unit/tuf/test_tasks.py b/tests/unit/tuf/test_tasks.py index 0a3d8973cf62..4936ec59b03a 100644 --- a/tests/unit/tuf/test_tasks.py +++ b/tests/unit/tuf/test_tasks.py @@ -46,7 +46,7 @@ def test_update_metadata(self, db_request, monkeypatch): assert rstuf.post_artifacts.calls == [ call( { - "targets": [ + "artifacts": [ { "path": project_name, "info": { diff --git a/warehouse/tuf/services.py b/warehouse/tuf/services.py index ae8dda0f092e..a1d28ee29bb9 100644 --- a/warehouse/tuf/services.py +++ b/warehouse/tuf/services.py @@ -66,7 +66,7 @@ def wait_for_success(self, task_id): case "SUCCESS": break - case "PENDING" | "RUNNING" | "RECEIVED" | "STARTED": + case "PENDING" | "PRE_RUN" | "RUNNING" | "RECEIVED" | "STARTED": time.sleep(self.delay) continue diff --git a/warehouse/tuf/tasks.py b/warehouse/tuf/tasks.py index 5562b575d4cc..dad09d57dc85 100644 --- a/warehouse/tuf/tasks.py +++ b/warehouse/tuf/tasks.py @@ -36,7 +36,7 @@ def update_metadata(request: Request, project_id: UUID): # that there is only one entry per target file, even if the content changes. digest, size = render_simple_detail(project, request, store=True) payload = { - "targets": [ + "artifacts": [ { "path": project.normalized_name, "info": { From a0816e2e0275f83dc492b2a5d220d62126196ec5 Mon Sep 17 00:00:00 2001 From: Kairo de Araujo Date: Wed, 9 Oct 2024 12:37:46 +0200 Subject: [PATCH 17/17] translations: make translations Signed-off-by: Kairo de Araujo --- warehouse/locale/messages.pot | 98 +++++++++++++++++------------------ 1 file changed, 49 insertions(+), 49 deletions(-) diff --git a/warehouse/locale/messages.pot b/warehouse/locale/messages.pot index 026e8878eced..76f6785fa667 100644 --- a/warehouse/locale/messages.pot +++ b/warehouse/locale/messages.pot @@ -152,7 +152,7 @@ msgstr "" msgid "Successful WebAuthn assertion" msgstr "" -#: warehouse/accounts/views.py:569 warehouse/manage/views/__init__.py:874 +#: warehouse/accounts/views.py:569 warehouse/manage/views/__init__.py:873 msgid "Recovery code accepted. The supplied code cannot be used again." msgstr "" @@ -286,7 +286,7 @@ msgid "You are now ${role} of the '${project_name}' project." msgstr "" #: warehouse/accounts/views.py:1548 warehouse/accounts/views.py:1791 -#: warehouse/manage/views/__init__.py:1418 +#: warehouse/manage/views/__init__.py:1417 msgid "" "Trusted publishing is temporarily disabled. See https://pypi.org/help" "#admin-intervention for details." @@ -306,19 +306,19 @@ msgstr "" msgid "You can't register more than 3 pending trusted publishers at once." msgstr "" -#: warehouse/accounts/views.py:1614 warehouse/manage/views/__init__.py:1473 -#: warehouse/manage/views/__init__.py:1586 -#: warehouse/manage/views/__init__.py:1698 -#: warehouse/manage/views/__init__.py:1808 +#: warehouse/accounts/views.py:1614 warehouse/manage/views/__init__.py:1472 +#: warehouse/manage/views/__init__.py:1585 +#: warehouse/manage/views/__init__.py:1697 +#: warehouse/manage/views/__init__.py:1807 msgid "" "There have been too many attempted trusted publisher registrations. Try " "again later." msgstr "" -#: warehouse/accounts/views.py:1625 warehouse/manage/views/__init__.py:1487 -#: warehouse/manage/views/__init__.py:1600 -#: warehouse/manage/views/__init__.py:1712 -#: warehouse/manage/views/__init__.py:1822 +#: warehouse/accounts/views.py:1625 warehouse/manage/views/__init__.py:1486 +#: warehouse/manage/views/__init__.py:1599 +#: warehouse/manage/views/__init__.py:1711 +#: warehouse/manage/views/__init__.py:1821 msgid "The trusted publisher could not be registered" msgstr "" @@ -446,157 +446,157 @@ msgid "" "less." msgstr "" -#: warehouse/manage/views/__init__.py:286 +#: warehouse/manage/views/__init__.py:285 msgid "Account details updated" msgstr "" -#: warehouse/manage/views/__init__.py:316 +#: warehouse/manage/views/__init__.py:315 msgid "Email ${email_address} added - check your email for a verification link" msgstr "" -#: warehouse/manage/views/__init__.py:822 +#: warehouse/manage/views/__init__.py:821 msgid "Recovery codes already generated" msgstr "" -#: warehouse/manage/views/__init__.py:823 +#: warehouse/manage/views/__init__.py:822 msgid "Generating new recovery codes will invalidate your existing codes." msgstr "" -#: warehouse/manage/views/__init__.py:932 +#: warehouse/manage/views/__init__.py:931 msgid "Verify your email to create an API token." msgstr "" -#: warehouse/manage/views/__init__.py:1032 +#: warehouse/manage/views/__init__.py:1031 msgid "API Token does not exist." msgstr "" -#: warehouse/manage/views/__init__.py:1064 +#: warehouse/manage/views/__init__.py:1063 msgid "Invalid credentials. Try again" msgstr "" -#: warehouse/manage/views/__init__.py:1183 +#: warehouse/manage/views/__init__.py:1182 msgid "Invalid alternate repository location details" msgstr "" -#: warehouse/manage/views/__init__.py:1220 +#: warehouse/manage/views/__init__.py:1219 msgid "Added alternate repository '${name}'" msgstr "" -#: warehouse/manage/views/__init__.py:1254 -#: warehouse/manage/views/__init__.py:2157 -#: warehouse/manage/views/__init__.py:2243 -#: warehouse/manage/views/__init__.py:2344 -#: warehouse/manage/views/__init__.py:2445 +#: warehouse/manage/views/__init__.py:1253 +#: warehouse/manage/views/__init__.py:2154 +#: warehouse/manage/views/__init__.py:2239 +#: warehouse/manage/views/__init__.py:2340 +#: warehouse/manage/views/__init__.py:2440 msgid "Confirm the request" msgstr "" -#: warehouse/manage/views/__init__.py:1266 +#: warehouse/manage/views/__init__.py:1265 msgid "Invalid alternate repository id" msgstr "" -#: warehouse/manage/views/__init__.py:1277 +#: warehouse/manage/views/__init__.py:1276 msgid "Invalid alternate repository for project" msgstr "" -#: warehouse/manage/views/__init__.py:1285 +#: warehouse/manage/views/__init__.py:1284 msgid "" "Could not delete alternate repository - ${confirm} is not the same as " "${alt_repo_name}" msgstr "" -#: warehouse/manage/views/__init__.py:1323 +#: warehouse/manage/views/__init__.py:1322 msgid "Deleted alternate repository '${name}'" msgstr "" -#: warehouse/manage/views/__init__.py:1454 +#: warehouse/manage/views/__init__.py:1453 msgid "" "GitHub-based trusted publishing is temporarily disabled. See " "https://pypi.org/help#admin-intervention for details." msgstr "" -#: warehouse/manage/views/__init__.py:1567 +#: warehouse/manage/views/__init__.py:1566 msgid "" "GitLab-based trusted publishing is temporarily disabled. See " "https://pypi.org/help#admin-intervention for details." msgstr "" -#: warehouse/manage/views/__init__.py:1679 +#: warehouse/manage/views/__init__.py:1678 msgid "" "Google-based trusted publishing is temporarily disabled. See " "https://pypi.org/help#admin-intervention for details." msgstr "" -#: warehouse/manage/views/__init__.py:1788 +#: warehouse/manage/views/__init__.py:1787 msgid "" "ActiveState-based trusted publishing is temporarily disabled. See " "https://pypi.org/help#admin-intervention for details." msgstr "" -#: warehouse/manage/views/__init__.py:2023 -#: warehouse/manage/views/__init__.py:2327 -#: warehouse/manage/views/__init__.py:2436 +#: warehouse/manage/views/__init__.py:2022 +#: warehouse/manage/views/__init__.py:2323 +#: warehouse/manage/views/__init__.py:2431 msgid "" "Project deletion temporarily disabled. See https://pypi.org/help#admin-" "intervention for details." msgstr "" -#: warehouse/manage/views/__init__.py:2169 +#: warehouse/manage/views/__init__.py:2166 msgid "Could not yank release - " msgstr "" -#: warehouse/manage/views/__init__.py:2255 +#: warehouse/manage/views/__init__.py:2251 msgid "Could not un-yank release - " msgstr "" -#: warehouse/manage/views/__init__.py:2356 +#: warehouse/manage/views/__init__.py:2352 msgid "Could not delete release - " msgstr "" -#: warehouse/manage/views/__init__.py:2457 +#: warehouse/manage/views/__init__.py:2452 msgid "Could not find file" msgstr "" -#: warehouse/manage/views/__init__.py:2461 +#: warehouse/manage/views/__init__.py:2456 msgid "Could not delete file - " msgstr "" -#: warehouse/manage/views/__init__.py:2611 +#: warehouse/manage/views/__init__.py:2606 msgid "Team '${team_name}' already has ${role_name} role for project" msgstr "" -#: warehouse/manage/views/__init__.py:2718 +#: warehouse/manage/views/__init__.py:2713 msgid "User '${username}' already has ${role_name} role for project" msgstr "" -#: warehouse/manage/views/__init__.py:2785 +#: warehouse/manage/views/__init__.py:2780 msgid "${username} is now ${role} of the '${project_name}' project." msgstr "" -#: warehouse/manage/views/__init__.py:2817 +#: warehouse/manage/views/__init__.py:2812 msgid "" "User '${username}' does not have a verified primary email address and " "cannot be added as a ${role_name} for project" msgstr "" -#: warehouse/manage/views/__init__.py:2830 +#: warehouse/manage/views/__init__.py:2825 #: warehouse/manage/views/organizations.py:878 msgid "User '${username}' already has an active invite. Please try again later." msgstr "" -#: warehouse/manage/views/__init__.py:2895 +#: warehouse/manage/views/__init__.py:2890 #: warehouse/manage/views/organizations.py:943 msgid "Invitation sent to '${username}'" msgstr "" -#: warehouse/manage/views/__init__.py:2928 +#: warehouse/manage/views/__init__.py:2923 msgid "Could not find role invitation." msgstr "" -#: warehouse/manage/views/__init__.py:2939 +#: warehouse/manage/views/__init__.py:2934 msgid "Invitation already expired." msgstr "" -#: warehouse/manage/views/__init__.py:2971 +#: warehouse/manage/views/__init__.py:2966 #: warehouse/manage/views/organizations.py:1130 msgid "Invitation revoked from '${username}'." msgstr ""