From af8f28d394c63996909c8c7f1caa7f78a7994dc6 Mon Sep 17 00:00:00 2001 From: Felix Fontein Date: Fri, 14 Apr 2023 06:40:08 +0200 Subject: [PATCH] Add support for Galaxy v3 API (#45) * Add support for Galaxy v3 API. * Add integration tests for the Galaxy API. The Galaxy v3 API versions list test is disabled since it is so incredibly slow. Just listing the first page takes 30 seconds (!). The total listing takes > 100 seconds. * Reorganize imports. * Improve docstrings; validate context.server == galaxy_server. * Fix check, update docstring. * According to ansible-galaxy CLI code, the result value can sometimes be called 'results' also for v3. --- changelogs/fragments/45-galaxy-v3.yml | 2 + src/antsibull_core/galaxy.py | 192 ++++++++++++++++++++++---- tests/functional/test_galaxy.py | 104 ++++++++++++++ 3 files changed, 270 insertions(+), 28 deletions(-) create mode 100644 changelogs/fragments/45-galaxy-v3.yml create mode 100644 tests/functional/test_galaxy.py diff --git a/changelogs/fragments/45-galaxy-v3.yml b/changelogs/fragments/45-galaxy-v3.yml new file mode 100644 index 0000000..77c9001 --- /dev/null +++ b/changelogs/fragments/45-galaxy-v3.yml @@ -0,0 +1,2 @@ +minor_changes: + - "Allow Galaxy client to communicate with the Galaxy v3 API (https://github.com/ansible-community/antsibull-core/pull/45)." diff --git a/src/antsibull_core/galaxy.py b/src/antsibull_core/galaxy.py index 61b21eb..161c4fe 100644 --- a/src/antsibull_core/galaxy.py +++ b/src/antsibull_core/galaxy.py @@ -7,9 +7,11 @@ from __future__ import annotations +import asyncio import os.path import shutil import typing as t +from enum import Enum from urllib.parse import urljoin import aiofiles @@ -49,23 +51,124 @@ class DownloadResults(t.NamedTuple): download_path: str +class GalaxyVersion(Enum): + V2 = 2 + V3 = 3 + + +class GalaxyContext: + server: str + version: GalaxyVersion + base_url: str + + def __init__(self, server: str, version: GalaxyVersion, base_url: str) -> None: + self.server = server + self.version = version + self.base_url = base_url + + @classmethod + async def create(cls, aio_session: aiohttp.client.ClientSession, galaxy_server: str + ) -> GalaxyContext: + api_url = urljoin(galaxy_server, 'api/') + async with retry_get(aio_session, api_url, + headers={'Accept': 'application/json'}) as response: + galaxy_info = await response.json() + available_versions: t.Mapping[str, str] = galaxy_info.get('available_versions') or {} + if 'v3' in available_versions: + version = GalaxyVersion.V3 + base_url = urljoin(galaxy_server, 'api/' + available_versions['v3']) + elif 'v2' in available_versions: + version = GalaxyVersion.V2 + base_url = urljoin(galaxy_server, 'api/' + available_versions['v2']) + else: + raise RuntimeError( + f'Information retrieved from {api_url} seems to indicate' + ' neither Galaxy v2 API nor Galaxy v3 API' + ) + return cls(galaxy_server, version, base_url) + + +_GALAXY_CONTEXT_CACHE: dict[str, t.Union[GalaxyContext, asyncio.Future]] = {} + + +async def _get_cached_galaxy_context(aio_session: aiohttp.client.ClientSession, + galaxy_server: str) -> GalaxyContext: + context_or_future = _GALAXY_CONTEXT_CACHE.get(galaxy_server) + if context_or_future is not None: + if asyncio.isfuture(context_or_future): + return await context_or_future + return t.cast(GalaxyContext, context_or_future) + + loop = asyncio.get_running_loop() + future = loop.create_future() + + async def _init(): + try: + context = await GalaxyContext.create(aio_session, galaxy_server) + future.set_result(context) + _GALAXY_CONTEXT_CACHE[galaxy_server] = context + except Exception as exc: # pylint: disable=broad-exception-caught + future.set_exception(exc) + + loop.create_task(_init()) + _GALAXY_CONTEXT_CACHE[galaxy_server] = future + return await future + + class GalaxyClient: """Class for querying the Galaxy REST API.""" def __init__(self, aio_session: aiohttp.client.ClientSession, - galaxy_server: str = _GALAXY_SERVER_URL) -> None: + galaxy_server: t.Optional[str] = None, + context: t.Optional[GalaxyContext] = None) -> None: """ Create a GalaxyClient object to query the Galaxy Server. :arg aio_session: :obj:`aiohttp.ClientSession` with which to perform all requests to galaxy. - :kwarg galaxy_server: URL to the galaxy server. + :kwarg galaxy_server: URL to the galaxy server. ``context`` must be provided instead + in the future. + :kwarg context: A ``GalaxyContext`` instance. Must be provided in the future. """ + if galaxy_server is None and context is None: + # TODO: deprecate + galaxy_server = _GALAXY_SERVER_URL + elif context is not None: + # TODO: deprecate + if galaxy_server is not None and galaxy_server != context.server: + raise ValueError( + f'galaxy_server ({galaxy_server}) does not coincide' + f' with context.server ({context.server})' + ) + galaxy_server = context.server self.galaxy_server = galaxy_server + self.context = context self.aio_session = aio_session - self.params = {'format': 'json'} + self.headers: t.Dict[str, str] = {'Accept': 'application/json'} + self.params: t.Dict[str, str] = {} + if context: + self._update_from_context(context) + + def _update_from_context(self, context: GalaxyContext) -> None: + if context.version == GalaxyVersion.V2: + self.params['format'] = 'json' - async def _get_galaxy_versions(self, versions_url: str) -> list[str]: + async def _ensure_context(self) -> GalaxyContext: + """ + Ensure that ``self.context`` is present. + """ + context = self.context + if context is not None: + return context + if self.galaxy_server is None: + raise RuntimeError('Unexpected None for GalaxyClient.galaxy_server') + context = await _get_cached_galaxy_context(self.aio_session, self.galaxy_server) + self.context = context + self._update_from_context(context) + return context + + async def _get_galaxy_versions(self, context: GalaxyContext, versions_url: str, + add_params: bool = True) -> list[str]: """ Retrieve the complete list of versions for a collection from a galaxy endpoint. @@ -73,23 +176,43 @@ async def _get_galaxy_versions(self, versions_url: str) -> list[str]: information is paged, it continues to retrieve linked pages until all of the information has been returned. + :arg context: the ``GalaxyContext`` to use. :arg version_url: url to the page to retrieve. + :arg add_params: used internally during recursion. Do not specify when calling this. :returns: List of the all the versions of the collection. """ - params = self.params.copy() - params['page_size'] = '100' + if add_params: + params = self.params.copy() + if context.version == GalaxyVersion.V2: + params['page_size'] = '100' + else: + params['limit'] = '50' + else: + params = None async with retry_get(self.aio_session, versions_url, params=params, - acceptable_error_codes=[404]) as response: + headers=self.headers, acceptable_error_codes=[404]) as response: if response.status == 404: raise NoSuchCollection(f'No collection found at: {versions_url}') collection_info = await response.json() versions = [] - for version_record in collection_info['results']: + if context.version == GalaxyVersion.V2: + results = collection_info['results'] + next_link = collection_info['next'] + else: + if 'data' in collection_info: + # Apparently 'data' isn't always used... + results = collection_info['data'] + else: + results = collection_info['results'] + next_link = collection_info['links']['next'] + add_params = False + for version_record in results: versions.append(version_record['version']) - - if collection_info['next']: - versions.extend(await self._get_galaxy_versions(collection_info['next'])) + if next_link: + if next_link.startswith('/'): + next_link = urljoin(context.server, next_link) + versions.extend(await self._get_galaxy_versions(context, next_link, add_params)) return versions @@ -100,9 +223,11 @@ async def get_versions(self, collection: str) -> list[str]: :arg collection: Name of the collection to get version info for. :returns: List of all the versions of this collection on galaxy. """ + context = await self._ensure_context() + collection = collection.replace('.', '/') - galaxy_url = urljoin(self.galaxy_server, f'api/v2/collections/{collection}/versions/') - retval = await self._get_galaxy_versions(galaxy_url) + galaxy_url = urljoin(context.base_url, f'collections/{collection}/versions/') + retval = await self._get_galaxy_versions(context, galaxy_url) return retval async def get_info(self, collection: str) -> dict[str, t.Any]: @@ -112,18 +237,23 @@ async def get_info(self, collection: str) -> dict[str, t.Any]: :arg collection: Namespace.collection to retrieve information about. :returns: Dictionary of information about the collection. - Please see the Galaxy REST API documentation for information on the structure of the - returned data. + Please see the Galaxy v2 and v3 REST API documentation for information on the + structure of the returned data. .. seealso:: An example return value from the - `Galaxy REST API `_ + `Galaxy v2 REST API + `_ + and the `Galaxy v3 REST API + `_ """ + context = await self._ensure_context() + collection = collection.replace('.', '/') - galaxy_url = urljoin(self.galaxy_server, f'api/v2/collections/{collection}/') + galaxy_url = urljoin(context.base_url, f'collections/{collection}/') async with retry_get(self.aio_session, galaxy_url, params=self.params, - acceptable_error_codes=[404]) as response: + headers=self.headers, acceptable_error_codes=[404]) as response: if response.status == 404: raise NoSuchCollection(f'No collection found at: {galaxy_url}') collection_info = await response.json() @@ -139,20 +269,23 @@ async def get_release_info(self, collection: str, :arg version: Version of the collection. :returns: Dictionary of information about the release. - Please see the Galaxy REST API documentation for information on the structure of the - returned data. + Please see the Galaxy v2 and v3 REST API documentation for information on the + structure of the returned data. .. seealso:: An example return value from the - `Galaxy REST API + `Galaxy v2 REST API `_ + and the `Galaxy v3 REST API + `_ """ + context = await self._ensure_context() + collection = collection.replace('.', '/') - galaxy_url = urljoin(self.galaxy_server, - f'api/v2/collections/{collection}/versions/{version}/') + galaxy_url = urljoin(context.base_url, f'collections/{collection}/versions/{version}/') async with retry_get(self.aio_session, galaxy_url, params=self.params, - acceptable_error_codes=[404]) as response: + headers=self.headers, acceptable_error_codes=[404]) as response: if response.status == 404: raise NoSuchCollection(f'No collection found at: {galaxy_url}') collection_info = await response.json() @@ -212,20 +345,23 @@ class CollectionDownloader(GalaxyClient): def __init__(self, aio_session: aiohttp.client.ClientSession, download_dir: str, - galaxy_server: str = _GALAXY_SERVER_URL, - collection_cache: str | None = None) -> None: + galaxy_server: t.Optional[str] = None, + collection_cache: str | None = None, + context: t.Optional[GalaxyContext] = None) -> None: """ Create an object to download collections from galaxy. :arg aio_session: :obj:`aiohttp.ClientSession` with which to perform all requests to galaxy. :arg download_dir: Directory to download into. - :kwarg galaxy_server: URL to the galaxy server. + :kwarg galaxy_server: URL to the galaxy server. ``context`` must be provided instead + in the future. + :kwarg context: A ``GalaxyContext`` instance. Must be provided in the future. :kwarg collection_cache: If given, a path to a directory containing collection tarballs. These tarballs will be used instead of downloading new tarballs provided that the versions match the criteria (latest compatible version known to galaxy). """ - super().__init__(aio_session, galaxy_server) + super().__init__(aio_session, galaxy_server=galaxy_server, context=context) self.download_dir = download_dir self.collection_cache: t.Final[str | None] = collection_cache diff --git a/tests/functional/test_galaxy.py b/tests/functional/test_galaxy.py new file mode 100644 index 0000000..b1bc6e8 --- /dev/null +++ b/tests/functional/test_galaxy.py @@ -0,0 +1,104 @@ +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later +# SPDX-FileCopyrightText: Ansible Project + +import hashlib + +import aiohttp +import pytest + +from antsibull_core.galaxy import CollectionDownloader, GalaxyClient, GalaxyContext, GalaxyVersion + +KNOWN_CG_VERSIONS = [ + '6.4.0', '6.3.0', '6.2.0', '6.1.0', '6.0.1', '6.0.0', '6.0.0-a1', + '5.8.6', '5.8.5', '5.8.4', '5.8.3', '5.8.2', '5.8.1', '5.8.0', '5.7.0', '5.6.0', '5.5.0', '5.4.0', '5.3.0', '5.2.0', '5.1.1', '5.1.0', '5.0.2', '5.0.1', '5.0.0', '5.0.0-a1', + '4.8.9', '4.8.8', '4.8.7', '4.8.6', '4.8.5', '4.8.4', '4.8.3', '4.8.2', '4.8.1', '4.8.0', '4.7.0', '4.6.1', '4.6.0', '4.5.0', '4.4.0', '4.3.0', '4.2.0', '4.1.0', '4.0.2', '4.0.1', '4.0.0', + '3.8.10', '3.8.9', '3.8.8', '3.8.7', '3.8.6', '3.8.5', '3.8.4', '3.8.3', '3.8.2', '3.8.1', '3.8.0', '3.7.0', '3.6.0', '3.5.0', '3.4.0', '3.3.2', '3.3.1', '3.3.0', '3.2.0', '3.1.0', '3.0.2', '3.0.1', '3.0.0', + '2.5.9', '2.5.8', '2.5.7', '2.5.6', '2.5.5', '2.5.4', '2.5.3', '2.5.2', '2.5.1', '2.5.0', '2.4.0', '2.3.0', '2.2.0', '2.1.1', '2.1.0', '2.0.1', '2.0.0', + '1.3.14', '1.3.13', '1.3.12', '1.3.11', '1.3.10', '1.3.9', '1.3.8', '1.3.7', '1.3.6', '1.3.5', '1.3.4', '1.3.3', '1.3.2', '1.3.1', '1.3.0', '1.2.0', '1.1.0', '1.0.0', + '0.3.0-experimental.meta.redirects-3', '0.3.0-experimental.meta.redirects-2', '0.3.0-experimental.meta.redirects', '0.2.1', '0.2.0', '0.1.4', '0.1.1', +] + + +async def galaxy_client_test(aio_session: aiohttp.ClientSession, context: GalaxyContext, + tmp_path_factory, + skip_versions_test: bool = False, + skip_download_test: bool = False, + ) -> None: + client = GalaxyClient(aio_session, context=context) + + # Check collection info + cg_info = await client.get_info('community.general') + print(cg_info) + assert cg_info['name'] == 'general' + assert cg_info['deprecated'] is False + + # Check info on specific collection version + cg_version = await client.get_release_info('community.general', '6.0.0') + cg_version.pop('files', None) # this is huge for Galaxy v3 API + cg_version.pop('metadata', None) # this is large for Galaxy v3 API + print(cg_version) + assert 'download_url' in cg_version + assert cg_version['artifact']['filename'] == 'community-general-6.0.0.tar.gz' + assert cg_version['artifact']['size'] == 2285782 + expected = '7c7ec856355078577b520f7432645754d75ad8e74a46e84d1ffee8fad80efc5c' + assert cg_version['artifact']['sha256'] == expected + assert cg_version['namespace']['name'] == 'community' + assert cg_version['collection']['name'] == 'general' + assert cg_version['version'] == '6.0.0' + + # Check collection list + if not skip_versions_test: + cg_versions = await client.get_versions('community.general') + print(cg_versions) + for known_version in KNOWN_CG_VERSIONS: + assert known_version in cg_versions + + # Download collection + if not skip_download_test: + download_path = tmp_path_factory.mktemp('download') + cache_path = tmp_path_factory.mktemp('cache') + downloader = CollectionDownloader( + aio_session, + str(download_path), + collection_cache=str(cache_path), + context=context, + ) + path = await downloader.download('community.dns', '0.1.0') + with open(path, 'rb') as f: + data = f.read() + length = len(data) + print(length) + assert length == 133242 + m = hashlib.sha256() + m.update(data) + digest = m.hexdigest() + expected = '2de9d40940536e65b35995a3f58dea7776de3f23f1a7ab865e0b3b8482d746b5' + assert digest == expected + + # Try again, should hit cache + path2 = await downloader.download('community.dns', '0.1.0') + with open(path2, 'rb') as f: + data2 = f.read() + assert path == path2 + assert data == data2 + + +@pytest.mark.asyncio +async def test_galaxy_v2(tmp_path_factory): + galaxy_url = 'https://galaxy.ansible.com' + async with aiohttp.ClientSession() as aio_session: + context = await GalaxyContext.create(aio_session, galaxy_url) + assert context.version == GalaxyVersion.V2 + assert context.base_url == galaxy_url + '/api/v2/' + await galaxy_client_test(aio_session, context, tmp_path_factory) + + +@pytest.mark.asyncio +async def test_galaxy_v3(tmp_path_factory): + galaxy_url = 'https://beta-galaxy.ansible.com' + async with aiohttp.ClientSession() as aio_session: + context = await GalaxyContext.create(aio_session, galaxy_url) + assert context.version == GalaxyVersion.V3 + assert context.base_url == galaxy_url + '/api/v3/' + await galaxy_client_test(aio_session, context, tmp_path_factory, skip_versions_test=True)