diff --git a/changelogs/fragments/45-galaxy-v3.yml b/changelogs/fragments/45-galaxy-v3.yml
new file mode 100644
index 0000000..77c9001
--- /dev/null
+++ b/changelogs/fragments/45-galaxy-v3.yml
@@ -0,0 +1,2 @@
+minor_changes:
+ - "Allow Galaxy client to communicate with the Galaxy v3 API (https://github.com/ansible-community/antsibull-core/pull/45)."
diff --git a/src/antsibull_core/galaxy.py b/src/antsibull_core/galaxy.py
index 61b21eb..161c4fe 100644
--- a/src/antsibull_core/galaxy.py
+++ b/src/antsibull_core/galaxy.py
@@ -7,9 +7,11 @@
from __future__ import annotations
+import asyncio
import os.path
import shutil
import typing as t
+from enum import Enum
from urllib.parse import urljoin
import aiofiles
@@ -49,23 +51,124 @@ class DownloadResults(t.NamedTuple):
download_path: str
+class GalaxyVersion(Enum):
+ V2 = 2
+ V3 = 3
+
+
+class GalaxyContext:
+ server: str
+ version: GalaxyVersion
+ base_url: str
+
+ def __init__(self, server: str, version: GalaxyVersion, base_url: str) -> None:
+ self.server = server
+ self.version = version
+ self.base_url = base_url
+
+ @classmethod
+ async def create(cls, aio_session: aiohttp.client.ClientSession, galaxy_server: str
+ ) -> GalaxyContext:
+ api_url = urljoin(galaxy_server, 'api/')
+ async with retry_get(aio_session, api_url,
+ headers={'Accept': 'application/json'}) as response:
+ galaxy_info = await response.json()
+ available_versions: t.Mapping[str, str] = galaxy_info.get('available_versions') or {}
+ if 'v3' in available_versions:
+ version = GalaxyVersion.V3
+ base_url = urljoin(galaxy_server, 'api/' + available_versions['v3'])
+ elif 'v2' in available_versions:
+ version = GalaxyVersion.V2
+ base_url = urljoin(galaxy_server, 'api/' + available_versions['v2'])
+ else:
+ raise RuntimeError(
+ f'Information retrieved from {api_url} seems to indicate'
+ ' neither Galaxy v2 API nor Galaxy v3 API'
+ )
+ return cls(galaxy_server, version, base_url)
+
+
+_GALAXY_CONTEXT_CACHE: dict[str, t.Union[GalaxyContext, asyncio.Future]] = {}
+
+
+async def _get_cached_galaxy_context(aio_session: aiohttp.client.ClientSession,
+ galaxy_server: str) -> GalaxyContext:
+ context_or_future = _GALAXY_CONTEXT_CACHE.get(galaxy_server)
+ if context_or_future is not None:
+ if asyncio.isfuture(context_or_future):
+ return await context_or_future
+ return t.cast(GalaxyContext, context_or_future)
+
+ loop = asyncio.get_running_loop()
+ future = loop.create_future()
+
+ async def _init():
+ try:
+ context = await GalaxyContext.create(aio_session, galaxy_server)
+ future.set_result(context)
+ _GALAXY_CONTEXT_CACHE[galaxy_server] = context
+ except Exception as exc: # pylint: disable=broad-exception-caught
+ future.set_exception(exc)
+
+ loop.create_task(_init())
+ _GALAXY_CONTEXT_CACHE[galaxy_server] = future
+ return await future
+
+
class GalaxyClient:
"""Class for querying the Galaxy REST API."""
def __init__(self, aio_session: aiohttp.client.ClientSession,
- galaxy_server: str = _GALAXY_SERVER_URL) -> None:
+ galaxy_server: t.Optional[str] = None,
+ context: t.Optional[GalaxyContext] = None) -> None:
"""
Create a GalaxyClient object to query the Galaxy Server.
:arg aio_session: :obj:`aiohttp.ClientSession` with which to perform all
requests to galaxy.
- :kwarg galaxy_server: URL to the galaxy server.
+ :kwarg galaxy_server: URL to the galaxy server. ``context`` must be provided instead
+ in the future.
+ :kwarg context: A ``GalaxyContext`` instance. Must be provided in the future.
"""
+ if galaxy_server is None and context is None:
+ # TODO: deprecate
+ galaxy_server = _GALAXY_SERVER_URL
+ elif context is not None:
+ # TODO: deprecate
+ if galaxy_server is not None and galaxy_server != context.server:
+ raise ValueError(
+ f'galaxy_server ({galaxy_server}) does not coincide'
+ f' with context.server ({context.server})'
+ )
+ galaxy_server = context.server
self.galaxy_server = galaxy_server
+ self.context = context
self.aio_session = aio_session
- self.params = {'format': 'json'}
+ self.headers: t.Dict[str, str] = {'Accept': 'application/json'}
+ self.params: t.Dict[str, str] = {}
+ if context:
+ self._update_from_context(context)
+
+ def _update_from_context(self, context: GalaxyContext) -> None:
+ if context.version == GalaxyVersion.V2:
+ self.params['format'] = 'json'
- async def _get_galaxy_versions(self, versions_url: str) -> list[str]:
+ async def _ensure_context(self) -> GalaxyContext:
+ """
+ Ensure that ``self.context`` is present.
+ """
+ context = self.context
+ if context is not None:
+ return context
+ if self.galaxy_server is None:
+ raise RuntimeError('Unexpected None for GalaxyClient.galaxy_server')
+ context = await _get_cached_galaxy_context(self.aio_session, self.galaxy_server)
+ self.context = context
+ self._update_from_context(context)
+ return context
+
+ async def _get_galaxy_versions(self, context: GalaxyContext, versions_url: str,
+ add_params: bool = True) -> list[str]:
"""
Retrieve the complete list of versions for a collection from a galaxy endpoint.
@@ -73,23 +176,43 @@ async def _get_galaxy_versions(self, versions_url: str) -> list[str]:
information is paged, it continues to retrieve linked pages until all of the information has
been returned.
+ :arg context: the ``GalaxyContext`` to use.
:arg version_url: url to the page to retrieve.
+ :arg add_params: used internally during recursion. Do not specify when calling this.
:returns: List of the all the versions of the collection.
"""
- params = self.params.copy()
- params['page_size'] = '100'
+ if add_params:
+ params = self.params.copy()
+ if context.version == GalaxyVersion.V2:
+ params['page_size'] = '100'
+ else:
+ params['limit'] = '50'
+ else:
+ params = None
async with retry_get(self.aio_session, versions_url, params=params,
- acceptable_error_codes=[404]) as response:
+ headers=self.headers, acceptable_error_codes=[404]) as response:
if response.status == 404:
raise NoSuchCollection(f'No collection found at: {versions_url}')
collection_info = await response.json()
versions = []
- for version_record in collection_info['results']:
+ if context.version == GalaxyVersion.V2:
+ results = collection_info['results']
+ next_link = collection_info['next']
+ else:
+ if 'data' in collection_info:
+ # Apparently 'data' isn't always used...
+ results = collection_info['data']
+ else:
+ results = collection_info['results']
+ next_link = collection_info['links']['next']
+ add_params = False
+ for version_record in results:
versions.append(version_record['version'])
-
- if collection_info['next']:
- versions.extend(await self._get_galaxy_versions(collection_info['next']))
+ if next_link:
+ if next_link.startswith('/'):
+ next_link = urljoin(context.server, next_link)
+ versions.extend(await self._get_galaxy_versions(context, next_link, add_params))
return versions
@@ -100,9 +223,11 @@ async def get_versions(self, collection: str) -> list[str]:
:arg collection: Name of the collection to get version info for.
:returns: List of all the versions of this collection on galaxy.
"""
+ context = await self._ensure_context()
+
collection = collection.replace('.', '/')
- galaxy_url = urljoin(self.galaxy_server, f'api/v2/collections/{collection}/versions/')
- retval = await self._get_galaxy_versions(galaxy_url)
+ galaxy_url = urljoin(context.base_url, f'collections/{collection}/versions/')
+ retval = await self._get_galaxy_versions(context, galaxy_url)
return retval
async def get_info(self, collection: str) -> dict[str, t.Any]:
@@ -112,18 +237,23 @@ async def get_info(self, collection: str) -> dict[str, t.Any]:
:arg collection: Namespace.collection to retrieve information about.
:returns: Dictionary of information about the collection.
- Please see the Galaxy REST API documentation for information on the structure of the
- returned data.
+ Please see the Galaxy v2 and v3 REST API documentation for information on the
+ structure of the returned data.
.. seealso::
An example return value from the
- `Galaxy REST API `_
+ `Galaxy v2 REST API
+ `_
+ and the `Galaxy v3 REST API
+ `_
"""
+ context = await self._ensure_context()
+
collection = collection.replace('.', '/')
- galaxy_url = urljoin(self.galaxy_server, f'api/v2/collections/{collection}/')
+ galaxy_url = urljoin(context.base_url, f'collections/{collection}/')
async with retry_get(self.aio_session, galaxy_url, params=self.params,
- acceptable_error_codes=[404]) as response:
+ headers=self.headers, acceptable_error_codes=[404]) as response:
if response.status == 404:
raise NoSuchCollection(f'No collection found at: {galaxy_url}')
collection_info = await response.json()
@@ -139,20 +269,23 @@ async def get_release_info(self, collection: str,
:arg version: Version of the collection.
:returns: Dictionary of information about the release.
- Please see the Galaxy REST API documentation for information on the structure of the
- returned data.
+ Please see the Galaxy v2 and v3 REST API documentation for information on the
+ structure of the returned data.
.. seealso::
An example return value from the
- `Galaxy REST API
+ `Galaxy v2 REST API
`_
+ and the `Galaxy v3 REST API
+ `_
"""
+ context = await self._ensure_context()
+
collection = collection.replace('.', '/')
- galaxy_url = urljoin(self.galaxy_server,
- f'api/v2/collections/{collection}/versions/{version}/')
+ galaxy_url = urljoin(context.base_url, f'collections/{collection}/versions/{version}/')
async with retry_get(self.aio_session, galaxy_url, params=self.params,
- acceptable_error_codes=[404]) as response:
+ headers=self.headers, acceptable_error_codes=[404]) as response:
if response.status == 404:
raise NoSuchCollection(f'No collection found at: {galaxy_url}')
collection_info = await response.json()
@@ -212,20 +345,23 @@ class CollectionDownloader(GalaxyClient):
def __init__(self, aio_session: aiohttp.client.ClientSession,
download_dir: str,
- galaxy_server: str = _GALAXY_SERVER_URL,
- collection_cache: str | None = None) -> None:
+ galaxy_server: t.Optional[str] = None,
+ collection_cache: str | None = None,
+ context: t.Optional[GalaxyContext] = None) -> None:
"""
Create an object to download collections from galaxy.
:arg aio_session: :obj:`aiohttp.ClientSession` with which to perform all
requests to galaxy.
:arg download_dir: Directory to download into.
- :kwarg galaxy_server: URL to the galaxy server.
+ :kwarg galaxy_server: URL to the galaxy server. ``context`` must be provided instead
+ in the future.
+ :kwarg context: A ``GalaxyContext`` instance. Must be provided in the future.
:kwarg collection_cache: If given, a path to a directory containing collection tarballs.
These tarballs will be used instead of downloading new tarballs provided that the
versions match the criteria (latest compatible version known to galaxy).
"""
- super().__init__(aio_session, galaxy_server)
+ super().__init__(aio_session, galaxy_server=galaxy_server, context=context)
self.download_dir = download_dir
self.collection_cache: t.Final[str | None] = collection_cache
diff --git a/tests/functional/test_galaxy.py b/tests/functional/test_galaxy.py
new file mode 100644
index 0000000..b1bc6e8
--- /dev/null
+++ b/tests/functional/test_galaxy.py
@@ -0,0 +1,104 @@
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+# SPDX-FileCopyrightText: Ansible Project
+
+import hashlib
+
+import aiohttp
+import pytest
+
+from antsibull_core.galaxy import CollectionDownloader, GalaxyClient, GalaxyContext, GalaxyVersion
+
+KNOWN_CG_VERSIONS = [
+ '6.4.0', '6.3.0', '6.2.0', '6.1.0', '6.0.1', '6.0.0', '6.0.0-a1',
+ '5.8.6', '5.8.5', '5.8.4', '5.8.3', '5.8.2', '5.8.1', '5.8.0', '5.7.0', '5.6.0', '5.5.0', '5.4.0', '5.3.0', '5.2.0', '5.1.1', '5.1.0', '5.0.2', '5.0.1', '5.0.0', '5.0.0-a1',
+ '4.8.9', '4.8.8', '4.8.7', '4.8.6', '4.8.5', '4.8.4', '4.8.3', '4.8.2', '4.8.1', '4.8.0', '4.7.0', '4.6.1', '4.6.0', '4.5.0', '4.4.0', '4.3.0', '4.2.0', '4.1.0', '4.0.2', '4.0.1', '4.0.0',
+ '3.8.10', '3.8.9', '3.8.8', '3.8.7', '3.8.6', '3.8.5', '3.8.4', '3.8.3', '3.8.2', '3.8.1', '3.8.0', '3.7.0', '3.6.0', '3.5.0', '3.4.0', '3.3.2', '3.3.1', '3.3.0', '3.2.0', '3.1.0', '3.0.2', '3.0.1', '3.0.0',
+ '2.5.9', '2.5.8', '2.5.7', '2.5.6', '2.5.5', '2.5.4', '2.5.3', '2.5.2', '2.5.1', '2.5.0', '2.4.0', '2.3.0', '2.2.0', '2.1.1', '2.1.0', '2.0.1', '2.0.0',
+ '1.3.14', '1.3.13', '1.3.12', '1.3.11', '1.3.10', '1.3.9', '1.3.8', '1.3.7', '1.3.6', '1.3.5', '1.3.4', '1.3.3', '1.3.2', '1.3.1', '1.3.0', '1.2.0', '1.1.0', '1.0.0',
+ '0.3.0-experimental.meta.redirects-3', '0.3.0-experimental.meta.redirects-2', '0.3.0-experimental.meta.redirects', '0.2.1', '0.2.0', '0.1.4', '0.1.1',
+]
+
+
+async def galaxy_client_test(aio_session: aiohttp.ClientSession, context: GalaxyContext,
+ tmp_path_factory,
+ skip_versions_test: bool = False,
+ skip_download_test: bool = False,
+ ) -> None:
+ client = GalaxyClient(aio_session, context=context)
+
+ # Check collection info
+ cg_info = await client.get_info('community.general')
+ print(cg_info)
+ assert cg_info['name'] == 'general'
+ assert cg_info['deprecated'] is False
+
+ # Check info on specific collection version
+ cg_version = await client.get_release_info('community.general', '6.0.0')
+ cg_version.pop('files', None) # this is huge for Galaxy v3 API
+ cg_version.pop('metadata', None) # this is large for Galaxy v3 API
+ print(cg_version)
+ assert 'download_url' in cg_version
+ assert cg_version['artifact']['filename'] == 'community-general-6.0.0.tar.gz'
+ assert cg_version['artifact']['size'] == 2285782
+ expected = '7c7ec856355078577b520f7432645754d75ad8e74a46e84d1ffee8fad80efc5c'
+ assert cg_version['artifact']['sha256'] == expected
+ assert cg_version['namespace']['name'] == 'community'
+ assert cg_version['collection']['name'] == 'general'
+ assert cg_version['version'] == '6.0.0'
+
+ # Check collection list
+ if not skip_versions_test:
+ cg_versions = await client.get_versions('community.general')
+ print(cg_versions)
+ for known_version in KNOWN_CG_VERSIONS:
+ assert known_version in cg_versions
+
+ # Download collection
+ if not skip_download_test:
+ download_path = tmp_path_factory.mktemp('download')
+ cache_path = tmp_path_factory.mktemp('cache')
+ downloader = CollectionDownloader(
+ aio_session,
+ str(download_path),
+ collection_cache=str(cache_path),
+ context=context,
+ )
+ path = await downloader.download('community.dns', '0.1.0')
+ with open(path, 'rb') as f:
+ data = f.read()
+ length = len(data)
+ print(length)
+ assert length == 133242
+ m = hashlib.sha256()
+ m.update(data)
+ digest = m.hexdigest()
+ expected = '2de9d40940536e65b35995a3f58dea7776de3f23f1a7ab865e0b3b8482d746b5'
+ assert digest == expected
+
+ # Try again, should hit cache
+ path2 = await downloader.download('community.dns', '0.1.0')
+ with open(path2, 'rb') as f:
+ data2 = f.read()
+ assert path == path2
+ assert data == data2
+
+
+@pytest.mark.asyncio
+async def test_galaxy_v2(tmp_path_factory):
+ galaxy_url = 'https://galaxy.ansible.com'
+ async with aiohttp.ClientSession() as aio_session:
+ context = await GalaxyContext.create(aio_session, galaxy_url)
+ assert context.version == GalaxyVersion.V2
+ assert context.base_url == galaxy_url + '/api/v2/'
+ await galaxy_client_test(aio_session, context, tmp_path_factory)
+
+
+@pytest.mark.asyncio
+async def test_galaxy_v3(tmp_path_factory):
+ galaxy_url = 'https://beta-galaxy.ansible.com'
+ async with aiohttp.ClientSession() as aio_session:
+ context = await GalaxyContext.create(aio_session, galaxy_url)
+ assert context.version == GalaxyVersion.V3
+ assert context.base_url == galaxy_url + '/api/v3/'
+ await galaxy_client_test(aio_session, context, tmp_path_factory, skip_versions_test=True)