From c5741c19a017194a7a075d55d4d83505e5ff4eb9 Mon Sep 17 00:00:00 2001 From: Matthias Dellweg Date: Tue, 3 May 2022 11:43:13 +0200 Subject: [PATCH] Activate streaming from object storage fixes #943 Required PR: https://github.com/ansible/galaxy-importer/pull/168 --- .github/workflows/ci.yml | 4 +++ .github/workflows/nightly.yml | 1 + .github/workflows/release.yml | 1 + .github/workflows/scripts/install.sh | 9 +++++++ CHANGES/943.feature | 1 + pulp_ansible/app/tasks/collections.py | 39 ++++++++++++--------------- template_config.yml | 2 +- 7 files changed, 34 insertions(+), 23 deletions(-) create mode 100644 CHANGES/943.feature diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d0b324a52..8dd108b23 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -94,8 +94,10 @@ jobs: - TEST: docs - TEST: azure - TEST: s3 + - TEST: stream outputs: deprecations-pulp: ${{ steps.deprecations.outputs.deprecations-pulp }} + deprecations-stream: ${{ steps.deprecations.outputs.deprecations-stream }} deprecations-azure: ${{ steps.deprecations.outputs.deprecations-azure }} deprecations-s3: ${{ steps.deprecations.outputs.deprecations-s3 }} @@ -345,6 +347,7 @@ jobs: - name: Fail on deprecations run: | test -z "${{ needs.test.outputs.deprecations-pulp }}" + test -z "${{ needs.test.outputs.deprecations-stream }}" test -z "${{ needs.test.outputs.deprecations-azure }}" test -z "${{ needs.test.outputs.deprecations-s3 }}" test -z "${{ needs.test.outputs.deprecations-upgrade }}" @@ -352,6 +355,7 @@ jobs: if: failure() run: | echo "${{ needs.test.outputs.deprecations-pulp }}" | base64 -d + echo "${{ needs.test.outputs.deprecations-stream }}" | base64 -d echo "${{ needs.test.outputs.deprecations-azure }}" | base64 -d echo "${{ needs.test.outputs.deprecations-s3 }}" | base64 -d echo "${{ needs.test.outputs.deprecations-upgrade }}" | base64 -d diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 324af6ccb..5e5c2fa13 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -26,6 +26,7 @@ jobs: - TEST: docs - TEST: azure - TEST: s3 + - TEST: stream - TEST: generate-bindings steps: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f1c64d2e2..a0ede1f0e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -84,6 +84,7 @@ jobs: - TEST: docs - TEST: azure - TEST: s3 + - TEST: stream - TEST: generate-bindings steps: diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index 4b5160ecb..6299c8660 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -89,6 +89,15 @@ VARSYAML if [ "$TEST" = "upgrade" ]; then sed -i "/^pulp_container_tag:.*/s//pulp_container_tag: upgrade-https/" vars/main.yaml fi +if [ "$TEST" == 'stream' ]; then + sed -i -e '/^services:/a \ + - name: ci-sftp\ + image: atmoz/sftp\ + volumes:\ + - ./ssh/id_ed25519.pub:/home/foo/.ssh/keys/id_ed25519.pub\ + command: "foo::::storage"' vars/main.yaml + sed -i -e '$a stream_test: true' vars/main.yaml +fi if [ "$TEST" = "s3" ]; then export MINIO_ACCESS_KEY=AKIAIT2Z5TDYPX3ARJBA diff --git a/CHANGES/943.feature b/CHANGES/943.feature new file mode 100644 index 000000000..f532e01bf --- /dev/null +++ b/CHANGES/943.feature @@ -0,0 +1 @@ +Enable support for the pulpcore setting ``REDIRECT_TO_OBJECT_STORAGE=False``. diff --git a/pulp_ansible/app/tasks/collections.py b/pulp_ansible/app/tasks/collections.py index 57291dc2e..b9515ad5d 100644 --- a/pulp_ansible/app/tasks/collections.py +++ b/pulp_ansible/app/tasks/collections.py @@ -1,27 +1,30 @@ import asyncio -from collections import defaultdict -from gettext import gettext as _ import hashlib import json import logging import tarfile -import yaml +from collections import defaultdict +from gettext import gettext as _ from operator import attrgetter from urllib.parse import urljoin +from uuid import uuid4 +import yaml from aiohttp.client_exceptions import ClientResponseError +from asgiref.sync import sync_to_async from async_lru import alru_cache from django.conf import settings from django.db import transaction from django.db.models import Q +from django.db.utils import IntegrityError from django.urls import reverse from django.utils.dateparse import parse_datetime -from galaxy_importer.collection import import_collection as process_collection from galaxy_importer.collection import CollectionFilename +from galaxy_importer.collection import import_collection as process_collection +from galaxy_importer.collection import sync_collection from galaxy_importer.exceptions import ImporterError +from git import GitCommandError, Repo from pkg_resources import Requirement -from rest_framework.serializers import ValidationError - from pulpcore.plugin.models import ( Artifact, ContentArtifact, @@ -39,12 +42,13 @@ DeclarativeArtifact, DeclarativeContent, DeclarativeVersion, + QueryExistingArtifacts, + QueryExistingContents, RemoteArtifactSaver, ResolveContentFutures, Stage, - QueryExistingArtifacts, - QueryExistingContents, ) +from rest_framework.serializers import ValidationError from semantic_version import Version from pulp_ansible.app.constants import PAGE_SIZE @@ -58,22 +62,13 @@ CollectionVersionSignature, Tag, ) -from pulp_ansible.app.serializers import CollectionVersionSerializer, CollectionRemoteSerializer +from pulp_ansible.app.serializers import CollectionRemoteSerializer, CollectionVersionSerializer from pulp_ansible.app.tasks.utils import ( + RequirementsFileEntry, get_file_obj_from_tarball, - parse_metadata, parse_collections_requirements_file, - RequirementsFileEntry, + parse_metadata, ) -from asgiref.sync import sync_to_async - -from django.db.utils import IntegrityError -from galaxy_importer.collection import sync_collection -from git import GitCommandError, Repo - - -from uuid import uuid4 - log = logging.getLogger(__name__) @@ -309,9 +304,9 @@ def import_collection( files_data = json.load( get_file_obj_from_tarball(tar, "FILES.json", temp_file.file.name) ) - url = _get_backend_storage_url(artifact_file) + artifact_file.seek(0) importer_result = process_collection( - artifact_file, filename=filename, file_url=url, logger=user_facing_logger + artifact_file, filename=filename, logger=user_facing_logger ) artifact = Artifact.from_pulp_temporary_file(temp_file) importer_result["artifact_url"] = reverse("artifacts-detail", args=[artifact.pk]) diff --git a/template_config.yml b/template_config.yml index f10bf7fbb..e82d3fac3 100644 --- a/template_config.yml +++ b/template_config.yml @@ -77,7 +77,7 @@ test_performance: false test_released_plugin_with_next_pulpcore_release: false test_reroute: true test_s3: true -test_stream: false +test_stream: true update_github: true update_redmine: false upgrade_range: