diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c84230..f74ed1e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,8 @@ # Changelog +## 3.2.0 +- Add support for custom S3ManifestStaticStorage subclasses with location set. +- Fix edge case where location is in the filename + ## 3.1.3 - fixed 2-pass to copy subdirectories diff --git a/collectfasta/__init__.py b/collectfasta/__init__.py index f749372..1173108 100644 --- a/collectfasta/__init__.py +++ b/collectfasta/__init__.py @@ -1 +1 @@ -__version__ = "3.1.3" +__version__ = "3.2.0" diff --git a/collectfasta/strategies/hashing.py b/collectfasta/strategies/hashing.py index 91c232c..15c84f0 100644 --- a/collectfasta/strategies/hashing.py +++ b/collectfasta/strategies/hashing.py @@ -112,9 +112,15 @@ class WithoutPrefixMixin(StrategyWithLocationProtocol): def copy_args_hook(self, args: Task) -> Task: assert isinstance(self.remote_storage, HasLocationProtocol) + if self.remote_storage.location == "" or self.remote_storage.location.endswith( + "/" + ): + location = self.remote_storage.location + else: + location = f"{self.remote_storage.location}/" return ( - args[0].replace(self.remote_storage.location, ""), - args[1].replace(self.remote_storage.location, ""), + args[0].replace(location, ""), + args[1].replace(location, ""), args[2], ) diff --git a/collectfasta/tests/conftest.py b/collectfasta/tests/conftest.py index 812b5f0..a0e38c1 100644 --- a/collectfasta/tests/conftest.py +++ b/collectfasta/tests/conftest.py @@ -19,6 +19,10 @@ def deco(f): S3_STORAGE_BACKEND = "storages.backends.s3.S3Storage" S3_STATIC_STORAGE_BACKEND = "storages.backends.s3.S3StaticStorage" S3_MANIFEST_STATIC_STORAGE_BACKEND = "storages.backends.s3.S3ManifestStaticStorage" +S3_CUSTOM_MANIFEST_STATIC_STORAGE_BACKEND = ( + "collectfasta.tests.utils.S3ManifestCustomStaticStorage" +) + GOOGLE_CLOUD_STORAGE_BACKEND = "collectfasta.tests.utils.GoogleCloudStorageTest" FILE_SYSTEM_STORAGE_BACKEND = "django.core.files.storage.FileSystemStorage" @@ -39,6 +43,7 @@ def deco(f): S3_STORAGE_BACKEND, S3_STATIC_STORAGE_BACKEND, S3_MANIFEST_STATIC_STORAGE_BACKEND, + S3_CUSTOM_MANIFEST_STATIC_STORAGE_BACKEND, ] BACKENDS = [ *S3_BACKENDS, @@ -63,6 +68,11 @@ def deco(f): BOTO3_MANIFEST_MEMORY_STRATEGY, BOTO3_MANIFEST_FILE_SYSTEM_STRATEGY, ], + S3_CUSTOM_MANIFEST_STATIC_STORAGE_BACKEND: [ + BOTO3_STRATEGY, + BOTO3_MANIFEST_MEMORY_STRATEGY, + BOTO3_MANIFEST_FILE_SYSTEM_STRATEGY, + ], GOOGLE_CLOUD_STORAGE_BACKEND: [GOOGLE_CLOUD_STRATEGY], FILE_SYSTEM_STORAGE_BACKEND: [FILE_SYSTEM_STRATEGY, CACHING_FILE_SYSTEM_STRATEGY], } @@ -90,13 +100,6 @@ def params_for_backends(): ) -S3_BACKENDS = [ - S3_STORAGE_BACKEND, - S3_STATIC_STORAGE_BACKEND, - S3_MANIFEST_STATIC_STORAGE_BACKEND, -] - - class StrategyFixture: def __init__(self, expected_copied_files, backend, strategy, two_pass): self.backend = backend @@ -111,7 +114,10 @@ def strategy(request): if strategy in ( BOTO3_MANIFEST_MEMORY_STRATEGY, BOTO3_MANIFEST_FILE_SYSTEM_STRATEGY, - ) and backend in (S3_MANIFEST_STATIC_STORAGE_BACKEND): + ) and backend in ( + S3_MANIFEST_STATIC_STORAGE_BACKEND, + S3_CUSTOM_MANIFEST_STATIC_STORAGE_BACKEND, + ): expected_copied_files = two_n_plus_1 else: expected_copied_files = n diff --git a/collectfasta/tests/utils.py b/collectfasta/tests/utils.py index 3b25279..a7be171 100644 --- a/collectfasta/tests/utils.py +++ b/collectfasta/tests/utils.py @@ -12,6 +12,7 @@ from django.conf import settings as django_settings from django.utils.module_loading import import_string from storages.backends.gcloud import GoogleCloudStorage +from storages.backends.s3boto3 import S3ManifestStaticStorage from typing_extensions import Final from collectfasta import settings @@ -49,6 +50,11 @@ def __init__(self, *args, **kwargs): self._client = get_fake_client() +class S3ManifestCustomStaticStorage(S3ManifestStaticStorage): + location = "prefix" + manifest_name = "prefixfiles.json" + + def create_two_referenced_static_files() -> tuple[pathlib.Path, pathlib.Path]: """Create a static file, then another file with a reference to the file""" path = create_static_file() @@ -84,6 +90,7 @@ def create_big_static_file() -> pathlib.Path: def clean_static_dir() -> None: clean_static_dir_recurse(static_dir.as_posix()) clean_static_dir_recurse(django_settings.AWS_LOCATION) + clean_static_dir_recurse(S3ManifestCustomStaticStorage.location) def clean_static_dir_recurse(location: str) -> None: