Merge pull request #4 from jasongi/2-fix-s3-prefix

Add support for custom S3ManifestStaticStorage subclasses with location set.

CHANGELOG.md

@@ -1,4 +1,8 @@
 # Changelog
+## 3.2.0
+- Add support for custom S3ManifestStaticStorage subclasses with location set.
+- Fix edge case where location is in the filename
+
 ## 3.1.3
 - fixed 2-pass to copy subdirectories
 

collectfasta/__init__.py

@@ -1 +1 @@
-__version__ = "3.1.3"
+__version__ = "3.2.0"

collectfasta/strategies/hashing.py

@@ -112,9 +112,15 @@ class WithoutPrefixMixin(StrategyWithLocationProtocol):
 
     def copy_args_hook(self, args: Task) -> Task:
         assert isinstance(self.remote_storage, HasLocationProtocol)
+        if self.remote_storage.location == "" or self.remote_storage.location.endswith(
+            "/"
+        ):
+            location = self.remote_storage.location
+        else:
+            location = f"{self.remote_storage.location}/"
         return (
-            args[0].replace(self.remote_storage.location, ""),
-            args[1].replace(self.remote_storage.location, ""),
+            args[0].replace(location, ""),
+            args[1].replace(location, ""),
             args[2],
         )
 

collectfasta/tests/conftest.py

@@ -19,6 +19,10 @@ def deco(f):
 S3_STORAGE_BACKEND = "storages.backends.s3.S3Storage"
 S3_STATIC_STORAGE_BACKEND = "storages.backends.s3.S3StaticStorage"
 S3_MANIFEST_STATIC_STORAGE_BACKEND = "storages.backends.s3.S3ManifestStaticStorage"
+S3_CUSTOM_MANIFEST_STATIC_STORAGE_BACKEND = (
+    "collectfasta.tests.utils.S3ManifestCustomStaticStorage"
+)
+
 GOOGLE_CLOUD_STORAGE_BACKEND = "collectfasta.tests.utils.GoogleCloudStorageTest"
 FILE_SYSTEM_STORAGE_BACKEND = "django.core.files.storage.FileSystemStorage"
 
@@ -39,6 +43,7 @@ def deco(f):
     S3_STORAGE_BACKEND,
     S3_STATIC_STORAGE_BACKEND,
     S3_MANIFEST_STATIC_STORAGE_BACKEND,
+    S3_CUSTOM_MANIFEST_STATIC_STORAGE_BACKEND,
 ]
 BACKENDS = [
     *S3_BACKENDS,
@@ -63,6 +68,11 @@ def deco(f):
         BOTO3_MANIFEST_MEMORY_STRATEGY,
         BOTO3_MANIFEST_FILE_SYSTEM_STRATEGY,
     ],
+    S3_CUSTOM_MANIFEST_STATIC_STORAGE_BACKEND: [
+        BOTO3_STRATEGY,
+        BOTO3_MANIFEST_MEMORY_STRATEGY,
+        BOTO3_MANIFEST_FILE_SYSTEM_STRATEGY,
+    ],
     GOOGLE_CLOUD_STORAGE_BACKEND: [GOOGLE_CLOUD_STRATEGY],
     FILE_SYSTEM_STORAGE_BACKEND: [FILE_SYSTEM_STRATEGY, CACHING_FILE_SYSTEM_STRATEGY],
 }
@@ -90,13 +100,6 @@ def params_for_backends():
             )
 
 
-S3_BACKENDS = [
-    S3_STORAGE_BACKEND,
-    S3_STATIC_STORAGE_BACKEND,
-    S3_MANIFEST_STATIC_STORAGE_BACKEND,
-]
-
-
 class StrategyFixture:
     def __init__(self, expected_copied_files, backend, strategy, two_pass):
         self.backend = backend
@@ -111,7 +114,10 @@ def strategy(request):
     if strategy in (
         BOTO3_MANIFEST_MEMORY_STRATEGY,
         BOTO3_MANIFEST_FILE_SYSTEM_STRATEGY,
-    ) and backend in (S3_MANIFEST_STATIC_STORAGE_BACKEND):
+    ) and backend in (
+        S3_MANIFEST_STATIC_STORAGE_BACKEND,
+        S3_CUSTOM_MANIFEST_STATIC_STORAGE_BACKEND,
+    ):
         expected_copied_files = two_n_plus_1
     else:
         expected_copied_files = n

collectfasta/tests/utils.py

@@ -12,6 +12,7 @@
 from django.conf import settings as django_settings
 from django.utils.module_loading import import_string
 from storages.backends.gcloud import GoogleCloudStorage
+from storages.backends.s3boto3 import S3ManifestStaticStorage
 from typing_extensions import Final
 
 from collectfasta import settings
@@ -49,6 +50,11 @@ def __init__(self, *args, **kwargs):
             self._client = get_fake_client()
 
 
+class S3ManifestCustomStaticStorage(S3ManifestStaticStorage):
+    location = "prefix"
+    manifest_name = "prefixfiles.json"
+
+
 def create_two_referenced_static_files() -> tuple[pathlib.Path, pathlib.Path]:
     """Create a static file, then another file with a reference to the file"""
     path = create_static_file()
@@ -84,6 +90,7 @@ def create_big_static_file() -> pathlib.Path:
 def clean_static_dir() -> None:
     clean_static_dir_recurse(static_dir.as_posix())
     clean_static_dir_recurse(django_settings.AWS_LOCATION)
+    clean_static_dir_recurse(S3ManifestCustomStaticStorage.location)
 
 
 def clean_static_dir_recurse(location: str) -> None: