Switch from deprecated django-storages module, install target

See https://github.com/jschneier/django-storages/blob/master/CHANGELOG.rst#s3-2

dandiapi/api/copy.py

@@ -8,10 +8,10 @@
 from dandiapi.api.storage import get_boto_client
 
 try:
-    from storages.backends.s3boto3 import S3Boto3Storage
+    from storages.backends.s3 import S3Storage
 except ImportError:
     # This should only be used for type interrogation, never instantiation
-    S3Boto3Storage = type('FakeS3Boto3Storage', (), {})
+    S3Storage = type('FakeS3Storage', (), {})
 try:
     from minio_storage.storage import MinioStorage
 except ImportError:

dandiapi/api/management/commands/cleanup_blobs.py

@@ -1,14 +1,14 @@
 from django.conf import settings
 import djclick as click
-from storages.backends.s3boto3 import S3Boto3Storage
+from storages.backends.s3 import S3Storage
 
 from dandiapi.api.models.upload import AssetBlob
 
 BUCKET = settings.DANDI_DANDISETS_BUCKET_NAME
 
 
 def s3_client():
-    storage = S3Boto3Storage(bucket_name=BUCKET)
+    storage = S3Storage(bucket_name=BUCKET)
     return storage.connection.meta.client
 
 

dandiapi/api/storage.py

@@ -17,7 +17,7 @@
 from minio_storage.storage import MinioStorage, create_minio_client_from_settings
 from s3_file_field._multipart_boto3 import Boto3MultipartManager
 from s3_file_field._multipart_minio import MinioMultipartManager
-from storages.backends.s3boto3 import S3Boto3Storage
+from storages.backends.s3 import S3Storage
 
 
 class ChecksumCalculatorFile:
@@ -95,7 +95,7 @@ def __init__(self, *args, **kwargs):
 class VerbatimNameStorageMixin:
     """A Storage mixin, storing files without transforming their original filename."""
 
-    # The basic S3Boto3Storage does not implement generate_filename or get_valid_name,
+    # The basic S3Storage does not implement generate_filename or get_valid_name,
     # so upon FileField save, the following call stack normally occurs:
     #   FieldFile.save
     #   FileField.generate_filename
@@ -110,7 +110,7 @@ def generate_filename(self, filename: str) -> str:
         return filename
 
 
-class TimeoutS3Boto3Storage(S3Boto3Storage):
+class TimeoutS3Storage(S3Storage):
     """Override boto3 default timeout values."""
 
     def __init__(self, **settings):
@@ -121,7 +121,7 @@ def __init__(self, **settings):
         )
 
 
-class VerbatimNameS3Storage(VerbatimNameStorageMixin, TimeoutS3Boto3Storage):
+class VerbatimNameS3Storage(VerbatimNameStorageMixin, TimeoutS3Storage):
     @property
     def multipart_manager(self):
         return DandiBoto3MultipartManager(self)
@@ -246,14 +246,14 @@ def create_s3_storage(bucket_name: str) -> Storage:
     """
     Return a new Storage instance, compatible with the default Storage class.
 
-    This abstracts over differences between S3Boto3Storage and MinioStorage,
+    This abstracts over differences between S3Storage and MinioStorage,
     allowing either to be used as an additional non-default Storage.
     """
     # For production, calling django.core.files.storage.get_storage_class is fine
-    # to return the storage class of S3Boto3Storage.
+    # to return the storage class of S3Storage.
     default_storage_class = get_storage_class()
 
-    if issubclass(default_storage_class, S3Boto3Storage):
+    if issubclass(default_storage_class, S3Storage):
         storage = VerbatimNameS3Storage(bucket_name=bucket_name)
         # Required to upload to the sponsored bucket
         storage.default_acl = 'bucket-owner-full-control'
@@ -280,7 +280,7 @@ def create_s3_storage(bucket_name: str) -> Storage:
         storage = VerbatimNameMinioStorage(
             bucket_name=bucket_name,
             base_url=base_url,
-            # All S3Boto3Storage URLs are presigned, and the bucket typically is not public
+            # All S3Storage URLs are presigned, and the bucket typically is not public
             presign_urls=True,
             auto_create_bucket=True,
             auto_create_policy=True,

dandiapi/api/views/asset.py

@@ -12,10 +12,10 @@
 from dandiapi.zarr.models import ZarrArchive
 
 try:
-    from storages.backends.s3boto3 import S3Boto3Storage
+    from storages.backends.s3 import S3Storage
 except ImportError:
     # This should only be used for type interrogation, never instantiation
-    S3Boto3Storage = type('FakeS3Boto3Storage', (), {})
+    S3Storage = type('FakeS3Storage', (), {})
 try:
     from minio_storage.storage import MinioStorage
 except ImportError:

dandiapi/conftest.py

@@ -4,7 +4,7 @@
 import pytest
 from pytest_factoryboy import register
 from rest_framework.test import APIClient
-from storages.backends.s3boto3 import S3Boto3Storage
+from storages.backends.s3 import S3Storage
 
 from dandiapi.api.storage import create_s3_storage
 from dandiapi.api.tests.factories import (
@@ -84,7 +84,7 @@ def authenticated_api_client(user) -> APIClient:
 # storage fixtures are copied from django-s3-file-field test fixtures
 
 
-def base_s3boto3_storage_factory(bucket_name: str) -> 'S3Boto3Storage':
+def base_s3boto3_storage_factory(bucket_name: str) -> 'S3Storage':
     return create_s3_storage(bucket_name)
 
 
@@ -109,12 +109,12 @@ def embargoed_minio_storage_factory() -> MinioStorage:
 
 
 @pytest.fixture
-def s3boto3_storage() -> 'S3Boto3Storage':
+def s3boto3_storage() -> 'S3Storage':
     return s3boto3_storage_factory()
 
 
 @pytest.fixture
-def embargoed_s3boto3_storage() -> 'S3Boto3Storage':
+def embargoed_s3boto3_storage() -> 'S3Storage':
     return s3boto3_storage_factory()
 
 
@@ -132,7 +132,7 @@ def embargoed_minio_storage() -> MinioStorage:
 def storage(request, settings) -> Storage:
     storage_factory = request.param
     if storage_factory == s3boto3_storage_factory:
-        settings.DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage'
+        settings.DEFAULT_FILE_STORAGE = 'storages.backends.s3.S3Storage'
         settings.AWS_S3_ACCESS_KEY_ID = settings.MINIO_STORAGE_ACCESS_KEY
         settings.AWS_S3_SECRET_ACCESS_KEY = settings.MINIO_STORAGE_SECRET_KEY
         settings.AWS_S3_REGION_NAME = 'test-region'

setup.py

@@ -66,7 +66,7 @@
         'django-composed-configuration[prod]>=0.22.0',
         # pin directly to a version since we're extending the private multipart interface
         'django-s3-file-field[boto3]==0.3.2',
-        'django-storages[boto3]>=1.14.2',
+        'django-storages[s3]>=1.14.2',
         'gunicorn',
         # Development-only, but required
         # TODO: starting with v0.5.0, django-minio-storage requires v7