diff --git a/dandiapi/api/copy.py b/dandiapi/api/copy.py index 2d4c03daf..ccec564d3 100644 --- a/dandiapi/api/copy.py +++ b/dandiapi/api/copy.py @@ -8,10 +8,10 @@ from dandiapi.api.storage import get_boto_client try: - from storages.backends.s3boto3 import S3Boto3Storage + from storages.backends.s3 import S3Storage except ImportError: # This should only be used for type interrogation, never instantiation - S3Boto3Storage = type('FakeS3Boto3Storage', (), {}) + S3Storage = type('FakeS3Storage', (), {}) try: from minio_storage.storage import MinioStorage except ImportError: diff --git a/dandiapi/api/management/commands/cleanup_blobs.py b/dandiapi/api/management/commands/cleanup_blobs.py index 4a6e06cc1..aecff99ca 100644 --- a/dandiapi/api/management/commands/cleanup_blobs.py +++ b/dandiapi/api/management/commands/cleanup_blobs.py @@ -1,6 +1,6 @@ from django.conf import settings import djclick as click -from storages.backends.s3boto3 import S3Boto3Storage +from storages.backends.s3 import S3Storage from dandiapi.api.models.upload import AssetBlob @@ -8,7 +8,7 @@ def s3_client(): - storage = S3Boto3Storage(bucket_name=BUCKET) + storage = S3Storage(bucket_name=BUCKET) return storage.connection.meta.client diff --git a/dandiapi/api/storage.py b/dandiapi/api/storage.py index 1ce1899f9..906d6feb7 100644 --- a/dandiapi/api/storage.py +++ b/dandiapi/api/storage.py @@ -17,7 +17,7 @@ from minio_storage.storage import MinioStorage, create_minio_client_from_settings from s3_file_field._multipart_boto3 import Boto3MultipartManager from s3_file_field._multipart_minio import MinioMultipartManager -from storages.backends.s3boto3 import S3Boto3Storage +from storages.backends.s3 import S3Storage class ChecksumCalculatorFile: @@ -95,7 +95,7 @@ def __init__(self, *args, **kwargs): class VerbatimNameStorageMixin: """A Storage mixin, storing files without transforming their original filename.""" - # The basic S3Boto3Storage does not implement generate_filename or get_valid_name, + # The basic S3Storage does not implement generate_filename or get_valid_name, # so upon FileField save, the following call stack normally occurs: # FieldFile.save # FileField.generate_filename @@ -110,7 +110,7 @@ def generate_filename(self, filename: str) -> str: return filename -class TimeoutS3Boto3Storage(S3Boto3Storage): +class TimeoutS3Storage(S3Storage): """Override boto3 default timeout values.""" def __init__(self, **settings): @@ -121,7 +121,7 @@ def __init__(self, **settings): ) -class VerbatimNameS3Storage(VerbatimNameStorageMixin, TimeoutS3Boto3Storage): +class VerbatimNameS3Storage(VerbatimNameStorageMixin, TimeoutS3Storage): @property def multipart_manager(self): return DandiBoto3MultipartManager(self) @@ -246,14 +246,14 @@ def create_s3_storage(bucket_name: str) -> Storage: """ Return a new Storage instance, compatible with the default Storage class. - This abstracts over differences between S3Boto3Storage and MinioStorage, + This abstracts over differences between S3Storage and MinioStorage, allowing either to be used as an additional non-default Storage. """ # For production, calling django.core.files.storage.get_storage_class is fine - # to return the storage class of S3Boto3Storage. + # to return the storage class of S3Storage. default_storage_class = get_storage_class() - if issubclass(default_storage_class, S3Boto3Storage): + if issubclass(default_storage_class, S3Storage): storage = VerbatimNameS3Storage(bucket_name=bucket_name) # Required to upload to the sponsored bucket storage.default_acl = 'bucket-owner-full-control' @@ -280,7 +280,7 @@ def create_s3_storage(bucket_name: str) -> Storage: storage = VerbatimNameMinioStorage( bucket_name=bucket_name, base_url=base_url, - # All S3Boto3Storage URLs are presigned, and the bucket typically is not public + # All S3Storage URLs are presigned, and the bucket typically is not public presign_urls=True, auto_create_bucket=True, auto_create_policy=True, diff --git a/dandiapi/api/views/asset.py b/dandiapi/api/views/asset.py index 8b4ea28ac..1cc3aa878 100644 --- a/dandiapi/api/views/asset.py +++ b/dandiapi/api/views/asset.py @@ -12,10 +12,10 @@ from dandiapi.zarr.models import ZarrArchive try: - from storages.backends.s3boto3 import S3Boto3Storage + from storages.backends.s3 import S3Storage except ImportError: # This should only be used for type interrogation, never instantiation - S3Boto3Storage = type('FakeS3Boto3Storage', (), {}) + S3Storage = type('FakeS3Storage', (), {}) try: from minio_storage.storage import MinioStorage except ImportError: diff --git a/dandiapi/conftest.py b/dandiapi/conftest.py index 228a9cbb4..b6dfe2978 100644 --- a/dandiapi/conftest.py +++ b/dandiapi/conftest.py @@ -4,7 +4,7 @@ import pytest from pytest_factoryboy import register from rest_framework.test import APIClient -from storages.backends.s3boto3 import S3Boto3Storage +from storages.backends.s3 import S3Storage from dandiapi.api.storage import create_s3_storage from dandiapi.api.tests.factories import ( @@ -84,7 +84,7 @@ def authenticated_api_client(user) -> APIClient: # storage fixtures are copied from django-s3-file-field test fixtures -def base_s3boto3_storage_factory(bucket_name: str) -> 'S3Boto3Storage': +def base_s3boto3_storage_factory(bucket_name: str) -> 'S3Storage': return create_s3_storage(bucket_name) @@ -109,12 +109,12 @@ def embargoed_minio_storage_factory() -> MinioStorage: @pytest.fixture -def s3boto3_storage() -> 'S3Boto3Storage': +def s3boto3_storage() -> 'S3Storage': return s3boto3_storage_factory() @pytest.fixture -def embargoed_s3boto3_storage() -> 'S3Boto3Storage': +def embargoed_s3boto3_storage() -> 'S3Storage': return s3boto3_storage_factory() @@ -132,7 +132,7 @@ def embargoed_minio_storage() -> MinioStorage: def storage(request, settings) -> Storage: storage_factory = request.param if storage_factory == s3boto3_storage_factory: - settings.DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' + settings.DEFAULT_FILE_STORAGE = 'storages.backends.s3.S3Storage' settings.AWS_S3_ACCESS_KEY_ID = settings.MINIO_STORAGE_ACCESS_KEY settings.AWS_S3_SECRET_ACCESS_KEY = settings.MINIO_STORAGE_SECRET_KEY settings.AWS_S3_REGION_NAME = 'test-region' diff --git a/setup.py b/setup.py index 9d95697c0..34fe429e4 100644 --- a/setup.py +++ b/setup.py @@ -66,8 +66,7 @@ 'django-composed-configuration[prod]>=0.22.0', # pin directly to a version since we're extending the private multipart interface 'django-s3-file-field[boto3]==0.3.2', - # TODO: unpin this when dandi is updating for breaking changes - 'django-storages[boto3]<=1.13.2', + 'django-storages[s3]>=1.14.2', 'gunicorn', # Development-only, but required # TODO: starting with v0.5.0, django-minio-storage requires v7