Merge pull request #88 from longguikeji/lh/file-storage

feat: add file storage config

infrastructure/views/file.py

@@ -14,6 +14,7 @@
     put_object,
     presign_get,
 )
+from oneid_meta.models.config import StorageConfig
 
 
 class FileCreateAPIView(generics.CreateAPIView):
@@ -31,14 +32,15 @@ def post(self, request, *args, **kwargs):
         uuid = uuid_utils.uuid4().hex
         suffix = os.path.splitext(file.name)[1]
         file_name = uuid + suffix
-        try:
+        storage_config = StorageConfig.get_current()
+        if storage_config.method == 'minio':
             put_object(
                 bucket_name=settings.MINIO_BUCKET,
                 object_name=file_name,
                 file_data=file,
                 length=file.size,
             )
-        except Exception:    # pylint: disable=broad-except
+        else:
             with open(settings.UPLOADFILES_PATH + file_name, 'wb') as f:
                 f.write(file.read())
         return Response({'file_name': file_name})
@@ -56,20 +58,20 @@ def get(self, request, filename):    # pylint: disable=unused-argument, no-self-
         '''
         download file
         '''
-
-        try:
+        storage_config = StorageConfig.get_current()
+        if storage_config.method == 'minio':
             url = presign_get(bucket_name=settings.MINIO_BUCKET,
                               object_name=filename,
                               response_headers={
                                   'response-content-disposition': 'attachment;filename=%s' % filename,
                               })
             return HttpResponseRedirect(url)
-        except Exception:    # pylint: disable=broad-except
-            filepath = settings.UPLOADFILES_PATH + filename
-            if os.path.exists(filepath):
-                data = open(filepath, 'rb').read()
-                res = HttpResponse(data)
-                res['Content-Type'] = 'application/octet-stream'
-                res['Content-Disposition'] = 'attachment;filename="{0}"'.format(filename)
-                return res
-            return Http404()
+
+        filepath = settings.UPLOADFILES_PATH + filename
+        if os.path.exists(filepath):
+            data = open(filepath, 'rb').read()
+            res = HttpResponse(data)
+            res['Content-Type'] = 'application/octet-stream'
+            res['Content-Disposition'] = 'attachment;filename="{0}"'.format(filename)
+            return res
+        return Http404()

oneid_meta/migrations/0066_minioconfig_storageconfig.py

@@ -0,0 +1,82 @@
+# Generated by Django 2.0.13 on 2019-11-25 11:40
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+import oneid_meta.models.config
+import uuid
+
+
+def init_storage_config(apps, schema_editor):
+
+    StorageConfig = apps.get_model('oneid_meta', 'StorageConfig')
+    MinioConfig = apps.get_model('oneid_meta', 'MinioConfig')
+    Site = apps.get_model('sites', 'Site')
+
+    site, _ = Site.objects.get_or_create(id=settings.SITE_ID)
+    storage_config, _ = StorageConfig.objects.get_or_create(site=site)
+    minio_config, _ = MinioConfig.objects.get_or_create(site=site)
+
+    if not settings.TESTING:
+        minio_config.end_point = getattr(settings, 'MINIO_ENDPOINT', '')
+        minio_config.access_key = getattr(settings, 'MINIO_ACCESS_KEY', '')
+        minio_config.secret_key = getattr(settings, 'MINIO_SECRET_KEY', '')
+        minio_config.secure = getattr(settings, 'MINIO_SECURE', '')
+        minio_config.location = getattr(settings, 'MINIO_LOCATION', '')
+        minio_config.bucket = getattr(settings, 'MINIO_BUCKET', '')
+        minio_config.save()
+
+        if minio_config.access_key != '' and minio_config.secret_key != '':
+            storage_config.method = 'minio'
+            storage_config.save()
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sites', '0002_alter_domain_unique'),
+        ('oneid_meta', '0065_auto_20191115_1422'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='MinioConfig',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('uuid', models.UUIDField(default=uuid.uuid4, unique=True)),
+                ('is_del', models.BooleanField(default=False, verbose_name='是否删除')),
+                ('is_active', models.BooleanField(default=True, verbose_name='是否可用')),
+                ('updated', models.DateTimeField(auto_now=True, null=True, verbose_name='更新时间')),
+                ('created', models.DateTimeField(auto_now_add=True, null=True, verbose_name='创建时间')),
+                ('end_point', models.CharField(blank=True, default='', max_length=225, verbose_name='END POINT')),
+                ('access_key', models.CharField(blank=True, default='', max_length=225, verbose_name='ACCESS KEY')),
+                ('secret_key', models.CharField(blank=True, default='', max_length=225, verbose_name='SECRET KEY')),
+                ('secure', models.BooleanField(default=True, max_length=225, verbose_name='SECURE')),
+                ('location', models.CharField(blank=True, default='', max_length=225, verbose_name='LOCATION')),
+                ('bucket', models.CharField(blank=True, default='', max_length=225, verbose_name='BUCKET')),
+                ('site', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='minio_config', to='sites.Site')),
+            ],
+            options={
+                'abstract': False,
+            },
+            bases=(models.Model, oneid_meta.models.config.SingletonConfigMixin),
+        ),
+        migrations.CreateModel(
+            name='StorageConfig',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('uuid', models.UUIDField(default=uuid.uuid4, unique=True)),
+                ('is_del', models.BooleanField(default=False, verbose_name='是否删除')),
+                ('is_active', models.BooleanField(default=True, verbose_name='是否可用')),
+                ('updated', models.DateTimeField(auto_now=True, null=True, verbose_name='更新时间')),
+                ('created', models.DateTimeField(auto_now_add=True, null=True, verbose_name='创建时间')),
+                ('method', models.CharField(blank=True, default='local', max_length=225, verbose_name='method')),
+                ('site', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='storage_config', to='sites.Site')),
+            ],
+            options={
+                'abstract': False,
+            },
+            bases=(models.Model, oneid_meta.models.config.SingletonConfigMixin),
+        ),
+        migrations.RunPython(init_storage_config),
+    ]

oneid_meta/models/__init__.py

@@ -35,7 +35,8 @@
 )
 
 from oneid_meta.models.config import (CompanyConfig, AccountConfig, SMSConfig, DingConfig, CustomField, NativeField,
-                                      EmailConfig, AlipayConfig, WorkWechatConfig, WechatConfig, QQConfig)
+                                      EmailConfig, AlipayConfig, WorkWechatConfig, WechatConfig, QQConfig,
+                                      StorageConfig, MinioConfig)
 
 from oneid_meta.models.event import (
     Invitation, )

oneid_meta/models/config.py

@@ -443,3 +443,38 @@ def check_valid(self):
 
     def __str__(self):
         return f'QQConfig[{self.id}]'    # pylint: disable=no-member
+
+
+class StorageConfig(BaseModel, SingletonConfigMixin):
+    '''
+    文件存储方式
+    '''
+    site = models.OneToOneField(Site, related_name='storage_config', on_delete=models.CASCADE)
+
+    method = models.CharField(max_length=225, blank=True, default='local', verbose_name='method')
+
+    def __str__(self):
+        return f'Method[{self.id}]: {self.display_name}'    # pylint: disable=no-member
+
+
+class MinioConfig(BaseModel, SingletonConfigMixin):
+    '''
+    Minio配置信息
+    '''
+    site = models.OneToOneField(Site, related_name='minio_config', on_delete=models.CASCADE)
+
+    end_point = models.CharField(max_length=225, blank=True, default='', verbose_name='END POINT')
+    access_key = models.CharField(max_length=225, blank=True, default='', verbose_name='ACCESS KEY')
+    secret_key = models.CharField(max_length=225, blank=True, default='', verbose_name='SECRET KEY')
+    secure = models.BooleanField(max_length=225, blank=True, default=True, verbose_name='SECURE')
+    location = models.CharField(max_length=225, blank=True, default='', verbose_name='LOCATION')
+    bucket = models.CharField(max_length=225, blank=True, default='', verbose_name='BUCKET')
+
+    def check_valid(self):    #pylint: disable=no-self-use
+        '''
+        检查配置是否有效
+        '''
+        return True
+
+    def __str__(self):
+        return f'Minio[{self.id}]: {self.display_name}'    # pylint: disable=no-member

siteapi/v1/blueprint.md

@@ -453,6 +453,16 @@ FORMAT: 1A
 + is_visible (boolean)
 + is_visible_editable (boolean)
 
+## StorageConfig (object)
++ method (string)
++ minio_config (object)
+    + end_point (string)
+    + access_key (string)
+    + secret_key (string)
+    + secure (boolean)
+    + location (string)
+    + bucket (string)
+
 ## CompanyMetaInfo (CompanyConfig)
 + display_name (string)
 
@@ -1940,6 +1950,18 @@ TODO: 可见权限的处理
 + Response 200 (application/json)
     + Attributes (NativeField)
 
+## 文件存储 [/config/storage/]
+
+### 获取文件存储方式和minio配置信息 [GET]
++ Response 200 (application/json)
+    + Attributes (StorageConfig)
+
+### 修改文件存储方式和minio配置信息 [PATCH]
++ request JSON Message
+    + Attributes (StorageConfig)
++ Response 200 (application/json)
+    + Attributes (StorageConfig)
+
 # Group Meta
 
 ## 基本信息 [/meta/]

siteapi/v1/serializers/config.py

@@ -6,7 +6,8 @@
 from rest_framework import serializers
 from rest_framework.exceptions import ValidationError
 from oneid_meta.models import (CompanyConfig, DingConfig, AlipayConfig, User, Dept, CustomField, NativeField,
-                               AccountConfig, SMSConfig, EmailConfig, WorkWechatConfig, WechatConfig, QQConfig)
+                               AccountConfig, SMSConfig, EmailConfig, WorkWechatConfig, WechatConfig, QQConfig,
+                               StorageConfig, MinioConfig)
 from common.django.drf.serializer import DynamicFieldsModelSerializer
 from infrastructure.serializers.sms import SMSClaimSerializer
 from siteapi.v1.views.utils import gen_uid
@@ -99,6 +100,68 @@ class Meta:    # pylint: disable=missing-docstring
         )
 
 
+class MinioConfigSerializer(DynamicFieldsModelSerializer):
+    '''
+    serializer for MinioConfig
+    '''
+    class Meta:    # pylint: disable=missing-docstring
+        model = MinioConfig
+
+        fields = (
+            'end_point',
+            'access_key',
+            'secret_key',
+            'secure',
+            'location',
+            'bucket',
+        )
+
+
+class StorageMethodSerializer(DynamicFieldsModelSerializer):
+    '''
+    serializer for StorageMethod
+    '''
+    class Meta:    # pylint: disable=missing-docstring
+        model = StorageConfig
+
+        fields = ('method', )
+
+
+class StorageConfigSerializer(DynamicFieldsModelSerializer):
+    '''
+    serializer for StorageConfig
+    '''
+    minio_config = MinioConfigSerializer(many=False, required=False)
+    method = serializers.CharField(source='storage_config.method')
+
+    class Meta:    # pylint: disable=missing-docstring
+        model = Site
+
+        fields = (
+            'minio_config',
+            'method',
+        )
+
+    @transaction.atomic()
+    def update(self, instance, validated_data):    # pylint: disable=too-many-locals, too-many-statements, too-many-branches
+
+        storage_config_data = validated_data.pop('storage_config', None)
+        if storage_config_data:
+            serializer = StorageMethodSerializer(StorageConfig.get_current(), storage_config_data, partial=True)
+            serializer.is_valid(raise_exception=True)
+            serializer.save()
+
+        minio_config_data = validated_data.pop('minio_config', None)
+        if minio_config_data:
+            serializer = MinioConfigSerializer(MinioConfig.get_current(), minio_config_data, partial=True)
+            serializer.is_valid(raise_exception=True)
+            serializer.save()
+
+        instance.refresh_from_db()
+
+        return instance
+
+
 class PublicAccountConfigSerializer(DynamicFieldsModelSerializer):
     '''
     serializer for AccountConfig

siteapi/v1/tests/test_config.py

@@ -399,3 +399,62 @@ def test_native_field(self):
         res = self.client.json_patch(reverse('siteapi:native_field_detail', args=('user', email_uuid)),
                                      data={'is_visible': False})
         self.assertEqual(res.status_code, 400)
+
+
+class ConfigStorageTestCase(TestCase):
+    def test_storage_field(self):
+        res = self.client.get(reverse('siteapi:storage_config'))
+        expect = {
+            'method': 'local',
+            'minio_config': {
+                'end_point': '',
+                'access_key': '',
+                'secret_key': '',
+                'secure': True,
+                'location': '',
+                'bucket': '',
+            },
+        }
+        self.assertEqual(res.json(), expect)
+
+        method = 'minio'
+        minio_config = {
+            'end_point': 'localhost:12345',
+            'access_key': '123',
+            'secret_key': '123',
+            'secure': False,
+            'location': 'us-east-4',
+            'bucket': 'arkid',
+        }
+
+        res = self.client.json_patch(reverse('siteapi:storage_config'),
+                                     data={
+                                         'method': method,
+                                         'minio_config': minio_config,
+                                     })
+        expect = {
+            'method': 'minio',
+            'minio_config': {
+                'end_point': 'localhost:12345',
+                'access_key': '123',
+                'secret_key': '123',
+                'secure': False,
+                'location': 'us-east-4',
+                'bucket': 'arkid',
+            }
+        }
+        self.assertEqual(res.json(), expect)
+
+        res = self.client.get(reverse('siteapi:storage_config'))
+        expect = {
+            'method': 'minio',
+            'minio_config': {
+                'end_point': 'localhost:12345',
+                'access_key': '123',
+                'secret_key': '123',
+                'secure': False,
+                'location': 'us-east-4',
+                'bucket': 'arkid',
+            }
+        }
+        self.assertEqual(res.json(), expect)

siteapi/v1/urls.py

@@ -20,6 +20,7 @@
     CustomFieldDetailAPIView,
     NativeFieldListAPIView,
     NativeFieldDetailAPIView,
+    StorageConfigAPIView,
 )
 
 from siteapi.v1.views import (
@@ -149,6 +150,7 @@
     # config
     url(r'^config/$', ConfigAPIView.as_view(), name='config'),
     url(r'^config/admin/$', AdminAPIView.as_view(), name='alter_admin'),
+    url(r'^config/storage/$', StorageConfigAPIView.as_view(), name='storage_config'),
     url(r'^config/custom/field/(?P<field_subject>[a-z_]+)/$',
         CustomFieldListCreateAPIView.as_view(),
         name='custom_field_list'),