Add support for Azure Blob Storage connection string authentication (#…

…4649)

This is a change to support Connection String authentication for Azure
Blob Storage. For example, Connection String Authentication allows for
Azure Blob Storage running on the IoT Edge of an edge device.

### How has this been tested?
<!-- Please describe in detail how you tested your changes.
Include details of your testing environment, and the tests you ran to
see how your change affects other areas of the code, etc. -->

I tested it by actually running it as shown in the next image, I could
not find where the E2E test for cloudstorage was done.
<img width="1001" alt="cvat_blob_connection_string"
src="https://user-images.githubusercontent.com/10334593/169458508-cfa4030a-578f-4aad-bfd9-fa01c9ca8230.png">

CHANGELOG.md

@@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## \[2.5.0] - Unreleased
 ### Added
-- TDB
+- Add support for Azure Blob Storage connection string authentication(<https://github.com/openvinotoolkit/cvat/pull/4649>)
 
 ### Changed
 - TDB

cvat-core/src/cloud-storage.ts

@@ -30,6 +30,7 @@ interface RawCloudStorageData {
     secret_key?: string,
     session_token?: string,
     key_file?: File,
+    connection_string?: string,
     specific_attributes?: string,
     owner?: any,
     created_date?: string,
@@ -47,6 +48,7 @@ export default class CloudStorage {
     public secretKey: string;
     public token: string;
     public keyFile: File;
+    public connectionString: string;
     public resource: string;
     public manifestPath: string;
     public provider_type: CloudStorageProviderType;
@@ -70,6 +72,7 @@ export default class CloudStorage {
             secret_key: undefined,
             session_token: undefined,
             key_file: undefined,
+            connection_string: undefined,
             specific_attributes: undefined,
             owner: undefined,
             created_date: undefined,
@@ -144,6 +147,13 @@ export default class CloudStorage {
                         }
                     },
                 },
+                connectionString: {
+                    get: () => data.connection_string,
+                    set: (value) => {
+                        validateNotEmptyString(value);
+                        data.connection_string = value;
+                    },
+                },
                 resource: {
                     get: () => data.resource,
                     set: (value) => {
@@ -282,6 +292,10 @@ Object.defineProperties(CloudStorage.prototype.save, {
                     data.key_file = cloudStorageInstance.keyFile;
                 }
 
+                if (cloudStorageInstance.connectionString) {
+                    data.connection_string = cloudStorageInstance.connectionString;
+                }
+
                 if (cloudStorageInstance.specificAttributes !== undefined) {
                     data.specific_attributes = cloudStorageInstance.specificAttributes;
                 }

cvat-ui/src/components/create-cloud-storage-page/cloud-storage-form.tsx

@@ -29,8 +29,8 @@ export interface Props {
     cloudStorage?: CloudStorage;
 }
 
-type CredentialsFormNames = 'key' | 'secret_key' | 'account_name' | 'session_token';
-type CredentialsCamelCaseNames = 'key' | 'secretKey' | 'accountName' | 'sessionToken';
+type CredentialsFormNames = 'key' | 'secret_key' | 'account_name' | 'session_token' | 'connection_string';
+type CredentialsCamelCaseNames = 'key' | 'secretKey' | 'accountName' | 'sessionToken' | 'connectionString';
 
 interface CloudStorageForm {
     credentials_type: CredentialsType;
@@ -43,6 +43,7 @@ interface CloudStorageForm {
     secret_key?: string;
     SAS_token?: string;
     key_file?: File;
+    connection_string?: string;
     description?: string;
     region?: string;
     prefix?: string;
@@ -77,12 +78,14 @@ export default function CreateCloudStorageForm(props: Props): JSX.Element {
         key: 'X'.repeat(128),
         secretKey: 'X'.repeat(40),
         keyFile: new File([], 'fakeKey.json'),
+        connectionString: 'X'.repeat(400),
     };
 
     const [keyVisibility, setKeyVisibility] = useState(false);
     const [secretKeyVisibility, setSecretKeyVisibility] = useState(false);
     const [sessionTokenVisibility, setSessionTokenVisibility] = useState(false);
     const [accountNameVisibility, setAccountNameVisibility] = useState(false);
+    const [connectionStringVisibility, setConnectionStringVisibility] = useState(false);
 
     const [manifestNames, setManifestNames] = useState<string[]>([]);
 
@@ -111,6 +114,8 @@ export default function CreateCloudStorageForm(props: Props): JSX.Element {
             fieldsValue.secret_key = fakeCredentialsData.secretKey;
         } else if (cloudStorage.credentialsType === CredentialsType.KEY_FILE_PATH) {
             setUploadedKeyFile(fakeCredentialsData.keyFile);
+        } else if (cloudStorage.credentialsType === CredentialsType.CONNECTION_STRING) {
+            fieldsValue.connection_string = fakeCredentialsData.connectionString;
         }
 
         if (cloudStorage.specificAttributes) {
@@ -261,6 +266,9 @@ export default function CreateCloudStorageForm(props: Props): JSX.Element {
             if (cloudStorageData.session_token === fakeCredentialsData.sessionToken) {
                 delete cloudStorageData.session_token;
             }
+            if (cloudStorageData.connection_string === fakeCredentialsData.connectionString) {
+                delete cloudStorageData.connection_string;
+            }
             dispatch(updateCloudStorageAsync(cloudStorageData));
         } else {
             dispatch(createCloudStorageAsync(cloudStorageData));
@@ -414,6 +422,25 @@ export default function CreateCloudStorageForm(props: Props): JSX.Element {
             );
         }
 
+        if (providerType === ProviderType.AZURE_CONTAINER && credentialsType === CredentialsType.CONNECTION_STRING) {
+            return (
+                <>
+                    <Form.Item
+                        label='Connection string'
+                        name='connection_string'
+                        rules={[{ required: true, message: 'Please, specify your connection string' }]}
+                        {...internalCommonProps}
+                    >
+                        <Input.Password
+                            maxLength={440}
+                            visibilityToggle={connectionStringVisibility}
+                            onChange={() => setConnectionStringVisibility(true)}
+                        />
+                    </Form.Item>
+                </>
+            );
+        }
+
         if (providerType === ProviderType.GOOGLE_CLOUD_STORAGE && credentialsType === CredentialsType.KEY_FILE_PATH) {
             return (
                 <Form.Item
@@ -541,6 +568,7 @@ export default function CreateCloudStorageForm(props: Props): JSX.Element {
                             Account name and SAS token
                         </Select.Option>
                         <Select.Option value={CredentialsType.ANONYMOUS_ACCESS}>Anonymous access</Select.Option>
+                        <Select.Option value={CredentialsType.CONNECTION_STRING}>Connection string</Select.Option>
                     </Select>
                 </Form.Item>
 

cvat-ui/src/utils/enums.tsx

@@ -12,6 +12,7 @@ export enum CredentialsType {
     KEY_SECRET_KEY_PAIR = 'KEY_SECRET_KEY_PAIR',
     ACCOUNT_NAME_TOKEN_PAIR = 'ACCOUNT_NAME_TOKEN_PAIR',
     ANONYMOUS_ACCESS = 'ANONYMOUS_ACCESS',
+    CONNECTION_STRING = 'CONNECTION_STRING',
     KEY_FILE_PATH = 'KEY_FILE_PATH',
 }
 

cvat/apps/engine/cloud_provider.py

@@ -16,7 +16,7 @@
 from botocore.exceptions import ClientError
 from botocore.handlers import disable_signing
 
-from azure.storage.blob import BlobServiceClient
+from azure.storage.blob import BlobServiceClient, ContainerClient
 from azure.core.exceptions import ResourceExistsError, HttpResponseError
 from azure.storage.blob import PublicAccess
 
@@ -26,6 +26,8 @@
 from cvat.apps.engine.log import slogger
 from cvat.apps.engine.models import CredentialsTypeChoice, CloudProviderChoice
 
+from typing import Optional
+
 class Status(str, Enum):
     AVAILABLE = 'AVAILABLE'
     NOT_FOUND = 'NOT_FOUND'
@@ -180,7 +182,8 @@ def get_cloud_storage_instance(cloud_provider, resource, credentials, specific_a
         instance = AzureBlobContainer(
             container=resource,
             account_name=credentials.account_name,
-            sas_token=credentials.session_token
+            sas_token=credentials.session_token,
+            connection_string=credentials.connection_string
         )
     elif cloud_provider == CloudProviderChoice.GOOGLE_CLOUD_STORAGE:
         instance = GoogleCloudStorage(
@@ -382,26 +385,36 @@ class AzureBlobContainer(_CloudStorage):
     class Effect:
         pass
 
-    def __init__(self, container, account_name, sas_token=None):
+    def __init__(
+        self,
+        container: str,
+        account_name: Optional[str] = None,
+        sas_token: Optional[str] = None,
+        connection_string: Optional[str] = None
+    ):
         super().__init__()
         self._account_name = account_name
-        if sas_token:
+        if connection_string:
+            self._blob_service_client = BlobServiceClient.from_connection_string(connection_string)
+        elif sas_token:
             self._blob_service_client = BlobServiceClient(account_url=self.account_url, credential=sas_token)
         else:
             self._blob_service_client = BlobServiceClient(account_url=self.account_url)
         self._container_client = self._blob_service_client.get_container_client(container)
 
     @property
-    def container(self):
+    def container(self) -> ContainerClient:
         return self._container_client
 
     @property
-    def name(self):
+    def name(self) -> str:
         return self._container_client.container_name
 
     @property
-    def account_url(self):
-        return "{}.blob.core.windows.net".format(self._account_name)
+    def account_url(self) -> Optional[str]:
+        if self._account_name:
+            return "{}.blob.core.windows.net".format(self._account_name)
+        return None
 
     def create(self):
         try:
@@ -603,7 +616,7 @@ def supported_actions(self):
         pass
 
 class Credentials:
-    __slots__ = ('key', 'secret_key', 'session_token', 'account_name', 'key_file_path', 'credentials_type')
+    __slots__ = ('key', 'secret_key', 'session_token', 'account_name', 'key_file_path', 'credentials_type', 'connection_string')
 
     def __init__(self, **credentials):
         self.key = credentials.get('key', '')
@@ -612,6 +625,7 @@ def __init__(self, **credentials):
         self.account_name = credentials.get('account_name', '')
         self.key_file_path = credentials.get('key_file_path', '')
         self.credentials_type = credentials.get('credentials_type', None)
+        self.connection_string = credentials.get('connection_string', None)
 
     def convert_to_db(self):
         converted_credentials = {
@@ -620,6 +634,7 @@ def convert_to_db(self):
             CredentialsTypeChoice.ACCOUNT_NAME_TOKEN_PAIR : " ".join([self.account_name, self.session_token]),
             CredentialsTypeChoice.KEY_FILE_PATH: self.key_file_path,
             CredentialsTypeChoice.ANONYMOUS_ACCESS: "" if not self.account_name else self.account_name,
+            CredentialsTypeChoice.CONNECTION_STRING: self.connection_string,
         }
         return converted_credentials[self.credentials_type]
 
@@ -634,6 +649,8 @@ def convert_from_db(self, credentials):
             self.account_name = credentials.get('value')
         elif self.credentials_type == CredentialsTypeChoice.KEY_FILE_PATH:
             self.key_file_path = credentials.get('value')
+        elif self.credentials_type == CredentialsTypeChoice.CONNECTION_STRING:
+            self.connection_string = credentials.get('value')
         else:
             raise NotImplementedError('Found {} not supported credentials type'.format(self.credentials_type))
 
@@ -657,6 +674,9 @@ def mapping_with_new_values(self, credentials):
         elif self.credentials_type == CredentialsTypeChoice.KEY_FILE_PATH:
             self.reset(exclusion={'key_file_path'})
             self.key_file_path = credentials.get('key_file_path', self.key_file_path)
+        elif self.credentials_type == CredentialsTypeChoice.CONNECTION_STRING:
+            self.reset(exclusion={'connection_string'})
+            self.connection_string = credentials.get('connection_string', self.connection_string)
         else:
             raise NotImplementedError('Mapping credentials: unsupported credentials type')
 

cvat/apps/engine/migrations/0066_alter_cloudstorage_credentials_type.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.18 on 2023-03-13 21:34
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('engine', '0065_auto_20230221_0931'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='cloudstorage',
+            name='credentials_type',
+            field=models.CharField(choices=[('KEY_SECRET_KEY_PAIR', 'KEY_SECRET_KEY_PAIR'), ('ACCOUNT_NAME_TOKEN_PAIR', 'ACCOUNT_NAME_TOKEN_PAIR'), ('KEY_FILE_PATH', 'KEY_FILE_PATH'), ('ANONYMOUS_ACCESS', 'ANONYMOUS_ACCESS'), ('CONNECTION_STRING', 'CONNECTION_STRING')], max_length=29),
+        ),
+    ]

cvat/apps/engine/models.py

@@ -763,6 +763,7 @@ class CredentialsTypeChoice(str, Enum):
     ACCOUNT_NAME_TOKEN_PAIR = 'ACCOUNT_NAME_TOKEN_PAIR' # nosec
     KEY_FILE_PATH = 'KEY_FILE_PATH'
     ANONYMOUS_ACCESS = 'ANONYMOUS_ACCESS'
+    CONNECTION_STRING = 'CONNECTION_STRING'
 
     @classmethod
     def choices(cls):

cvat/apps/engine/serializers.py

@@ -1415,13 +1415,14 @@ class CloudStorageWriteSerializer(serializers.ModelSerializer):
     key_file = serializers.FileField(required=False)
     account_name = serializers.CharField(max_length=24, allow_blank=True, required=False)
     manifests = ManifestSerializer(many=True, default=[])
+    connection_string = serializers.CharField(max_length=440, allow_blank=True, required=False)
 
     class Meta:
         model = models.CloudStorage
         fields = (
             'provider_type', 'resource', 'display_name', 'owner', 'credentials_type',
             'created_date', 'updated_date', 'session_token', 'account_name', 'key',
-            'secret_key', 'key_file', 'specific_attributes', 'description', 'id',
+            'secret_key', 'connection_string', 'key_file', 'specific_attributes', 'description', 'id',
             'manifests', 'organization'
         )
         read_only_fields = ('created_date', 'updated_date', 'owner', 'organization')
@@ -1439,8 +1440,8 @@ def validate_specific_attributes(self, value):
     def validate(self, attrs):
         provider_type = attrs.get('provider_type')
         if provider_type == models.CloudProviderChoice.AZURE_CONTAINER:
-            if not attrs.get('account_name', ''):
-                raise serializers.ValidationError('Account name for Azure container was not specified')
+            if not attrs.get('account_name', '') and not attrs.get('connection_string', ''):
+                raise serializers.ValidationError('Account name or connection string for Azure container was not specified')
         return attrs
 
     @staticmethod
@@ -1478,7 +1479,8 @@ def create(self, validated_data):
             secret_key=validated_data.pop('secret_key', ''),
             session_token=validated_data.pop('session_token', ''),
             key_file_path=temporary_file,
-            credentials_type = validated_data.get('credentials_type')
+            credentials_type = validated_data.get('credentials_type'),
+            connection_string = validated_data.pop('connection_string', '')
         )
         details = {
             'resource': validated_data.get('resource'),

cvat/schema.yml

@@ -417,6 +417,7 @@ paths:
           - ACCOUNT_NAME_TOKEN_PAIR
           - KEY_FILE_PATH
           - ANONYMOUS_ACCESS
+          - CONNECTION_STRING
       - name: filter
         required: false
         in: query
@@ -7186,6 +7187,9 @@ components:
         secret_key:
           type: string
           maxLength: 44
+        connection_string:
+          type: string
+          maxLength: 440
         key_file:
           type: string
           format: binary
@@ -7256,6 +7260,7 @@ components:
       - ACCOUNT_NAME_TOKEN_PAIR
       - KEY_FILE_PATH
       - ANONYMOUS_ACCESS
+      - CONNECTION_STRING
       type: string
     DataMetaRead:
       type: object
@@ -8651,6 +8656,9 @@ components:
         secret_key:
           type: string
           maxLength: 44
+        connection_string:
+          type: string
+          maxLength: 440
         key_file:
           type: string
           format: binary