Migrate msrest to azure-core

plugins/module_utils/azure_rm_common.py

@@ -221,13 +221,6 @@ def default_api_version(self):
     HAS_PACKAGING_VERSION = False
     HAS_PACKAGING_VERSION_EXC = traceback.format_exc()
 
-# NB: packaging issue sometimes cause msrestazure not to be installed, check it separately
-try:
-    from msrest.serialization import Serializer
-except ImportError:
-    HAS_MSRESTAZURE_EXC = traceback.format_exc()
-    HAS_MSRESTAZURE = False
-
 try:
     from enum import Enum
     from msrestazure.azure_active_directory import AADTokenCredentials
@@ -268,9 +261,7 @@ def default_api_version(self):
     import azure.mgmt.automation.models as AutomationModel
     from azure.mgmt.iothub import IotHubClient
     from azure.mgmt.iothub import models as IoTHubModels
-    from msrest.service_client import ServiceClient
     from msrestazure import AzureConfiguration
-    from msrest.authentication import Authentication
     from azure.mgmt.resource.locks import ManagementLockClient
     from azure.mgmt.recoveryservicesbackup import RecoveryServicesBackupClient
     import azure.mgmt.recoveryservicesbackup.models as RecoveryServicesBackupModels
@@ -619,18 +610,7 @@ def serialize_obj(self, obj, class_name, enum_modules=None):
         :param enum_modules: List of module names to build enum dependencies from.
         :return: serialized result
         '''
-        enum_modules = [] if enum_modules is None else enum_modules
-
-        dependencies = dict()
-        if enum_modules:
-            for module_name in enum_modules:
-                mod = importlib.import_module(module_name)
-                for mod_class_name, mod_class_obj in inspect.getmembers(mod, predicate=inspect.isclass):
-                    dependencies[mod_class_name] = mod_class_obj
-            self.log("dependencies: ")
-            self.log(str(dependencies))
-        serializer = Serializer(classes=dependencies)
-        return serializer.body(obj, class_name, keep_readonly=True)
+        return obj.as_dict()
 
     def get_poller_result(self, poller, wait=5):
         '''
@@ -998,13 +978,6 @@ def generate_sas_token(self, **kwags):
             result['skn'] = policy
         return 'SharedAccessSignature ' + urlencode(result)
 
-    def get_data_svc_client(self, **kwags):
-        url = kwags.get('base_url', None)
-        config = AzureConfiguration(base_url='https://{0}'.format(url))
-        config.credentials = AzureSASAuthentication(token=self.generate_sas_token(**kwags))
-        config = self.add_user_agent(config)
-        return ServiceClient(creds=config.credentials, config=config)
-
     def get_subnet_detail(self, subnet_id):
         vnet_detail = subnet_id.split('/Microsoft.Network/virtualNetworks/')[1].split('/subnets/')
         return dict(
@@ -1454,22 +1427,6 @@ def datafactory_model(self):
         return DataFactoryModel
 
 
-class AzureSASAuthentication(Authentication):
-    """Simple SAS Authentication.
-    An implementation of Authentication in
-    https://github.com/Azure/msrest-for-python/blob/0732bc90bdb290e5f58c675ffdd7dbfa9acefc93/msrest/authentication.py
-
-    :param str token: SAS token
-    """
-    def __init__(self, token):
-        self.token = token
-
-    def signed_session(self):
-        session = super(AzureSASAuthentication, self).signed_session()
-        session.headers['Authorization'] = self.token
-        return session
-
-
 class AzureRMAuthException(Exception):
     pass
 

plugins/module_utils/azure_rm_common_rest.py

@@ -13,44 +13,46 @@
 
 try:
     from msrestazure.azure_exceptions import CloudError
-    from msrestazure.azure_configuration import AzureConfiguration
-    from msrest.service_client import ServiceClient
-    from msrest.pipeline import ClientRawResponse
-    from msrest.polling import LROPoller
-    from msrestazure.polling.arm_polling import ARMPolling
+    from azure.core._pipeline_client import PipelineClient
+    from azure.core.polling import LROPoller
+    from azure.core.pipeline import PipelineResponse
+    from azure.core.pipeline.policies import BearerTokenCredentialPolicy
+    from azure.mgmt.core.polling.arm_polling import ARMPolling
     import uuid
     import json
+    from azure.core.configuration import Configuration
 except ImportError:
     # This is handled in azure_rm_common
-    AzureConfiguration = object
+    Configuration = object
 
 ANSIBLE_USER_AGENT = 'Ansible/{0}'.format(ANSIBLE_VERSION)
 
 
-class GenericRestClientConfiguration(AzureConfiguration):
+class GenericRestClientConfiguration(Configuration):
 
-    def __init__(self, credentials, subscription_id, base_url=None):
+    def __init__(self, credential, subscription_id, credential_scopes=None, base_url=None):
 
-        if credentials is None:
+        if credential is None:
             raise ValueError("Parameter 'credentials' must not be None.")
         if subscription_id is None:
             raise ValueError("Parameter 'subscription_id' must not be None.")
         if not base_url:
             base_url = 'https://management.azure.com'
+        if not credential_scopes:
+            credential_scopes = 'https://management.azure.com/.default'
 
-        super(GenericRestClientConfiguration, self).__init__(base_url)
+        super(GenericRestClientConfiguration, self).__init__()
 
-        self.add_user_agent(ANSIBLE_USER_AGENT)
-
-        self.credentials = credentials
+        self.credentials = credential
         self.subscription_id = subscription_id
+        self.authentication_policy = BearerTokenCredentialPolicy(credential, credential_scopes)
 
 
 class GenericRestClient(object):
 
-    def __init__(self, credentials, subscription_id, base_url=None):
-        self.config = GenericRestClientConfiguration(credentials, subscription_id, base_url)
-        self._client = ServiceClient(self.config.credentials, self.config)
+    def __init__(self, credential, subscription_id, base_url=None, credential_scopes=None):
+        self.config = GenericRestClientConfiguration(credential, subscription_id, credential_scopes[0])
+        self._client = PipelineClient(base_url, config=self.config)
         self.models = None
 
     def query(self, url, method, query_parameters, header_parameters, body, expected_status_codes, polling_timeout, polling_interval):
@@ -65,21 +67,21 @@ def query(self, url, method, query_parameters, header_parameters, body, expected
         header_parameters['x-ms-client-request-id'] = str(uuid.uuid1())
 
         if method == 'GET':
-            request = self._client.get(url, query_parameters)
+            request = self._client.get(url, query_parameters, header_parameters, body)
         elif method == 'PUT':
-            request = self._client.put(url, query_parameters)
+            request = self._client.put(url, query_parameters, header_parameters, body)
         elif method == 'POST':
-            request = self._client.post(url, query_parameters)
+            request = self._client.post(url, query_parameters, header_parameters, body)
         elif method == 'HEAD':
-            request = self._client.head(url, query_parameters)
+            request = self._client.head(url, query_parameters, header_parameters, body)
         elif method == 'PATCH':
-            request = self._client.patch(url, query_parameters)
+            request = self._client.patch(url, query_parameters, header_parameters, body)
         elif method == 'DELETE':
-            request = self._client.delete(url, query_parameters)
+            request = self._client.delete(url, query_parameters, header_parameters, body)
         elif method == 'MERGE':
-            request = self._client.merge(url, query_parameters)
+            request = self._client.merge(url, query_parameters, header_parameters, body)
 
-        response = self._client.send(request, header_parameters, body, **operation_config)
+        response = self._client.send_request(request, **operation_config)
 
         if response.status_code not in expected_status_codes:
             exp = CloudError(response)
@@ -89,7 +91,7 @@ def query(self, url, method, query_parameters, header_parameters, body, expected
             def get_long_running_output(response):
                 return response
             poller = LROPoller(self._client,
-                               ClientRawResponse(None, response),
+                               PipelineResponse(None, response, None),
                                get_long_running_output,
                                ARMPolling(polling_interval, **operation_config))
             response = self.get_poller_result(poller, polling_timeout)

plugins/modules/azure_rm_apimanagement.py

@@ -544,6 +544,7 @@ def exec_module(self, **kwargs):
         response = None
 
         self.mgmt_client = self.get_mgmt_svc_client(GenericRestClient,
+                                                    is_track2=True,
                                                     base_url=self._cloud_environment.endpoints.resource_manager)
 
         old_response = self.get_resource()
@@ -613,9 +614,9 @@ def create_and_update_resource(self):
             self.log('Error while creating/updating the Api instance.')
             self.fail('Error creating the Api instance: {0}'.format(str(exc)))
         try:
-            response = json.loads(response.text)
+            response = json.loads(response.body())
         except Exception:
-            response = {'text': response.text}
+            response = {'text': response.context['deserialized_data']}
 
         return response
 
@@ -653,7 +654,7 @@ def get_resource(self):
                 30,
             )
             isFound = True
-            response = json.loads(response.text)
+            response = json.loads(response.body())
             self.log("Response : {0}".format(response))
         except CloudError as e:
             self.log('Could not find the Api instance from the given parameters.')

plugins/modules/azure_rm_apimanagement_info.py

@@ -163,6 +163,7 @@ def exec_module(self, **kwargs):
                 self.body[key] = kwargs[key]
 
         self.mgmt_client = self.get_mgmt_svc_client(GenericRestClient,
+                                                    is_track2=True,
                                                     base_url=self._cloud_environment.endpoints.resource_manager)
 
         if (self.resource_group is not None and
@@ -213,7 +214,7 @@ def get_api_info(self):
         except CloudError as e:
             self.log('Could not get the information.{0}'.format(e))
         try:
-            response = json.loads(response.text)
+            response = json.loads(response.body())
         except Exception:
             return None
 
@@ -236,7 +237,7 @@ def listbytags(self):
         except CloudError as e:
             self.log('Could not get info for the given api tags {0}'.format(e))
         try:
-            response = json.loads(response.text)
+            response = json.loads(response.body())
         except Exception:
             return None
 
@@ -256,7 +257,7 @@ def listbyservice(self):
                 600,
                 30,
             )
-            response = json.loads(response.text)
+            response = json.loads(response.body())
         except CloudError as e:
             self.log('Could not get info for a given services.{0}'.format(e))
         try:

plugins/modules/azure_rm_apimanagementservice.py

@@ -197,6 +197,7 @@ def exec_module(self, **kwargs):
         response = None
 
         self.mgmt_client = self.get_mgmt_svc_client(GenericRestClient,
+                                                    is_track2=True,
                                                     base_url=self._cloud_environment.endpoints.resource_manager)
 
         resource_group = self.get_resource_group(self.resource_group)
@@ -291,9 +292,9 @@ def create_update_resource(self):
             self.fail('Error creating the ApiManagementService instance: {0}'.format(str(exc)))
 
         try:
-            response = json.loads(response.text)
+            response = json.loads(response.body())
         except Exception:
-            response = {'text': response.text}
+            response = {'text': response.context['deserialized_data']}
             pass
 
         return response
@@ -332,7 +333,7 @@ def get_resource(self):
         except CloudError as e:
             self.log('Did not find the ApiManagementService instance.')
         if found is True:
-            return json.loads(response.text)
+            return json.loads(response.body())
 
         return False
 

plugins/modules/azure_rm_apimanagementservice_info.py

@@ -163,6 +163,7 @@ def exec_module(self, **kwargs):
             setattr(self, key, kwargs[key])
 
         self.mgmt_client = self.get_mgmt_svc_client(GenericRestClient,
+                                                    is_track2=True,
                                                     base_url=self._cloud_environment.endpoints.resource_manager)
 
         if (self.resource_group is not None and self.name is not None):
@@ -197,7 +198,7 @@ def get(self):
                                               self.status_code,
                                               600,
                                               30)
-            results = json.loads(response.text)
+            results = json.loads(response.body())
         except CloudError as e:
             self.log('Could not get info for @(Model.ModuleOperationNameUpper).')
 
@@ -226,7 +227,7 @@ def listbyresourcegroup(self):
                                               self.status_code,
                                               600,
                                               30)
-            results = json.loads(response.text)
+            results = json.loads(response.body())
         except CloudError as e:
             self.log('Could not get info for @(Model.ModuleOperationNameUpper).')
 
@@ -252,7 +253,7 @@ def list(self):
                                               self.status_code,
                                               600,
                                               30)
-            results = json.loads(response.text)
+            results = json.loads(response.body())
         except CloudError as e:
             self.log('Could not get info for @(Model.ModuleOperationNameUpper).')
 

plugins/modules/azure_rm_appserviceplan_info.py

@@ -217,14 +217,14 @@ def construct_curated_plan(self, plan):
         curated_output = dict()
         curated_output['id'] = plan_facts['id']
         curated_output['name'] = plan_facts['name']
-        curated_output['resource_group'] = plan_facts['properties']['resourceGroup']
+        curated_output['resource_group'] = plan_facts['resource_group']
         curated_output['location'] = plan_facts['location']
         curated_output['tags'] = plan_facts.get('tags', None)
         curated_output['is_linux'] = False
         curated_output['kind'] = plan_facts['kind']
         curated_output['sku'] = plan_facts['sku']
 
-        if plan_facts['properties'].get('reserved', None):
+        if plan_facts.get('reserved', None):
             curated_output['is_linux'] = True
 
         return curated_output

plugins/modules/azure_rm_azurefirewall.py

@@ -571,6 +571,7 @@ def exec_module(self, **kwargs):
         response = None
 
         self.mgmt_client = self.get_mgmt_svc_client(GenericRestClient,
+                                                    is_track2=True,
                                                     base_url=self._cloud_environment.endpoints.resource_manager)
 
         resource_group = self.get_resource_group(self.resource_group)
@@ -666,9 +667,9 @@ def create_update_resource(self):
             self.fail('Error creating the AzureFirewall instance: {0}'.format(str(exc)))
 
         try:
-            response = json.loads(response.text)
+            response = json.loads(response.body())
         except Exception:
-            response = {'text': response.text}
+            response = {'text': response.context['deserialized_data']}
 
         return response
 
@@ -701,7 +702,7 @@ def get_resource(self):
                                               self.status_code,
                                               600,
                                               30)
-            response = json.loads(response.text)
+            response = json.loads(response.body())
             found = True
             self.log("Response : {0}".format(response))
             # self.log("AzureFirewall instance : {0} found".format(response.name))

plugins/modules/azure_rm_azurefirewall_info.py

@@ -148,6 +148,7 @@ def exec_module(self, **kwargs):
             setattr(self, key, kwargs[key])
 
         self.mgmt_client = self.get_mgmt_svc_client(GenericRestClient,
+                                                    is_track2=True,
                                                     base_url=self._cloud_environment.endpoints.resource_manager)
 
         if (self.resource_group is not None and self.name is not None):
@@ -183,7 +184,7 @@ def get(self):
                                               self.status_code,
                                               600,
                                               30)
-            results = json.loads(response.text)
+            results = json.loads(response.body())
             # self.log('Response : {0}'.format(response))
         except CloudError as e:
             self.log('Could not get info for @(Model.ModuleOperationNameUpper).')
@@ -213,7 +214,7 @@ def list(self):
                                               self.status_code,
                                               600,
                                               30)
-            results = json.loads(response.text)
+            results = json.loads(response.body())
             # self.log('Response : {0}'.format(response))
         except CloudError as e:
             self.log('Could not get info for @(Model.ModuleOperationNameUpper).')
@@ -240,7 +241,7 @@ def listall(self):
                                               self.status_code,
                                               600,
                                               30)
-            results = json.loads(response.text)
+            results = json.loads(response.body())
             # self.log('Response : {0}'.format(response))
         except CloudError as e:
             self.log('Could not get info for @(Model.ModuleOperationNameUpper).')