Azure · zihzhan-msft · Nov 15, 2021 · Oct 15, 2021 · Oct 15, 2021 · Oct 15, 2021
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -41,6 +41,7 @@
 /sdk/communication/                                                  @acsdevx-msft
 /sdk/communication/azure-communication-phonenumbers/                 @RoyHerrod @danielav7 @whisper6284 @AlonsoMondal
 /sdk/communication/azure-communication-sms/                          @RoyHerrod @arifibrahim4
+/sdk/communication/azure-communication-identity/                     @Azure/acs-identity-sdk
 
 # PRLabel: %KeyVault
 /sdk/keyvault/                                                       @schaabs @chlowell @mccoyp @YalinLi0312
@@ -65,7 +66,7 @@
 /sdk/containerservice/                                               @samkreter @zqingqing1 @GaneshaThirumurthi
 
 # PRLabel: %Cosmos
-/sdk/cosmos/                                                         @southpolesteve @zfoster
+/sdk/cosmos/                                                         @kushagraThapar @simorenoh @simplynaveen20 @xinlian12 @moderakh 
 
 # PRLabel: %Data Factory
 /sdk/datafactory/                                                    @hvermis

diff --git a/.gitignore b/.gitignore
@@ -95,12 +95,17 @@ venv/
 code_reports
 
 # Azure Storage test credentials
+sdk/storage/azure-storage-blob/tests/settings/settings_real.py
+sdk/storage/azure-storage-queue/tests/settings/settings_real.py
+sdk/storage/azure-storage-file-share/tests/settings/settings_real.py
+sdk/storage/azure-storage-file-datalake/tests/settings/settings_real.py
+
+
+# The locations below are deprecated - keep to prevent any accidental secrets leakage ==========
 sdk/storage/azure-storage-blob/tests/_shared/settings_real.py
 sdk/storage/azure-storage-queue/tests/_shared/settings_real.py
 sdk/storage/azure-storage-file-share/tests/_shared/settings_real.py
 sdk/storage/azure-storage-file-datalake/tests/settings_real.py
-
-# The locations below are deprecated - keep to prevent any accidental secrets leakage ==========
 sdk/storage/azure-storage-blob/tests/settings_real.py
 sdk/storage/azure-storage-queue/tests/settings_real.py
 sdk/storage/azure-storage-file-share/tests/settings_real.py

diff --git a/README.md b/README.md
@@ -36,7 +36,7 @@ You can find the [most up to date list of all of the new packages on our page](h
 
 ### Client: Previous Versions
 
-Last stable versions of packages that have been provided for usage with Azure and are production-ready. These libraries provide you with similar functionalities to the Preview ones as they allow you to use and consume existing resources and interact with them, for example: upload a blob. They might not implement the [guidelines](https://azure.github.io/azure-sdk/python/guidelines/index.html) or have the same feature set as the Novemeber releases. They do however offer wider coverage of services.
+Last stable versions of packages that have been provided for usage with Azure and are production-ready. These libraries provide you with similar functionalities to the Preview ones as they allow you to use and consume existing resources and interact with them, for example: upload a blob. They might not implement the [guidelines](https://azure.github.io/azure-sdk/python/guidelines/index.html) or have the same feature set as the November releases. They do however offer wider coverage of services.
 
 ### Management: New Releases
 A new set of management libraries that follow the [Azure SDK Design Guidelines for Python](https://azure.github.io/azure-sdk/python/guidelines/) are now available. These new libraries provide a number of core capabilities that are shared amongst all Azure SDKs, including the intuitive Azure Identity library, an HTTP Pipeline with custom policies, error-handling, distributed tracing, and much more.

diff --git a/doc/dev/README.md b/doc/dev/README.md
@@ -7,7 +7,7 @@ Overview of the documents:
 - [Release](https://github.com/Azure/azure-sdk-for-python/blob/main/doc/dev/release.md) : How to release a package when ready
 - [Packaging](https://github.com/Azure/azure-sdk-for-python/blob/main/doc/dev/packaging.md) : How to organize packaging information for packages under `azure`
 - [Testing](https://github.com/Azure/azure-sdk-for-python/blob/main/doc/dev/tests.md): How to write unit and functional tests for a library
-- [Docstrings and Type hints](https://github.com/Azure/azure-sdk-for-python/blob/main/doc/dev/docstring_typehint.md): How to document an SDK for compatability with internal tools (API View) and our documentation at [MS Docs][ms_docs] and the [azure.github.io][azure_github_io] site.
+- [Docstrings and Type hints](https://github.com/Azure/azure-sdk-for-python/blob/main/doc/dev/docstring_typehint.md): How to document an SDK for compatibility with internal tools (API View) and our documentation at [MS Docs][ms_docs] and the [azure.github.io][azure_github_io] site.
 
 The [mgmt](https://github.com/Azure/azure-sdk-for-python/blob/main/doc/dev/mgmt) folder contains information specific to management packages (i.e. packages prefixed by `azure-mgmt`)
 

diff --git a/doc/dev/get_unreleased_package_guide.md b/doc/dev/get_unreleased_package_guide.md
@@ -16,4 +16,4 @@ The following figure shows the wheel and zip of the package.Click to download th
 (1) If there is no link in the figure above, it may be folded. You can also find it in the check.
 ![img.png](unreleased_package_guide_example3.png)
 
-(2) [Private repo](https://github.com/Azure/azure-rest-api-specs-pr) can only be triggered when the target branch is `main`
+(2) The private Azure/azure-rest-api-specs-pr repo can only be triggered when the target branch is `main`
diff --git a/doc/dev/package_version/package_version_rule.md b/doc/dev/package_version/package_version_rule.md
@@ -0,0 +1,44 @@
+This file claims the rules how Python decide next version number for package.
+
+The package version contains two part:
+1. the package is preview or stable?
+2. version number
+
+# How to judge preview or stable?
+Python SDK is generated with [swagger](https://github.com/Azure/azure-rest-api-specs), so if swagger content is preview, 
+the package is preview; if swagger content is stable, the package is stable.
+
+(1) For single api package(for example: [datadog](https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/datadog)),
+ as long as the current tag is preview, the package version should be preview
+
+(2) For multi api package(for example: [network](https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/network/azure-mgmt-network)), 
+there will be `DEFAULT_API_VERSION`(for example: [`DEFAULT_API_VERSION` of network](https://github.com/Azure/azure-sdk-for-python/blob/59709af16b7cd29a51d562137bc5bbfdf53f9327/sdk/network/azure-mgmt-network/azure/mgmt/network/_network_management_client.py#L60)). 
+As long as it is preview, then the package version is preview.
+
+(note1: If the name of tag contains 'preview' or the tag contains files of 'preview' folder, then the tag is preview tag. 
+For exampe: [preview tag](https://github.com/Azure/azure-rest-api-specs/tree/main/specification/compute/resource-manager#tag-package-2021-06-01-preview))
+
+(note2: If the api-version contains 'preview', then it is preview api-version. for example: [preview api-version](https://github.com/Azure/azure-rest-api-specs/blob/69eacf00a36d565d3220d5dd6f4a5293664f1ae9/specification/network/resource-manager/Microsoft.Network/preview/2015-05-01-preview/network.json#L6))
+
+(note3: The difference about single api and multi api, please see the detailed file)
+
+(note4: preview package version contains `b`, for example: `1.0.0b1`)
+
+# How to decide next version number
+1\. If current version is preview version, the new tag is preview tag, then next version is `x.x.xbx+1`
+
+2\. If current version is stable version, the new tag is stable tag, then :
+ * if there is breaking change, next version is `x+1.x.x`
+ * if there is new feature but no breaking change, next version is `x.x+1.x`
+ * if there is only bugfix, next version is `x.x.x+1`
+
+3\. If current version is stable version, the new tag is preview tag, calculate version number according to `2` 
+and then append `b1` in the result
+
+
+According to the up rules, we could summarize all the possibilities in the following table:
+
+![img.png](version_summary.png)
+
+(`-` means that this item doesn't influence result)
+
diff --git a/doc/dev/package_version/version summary.xlsx b/doc/dev/package_version/version summary.xlsx
diff --git a/doc/dev/package_version/version_summary.png b/doc/dev/package_version/version_summary.png
diff --git a/doc/dev/perfstress_tests.md b/doc/dev/perfstress_tests.md
@@ -82,7 +82,7 @@ The framework has a series of common command line options built in:
 - `-w --warm-up=5` Number of seconds to spend warming up the connection before measuring begins. Default is 5.
 - `--sync` Whether to run the tests in sync or async. Default is False (async).
 - `--no-cleanup` Whether to keep newly created resources after test run. Default is False (resources will be deleted).
-- `-x --test-proxy` Whether to run the tests against the test proxy server. Specfiy the URL for the proxy endpoint (e.g. "https://localhost:5001").
+- `-x --test-proxies` Whether to run the tests against the test proxy server. Specify the URL(s) for the proxy endpoint(s) (e.g. "https://localhost:5001").
 - `--profile` Whether to run the perftest with cProfile. If enabled (default is False), the output file of the **last completed single iteration** will be written to the current working directory in the format `"cProfile-<TestClassName>-<TestID>-<sync/async>.pstats"`.
 
 

diff --git a/doc/dev/test_proxy_migration_guide.md b/doc/dev/test_proxy_migration_guide.md
@@ -28,25 +28,25 @@ class TestExample(AzureTestCase):
 ### New test structure
 
 To use the proxy, test classes should inherit from AzureRecordedTestCase and recorded test methods should use a
-RecordedByProxy decorator:
+`recorded_by_proxy` decorator:
 
 ```py
-from devtools_testutils import AzureRecordedTestCase, RecordedByProxy
+from devtools_testutils import AzureRecordedTestCase, recorded_by_proxy
 
 class TestExample(AzureRecordedTestCase):
 
-    @RecordedByProxy
+    @recorded_by_proxy
     def test_example(self):
         ...
 
     @ExamplePreparer()
-    @RecordedByProxy
+    @recorded_by_proxy
     def test_example_with_preparer(self):
         ...
 ```
 
-For async tests, import the RecordedByProxyAsync decorator from `devtools_testutils.aio` and use it in the same
-way as RecordedByProxy.
+For async tests, import the `recorded_by_proxy_async` decorator from `devtools_testutils.aio` and use it in the same
+way as `recorded_by_proxy`.
 
 > **Note:** since AzureRecordedTestCase doesn't inherit from `unittest.TestCase`, test class names need to start
 > with "Test" in order to be properly collected by pytest by default. For more information, please refer to
@@ -135,6 +135,43 @@ made to `https://fakeendpoint-secondary.table.core.windows.net`, and URIs will a
 
 For more details about sanitizers and their options, please refer to [devtools_testutils/sanitizers.py][py_sanitizers].
 
+### Record test variables
+
+To run recorded tests successfully when there's an element of non-secret randomness to them, the test proxy provides a
+[`variables` API](https://github.com/Azure/azure-sdk-tools/tree/main/tools/test-proxy/Azure.Sdk.Tools.TestProxy#storing-variables).
+This makes it possible for a test to record the values of variables that were used during recording and use the same
+values in playback mode without a sanitizer.
+
+For example, imagine that a test uses a randomized `table_name` variable when creating resources. The same random value
+for `table_name` can be used in playback mode by using this `variables` API.
+
+There are two requirements for a test to use recorded variables. First, the test method should accept `**kwargs` and/or
+a `variables` parameter. Second, the test method should `return` a dictionary with any test variables that it wants to
+record. This dictionary will be stored in the recording when the test is run live, and will be passed to the test as a
+`variables` keyword argument when the test is run in playback.
+
+Below is a code example of how a test method could use recorded variables:
+
+```python
+from devtools_testutils import AzureRecordedTestCase, recorded_by_proxy
+
+class TestExample(AzureRecordedTestCase):
+
+    @recorded_by_proxy
+    def test_example(self, variables):
+        # in live mode, variables is an empty dictionary
+        # in playback mode, the value of variables is {"table_name": "random-value"}
+        if self.is_live:
+            table_name = "random-value"
+            variables = {"table_name": table_name}
+
+        # use variables["table_name"] when using the table name throughout the test
+        ...
+
+        # return the variables at the end of the test
+        return variables
+```
+
 ## Implementation details
 
 ### What does the test proxy do?
@@ -147,7 +184,7 @@ For example, if an operation would typically make a GET request to
 `https://localhost:5001/Tables` instead. The original endpoint should be stored in an `x-recording-upstream-base-uri` --
 the proxy will send the original request and record the result.
 
-The RecordedByProxy and RecordedByProxyAsync decorators patch test requests to do this for you.
+The `recorded_by_proxy` and `recorded_by_proxy_async` decorators patch test requests to do this for you.
 
 ### How does the test proxy know when and what to record or play back?
 
@@ -179,7 +216,7 @@ Running tests in playback follows the same pattern, except that requests will be
 `/playback/stop` instead. A header, `x-recording-mode`, should be set to `record` for all requests when recording and
 `playback` when playing recordings back. More details can be found [here][detailed_docs].
 
-The RecordedByProxy and RecordedByProxyAsync decorators send the appropriate requests at the start and end of each test
+The `recorded_by_proxy` and `recorded_by_proxy_async` decorators send the appropriate requests at the start and end of each test
 case.
 
 [detailed_docs]: https://github.com/Azure/azure-sdk-tools/tree/main/tools/test-proxy/Azure.Sdk.Tools.TestProxy/README.md

@@ -94,6 +94,7 @@ known_content_issues:
   - ['sdk/purview/azure-purview-scanning/swagger/README.md',  '#4554']
   - ['sdk/agrifood/azure-agrifood-farming/swagger/README.md',  '#4554']
   - ['sdk/purview/azure-purview-account/swagger/README.md', '#4554']
+  - ['sdk/purview/azure-purview-administration/swagger/README.md', '#4554']
   - ['sdk/containerregistry/azure-containerregistry/swagger/README.md', '#4554']
   - ['sdk/appconfiguration/azure-appconfiguration/swagger/README.md', '#4554']
   - ['sdk/attestation/azure-security-attestation/swagger/README.md', '#4554']

@@ -46,7 +46,7 @@
                 "sdk/identity/azure-identity/tests/ec-certificate.pem",
                 "sdk/core/azure-servicemanagement-legacy/tests/legacy_mgmt_settings_fake.py",
                 "sdk/storage/azure-storage-blob/tests/fake_credentials.py",
-                "sdk/storage/azure-storage-file-datalake/tests/fake_credentials.py",
+                "tools/azure-sdk-tools/devtools_testutils/fake_credentials.py",
                 "tools/azure-sdk-tools/devtools_testutils/mgmt_settings_fake.py"
             ],
             "_justification": "File contains private key used by test code."

@@ -3,15 +3,15 @@ cryptography==3.1
 setuptools==44.1.0; python_version == '2.7'
 setuptools==46.4.0; python_version >= '3.5'
 virtualenv==20.0.23
-wheel==0.34.2
+wheel==0.37.0
 Jinja2==2.11.2
 packaging==20.4
 tox==3.15.0
 tox-monorepo==0.1.2
 twine==1.15.0; python_version == '2.7' or python_version == '3.5'
 twine==3.1.1; python_version >= '3.6'
 pathlib2==2.3.5
-readme-renderer[md]==25.0
+readme-renderer[md]==25.0; python_version < '3.10'
 doc-warden==0.7.1
 # we pin coverage to 4.5.4 because there is an bug with `pytest-cov`. the generated coverage files cannot be `coverage combine`ed
 coverage==4.5.4
@@ -22,6 +22,9 @@ pip==20.3
 black==21.6b0; python_version >= '3.6'
 wrapt<=1.12.1; python_version == '2.7'
 
+# python 3.10 pinned packages
+cffi==1.15.0rc2; python_version >= '3.10'
+
 # locking packages defined as deps from azure-sdk-tools or azure-devtools
 pytoml==0.1.21
 pyOpenSSL==19.1.0

@@ -84,11 +84,13 @@ if (!$PSBoundParameters.ContainsKey('ErrorAction')) {
     $ErrorActionPreference = 'Stop'
 }
 
-function Log($Message) {
+function Log($Message)
+{
     Write-Host ('{0} - {1}' -f [DateTime]::Now.ToLongTimeString(), $Message)
 }
 
-function Retry([scriptblock] $Action, [int] $Attempts = 5) {
+function Retry([scriptblock] $Action, [int] $Attempts = 5)
+{
     $attempt = 0
     $sleep = 5
 
@@ -109,7 +111,20 @@ function Retry([scriptblock] $Action, [int] $Attempts = 5) {
     }
 }
 
-function MergeHashes([hashtable] $source, [psvariable] $dest) {
+function LoadCloudConfig([string] $env)
+{
+    $configPath = "$PSScriptRoot/clouds/$env.json"
+    if (!(Test-Path $configPath)) {
+        Write-Warning "Could not find cloud configuration for environment '$env'"
+        return @{}
+    }
+
+    $config = Get-Content $configPath | ConvertFrom-Json -AsHashtable
+    return $config
+}
+
+function MergeHashes([hashtable] $source, [psvariable] $dest)
+{
     foreach ($key in $source.Keys) {
         if ($dest.Value.ContainsKey($key) -and $dest.Value[$key] -ne $source[$key]) {
             Write-Warning ("Overwriting '$($dest.Name).$($key)' with value '$($dest.Value[$key])' " +
@@ -119,7 +134,8 @@ function MergeHashes([hashtable] $source, [psvariable] $dest) {
     }
 }
 
-function BuildBicepFile([System.IO.FileSystemInfo] $file) {
+function BuildBicepFile([System.IO.FileSystemInfo] $file)
+{
     if (!(Get-Command bicep -ErrorAction Ignore)) {
         Write-Error "A bicep file was found at '$($file.FullName)' but the Azure Bicep CLI is not installed. See https://aka.ms/install-bicep-pwsh"
         throw
@@ -334,7 +350,7 @@ try {
     $serviceName = if (Split-Path $ServiceDirectory) {
         Split-Path -Leaf $ServiceDirectory
     } else {
-        $ServiceDirectory
+        $ServiceDirectory.Trim('/')
     }
 
     $ResourceGroupName = if ($ResourceGroupName) {
@@ -492,6 +508,8 @@ try {
         $templateParameters.Add('testApplicationSecret', $TestApplicationSecret)
     }
 
+    $defaultCloudParameters = LoadCloudConfig $Environment
+    MergeHashes $defaultCloudParameters $(Get-Variable templateParameters)
     MergeHashes $ArmTemplateParameters $(Get-Variable templateParameters)
     MergeHashes $AdditionalParameters $(Get-Variable templateParameters)
 
@@ -600,7 +618,7 @@ try {
             $outputFile = "$($templateFile.originalFilePath).env"
 
             $environmentText = $deploymentOutputs | ConvertTo-Json;
-            $bytes = ([System.Text.Encoding]::UTF8).GetBytes($environmentText)
+            $bytes = [System.Text.Encoding]::UTF8.GetBytes($environmentText)
             $protectedBytes = [Security.Cryptography.ProtectedData]::Protect($bytes, $null, [Security.Cryptography.DataProtectionScope]::CurrentUser)
 
             Set-Content $outputFile -Value $protectedBytes -AsByteStream -Force

@@ -0,0 +1,15 @@
+{
+    "azConfigEndpointSuffix": ".azconfig.azure.cn",
+    "azureAuthorityHost": "https://login.chinacloudapi.cn/",
+    "cognitiveServicesEndpointSuffix": ".cognitiveservices.azure.cn",
+    "containerRegistryEndpointSuffix": ".azurecr.cn",
+    "cosmosEndpointSuffix": "cosmos.azure.cn",
+    "enableStorageVersioning": false,
+    "keyVaultDomainSuffix": ".vault.azure.cn",
+    "keyVaultEndpointSuffix": ".vault.azure.cn",
+    "keyVaultSku": "standard",
+    "searchEndpointSuffix": "search.azure.cn",
+    "serviceBusEndpointSuffix": ".servicebus.chinacloudapi.cn",
+    "storageEndpointSuffix": "core.chinacloudapi.cn",
+    "textAnalyticsSku": "S"
+}
@@ -0,0 +1,8 @@
+{
+    "azureAuthorityHost": "https://login.microsoftonline.com/",
+    "cognitiveServicesEndpointSuffix": ".cognitiveservices.azure.com",
+    "communicationServicesEndpointSuffix": ".communication.azure.com",
+    "keyVaultDomainSuffix": ".vault.azure.net",
+    "keyVaultEndpointSuffix": ".vault.azure.net",
+    "storageEndpointSuffix": "core.windows.net"
+}
@@ -0,0 +1,15 @@
+{
+    "azConfigEndpointSuffix": ".azconfig.azure.us",
+    "azureAuthorityHost": "https://login.microsoftonline.us/",
+    "cognitiveServicesEndpointSuffix": ".cognitiveservices.azure.us",
+    "containerRegistryEndpointSuffix": ".azurecr.us",
+    "cosmosEndpointSuffix": "cosmos.azure.us",
+    "enableStorageVersioning": false,
+    "formRecognizerLocation": "usgovvirginia",
+    "keyVaultDomainSuffix": ".vault.usgovcloudapi.net",
+    "keyVaultEndpointSuffix": ".vault.usgovcloudapi.net",
+    "keyVaultSku": "premium",
+    "searchEndpointSuffix": "search.azure.us",
+    "serviceBusEndpointSuffix": ".servicebus.usgovcloudapi.net",
+    "storageEndpointSuffix": "core.usgovcloudapi.net"
+}