Merge branch 'master' of https://github.com/Azure/azure-sdk-for-python …

…into fr-business-cards

* 'master' of https://github.com/Azure/azure-sdk-for-python:
  Adding one more item to the limitations list (#13953)
  [Cosmos] 4.2.0 (#14358)
  [formrecognizer] composed model (#14029)
  [form recognizer] add receipt locale tests (#14327)
  Sync eng/common directory with azure-sdk-tools for PR 1077 (#14329)
  add metrics advisor to package mapping for docs (#14350)
  Increment package version after release of azure_identity (#14351)
  Sync eng/common directory with azure-sdk-tools for PR 1066 (#14333)
  Communication SMS package - nspkg dependency update (#14346)
  Add tests.yml file to template sdk directory. (#14324)
  Increment version for keyvault releases (#14312)
  Merge azure-identity 1.4.1 hotfix into master (#14349)
  Increment version for metricsadvisor releases (#14343)
  Pin Deps in "TestOldest" Regression (#14316)

doc/sphinx/package_service_mapping.json

@@ -916,6 +916,11 @@
     "service_name": "Form Recognizer",
     "manually_generated": true
   },
+  "azure-ai-metricsadvisor": {
+    "category": "Client",
+    "service_name": "Metrics Advisor",
+    "manually_generated": true
+  },
   "azure-storage-blob": {
     "category": "Client",
     "service_name": "Storage",

eng/common/TestResources/New-TestResources.ps1

@@ -11,17 +11,17 @@
 [CmdletBinding(DefaultParameterSetName = 'Default', SupportsShouldProcess = $true, ConfirmImpact = 'Medium')]
 param (
     # Limit $BaseName to enough characters to be under limit plus prefixes, and https://docs.microsoft.com/azure/architecture/best-practices/resource-naming.
-    [Parameter(Mandatory = $true, Position = 0)]
+    [Parameter()]
     [ValidatePattern('^[-a-zA-Z0-9\.\(\)_]{0,80}(?<=[a-zA-Z0-9\(\)])$')]
     [string] $BaseName,
 
     [ValidatePattern('^[-\w\._\(\)]+$')]
     [string] $ResourceGroupName,
 
-    [Parameter(Mandatory = $true)]
+    [Parameter(Mandatory = $true, Position = 0)]
     [string] $ServiceDirectory,
 
-    [Parameter(Mandatory = $true)]
+    [Parameter()]
     [ValidatePattern('^[0-9a-f]{8}(-[0-9a-f]{4}){3}-[0-9a-f]{12}$')]
     [string] $TestApplicationId,
 
@@ -119,6 +119,8 @@ $root = [System.IO.Path]::Combine($repositoryRoot, "sdk", $ServiceDirectory) | R
 $templateFileName = 'test-resources.json'
 $templateFiles = @()
 $environmentVariables = @{}
+# Azure SDK Developer Playground
+$defaultSubscription = "faa080af-c1d8-40ad-9cce-e1a450ca5b57"
 
 Write-Verbose "Checking for '$templateFileName' files under '$root'"
 Get-ChildItem -Path $root -Filter $templateFileName -Recurse | ForEach-Object {
@@ -133,6 +135,26 @@ if (!$templateFiles) {
     exit
 }
 
+$UserName =  if ($env:USER) { $env:USER } else { "${env:USERNAME}" }
+
+# If no base name is specified use current user name
+if (!$BaseName) {
+    $BaseName = "$UserName$ServiceDirectory"
+
+    Log "BaseName was not set. Using default base name: '$BaseName'"
+}
+
+# Try detecting repos that support OutFile and defaulting to it
+if (!$CI -and !$PSBoundParameters.ContainsKey('OutFile') -and $IsWindows)
+{
+    # TODO: find a better way to detect the language
+    if (Test-Path "$repositoryRoot/eng/service.proj")
+    {
+        $OutFile = $true
+        Log "Detected .NET repository. Defaulting OutFile to true. Test environment settings would be stored into the file so you don't need to set environment variables manually."
+    }
+}
+
 # If no location is specified use safe default locations for the given
 # environment. If no matching environment is found $Location remains an empty
 # string.
@@ -147,7 +169,7 @@ if (!$Location) {
     Write-Verbose "Location was not set. Using default location for environment: '$Location'"
 }
 
-# Log in if requested; otherwise, the user is expected to already be authenticated via Connect-AzAccount.
+# Log in if requested; otherwise, try to login into playground subscription.
 if ($ProvisionerApplicationId) {
     $null = Disable-AzContextAutosave -Scope Process
 
@@ -176,6 +198,25 @@ if ($ProvisionerApplicationId) {
         }
     }
 }
+elseif (!$CI)
+{
+    # check if user is logged in and login into
+    $context = Get-AzContext;
+    if (!$context)
+    {
+        Log "You are not logged in, connecting to 'Azure SDK Developer Playground'"
+        Connect-AzAccount -Subscription $defaultSubscription
+    }
+
+    # If no test application id is specified create a new service principal
+    if (!$TestApplicationId) {        
+        Log "TestApplicationId was not specified, creating a new service principal."
+        $servicePrincipal = New-AzADServicePrincipal -Role Owner
+
+        $TestApplicationId = $servicePrincipal.ApplicationId
+        $TestApplicationSecret = (ConvertFrom-SecureString $servicePrincipal.Secret -AsPlainText);
+    }
+}
 
 # Get test application OID from ID if not already provided.
 if ($TestApplicationId -and !$TestApplicationOid) {
@@ -215,7 +256,7 @@ $ResourceGroupName = if ($ResourceGroupName) {
 
 # Tag the resource group to be deleted after a certain number of hours if specified.
 $tags = @{
-    Creator = if ($env:USER) { $env:USER } else { "${env:USERNAME}" }
+    Creator = $UserName
     ServiceDirectory = $ServiceDirectory
 }
 
@@ -403,7 +444,11 @@ foreach ($templateFile in $templateFiles) {
 
 $exitActions.Invoke()
 
-return $environmentVariables
+# Suppress output locally
+if ($CI)
+{
+    return $environmentVariables
+}
 
 <#
 .SYNOPSIS
@@ -466,7 +511,7 @@ test resources (e.g. Role Assignments on resources). It is passed as to the ARM
 template as 'testApplicationOid'
 
 For more information on the relationship between AAD Applications and Service
-Principals see: https://docs.microsoft.com/en-us/azure/active-directory/develop/app-objects-and-service-principals
+Principals see: https://docs.microsoft.com/azure/active-directory/develop/app-objects-and-service-principals
 
 .PARAMETER TenantId
 The tenant ID of a service principal when a provisioner is specified. The same
@@ -568,4 +613,4 @@ log redaction).
 
 .LINK
 Remove-TestResources.ps1
-#>
+#>

eng/common/TestResources/New-TestResources.ps1.md

@@ -14,16 +14,16 @@ Deploys live test resources defined for a service directory to Azure.
 
 ### Default (Default)
 ```
-New-TestResources.ps1 [-BaseName] <String> [-ResourceGroupName <String>] -ServiceDirectory <String>
- -TestApplicationId <String> [-TestApplicationSecret <String>] [-TestApplicationOid <String>]
+New-TestResources.ps1 [-BaseName <String>] [-ResourceGroupName <String>] [-ServiceDirectory] <String>
+ [-TestApplicationId <String>] [-TestApplicationSecret <String>] [-TestApplicationOid <String>]
  [-DeleteAfterHours <Int32>] [-Location <String>] [-Environment <String>] [-AdditionalParameters <Hashtable>]
  [-CI] [-Force] [-OutFile] [-WhatIf] [-Confirm] [<CommonParameters>]
 ```
 
 ### Provisioner
 ```
-New-TestResources.ps1 [-BaseName] <String> [-ResourceGroupName <String>] -ServiceDirectory <String>
- -TestApplicationId <String> [-TestApplicationSecret <String>] [-TestApplicationOid <String>]
+New-TestResources.ps1 [-BaseName <String>] [-ResourceGroupName <String>] [-ServiceDirectory] <String>
+ [-TestApplicationId <String>] [-TestApplicationSecret <String>] [-TestApplicationOid <String>]
  -TenantId <String> [-SubscriptionId <String>] -ProvisionerApplicationId <String>
  -ProvisionerApplicationSecret <String> [-DeleteAfterHours <Int32>] [-Location <String>]
  [-Environment <String>] [-AdditionalParameters <Hashtable>] [-CI] [-Force] [-OutFile] [-WhatIf] [-Confirm]
@@ -106,8 +106,8 @@ Type: String
 Parameter Sets: (All)
 Aliases:
 
-Required: True
-Position: 1
+Required: False
+Position: Named
 Default value: None
 Accept pipeline input: False
 Accept wildcard characters: False
@@ -140,7 +140,7 @@ Parameter Sets: (All)
 Aliases:
 
 Required: True
-Position: Named
+Position: 1
 Default value: None
 Accept pipeline input: False
 Accept wildcard characters: False
@@ -159,7 +159,7 @@ Type: String
 Parameter Sets: (All)
 Aliases:
 
-Required: True
+Required: False
 Position: Named
 Default value: None
 Accept pipeline input: False

eng/common/TestResources/README.md

@@ -27,16 +27,10 @@ is a member of multiple subscriptions.
 
 ```powershell
 Connect-AzAccount -Subscription 'YOUR SUBSCRIPTION ID'
-$sp = New-AzADServicePrincipal -Role Owner
-eng\common\TestResources\New-TestResources.ps1 `
-  -BaseName 'myusername' `
-  -ServiceDirectory 'search' `
-  -TestApplicationId $sp.ApplicationId `
-  -TestApplicationSecret (ConvertFrom-SecureString $sp.Secret -AsPlainText)
+eng\common\TestResources\New-TestResources.ps1 -ServiceDirectory 'search'
 ```
 
-If you are running this for a .NET project on Windows, the recommended method is to
-add the `-OutFile` switch to the above command. This will save test environment settings
+The `OutFile` switch would be set if you are running this for a .NET project on Windows. This will save test environment settings
 into a test-resources.json.env file next to test-resources.json. The file is protected via DPAPI.
 The environment file would be scoped to the current repository directory and avoids the need to
 set environment variables or restart your IDE to recognize them.

eng/common/scripts/copy-docs-to-blobstorage.ps1

@@ -177,10 +177,9 @@ function Update-Existing-Versions
     $sortedVersionObj.LatestGAPackage | Out-File -File "$($DocLocation)/latest-ga" -Force -NoNewLine
     $sortedVersionObj.LatestPreviewPackage | Out-File -File "$($DocLocation)/latest-preview" -Force -NoNewLine
 
-    & $($AzCopy) cp "$($DocLocation)/versions" "$($DocDest)/$($PkgName)/versioning/versions$($SASKey)"
-    & $($AzCopy) cp "$($DocLocation)/latest-preview" "$($DocDest)/$($PkgName)/versioning/latest-preview$($SASKey)"
-    & $($AzCopy) cp "$($DocLocation)/latest-ga" "$($DocDest)/$($PkgName)/versioning/latest-ga$($SASKey)"
-
+    & $($AzCopy) cp "$($DocLocation)/versions" "$($DocDest)/$($PkgName)/versioning/versions$($SASKey)" --cache-control "max-age=300, must-revalidate"
+    & $($AzCopy) cp "$($DocLocation)/latest-preview" "$($DocDest)/$($PkgName)/versioning/latest-preview$($SASKey)" --cache-control "max-age=300, must-revalidate"
+    & $($AzCopy) cp "$($DocLocation)/latest-ga" "$($DocDest)/$($PkgName)/versioning/latest-ga$($SASKey)" --cache-control "max-age=300, must-revalidate"
     return $sortedVersionObj
 }
 
@@ -218,7 +217,7 @@ function Upload-Blobs
     } 
 
     Write-Host "Uploading $($PkgName)/$($DocVersion) to $($DocDest)..."
-    & $($AzCopy) cp "$($DocDir)/**" "$($DocDest)/$($PkgName)/$($DocVersion)$($SASKey)" --recursive=true
+    & $($AzCopy) cp "$($DocDir)/**" "$($DocDest)/$($PkgName)/$($DocVersion)$($SASKey)" --recursive=true --cache-control "max-age=300, must-revalidate"
 
     Write-Host "Handling versioning files under $($DocDest)/$($PkgName)/versioning/"
     $versionsObj = (Update-Existing-Versions -PkgName $PkgName -PkgVersion $DocVersion -DocDest $DocDest)
@@ -229,7 +228,7 @@ function Upload-Blobs
     if ($UploadLatest -and ($latestVersion -eq $DocVersion))
     {
         Write-Host "Uploading $($PkgName) to latest folder in $($DocDest)..."
-        & $($AzCopy) cp "$($DocDir)/**" "$($DocDest)/$($PkgName)/latest$($SASKey)" --recursive=true
+        & $($AzCopy) cp "$($DocDir)/**" "$($DocDest)/$($PkgName)/latest$($SASKey)" --recursive=true --cache-control "max-age=300, must-revalidate"
     }
 }
 

scripts/devops_tasks/build_packages.py

@@ -12,7 +12,7 @@
 import sys
 import os
 
-from common_tasks import process_glob_string, run_check_call
+from common_tasks import process_glob_string, run_check_call, str_to_bool
 
 root_dir = os.path.abspath(os.path.join(os.path.abspath(__file__), "..", "..", ".."))
 build_packing_script_location = os.path.join(root_dir, "build_package.py")
@@ -22,17 +22,6 @@
 sys.path.append(tox_path)
 from sanitize_setup import process_requires
 
-
-def str_to_bool(input_string):
-    if isinstance(input_string, bool):
-        return input_string
-    elif input_string.lower() in ("true", "t", "1"):
-        return True
-    elif input_string.lower() in ("false", "f", "0"):
-        return False
-    else:
-        return False
-
 def build_packages(targeted_packages, distribution_directory, is_dev_build=False):
     # run the build and distribution
     for package_root in targeted_packages:

scripts/devops_tasks/common_tasks.py

@@ -119,6 +119,15 @@ def clean_coverage(coverage_dir):
         else:
             raise
 
+def str_to_bool(input_string):
+    if isinstance(input_string, bool):
+        return input_string
+    elif input_string.lower() in ("true", "t", "1"):
+        return True
+    elif input_string.lower() in ("false", "f", "0"):
+        return False
+    else:
+        return False
 
 def parse_setup(setup_path):
     setup_filename = os.path.join(setup_path, "setup.py")
@@ -399,7 +408,7 @@ def extend_dev_requirements(dev_req_path, packages_to_include):
     # include any package given in included list. omit duplicate
     for requirement in packages_to_include:
         if requirement not in requirements:
-            requirements.append(requirement)
+            requirements.insert(0, requirement.rstrip() + '\n')
 
     logging.info("Extending dev requirements. New result:: {}".format(requirements))
     # create new dev requirements file with different name for filtered requirements

scripts/devops_tasks/test_regression.py

@@ -23,7 +23,9 @@
     find_packages_missing_on_pypi,
     find_whl,
     find_tools_packages,
-    get_installed_packages
+    get_installed_packages,
+    extend_dev_requirements,
+    str_to_bool
 )
 from git_helper import get_release_tag, git_checkout_tag, git_checkout_branch, clone_repo
 
@@ -38,6 +40,8 @@
 AZURE_SDK_FOR_PYTHON_GIT_URL = "https://github.com/Azure/azure-sdk-for-python.git"
 TEMP_FOLDER_NAME = ".tmp_code_path"
 
+OLDEST_EXTENSION_PKGS = ['msrestazure','adal']
+
 logging.getLogger().setLevel(logging.INFO)
 
 class CustomVirtualEnv:
@@ -114,6 +118,7 @@ def run(self):
             logging.info("Dependent packages for [{0}]: {1}".format(pkg_name, dep_packages))
             for dep_pkg_path in dep_packages:
                 dep_pkg_name, _, _, _ = parse_setup(dep_pkg_path)
+
                 logging.info(
                     "Starting regression test of {0} against released {1}".format(
                         pkg_name, dep_pkg_name
@@ -235,6 +240,18 @@ def _install_packages(self, dependent_pkg_path, pkg_to_exclude):
             dependent_pkg_path, list_to_exclude, dependent_pkg_path
         )
 
+        # early versions of azure-sdk-tools had an unpinned version of azure-mgmt packages. 
+        # that unpinned version hits an a code path in azure-sdk-tools that hits this error.
+        if filtered_dev_req_path and self.context.is_latest_depend_test == False:
+            logging.info(
+                "Extending dev requirements with {}".format(OLDEST_EXTENSION_PKGS)
+            )
+            extend_dev_requirements(
+                filtered_dev_req_path, OLDEST_EXTENSION_PKGS
+            )
+        else:
+            logging.info("Not extending dev requirements {} {}".format(filtered_dev_req_path, self.context.is_latest_depend_test))
+
         if filtered_dev_req_path:
             logging.info(
                 "Installing filtered dev requirements from {}".format(filtered_dev_req_path)
@@ -289,7 +306,6 @@ def find_package_dependency(glob_string, repo_root_dir):
 
 # This is the main function which identifies packages to test, find dependency matrix and trigger test
 def run_main(args):
-
     temp_dir = ""
     if args.temp_dir:
         temp_dir = args.temp_dir
@@ -318,7 +334,7 @@ def run_main(args):
     if len(targeted_packages) == 0:
         exit(0)
 
-    # clone code repo only if it doesn't exists
+    # clone code repo only if it doesn't exist
     if not os.path.exists(code_repo_root):
         clone_repo(temp_dir, AZURE_SDK_FOR_PYTHON_GIT_URL)
     else:
@@ -333,7 +349,7 @@ def run_main(args):
 
     # Create regression text context. One context object will be reused for all packages
     context = RegressionContext(
-        args.whl_dir, temp_dir, args.verify_latest, args.mark_arg
+        args.whl_dir, temp_dir, str_to_bool(args.verify_latest), args.mark_arg
     )
 
     for pkg_path in targeted_packages:

sdk/communication/azure-communication-sms/CHANGELOG.md

@@ -1,5 +1,8 @@
 # Release History
 
+## 1.0.0b3 (2020-10-07)
+- Add dependency to `azure-communication-nspkg` package, to support py2
+
 ## 1.0.0b2 (2020-10-06)
 - Updated `azure-communication-sms` version.
 

sdk/communication/azure-communication-sms/azure/communication/sms/_version.py

@@ -4,6 +4,6 @@
 # license information.
 # --------------------------------------------------------------------------
 
-VERSION = "1.0.0b2"
+VERSION = "1.0.0b3"
 
 SDK_MONIKER = "communication-sms/{}".format(VERSION)  # type: str

sdk/communication/azure-communication-sms/setup.py

@@ -68,7 +68,7 @@
         'six>=1.6'
     ],
     extras_require={
-        ":python_version<'3.0'": ['azure-nspkg'],
+        ":python_version<'3.0'": ['azure-communication-nspkg'],
         ":python_version<'3.5'": ["typing"]
     }
 )