Merge branch 'master' of https://github.com/Azure/azure-sdk-for-python …

…into ta_opinion_mining_sample

* 'master' of https://github.com/Azure/azure-sdk-for-python: (124 commits)
  [formrecognizer] Add type to FormField (Azure#12561)
  Add example summary for azure-identity readme.md (Azure#12509)
  Add logging to credentials (Azure#12319)
  Sdk automation/track2 azure mgmt keyvault (Azure#12638)
  Remove unnecessary coroutine declaration (Azure#12602)
  [Cosmos] Fix type comment (Azure#12598)
  replace aka link (Azure#12597)
  [ServiceBus] Message/ReceivedMessage Properties alignment with other languages (Azure#12451)
  Find list of installed packages using pkg_resources (Azure#12591)
  token refresh offset (Azure#12136)
  updates (Azure#12595)
  User authentication samples (Azure#11343)
  Remove unnecessary base class (Azure#12374)
  Sequence -> Iterable for scopes (Azure#12579)
  Disable apistubgen step until issue is fixed (Azure#12594)
  fix pylint issue (Azure#12578)
  fix name in example (Azure#12572)
  Update tests.md (Azure#12574)
  Add stress tests for max batch size/prefetch, and for unsettled message receipt.  Add capability to not auto-complete and adjust max_batch_size into the base stress tester. (Azure#12344)
  [formrecognizer] Capitalize enum values (Azure#12540)
  ...

.github/CODEOWNERS

@@ -9,7 +9,7 @@
 /sdk/ @mayurid @lmazuel
 
 # Core
-/sdk/core/                                                           @lmazuel @xiangyan99 @johanste
+/sdk/core/                                                           @lmazuel @xiangyan99 @johanste @rakshith91
 /sdk/core/azure-core/                                                @xiangyan99 @lmazuel
 
 # Service team
@@ -33,12 +33,13 @@
 /sdk/hdinsight/                                                      @idear1203
 /sdk/machinelearningcompute/                                         @shutchings
 /sdk/recoveryservices/                                               @DheerendraRathor
+/sdk/search/                                                         @xiangyan99 @rakshith91
 /sdk/servicefabric/                                                  @QingChenmsft @samedder
 /sql/sql/                                                            @jaredmoo
 /sdk/servicebus/                                                     @annatisch @yunhaoling @YijunXieMS @KieranBrantnerMagee
 /sdk/synapse/                                                        @aim-for-better @idear1203
 /sdk/textanalytics/                                                  @kristapratico @iscai-msft
-/sdk/formrecognizer/                                                 @kristapratico @iscai-msft
+/sdk/formrecognizer/                                                 @kristapratico @iscai-msft @rakshith91
 
 # Smoke Tests
 /common/smoketest/                                                   @lmazuel @chlowell @annatisch @rakshith91 @shurd @southpolesteve

SECURITY.md

@@ -1,8 +1,41 @@
-# Security Policy
+<!-- BEGIN MICROSOFT SECURITY.MD V0.0.5 BLOCK -->
 
-## Reporting a Vulnerability
+## Security
 
-Security issues and bugs should be reported privately, via email, to the Microsoft Security Response Center (MSRC) <secure@microsoft.com>. You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Further information, including the MSRC PGP key, can be found in the [Security TechCenter](https://www.microsoft.com/msrc/faqs-report-an-issue).
+Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/Microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/).
 
-Please do not open issues for anything you think might have a security implication.
+If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://docs.microsoft.com/en-us/previous-versions/tn-archive/cc751383(v=technet.10)), please report it to us as described below.
 
+## Reporting Security Issues
+
+**Please do not report security vulnerabilities through public GitHub issues.**
+
+Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://msrc.microsoft.com/create-report).
+
+If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com).  If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://www.microsoft.com/en-us/msrc/pgp-key-msrc).
+
+You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://www.microsoft.com/msrc). 
+
+Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue:
+
+  * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.)
+  * Full paths of source file(s) related to the manifestation of the issue
+  * The location of the affected source code (tag/branch/commit or direct URL)
+  * Any special configuration required to reproduce the issue
+  * Step-by-step instructions to reproduce the issue
+  * Proof-of-concept or exploit code (if possible)
+  * Impact of the issue, including how an attacker might exploit the issue
+
+This information will help us triage your report more quickly.
+
+If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://microsoft.com/msrc/bounty) page for more details about our active programs.
+
+## Preferred Languages
+
+We prefer all communications to be in English.
+
+## Policy
+
+Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://www.microsoft.com/en-us/msrc/cvd).
+
+<!-- END MICROSOFT SECURITY.MD BLOCK -->

doc/dev/mgmt/mgmt_release.md

@@ -18,6 +18,22 @@ Once you have a PR that contains accurate with correct tests (or no tests at all
 
 IMPORTANT NOTE: All the commands in this page assumes you have loaded the [dev_setup](../dev_setup.md) in your currently loaded virtual environment.
 
+## Manual generation
+
+If the automation is not doing its job to create an auto PR, Python has a SwaggerToSdk CLI that can be used to generate a specific Readme. You need
+a virtual environment loaded with at least `tools/azure-sdk-tools` installed.
+
+```shell
+# Using default configuration (this can be a Github raw link)
+generate_sdk -v -m ..\azure-rest-api-specs\specification\compute\resource-manager\readme.md
+
+# Forcing Track1 generation
+generate_sdk -v -c eng\swagger_to_sdk_config_v4.json -m ..\azure-rest-api-specs\specification\cognitiveservices\data-plane\Face\readme.md
+
+# For more details about the available options
+generate_sdk --help
+```
+
 ## Building the packaging information
 
 If the automation is doing its job correctly, there is a pipeline called "update PR" that is supposed to update the package on the branch.

doc/dev/mgmt/tests.md

@@ -176,7 +176,7 @@ providing several features useful for the SDK tests, for example:
 *   Patches for overriding functions and methods that don't work well with tests
     (such as long-running operations)
 
-Code in the [`azure-sdk-tools/devtools_testutils`](tools/azure-sdk-tools/devtools_testutils) directory
+Code in the [`azure-sdk-tools/devtools_testutils`](https://github.com/Azure/azure-sdk-for-python/tree/master/tools/azure-sdk-tools/devtools_testutils) directory
 provides concrete implementations of the features provided in `scenario_tests`
 that are oriented around use in SDK testing
 and that you can use directly in your unit tests.

doc/sphinx/package_service_mapping.json

@@ -901,6 +901,11 @@
     "service_name": "Storage",
     "manually_generated": true
   },
+  "azure-storage-blob-changefeed": {
+    "category": "Client",
+    "service_name": "Storage",
+    "manually_generated": true
+  },
   "azure-storage-file-share": {
     "category": "Client",
     "service_name": "Storage",

eng/.docsettings.yml

@@ -43,6 +43,10 @@ known_content_issues:
   # data plane. common issue filed to azure-sdk-for-python
   - ['README.rst', '#4554']
   - ['sdk/synapse/azure-synapse/README.md', '#4554']
+  - ['sdk/synapse/azure-synapse-accesscontrol/README.md', '#4554']
+  - ['sdk/synapse/azure-synapse-spark/README.md', '#4554']
+  - ['sdk/synapse/azure-synapse-artifacts/README.md', '#4554']
+  - ['sdk/synapse/azure-synapse-nspkg/README.md', '#4554']
   - ['sdk/applicationinsights/azure-applicationinsights/README.md', '#4554']
   - ['sdk/batch/azure-batch/README.md', '#4554']
   - ['sdk/cognitiveservices/azure-cognitiveservices-anomalydetector/README.md', '#4554']
@@ -110,6 +114,7 @@ known_content_issues:
   - ['sdk/search/azure-search-nspkg/README.md', 'nspkg and common']
   - ['sdk/storage/azure-storage-blob/samples/README.md', 'nspkg and common']
   - ['sdk/storage/azure-storage-file-datalake/samples/README.md', 'nspkg and common']
+  - ['sdk/storage/azure-storage-blob-changefeed/samples/README.md', 'nspkg and common']
   - ['sdk/storage/azure-storage-file-share/samples/README.md', 'nspkg and common']
   - ['sdk/storage/azure-storage-queue/samples/README.md', 'nspkg and common']
   - ['sdk/textanalytics/azure-ai-nspkg/README.md', 'nspkg and common']

eng/ci_tools.txt

@@ -1,36 +1,35 @@
 # requirements leveraged by ci tools
 setuptools==44.1.0; python_version == '2.7'
-setuptools==45.1.0; python_version >= '3.5'
+setuptools==46.4.0; python_version >= '3.5'
 virtualenv==20.0.23
 wheel==0.34.2 
-Jinja2==2.11.1
+Jinja2==2.11.2
 packaging==20.4
-tox==3.14.6
+tox==3.15.0
 tox-monorepo==0.1.2
-twine==1.15.0
+twine==1.15.0; python_version == '2.7' or python_version == '3.5'
+twine==3.1.1; python_version >= '3.6'
 pathlib2==2.3.5
 readme-renderer[md]==25.0
-doc-warden==0.5.4
+doc-warden==0.7.1
+# we pin coverage to 4.5.4 because there is an bug with `pytest-cov`. the generated coverage files cannot be `coverage combine`ed
 coverage==4.5.4
-codecov==2.0.22 
-beautifulsoup4==4.8.2
+codecov==2.1.0
+beautifulsoup4==4.9.1
 pkginfo==1.5.0.1
 
 # locking packages defined as deps from azure-sdk-tools or azure-devtools
 pytoml==0.1.21
 pyOpenSSL==19.1.0
 json-delta==2.0
-ConfigArgParse==1.1
+ConfigArgParse==1.2.3
 six==1.14.0
 vcrpy==3.0.0
 pyyaml==5.3.1
-pytest==5.4.1; python_version >= '3.5'
+pytest==5.4.2; python_version >= '3.5'
 pytest==4.6.9; python_version == '2.7'
 pytest-cov==2.8.1
 
 # local dev packages
 ./tools/azure-devtools
 ./tools/azure-sdk-tools
-
-
-

eng/common/TestResources/New-TestResources.ps1.md

@@ -412,6 +412,23 @@ Accept pipeline input: False
 Accept wildcard characters: False
 ```
 
+### -OutFile
+save test environment settings into a test-resources.json.env file next to test-resources.json. 
+The file is protected via DPAPI. The environment file would be scoped to the current repository directory.
+Note: Supported only on Windows.
+
+```yaml
+Type: SwitchParameter
+Parameter Sets: (All)
+Aliases:
+
+Required: False
+Position: Named
+Default value: None
+Accept pipeline input: False
+Accept wildcard characters: False
+```
+
 ### CommonParameters
 This cmdlet supports the common parameters: -Debug, -ErrorAction, -ErrorVariable, -InformationAction, -InformationVariable, -OutVariable, -OutBuffer, -PipelineVariable, -Verbose, -WarningAction, and -WarningVariable. For more information, see [about_CommonParameters](http://go.microsoft.com/fwlink/?LinkID=113216).
 

eng/common/TestResources/README.md

@@ -35,6 +35,12 @@ eng\common\TestResources\New-TestResources.ps1 `
   -TestApplicationSecret (ConvertFrom-SecureString $sp.Secret -AsPlainText)
 ```
 
+If you are running this for a .NET project on Windows, the recommended method is to 
+add the `-OutFile` switch to the above command. This will save test environment settings 
+into a test-resources.json.env file next to test-resources.json. The file is protected via DPAPI. 
+The environment file would be scoped to the current repository directory and avoids the need to 
+set environment variables or restart your IDE to recognize them.
+
 Along with some log messages, this will output environment variables based on
 your current shell like in the following example:
 

eng/common/Update-Change-Log.ps1

@@ -27,10 +27,10 @@ function Get-ChangelogPath($Path)
 { 
    # Check if CHANGELOG.md is present in path 
    $ChangeLogPath = Join-Path -Path $Path -ChildPath "CHANGELOG.md"
-   if ((Test-Path -Path $ChangeLogPath) -eq $False){
+   if ((Test-Path -Path $ChangeLogPath) -eq $False) {
       # Check if change log exists with name HISTORY.md
       $ChangeLogPath = Join-Path -Path $Path -ChildPath "HISTORY.md"
-      if ((Test-Path -Path $ChangeLogPath) -eq $False){
+      if ((Test-Path -Path $ChangeLogPath) -eq $False) {
          Write-Host "Change log is not found in path[$Path]"
          exit(1)
       }
@@ -45,7 +45,7 @@ function Get-VersionTitle($Version, $Unreleased)
 {
    # Generate version title
    $newVersionTitle = "## $Version $UNRELEASED_TAG"
-   if ($Unreleased -eq $False){
+   if ($Unreleased -eq $False) {
       $releaseDate = Get-Date -Format "(yyyy-MM-dd)"
       $newVersionTitle = "## $Version $releaseDate"
    }
@@ -67,10 +67,10 @@ function Get-NewChangeLog( [System.Collections.ArrayList]$ChangelogLines, $Versi
    # Version increment tool passes replaceversion as False and Unreleased as True
    $is_version_increment = $ReplaceVersion -eq $False -and $Unreleased -eq $True
 
-   for(; $Index -lt $ChangelogLines.Count; $Index++){
-      if (Version-Matches($ChangelogLines[$Index])){
+   for (; $Index -lt $ChangelogLines.Count; $Index++) {
+      if (Version-Matches($ChangelogLines[$Index])) {
          # Find current title in change log
-         if( -not $CurrentTitle){
+         if( -not $CurrentTitle) {
             $CurrentTitle = $ChangelogLines[$Index]
             $CurrentIndex = $Index
             Write-Host "Current Version title: $CurrentTitle"
@@ -80,7 +80,7 @@ function Get-NewChangeLog( [System.Collections.ArrayList]$ChangelogLines, $Versi
          # update change log script is triggered for all packages with current version for Java ( or any language where version is maintained in common file)
          # and this can cause an issue if someone changes changelog manually to prepare for release without updating actual version in central version file
          # Do not add new line or replace existing title when version is already present and script is triggered to add new line
-         if ($is_version_increment -and $ChangelogLines[$Index].Contains($Version)){
+         if ($is_version_increment -and $ChangelogLines[$Index].Contains($Version)) {
             Write-Host "Version is already present in change log."
             exit(0)
          }
@@ -90,26 +90,24 @@ function Get-NewChangeLog( [System.Collections.ArrayList]$ChangelogLines, $Versi
    # Generate version title
    $newVersionTitle = Get-VersionTitle -Version $Version -Unreleased $Unreleased
 
-   if( $newVersionTitle -eq $CurrentTitle){
+   if( $newVersionTitle -eq $CurrentTitle) {
       Write-Host "No change is required in change log. Version is already present."
       exit(0)
    }
 
-
-
-   if (($ReplaceVersion -eq $True) -and ($Unreleased -eq $False) -and (-not $CurrentTitle.Contains($UNRELEASED_TAG))){
+   if (($ReplaceVersion -eq $True) -and ($Unreleased -eq $False) -and $CurrentTitle.Contains($version) -and (-not $CurrentTitle.Contains($UNRELEASED_TAG))) {
       Write-Host "Version is already present in change log with a release date."
       exit(0)
    }
 
    # if current version title already has new version then we should replace title to update it
-   if ($CurrentTitle.Contains($Version) -and $ReplaceVersion -eq $False){
+   if ($CurrentTitle.Contains($Version) -and $ReplaceVersion -eq $False) {
       Write-Host "Version is already present in title. Updating version title"
       $ReplaceVersion = $True
    }
 
    # if version is already found and not replacing then nothing to do
-   if ($ReplaceVersion -eq $False){
+   if ($ReplaceVersion -eq $False) {
       Write-Host "Adding version title $newVersionTitle"
       $ChangelogLines.insert($CurrentIndex, "")
       $ChangelogLines.insert($CurrentIndex, "")
@@ -121,24 +119,28 @@ function Get-NewChangeLog( [System.Collections.ArrayList]$ChangelogLines, $Versi
       $ChangelogLines[$CurrentIndex] = $newVersionTitle
    }
 
-   return $ChangelogLines      
+   return $ChangelogLines
 }
 
 
 # Make sure path is valid
-if ((Test-Path -Path $ChangeLogPath) -eq $False){
+if ((Test-Path -Path $ChangeLogPath) -eq $False) {
    Write-Host "Change log path is invalid. [$ChangeLogPath]"
    exit(1)
 }
 
 # probe change log path if path is directory 
-if (Test-Path -Path $ChangeLogPath -PathType Container)
-{   
+if (Test-Path -Path $ChangeLogPath -PathType Container) {
    $ChangeLogPath = Get-ChangelogPath -Path $ChangeLogPath
 }
 
 # Read current change logs and add/update version
 $ChangelogLines = [System.Collections.ArrayList](Get-Content -Path $ChangeLogPath)
+
+if ($null -eq $ChangelogLines) { 
+   $ChangelogLines = @() 
+}
+
 $NewContents = Get-NewChangeLog -ChangelogLines $ChangelogLines -Version $Version -Unreleased $Unreleased -ReplaceVersion $ReplaceVersion
 
 Write-Host "Writing change log to file [$ChangeLogPath]"