Merging Upstream

.github/CODEOWNERS

@@ -191,3 +191,7 @@
 /src/cloudservice/ @qwordy
 
 /src/webpubsub/ @zackliu
+
+/src/connectedvmware/ @sanmishra18
+
+/src/diskpool/ @Juliehzl

k8s-custom-pipelines.yml

@@ -27,7 +27,7 @@ stages:
   - job: K8sExtensionTestSuite
     displayName: "Run the Test Suite"
     pool:
-      vmImage: 'ubuntu-16.04'
+      vmImage: 'ubuntu-latest'
     steps:
       - checkout: self
       - bash: |
@@ -56,15 +56,16 @@ stages:
           source env/bin/activate
 
           # clone azure-cli
+          git clone -q --single-branch -b dev https://github.com/Azure/azure-cli.git ../azure-cli
+
           pip install --upgrade pip
-          pip install azdev
+          pip install -q azdev
 
           ls $(CLI_REPO_PATH)
 
           azdev --version
-          azdev setup -r $(CLI_REPO_PATH) -e $(EXTENSION_NAME)
+          azdev setup -c ../azure-cli -r $(CLI_REPO_PATH) -e $(EXTENSION_NAME)
           azdev extension build $(EXTENSION_NAME)
-
         workingDirectory: $(CLI_REPO_PATH)
         displayName: "Setup and Build Extension with azdev"
 
@@ -98,6 +99,10 @@ stages:
           chmod +x ./kind
           ./kind create cluster
         displayName: "Create and Start the Kind cluster"
+
+      - bash: |
+          curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
+        displayName: "Upgrade az to latest version"
 
       - task: AzureCLI@2
         displayName: Bootstrap
@@ -108,7 +113,7 @@ stages:
           inlineScript: |
             .\Bootstrap.ps1 -CI
           workingDirectory: $(TEST_PATH)
-
+            
       - task: AzureCLI@2
         displayName: Run the Test Suite Public Extensions Only
         inputs:
@@ -153,7 +158,7 @@ stages:
 
   - job: BuildPublishExtension
     pool:
-      vmImage: 'ubuntu-16.04'
+      vmImage: 'ubuntu-latest'
     displayName: "Build and Publish the Extension Artifact"
     variables: 
       CLI_REPO_PATH: $(Agent.BuildDirectory)/s
@@ -197,13 +202,15 @@ stages:
         source env/bin/activate
 
         # clone azure-cli
+        git clone -q --single-branch -b dev https://github.com/Azure/azure-cli.git ../azure-cli
+
         pip install --upgrade pip
-        pip install azdev
+        pip install -q azdev
 
         ls $(CLI_REPO_PATH)
 
         azdev --version
-        azdev setup -r $(CLI_REPO_PATH) -e $(EXTENSION_NAME)
+        azdev setup -c ../azure-cli -r $(CLI_REPO_PATH) -e $(EXTENSION_NAME)
         azdev extension build $(EXTENSION_NAME)
       workingDirectory: $(CLI_REPO_PATH)
       displayName: "Setup and Build Extension with azdev"
@@ -218,7 +225,7 @@ stages:
   - job: CheckLicenseHeader
     displayName: "Check License"
     pool:
-      vmImage: 'ubuntu-16.04'
+      vmImage: 'ubuntu-latest'
     steps:
     - task: UsePythonVersion@0
       displayName: 'Use Python 3.6'
@@ -249,7 +256,7 @@ stages:
   - job: StaticAnalysis
     displayName: "Static Analysis"
     pool:
-      vmImage: 'ubuntu-16.04'
+      vmImage: 'ubuntu-latest'
     steps:
       - task: UsePythonVersion@0
         displayName: 'Use Python 3.6'
@@ -263,7 +270,7 @@ stages:
   - job: IndexVerify
     displayName: "Verify Extensions Index"
     pool:
-      vmImage: 'ubuntu-16.04'
+      vmImage: 'ubuntu-latest'
     steps:
     - task: UsePythonVersion@0
       displayName: 'Use Python 3.7'
@@ -280,7 +287,7 @@ stages:
   - job: SourceTests
     displayName: "Integration Tests, Build Tests"
     pool:
-      vmImage: 'ubuntu-16.04'
+      vmImage: 'ubuntu-latest'
     strategy:
       matrix:
         Python36:
@@ -303,7 +310,7 @@ stages:
   - job: LintModifiedExtensions
     displayName: "CLI Linter on Modified Extensions"
     pool:
-      vmImage: 'ubuntu-16.04'
+      vmImage: 'ubuntu-latest'
     steps:
       - task: UsePythonVersion@0
         displayName: 'Use Python 3.6'
@@ -334,4 +341,4 @@ stages:
         displayName: "CLI Linter on Modified Extension"
         env:
           ADO_PULL_REQUEST_LATEST_COMMIT: $(System.PullRequest.SourceCommitId)
-          ADO_PULL_REQUEST_TARGET_BRANCH: $(System.PullRequest.TargetBranch)
+          ADO_PULL_REQUEST_TARGET_BRANCH: $(System.PullRequest.TargetBranch)

linter_exclusions.yml

@@ -920,14 +920,39 @@ import-export update:
     backup_drive_manifest:
       rule_exclusions:
       - option_length_too_long
-iot pnp capability-model update:
+iot hub configuration update:
   parameters:
-    repo_endpoint:
+    auth_type_dataplane:
       rule_exclusions:
       - no_parameter_defaults_for_update_commands
-iot pnp interface update:
+iot hub device-identity update:
   parameters:
-    repo_endpoint:
+    auth_type_dataplane:
+      rule_exclusions:
+      - no_parameter_defaults_for_update_commands
+iot hub device-twin update:
+  parameters:
+    auth_type_dataplane:
+      rule_exclusions:
+      - no_parameter_defaults_for_update_commands
+iot hub distributed-tracing update:
+  parameters:
+    auth_type_dataplane:
+      rule_exclusions:
+      - no_parameter_defaults_for_update_commands
+iot hub module-identity update:
+  parameters:
+    auth_type_dataplane:
+      rule_exclusions:
+      - no_parameter_defaults_for_update_commands
+iot hub module-twin update:
+  parameters:
+    auth_type_dataplane:
+      rule_exclusions:
+      - no_parameter_defaults_for_update_commands
+iot edge deployment update:
+  parameters:
+    auth_type_dataplane:
       rule_exclusions:
       - no_parameter_defaults_for_update_commands
 k8sconfiguration create:

scripts/ci/avail-ext-doc/list-template.md

@@ -9,18 +9,19 @@ ms.topic: article
 ms.prod: azure
 ms.technology: azure-cli
 ms.devlang: azure-cli
+ms.custom: devx-track-azurecli
 ---
 
 # Available extensions for the Azure CLI
 
 This article is a complete list of the available extensions for the Azure CLI which are supported by Microsoft.
 
-The list of extensions is also available  from the CLI. To get it, run [az extension list-available](/cli/azure/extension#az_extension_list_available):
+The list of extensions is also available from the CLI. To get it, run [az extension list-available](/cli/azure/extension#az_extension_list_available):
 
 ```azurecli-interactive
 az extension list-available --output table
 ```
 
-| Name | Version | Summary | Preview |
-|------|---------|---------|---------|{% for extension in extensions %}
-| [{{ extension.name }}]({{ extension.project_url }}) | {{ extension.version }} | {{ extension.desc }} | {{ extension.preview }} |{% endfor %}
+| Extension | Required Minimum CLI Version | Description | Status | Release Notes |
+|----|-----------------|-------------|---------|---------------|{% for extension in extensions %}
+|[{{ extension.name }}]({{ extension.project_url }}) | {{ extension.min_cli_core_version }} | {{ extension.desc }} | {{ extension.status }} | [{{extension.version}}]({{extension.history}}) |{% endfor %}

scripts/ci/avail-ext-doc/requirements.txt

@@ -1,2 +1,3 @@
 Jinja2~=2.10.1
-wheel==0.31.1
+wheel==0.31.1
+requests

scripts/ci/avail-ext-doc/update_extension_list.py

@@ -18,7 +18,7 @@
 from pkg_resources import parse_version
 
 from jinja2 import Template  # pylint: disable=import-error
-
+import requests
 
 SCRIPTS_LOCATION = os.path.abspath(os.path.join('.', 'scripts'))
 
@@ -36,12 +36,26 @@ def get_extensions():
     for _, exts in index_extensions.items():
         # Get latest version
         exts = sorted(exts, key=lambda c: parse_version(c['metadata']['version']), reverse=True)
+
+        # some extension modules may not include 'HISTORY.rst'
+        project_url = exts[0]['metadata']['extensions']['python.details']['project_urls']['Home']
+        history_tmp = project_url + '/HISTORY.rst'
+        history = project_url if str(requests.get(history_tmp).status_code) == '404' else history_tmp
+        if exts[0]['metadata'].get('azext.isPreview'):
+            status = 'Preview'
+        elif exts[0]['metadata'].get('azext.isExperimental'):
+            status = 'Experimental'
+        else:
+            status = 'GA'
+
         extensions.append({
             'name': exts[0]['metadata']['name'],
             'desc': exts[0]['metadata']['summary'],
+            'min_cli_core_version': exts[0]['metadata']['azext.minCliCoreVersion'],
             'version': exts[0]['metadata']['version'],
-            'project_url': exts[0]['metadata']['extensions']['python.details']['project_urls']['Home'],
-            'preview': 'Yes' if exts[0]['metadata'].get('azext.isPreview') else ''
+            'project_url': project_url,
+            'history': history,
+            'status': status
         })
     return extensions
 

scripts/ci/build_ext_cmd_tree.sh

@@ -18,8 +18,8 @@ export AZURE_CORE_COLLECT_TELEMETRY=False
 export AZURE_EXTENSION_INDEX_URL=https://raw.githubusercontent.com/Azure/azure-cli-extensions/master/src/index.json
 
 output=$(az extension list-available --query [].name -otsv)
-# azure-cli-iot-ext is the deprecated old versions of the renamed azure-iot extension
-blocklist=("azure-cli-iot-ext")
+# azure-cli-ml is replaced by ml
+blocklist=("azure-cli-ml")
 
 rm -f ~/.azure/extCmdTreeToUpload.json
 

scripts/ci/credscan/CredScanSuppressions.json

@@ -99,6 +99,18 @@
         "src\\stream-analytics\\azext_stream_analytics\\_help.py"
       ],
       "_justification": "dummy passwords for one-off resources"
+    },
+    {
+      "file": [
+        "src\\storage-blob-preview\\azext_storage_blob_preview\\tests\\latest\\recordings\\test_storage_blob_incremental_copy.yaml"
+      ],
+      "_justification": "[Storage] response body contains random value recognized as secret in outdated recoding files of storage may remove in the future"
+    },
+    {
+      "file": [
+        "src\\diskpool\\README.md"
+      ],
+      "_justification": "Faked password"
     }
   ]
 }

src/aks-preview/HISTORY.md

@@ -2,10 +2,24 @@
 
 Release History
 ===============
+
+0.5.15
++++++
+* Update to use 2021-05-01 api-version
+
+0.5.14
++++++
+* Add os-sku argument for cluster and nodepool creation
+
+0.5.13
++++++
+* Add compatible logic for the track 2 migration of resource dependence
+
 0.5.12
 +++++
 * Add --enable-azure-rbac and --disable-azure-rbac in aks update
 * Support disabling local accounts
+* Add addon `azure-defender` to list of available addons under `az aks enable-addons` command
 
 0.5.11
 +++++

src/aks-preview/az_aks_tool/.gitignore

@@ -0,0 +1 @@
+__pycache__

src/aks-preview/az_aks_tool/cli.py

@@ -0,0 +1,42 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+import glob
+import os
+import logging
+
+import az_aks_tool.const as const
+import az_aks_tool.index as index
+logger = logging.getLogger(__name__)
+
+
+def get_cli_mod_data(mod_name=const.ACS_MOD_NAME, profile="latest"):
+    profile_split = profile.split('-')
+    profile_namespace = '_'.join([profile_split[-1]] + profile_split[:-1])
+
+    # key value pairs of all modules(in azcli & extention) and its absolute path, used later to find test indexes
+    path_table = index.get_path_table()
+    command_modules = path_table["mod"]
+    inverse_name_table = index.get_name_index(invert=True)
+
+    # construct 'import_name' & mod_data', used later to find test indexes
+    acs_mod_path = command_modules[mod_name]
+    mod_data = {
+        "alt_name": "{}{}".format(const.COMMAND_MODULE_PREFIX, mod_name),
+        "filepath": os.path.join(acs_mod_path, "tests", profile_namespace),
+        "base_path": "azure.cli.command_modules.{}.tests.{}".format(mod_name, profile_namespace),
+        "files": {}
+    }
+
+    cli_test = index.discover_module_tests(mod_name, mod_data)
+    return cli_test
+
+
+def get_cli_test_index(module_data=None, mod_name=const.ACS_MOD_NAME, profile="latest"):
+    if mod_name in module_data:
+        mod_data = module_data[mod_name]
+    else:
+        mod_data = get_cli_mod_data(mod_name=mod_name, profile=profile)    
+    return mod_data["files"]

src/aks-preview/az_aks_tool/const.py

@@ -0,0 +1,17 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+import sys
+
+IS_WINDOWS = sys.platform.lower() in ['windows', 'win32']
+
+CLI_REPO_NAME = "azure-cli"
+EXT_REPO_NAME = 'azure-cli-extensions'
+COMMAND_MODULE_PREFIX = 'azure-cli-'
+EXTENSION_PREFIX = 'azext_'
+ACS_MOD_NAME = "acs"
+AKS_PREVIEW_MOD_NAME = EXTENSION_PREFIX + "aks_preview"  # azext_aks_preview
+
+ENV_VAR_TEST_LIVE = 'AZURE_TEST_RUN_LIVE'               # denotes that tests should be run live instead of played back