Merge branch 'main' of https://github.com/Azure/azure-sdk-for-python …

…into feature/conversations/initial * 'main' of https://github.com/Azure/azure-sdk-for-python: [cognitivelanguage] add live pipeline (#20986) [qna] address heath's comments (#20998) [qna] regen with latest and implement design changes (#20975) Add PoliCheck (#20776) Add credential Scope keyword (#20987) remove TODOs from samples (#20994) [KV] Add Py 2.7 disclaimer, 3.10 classifier (#20990) [formrecognizer] update sample readme (#20961) [formrecognizer] Update field schema links in v3 samples (#20988) Prep for release (#20991) reenable tests (#20978) Update 3.10 and 2.7 disclaimer (#20979) Increment package version after release of azure-core (#20980) [Monitor exporter] Update OT Exporter to latest spec changes (#20770) [formrecognizer] sample fixes (#20964)
Azure · Oct 1, 2021 · 9a00c6b · 9a00c6b
2 parents ab375e5 + 3f2a011
commit 9a00c6b
Show file tree

Hide file tree

Showing 169 changed files with 2,400 additions and 1,545 deletions.
diff --git a/eng/guardian-tools/policheck/PolicheckExclusions.xml b/eng/guardian-tools/policheck/PolicheckExclusions.xml
@@ -0,0 +1,12 @@
+<PoliCheckExclusions>
+<!-- All strings must be UPPER CASE -->
+<!-- Each of these exclusions is a folder name - if \[name]\ exists in the file path, it will be skipped -->
+<!--<Exclusion Type="FolderPathFull">ABC|XYZ</Exclusion>-->
+<!-- Each of these exclusions is a folder name - if any folder or file starts with "\[name]", it will be 
+skipped -->
+<!--<Exclusion Type="FolderPathStart">ABC|XYZ</Exclusion>-->
+<!-- Each of these file types will be completely skipped for the entire scan -->
+<!--<Exclusion Type="FileType">.ABC|.XYZ</Exclusion>-->
+<!-- The specified file names will be skipped during the scan regardless which folder they are in -->
+<!--<Exclusion Type="FileName">ABC.TXT|XYZ.CS</Exclusion>-->
+</PoliCheckExclusions>
diff --git a/eng/pipelines/aggregate-reports.yml b/eng/pipelines/aggregate-reports.yml
@@ -8,60 +8,105 @@ pr:
     include:
       - eng/pipelines/aggregate-reports.yml
 
-jobs:
-- job: 'ValidateDependencies'
-  variables:
+pool:
+  name: azsdk-pool-mms-win-2019-general
+  vmImage: MMS2019
+
+variables:
   - template: ./templates/variables/globals.yml
 
-  pool:
-    name: azsdk-pool-mms-win-2019-general
-    vmImage: MMS2019
-
-  steps:
-  - template: /eng/pipelines/templates/steps/analyze_dependency.yml
-
-  - task: AzureFileCopy@2
-    displayName: 'Upload dependency report'
-    condition: and(succeededOrFailed(), eq(variables['System.TeamProject'], 'internal'))
-    inputs:
-      sourcePath: '$(Build.ArtifactStagingDirectory)/reports'
-      azureSubscription: 'Azure SDK Artifacts'
-      destination: AzureBlob
-      storage: azuresdkartifacts
-      containerName: 'azure-sdk-for-python'
-      blobPrefix: dependencies
-
-  - task: PowerShell@2
-    displayName: "Verify Repository Resource Refs"
-    inputs:
-      pwsh: true
-      workingDirectory: $(Build.SourcesDirectory)
-      filePath: eng/common/scripts/Verify-Resource-Ref.ps1
-
-  - task: securedevelopmentteam.vss-secure-development-tools.build-task-credscan.CredScan@3
-    displayName: 'Run CredScan'
-    condition: succeededOrFailed()
-    inputs:
-      suppressionsFile: 'eng\CredScanSuppression.json'
-  - task: securedevelopmentteam.vss-secure-development-tools.build-task-postanalysis.PostAnalysis@2
-    displayName: 'Post Analysis'
-    condition: succeededOrFailed()
-    inputs:
-      GdnBreakAllTools: false
-      GdnBreakGdnToolCredScan: true
-      GdnBreakGdnToolCredScanSeverity: Error
-      GdnBreakBaselineFiles: $(Build.SourcesDirectory)\eng\python.gdnbaselines
-      GdnBreakBaselines: baseline
-      # Used for generating baseline file.
-      # GdnBreakOutputBaselineFile: python
-      # GdnBreakOutputBaseline: baseline
-    continueOnError: true
-  - task: securedevelopmentteam.vss-secure-development-tools.build-task-publishsecurityanalysislogs.PublishSecurityAnalysisLogs@3
-    displayName: 'Publish Security Analysis Logs'
-    continueOnError: true
-    condition: succeededOrFailed()
-  - template: ../common/pipelines/templates/steps/verify-links.yml
-    parameters:
-      Directory: ""
-      CheckLinkGuidance: $true
-      Condition: succeededOrFailed()
+stages:
+  - stage: ValidateDependencies
+    displayName: Validate Dependencies
+
+    jobs:
+      - job: ValidateDependencies
+        timeoutInMinutes: 120
+        steps:
+
+          - template: /eng/pipelines/templates/steps/analyze_dependency.yml
+
+          - task: AzureFileCopy@2
+            displayName: 'Upload dependency report'
+            condition: and(succeededOrFailed(), eq(variables['System.TeamProject'], 'internal'))
+            inputs:
+              sourcePath: '$(Build.ArtifactStagingDirectory)/reports'
+              azureSubscription: 'Azure SDK Artifacts'
+              destination: AzureBlob
+              storage: azuresdkartifacts
+              containerName: 'azure-sdk-for-python'
+              blobPrefix: dependencies
+
+          - task: PowerShell@2
+            displayName: "Verify Repository Resource Refs"
+            inputs:
+              pwsh: true
+              workingDirectory: $(Build.SourcesDirectory)
+              filePath: eng/common/scripts/Verify-Resource-Ref.ps1
+
+          - template: ../common/pipelines/templates/steps/verify-links.yml
+            parameters:
+              Directory: ""
+              CheckLinkGuidance: $true
+              Condition: succeededOrFailed()
+
+  - stage: ComplianceTools
+    displayName: Compliance Tools
+    dependsOn: []
+
+    jobs:
+      - job: ComplianceTools
+        timeoutInMinutes: 120
+        steps:
+          - task: securedevelopmentteam.vss-secure-development-tools.build-task-credscan.CredScan@3
+            displayName: 'Run CredScan'
+            condition: succeededOrFailed()
+            inputs:
+              suppressionsFile: 'eng\CredScanSuppression.json'
+
+          - task: securedevelopmentteam.vss-secure-development-tools.build-task-postanalysis.PostAnalysis@2
+            displayName: 'Post Analysis'
+            condition: succeededOrFailed()
+            inputs:
+              GdnBreakAllTools: false
+              GdnBreakGdnToolCredScan: true
+              GdnBreakGdnToolCredScanSeverity: Error
+              GdnBreakBaselineFiles: $(Build.SourcesDirectory)\eng\python.gdnbaselines
+              GdnBreakBaselines: baseline
+              # Used for generating baseline file.
+              # GdnBreakOutputBaselineFile: python
+              # GdnBreakOutputBaseline: baseline
+            continueOnError: true
+
+          - pwsh: |
+              azcopy copy "https://azuresdkartifacts.blob.core.windows.net/policheck/PythonPoliCheckExclusion.mdb?$(azuresdk-policheck-blob-SAS)" `
+              "$(Build.BinariesDirectory)"
+            displayName: 'Download PoliCheck Exclusion Database'
+            condition: succeededOrFailed()
+
+          - task: securedevelopmentteam.vss-secure-development-tools.build-task-policheck.PoliCheck@2
+            displayName: 'Run PoliCheck'
+            inputs:
+              targetType: F
+              targetArgument: '$(Build.SourcesDirectory)'
+              result: PoliCheck.sarif
+              optionsFC: 0
+              optionsXS: 1
+              optionsPE: 1|2|3|4
+              optionsRulesDBPath: "$(Build.BinariesDirectory)/PythonPoliCheckExclusion.mdb"
+              optionsUEPATH: "$(Build.SourcesDirectory)/eng/guardian-tools/policheck/PolicheckExclusions.xml"
+            condition: succeededOrFailed()
+
+          - task: securedevelopmentteam.vss-secure-development-tools.build-task-postanalysis.PostAnalysis@2
+            displayName: 'Post Analysis (PoliCheck)'
+            inputs:
+              GdnBreakAllTools: false
+              GdnBreakGdnToolPoliCheck: true
+              GdnBreakGdnToolPoliCheckSeverity: Warning
+            condition: succeededOrFailed()
+            continueOnError: true
+
+          - task: securedevelopmentteam.vss-secure-development-tools.build-task-publishsecurityanalysislogs.PublishSecurityAnalysisLogs@3
+            displayName: 'Publish Security Analysis Logs'
+            continueOnError: true
+            condition: succeededOrFailed()
diff --git a/scripts/devops_tasks/test_run_samples.py b/scripts/devops_tasks/test_run_samples.py
@@ -125,6 +125,10 @@
         "sample_list_translations_with_filters.py",
         "sample_translation_with_custom_model.py",
         "sample_translation_with_custom_model_async.py",
+    ],
+    "azure-ai-formrecognizer": [
+        "sample_manage_custom_models.py",
+        "sample_manage_custom_models_async.py",
     ]
 }
 

diff --git a/sdk/cognitivelanguage/azure-ai-language-questionanswering/CHANGELOG.md b/sdk/cognitivelanguage/azure-ai-language-questionanswering/CHANGELOG.md
@@ -5,10 +5,20 @@
 * We are now targeting service version `2021-07-15-preview`
 
 ### Breaking changes
+
 * The method `QuestionAnsweringClient.query_knowledgebase` has been renamed to `query_knowledge_base`.
+* Options bag model `KnowledgeBaseQueryOptions` for `query_knowledge_base` is renamed to `QueryKnowledgeBaseOptions`
+* Options bag model `TextQueryOptions` for `query_text` is renamed to `QueryTextOptions`
+* The filters model `StrictFilters` is renamed to `QueryFilters`
+* Enum `CompoundOperationKind` is renamed to `LogicalOperationKind`
+* We have removed the `string_index_type` input to all models and operations. We have also removed the `StringIndexType` enum.
+* The type of input `metadata` to `MetadataFilter` has changed from a dictionary of strings to a list of key-value tuples. For example, the input has changed from `{"key": "value"}` to `[("key", "value")]`.
+* The input to the `query_knowledge_base` and `query_text` overloads that take in a positional model for the body should be considered positional only.
 
 ### Features Added
+
 * The method `QuestionAnsweringClient.query_text` now supports a list of records as strings, where the ID value will be automatically populated.
+* Added keyword argument `default_language` onto `QuestionAnsweringClient`, which has default value `'en'`. The default language for any operation call will be this default language value.
 
 
 ## 1.0.0b1 (2021-07-27)

diff --git a/sdk/cognitivelanguage/azure-ai-language-questionanswering/README.md b/sdk/cognitivelanguage/azure-ai-language-questionanswering/README.md
@@ -6,13 +6,17 @@ Question Answering is a cloud-based API service that lets you create a conversat
 
 [Source code][questionanswering_client_src] | [Package (PyPI)][questionanswering_pypi_package] | [API reference documentation][questionanswering_refdocs] | [Product documentation][questionanswering_docs] | [Samples][questionanswering_samples]
 
+## _Disclaimer_
+
+_Azure SDK Python packages support for Python 2.7 is ending 01 January 2022. For more information and questions, please refer to https://github.com/Azure/azure-sdk-for-python/issues/20691_
+
 ## Getting started
 
 ### Prerequisites
 
-* Python 2.7, or 3.6 or later is required to use this package.
-* An [Azure subscription][azure_subscription]
-* An existing Question Answering resource
+- Python 2.7, or 3.6 or later is required to use this package.
+- An [Azure subscription][azure_subscription]
+- An existing Question Answering resource
 
 > Note: the new unified Cognitive Language Services are not currently available for deployment.
 
@@ -56,14 +60,15 @@ client = QuestionAnsweringClient(endpoint, credential)
 
 ### QuestionAnsweringClient
 
-The [QuestionAnsweringClient][questionanswering_client_class] is the primary interface for asking questions using a knowledge base with your own information, or text input using pre-trained models. 
+The [QuestionAnsweringClient][questionanswering_client_class] is the primary interface for asking questions using a knowledge base with your own information, or text input using pre-trained models.
 For asynchronous operations, an async `QuestionAnsweringClient` is in the `azure.ai.language.questionanswering.aio` namespace.
 
 ## Examples
 
 The `azure-ai-language-questionanswering` client library provides both synchronous and asynchronous APIs.
 
 The following examples show common scenarios using the `client` [created above](#create-questionansweringclient).
+
 - [Ask a question](#ask-a-question)
 - [Ask a follow-up question](#ask-a-follow-up-question)
 - [Asynchronous operations](#asynchronous-operations)
@@ -75,7 +80,7 @@ The only input required to ask a question using a knowledge base is just the que
 ```python
 from azure.ai.language.questionanswering import models as qna
 
-params = qna.KnowledgeBaseQueryOptions(
+params = qna.QueryKnowledgeBaseOptions(
     question="How long should my Surface battery last?"
 )
 
@@ -89,14 +94,14 @@ for candidate in output.answers:
 
 ```
 
-You can set additional properties on `KnowledgeBaseQueryOptions` to limit the number of answers, specify a minimum confidence score, and more.
+You can set additional properties on `QueryKnowledgeBaseOptions` to limit the number of answers, specify a minimum confidence score, and more.
 
 ### Ask a follow-up question
 
 If your knowledge base is configured for [chit-chat][questionanswering_docs_chat], the answers from the knowledge base may include suggested [prompts for follow-up questions][questionanswering_refdocs_prompts] to initiate a conversation. You can ask a follow-up question by providing the ID of your chosen answer as the context for the continued conversation:
 
 ```python
-params = qna.models.KnowledgeBaseQueryOptions(
+params = qna.models.QueryKnowledgeBaseOptions(
     question="How long should charging take?"
     context=qna.models.KnowledgeBaseAnswerRequestContext(
         previous_qna_id=previous_answer.id
@@ -112,17 +117,19 @@ for candidate in output.answers:
     print("Source: {}".format(candidate.source))
 
 ```
+
 ### Asynchronous operations
 
 The above examples can also be run asynchronously using the client in the `aio` namespace:
+
 ```python
 from azure.core.credentials import AzureKeyCredential
 from azure.ai.language.questionanswering.aio import QuestionAnsweringClient
 from azure.ai.language.questionanswering import models as qna
 
 client = QuestionAnsweringClient(endpoint, credential)
 
-params = qna.KnowledgeBaseQueryOptions(
+params = qna.QueryKnowledgeBaseOptions(
     question="How long should my Surface battery last?"
 )
 
@@ -133,11 +140,13 @@ output = await client.query_knowledgebase(
 ```
 
 ## Optional Configuration
+
 Optional keyword arguments can be passed in at the client and per-operation level. The azure-core [reference documentation][azure_core_ref_docs] describes available configurations for retries, logging, transport protocols, and more.
 
 ## Troubleshooting
 
 ### General
+
 Azure QuestionAnswering clients raise exceptions defined in [Azure Core][azure_core_readme].
 When you interact with the Cognitive Language Services Question Answering client library using the Python SDK, errors returned by the service correspond to the same HTTP status codes returned for [REST API][questionanswering_rest_docs] requests.
 
@@ -156,6 +165,7 @@ except HttpResponseError as error:
 ```
 
 ### Logging
+
 This library uses the standard
 [logging][python_logging] library for logging.
 Basic information about HTTP sessions (URLs, headers, etc.) is logged at INFO
@@ -168,9 +178,9 @@ See full SDK logging documentation with examples [here][sdk_logging_docs].
 
 ## Next steps
 
-* View our [samples][questionanswering_samples].
-* Read about the different [features][questionanswering_docs_features] of the Question Answering service.
-* Try our service [demos][questionanswering_docs_demos].
+- View our [samples][questionanswering_samples].
+- Read about the different [features][questionanswering_docs_features] of the Question Answering service.
+- Try our service [demos][questionanswering_docs_demos].
 
 ## Contributing
 
@@ -183,6 +193,7 @@ When you submit a pull request, a CLA-bot will automatically determine whether y
 This project has adopted the [Microsoft Open Source Code of Conduct][code_of_conduct]. For more information see the [Code of Conduct FAQ][coc_faq] or contact [opencode@microsoft.com][coc_contact] with any additional questions or comments.
 
 <!-- LINKS -->
+
 [azure_cli]: https://docs.microsoft.com/cli/azure/
 [azure_portal]: https://portal.azure.com/
 [azure_subscription]: https://azure.microsoft.com/free/
@@ -196,7 +207,7 @@ This project has adopted the [Microsoft Open Source Code of Conduct][code_of_con
 [sdk_logging_docs]: https://docs.microsoft.com/azure/developer/python/azure-sdk-logging
 [azure_core_ref_docs]: https://azuresdkdocs.blob.core.windows.net/$web/python/azure-core/latest/azure.core.html
 [azure_core_readme]: https://github.com/Azure/azure-sdk-for-python/blob/main/sdk/core/azure-core/README.md
-[pip_link]:https://pypi.org/project/pip/
+[pip_link]: https://pypi.org/project/pip/
 [questionanswering_client_class]: https://azuresdkdocs.blob.core.windows.net/$web/python/azure-ai-language-questionanswering/1.0.0b1/azure.ai.language.questionanswering.html#azure.ai.language.questionanswering.QuestionAnsweringClient
 [questionanswering_refdocs_prompts]: https://azuresdkdocs.blob.core.windows.net/$web/python/azure-ai-language-questionanswering/1.0.0b1/azure.ai.language.questionanswering.models.html#azure.ai.language.questionanswering.models.KnowledgeBaseAnswerDialog
 [questionanswering_client_src]: https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/cognitivelanguage/azure-ai-language-questionanswering/

diff --git a/...anguage/azure-ai-language-questionanswering/azure/ai/language/questionanswering/_patch.py b/...anguage/azure-ai-language-questionanswering/azure/ai/language/questionanswering/_patch.py
@@ -36,3 +36,29 @@ def _validate_text_records(records):
         else:
             request_batch.append(doc)
     return request_batch
+
+def _get_positional_body(*args, **kwargs):
+    """Verify args and kwargs are valid, and then return the positional body, if users passed it in."""
+    if len(args) > 1:
+        raise TypeError("There can only be one positional argument, which is the POST body of this request.")
+    if args and "options" in kwargs:
+        raise TypeError(
+            "You have already supplied the request body as a positional parameter, "
+            "you can not supply it as a keyword argument as well."
+        )
+    return args[0] if args else None
+
+def _verify_qna_id_and_question(query_knowledgebase_options):
+    """For query_knowledge_base we require either `question` or `qna_id`."""
+    try:
+        qna_id = query_knowledgebase_options.qna_id
+        question = query_knowledgebase_options.question
+    except AttributeError:
+        qna_id = query_knowledgebase_options.get("qna_id") or query_knowledgebase_options.get("qnaId")
+        question = query_knowledgebase_options.get("question")
+    if not (qna_id or question):
+        raise TypeError(
+            "You need to pass in either `qna_id` or `question`."
+        )
+    if qna_id and question:
+        raise TypeError("You can not specify both `qna_id` and `question`.")
diff --git a/...guage-questionanswering/azure/ai/language/questionanswering/_question_answering_client.py b/...guage-questionanswering/azure/ai/language/questionanswering/_question_answering_client.py
@@ -32,6 +32,7 @@ class QuestionAnsweringClient(QuestionAnsweringClientOperationsMixin):
     :type endpoint: str
     :param credential: Credential needed for the client to connect to Azure.
     :type credential: ~azure.core.credentials.AzureKeyCredential
+    :keyword str default_language: Sets the default language to use for all operations.
     """
 
     def __init__(
@@ -49,6 +50,7 @@ def __init__(
         self._serialize = Serializer(client_models)
         self._deserialize = Deserializer(client_models)
         self._serialize.client_side_validation = False
+        self._default_language = kwargs.pop("default_language", None)
 
     def send_request(
         self,