[Package mode] add new flag --package-mode (#1154)

ChangeLog.md

@@ -10,6 +10,10 @@
 | `msrest` dep of generated code                                          | `0.6.21`    |
 | `azure-mgmt-core` dep of generated code (If generating mgmt plane code) | `1.3.0`     |
 
+**New Features**
+
+- Add flag `--package-mode` to generate necessary files for package #1154
+
 **Bug Fixes**
 
 - Improve operation group documentation to prevent users from initializing operation groups themselves #1179

autorest/codegen/__init__.py

@@ -6,6 +6,7 @@
 import logging
 import sys
 from typing import Dict, Any, Set, Union, List, Type
+from pathlib import Path
 import yaml
 
 from .. import Plugin
@@ -70,12 +71,21 @@ def _validate_code_model_options(options: Dict[str, Any]) -> None:
     if options["basic_setup_py"] and not options["package_version"]:
         raise ValueError("--basic-setup-py must be used with --package-version")
 
+    if options["package_mode"] and not options["package_version"]:
+        raise ValueError("--package-mode must be used with --package-version")
+
     if not options["show_operations"] and options["combine_operation_files"]:
         raise ValueError(
             "Can not combine operation files if you are not showing operations. "
             "If you want operation files, pass in flag --show-operations"
         )
 
+    if options["package_mode"]:
+        if options["package_mode"] not in ("mgmtplane", "dataplane") and not Path(options["package_mode"]).exists():
+            raise ValueError(
+                "--package-mode can only be 'mgmtplane' or 'dataplane' or directory which contains template files"
+            )
+
     if options["reformat_next_link"] and options["version_tolerant"]:
         raise ValueError(
             "--reformat-next-link can not be true for version tolerant generations. "
@@ -117,6 +127,14 @@ def _build_exceptions_set(yaml_data: List[Dict[str, Any]]) -> Set[int]:
                     exceptions_set.add(id(exception["schema"]))
         return exceptions_set
 
+    @staticmethod
+    def _build_package_dependency() -> Dict[str, str]:
+        return {
+            "dependency_azure_mgmt_core": "azure-mgmt-core>=1.3.0,<2.0.0",
+            "dependency_azure_core": "azure-core<2.0.0,>=1.20.1",
+            "dependency_msrest": "msrest>=0.6.21",
+        }
+
     def _create_code_model(self, yaml_data: Dict[str, Any], options: Dict[str, Union[str, bool]]) -> CodeModel:
         # Create a code model
 
@@ -161,6 +179,7 @@ def _create_code_model(self, yaml_data: Dict[str, Any], options: Dict[str, Union
         if options["credential"]:
             code_model.global_parameters.add_credential_global_parameter()
 
+        code_model.package_dependency = self._build_package_dependency()
         return code_model
 
     def _get_credential_scopes(self, credential):
@@ -295,6 +314,9 @@ def _build_code_model_options(self) -> Dict[str, Any]:
             "low_level_client": low_level_client,
             "combine_operation_files": self._autorestapi.get_boolean_value("combine-operation-files", version_tolerant),
             "python3_only": python3_only,
+            "package_mode": self._autorestapi.get_value("package-mode"),
+            "package_pprint_name": self._autorestapi.get_value("package-pprint-name"),
+            "package_configuration": self._autorestapi.get_value("package-configuration"),
             "default_optional_constants_to_none": self._autorestapi.get_boolean_value(
                 "default-optional-constants-to-none", low_level_client or version_tolerant
             ),

autorest/codegen/models/code_model.py

@@ -51,6 +51,8 @@ class CodeModel:  # pylint: disable=too-many-instance-attributes, too-many-publi
     :type primitives: Dict[int, ~autorest.models.BaseSchema]
     :param operation_groups: The operation groups we are going to serialize
     :type operation_groups: list[~autorest.models.OperationGroup]
+    :param package_dependency: All the dependencies needed in setup.py
+    :type package_dependency: Dict[str, str]
     """
 
     def __init__(
@@ -76,6 +78,7 @@ def __init__(
         self._rest: Optional[Rest] = None
         self.request_builder_ids: Dict[int, RequestBuilder] = {}
         self._credential_schema_policy: Optional[CredentialSchemaPolicy] = None
+        self.package_dependency: Dict[str, str] = {}
 
     @property
     def global_parameters(self) -> GlobalParameterList:

autorest/codegen/serializers/__init__.py

@@ -3,16 +3,17 @@
 # Licensed under the MIT License. See License.txt in the project root for
 # license information.
 # --------------------------------------------------------------------------
-from typing import List, Optional
+from typing import Dict, List, Optional, Any
 from pathlib import Path
-from jinja2 import PackageLoader, Environment
+from jinja2 import PackageLoader, Environment, FileSystemLoader, StrictUndefined
 from autorest.codegen.models.operation_group import OperationGroup
 
 from ...jsonrpc import AutorestAPI
 from ..models import (
     CodeModel,
     OperationGroup,
     RequestBuilder,
+    TokenCredentialSchema
 )
 from .enum_serializer import EnumSerializer
 from .general_serializer import GeneralSerializer
@@ -29,6 +30,20 @@
     "JinjaSerializer",
 ]
 
+_PACKAGE_FILES = [
+    "CHANGELOG.md.jinja2",
+    "dev_requirements.txt.jinja2",
+    "LICENSE.jinja2",
+    "MANIFEST.in.jinja2",
+    "README.md.jinja2",
+    "setup.py.jinja2",
+]
+
+_REGENERATE_FILES = {
+    "setup.py",
+    "MANIFEST.in"
+}
+
 class JinjaSerializer:
     def __init__(self, autorestapi: AutorestAPI, code_model: CodeModel) -> None:
         self._autorestapi = autorestapi
@@ -74,6 +89,60 @@ def serialize(self) -> None:
         if self.code_model.options["models_mode"] and (self.code_model.schemas or self.code_model.enums):
             self._serialize_and_write_models_folder(env=env, namespace_path=namespace_path)
 
+        if self.code_model.options["package_mode"]:
+            self._serialize_and_write_package_files(out_path=namespace_path)
+
+
+    def _serialize_and_write_package_files(self, out_path: Path) -> None:
+        def _serialize_and_write_package_files_proc(**kwargs: Any):
+            for template_name in package_files:
+                file = template_name.replace(".jinja2", "")
+                output_name = out_path / file
+                if not self._autorestapi.read_file(output_name) or file in _REGENERATE_FILES:
+                    template = env.get_template(template_name)
+                    render_result = template.render(**kwargs)
+                    self._autorestapi.write_file(output_name, render_result)
+
+        def _prepare_params() -> Dict[Any, Any]:
+            package_parts = self.code_model.options["package_name"].split("-")[:-1]
+            try:
+                token_cred = isinstance(self.code_model.credential_schema_policy.credential, TokenCredentialSchema)
+            except ValueError:
+                token_cred = False
+            version = self.code_model.options["package_version"]
+            if any(x in version for x in ["a", "b", "rc"]) or version[0] == '0':
+                dev_status = "4 - Beta"
+            else:
+                dev_status = "5 - Production/Stable"
+            params = {
+                "token_credential": token_cred,
+                "pkgutil_names": [".".join(package_parts[: i + 1]) for i in range(len(package_parts))],
+                "init_names": ["/".join(package_parts[: i + 1]) + "/__init__.py" for i in range(len(package_parts))],
+                "dev_status": dev_status
+            }
+            params.update(self.code_model.options)
+            params.update(self.code_model.package_dependency)
+            return params
+
+        count = self.code_model.options["package_name"].count("-") + 1
+        for _ in range(count):
+            out_path = out_path / Path("..")
+
+        if self.code_model.options["package_mode"] in ("dataplane", "mgmtplane"):
+            env = Environment(
+                loader=PackageLoader("autorest.codegen", "templates"),
+                undefined=StrictUndefined)
+            package_files = _PACKAGE_FILES
+            _serialize_and_write_package_files_proc(**_prepare_params())
+        elif Path(self.code_model.options["package_mode"]).exists():
+            env = Environment(
+                loader=FileSystemLoader(str(Path(self.code_model.options["package_mode"]))),
+                keep_trailing_newline=True,
+                undefined=StrictUndefined
+            )
+            package_files = env.list_templates()
+            params = self.code_model.options["package_configuration"] or {}
+            _serialize_and_write_package_files_proc(**params)
 
 
     def _keep_patch_file(self, path_file: Path, env: Environment):

autorest/codegen/serializers/general_serializer.py

@@ -119,4 +119,7 @@ def serialize_version_file(self) -> str:
 
     def serialize_setup_file(self) -> str:
         template = self.env.get_template("setup.py.jinja2")
-        return template.render(code_model=self.code_model)
+        params = {}
+        params.update(self.code_model.options)
+        params.update(self.code_model.package_dependency)
+        return template.render(code_model=self.code_model, **params)

autorest/codegen/templates/CHANGELOG.md.jinja2

@@ -0,0 +1,6 @@
+# Release History
+
+## 1.0.0b1 (1970-01-01)
+
+- Initial version
+

autorest/codegen/templates/LICENSE.jinja2

@@ -0,0 +1,21 @@
+Copyright (c) Microsoft Corporation.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

autorest/codegen/templates/MANIFEST.in.jinja2

@@ -0,0 +1,7 @@
+include *.md
+include LICENSE
+recursive-include tests *.py
+recursive-include samples *.py *.md
+{%- for init_name in init_names %}
+include {{ init_name }}
+{%- endfor %}

autorest/codegen/templates/README.md.jinja2

@@ -0,0 +1,105 @@
+{% if package_mode == "mgmtplane" -%}
+# Microsoft Azure SDK for Python
+
+This is the Microsoft {{package_pprint_name}} Client Library.
+This package has been tested with Python 3.6+.
+For a more complete view of Azure libraries, see the [azure sdk python release](https://aka.ms/azsdk/python/all).
+
+# Usage
+
+To learn how to use this package, see the [quickstart guide](https://aka.ms/azsdk/python/mgmt)
+
+For docs and references, see [Python SDK References](https://docs.microsoft.com/python/api/overview/azure)
+Code samples for this package can be found at [{{package_pprint_name}}](https://docs.microsoft.com/samples/browse/?languages=python&term=Getting%20started%20-%20Managing&terms=Getting%20started%20-%20Managing) on docs.microsoft.com.
+Additional code samples for different Azure services are available at [Samples Repo](https://aka.ms/azsdk/python/mgmt/samples)
+
+# Provide Feedback
+
+If you encounter any bugs or have suggestions, please file an issue in the
+[Issues](https://github.com/Azure/azure-sdk-for-python/issues)
+section of the project. 
+
+
+![Impressions](https://azure-sdk-impressions.azurewebsites.net/api/impressions/azure-sdk-for-python%2F{{package_name}}%2FREADME.png)
+{% else %}
+# {{ package_pprint_name }} client library for Python
+<!-- write necessary description of service -->
+
+## Getting started
+
+### Installating the package
+
+```bash
+python -m pip install {{ package_name }}
+```
+
+#### Prequisites
+
+- Python 3.6 or later is required to use this package.
+- You need an [Azure subscription][azure_sub] to use this package.
+- An existing {{ package_pprint_name }} instance.
+
+{%- if token_credential %}
+#### Create with an Azure Active Directory Credential
+To use an [Azure Active Directory (AAD) token credential][authenticate_with_token],
+provide an instance of the desired credential type obtained from the
+[azure-identity][azure_identity_credentials] library.
+
+To authenticate with AAD, you must first [pip][pip] install [`azure-identity`][azure_identity_pip]
+
+After setup, you can choose which type of [credential][azure_identity_credentials] from azure.identity to use.
+As an example, [DefaultAzureCredential][default_azure_credential] can be used to authenticate the client:
+
+Set the values of the client ID, tenant ID, and client secret of the AAD application as environment variables:
+`AZURE_CLIENT_ID`, `AZURE_TENANT_ID`, `AZURE_CLIENT_SECRET`
+
+Use the returned token credential to authenticate the client:
+
+```python
+>>> from {{ namespace }} import {{ client_name }}
+>>> from azure.identity import DefaultAzureCredential
+>>> client = {{ client_name }}(endpoint='<endpoint>', credential=DefaultAzureCredential())
+```
+
+## Examples
+
+```python
+>>> from {{ namespace }} import {{ client_name }}
+>>> from azure.identity import DefaultAzureCredential
+>>> from azure.core.exceptions import HttpResponseError
+
+>>> client = {{ client_name }}(endpoint='<endpoint>', credential=DefaultAzureCredential())
+>>> try:
+        <!-- write test code here -->
+    except HttpResponseError as e:
+        print('service responds error: {}'.format(e.response.json()))
+
+```
+{%- endif %}
+
+## Contributing
+
+This project welcomes contributions and suggestions. Most contributions require
+you to agree to a Contributor License Agreement (CLA) declaring that you have
+the right to, and actually do, grant us the rights to use your contribution.
+For details, visit https://cla.microsoft.com.
+
+When you submit a pull request, a CLA-bot will automatically determine whether
+you need to provide a CLA and decorate the PR appropriately (e.g., label,
+comment). Simply follow the instructions provided by the bot. You will only
+need to do this once across all repos using our CLA.
+
+This project has adopted the
+[Microsoft Open Source Code of Conduct][code_of_conduct]. For more information,
+see the Code of Conduct FAQ or contact opencode@microsoft.com with any
+additional questions or comments.
+
+<!-- LINKS -->
+[code_of_conduct]: https://opensource.microsoft.com/codeofconduct/
+[authenticate_with_token]: https://docs.microsoft.com/azure/cognitive-services/authentication?tabs=powershell#authenticate-with-an-authentication-token
+[azure_identity_credentials]: https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/identity/azure-identity#credentials
+[azure_identity_pip]: https://pypi.org/project/azure-identity/
+[default_azure_credential]: https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/identity/azure-identity#defaultazurecredential
+[pip]: https://pypi.org/project/pip/
+[azure_sub]: https://azure.microsoft.com/free/
+{% endif %}

autorest/codegen/templates/dev_requirements.txt.jinja2

@@ -0,0 +1,10 @@
+-e ../../../tools/azure-devtools
+-e ../../../tools/azure-sdk-tools
+../../core/azure-core
+{% if token_credential -%}
+../../identity/azure-identity
+{% endif -%}
+{% if azure_arm -%}
+../../core/azure-mgmt-core
+{% endif -%}
+aiohttp