Upload metadata function (Azure#17108)

sdk/template/azure-template/MANIFEST.in

@@ -1,3 +1,4 @@
+include _meta.json
 include *.md
 include azure/__init__.py
 recursive-include tests *.py

tools/azure-sdk-tools/packaging_tools/auto_codegen.py

@@ -1,14 +1,17 @@
 import argparse
 import json
 import logging
+import os
 from pathlib import Path
 import re
 from subprocess import check_call
 
 from .swaggertosdk.SwaggerToSdkCore import (
+    read_config,
     CONFIG_FILE,
 )
 from azure_devtools.ci_tools.git_tools import get_diff_file_list
+from .swaggertosdk.autorest_tools  import build_autorest_options
 from .generate_sdk import generate
 
 _LOGGER = logging.getLogger(__name__)
@@ -37,6 +40,57 @@ def init_new_service(package_name, folder_name):
             with open(str(ci), 'w') as file_out:
                 file_out.writelines(content)
 
+def update_servicemetadata(sdk_folder, data, config, folder_name, package_name, spec_folder, input_readme):
+
+    readme_file = str(Path(spec_folder, input_readme))
+    global_conf = config["meta"]
+    local_conf = config["projects"][readme_file]
+
+    cmd = ["autorest", input_readme]
+    cmd += build_autorest_options(global_conf, local_conf)
+
+    # metadata
+    metadata = {
+        "autorest": global_conf["autorest_options"]["version"],
+        "use": global_conf["autorest_options"]["use"],
+        "commit": data["headSha"],
+        "repository_url": data["repoHttpsUrl"],
+        "autorest_command": " ".join(cmd),
+        "readme": input_readme
+    }
+
+    _LOGGER.info("Metadata json:\n {}".format(json.dumps(metadata, indent=2)))
+
+    package_folder = Path(sdk_folder, folder_name, package_name).expanduser()
+    if not os.path.exists(package_folder):
+        _LOGGER.info(f"Package folder doesn't exist: {package_folder}")
+        _LOGGER.info("Failed to save metadata.")
+        return
+
+    metadata_file_path = os.path.join(package_folder, "_meta.json")
+    with open(metadata_file_path, "w") as writer:
+        json.dump(metadata, writer, indent=2)
+    _LOGGER.info(f"Saved metadata to {metadata_file_path}")
+
+    # Check whether MANIFEST.in includes _meta.json
+    require_meta = "include _meta.json\n"
+    manifest_file = os.path.join(package_folder, "MANIFEST.in")
+    if not os.path.exists(manifest_file):
+        _LOGGER.info(f"MANIFEST.in doesn't exist: {manifest_file}")
+        return
+
+    includes = []
+    write_flag = False
+    with open(manifest_file, "r") as f:
+        includes = f.readlines()
+        if require_meta not in includes:
+            includes = [require_meta] + includes
+            write_flag = True
+
+    if write_flag:
+        with open(manifest_file, "w") as f:
+            f.write("".join(includes))
+
 
 def main(generate_input, generate_output):
     with open(generate_input, "r") as reader:
@@ -49,7 +103,7 @@ def main(generate_input, generate_output):
     for input_readme in data["relatedReadmeMdFiles"]:
         relative_path_readme = str(Path(spec_folder, input_readme))
         _LOGGER.info(f'[CODEGEN]({input_readme})codegen begin')
-        generate(CONFIG_FILE,
+        config = generate(CONFIG_FILE,
                  sdk_folder,
                  [],
                  relative_path_readme,
@@ -59,6 +113,7 @@ def main(generate_input, generate_output):
         package_names = get_package_names(sdk_folder)
         _LOGGER.info(f'[CODEGEN]({input_readme})codegen end. [(packages:{str(package_names)})]')
 
+
         for folder_name, package_name in package_names:
             if package_name in package_total:
                 continue
@@ -77,6 +132,12 @@ def main(generate_input, generate_output):
             # Generate some necessary file for new service
             init_new_service(package_name, folder_name)
 
+            # Update metadata
+            try:
+                update_servicemetadata(sdk_folder, data, config, folder_name, package_name, spec_folder, input_readme)
+            except Exception as e:
+                _LOGGER.info(str(e))
+
             # Setup package locally
             check_call(f'pip install --ignore-requires-python -e {str(Path(sdk_folder, folder_name, package_name))}',
                        shell=True)

tools/azure-sdk-tools/packaging_tools/generate_sdk.py

@@ -82,6 +82,7 @@ def generate(config_path, sdk_folder, project_pattern, readme, restapi_git_folde
                 local_conf,
                 autorest_bin
             )
+    return config
 
 
 def generate_main():

tools/azure-sdk-tools/tests/test_servicemetadata.py

@@ -0,0 +1,154 @@
+import os
+import json
+import tempfile
+import unittest
+from pathlib import Path
+
+import pytest
+from packaging_tools.auto_codegen import update_servicemetadata
+
+
+"""
+Create metadata file
+
+Update metadata file
+
+Update MANIFETS.IN
+
+No need to update MANIFETS.IN
+"""
+
+MANIFEST_TEMP = """recursive-include tests *.py *.yaml
+include *.md
+include azure/__init__.py
+include azure/mgmt/__init__.py
+"""
+
+class TestServiceMetadata(unittest.TestCase):
+
+    def setUp(self):
+        self.sdk_folder = "sdk"
+        self.data = {
+            "specFolder": "../azure-rest-api-specs",
+            "headSha": "e295fe97eee3709668d3e5f7f8b434026b814ef9",
+            "headRef": "master",
+            "repoHttpsUrl": "https://github.com/Azure/azure-rest-api-specs",
+        }
+        self.config = {
+            "meta": {
+                "autorest_options": {
+                    "version": "3.0.6369",
+                    "use": "@autorest/python@5.4.3",
+                    "python": "",
+                    "python-mode": "update",
+                    "sdkrel:python-sdks-folder": "./sdk/.",
+                    "multiapi": "",
+                    "track2": ""
+                },
+                "advanced_options": {
+                    "create_sdk_pull_requests": True,
+                    "sdk_generation_pull_request_base": "integration_branch"
+                },
+                "repotag": "azure-sdk-for-python-track2",
+                "version": "0.2.0"
+            },
+            "projects": {
+                "./azure-rest-api-specs/specification/operationalinsights/resource-manager/readme.md": {}
+            }
+
+        }
+        self.folder_name = "monitor"
+        self.package_name = "azure-mgmt-monitor"
+        self.spec_folder = "./"
+        self.input_readme = "azure-rest-api-specs/specification/operationalinsights/resource-manager/readme.md"
+
+    def test_metadata(self):
+
+        with tempfile.TemporaryDirectory() as temp_dir:
+            # Init directories
+            self.sdk_folder = str(Path(temp_dir, "sdk"))
+            self.spec_folder = str(Path(temp_dir, "spec"))
+            os.makedirs(self.sdk_folder)
+            os.makedirs(self.spec_folder)
+
+            readme_file = str(Path(self.spec_folder, self.input_readme))
+            self.config["projects"][readme_file] = {}
+
+            package_folder = Path(self.sdk_folder, self.folder_name, self.package_name).expanduser()
+            metadata_file_path = os.path.join(package_folder, "_meta.json")
+            manifest_file = os.path.join(package_folder, "MANIFEST.in")
+            os.makedirs(package_folder)
+
+            # Test MANIFEST.in does not exist
+            update_servicemetadata(
+                sdk_folder=self.sdk_folder,
+                data=self.data,
+                config=self.config,
+                folder_name=self.folder_name,
+                package_name=self.package_name,
+                spec_folder=self.spec_folder,
+                input_readme=self.input_readme
+            )
+
+            assert os.path.isfile(metadata_file_path) == True
+            # Do not create MANIFEST.in, if it does not exist
+            assert os.path.isfile(manifest_file) == False
+
+            # Test update metadata
+            with open(metadata_file_path, "w") as f:
+                json.dump({"autorest": "3.0.0"}, f, indent=2)
+            update_servicemetadata(
+                sdk_folder=self.sdk_folder,
+                data=self.data,
+                config=self.config,
+                folder_name=self.folder_name,
+                package_name=self.package_name,
+                spec_folder=self.spec_folder,
+                input_readme=self.input_readme
+            )
+
+            assert os.path.isfile(metadata_file_path) == True
+            assert os.path.isfile(manifest_file) == False
+            with open(metadata_file_path, "r") as f:
+                md = json.load(f)
+                assert md["autorest"] == "3.0.6369"
+
+            # Test update MANIFEST.in
+            with open(manifest_file, "w") as f:
+                f.write(MANIFEST_TEMP)
+            update_servicemetadata(
+                sdk_folder=self.sdk_folder,
+                data=self.data,
+                config=self.config,
+                folder_name=self.folder_name,
+                package_name=self.package_name,
+                spec_folder=self.spec_folder,
+                input_readme=self.input_readme
+            )
+
+            assert os.path.isfile(metadata_file_path) == True
+            assert os.path.isfile(manifest_file) == True
+            with open(manifest_file, "r") as f:
+                meta_line = "include _meta.json\n"
+                line = f.readline()
+                assert meta_line == line
+
+            # Test update MANIFEST.in again
+            update_servicemetadata(
+                sdk_folder=self.sdk_folder,
+                data=self.data,
+                config=self.config,
+                folder_name=self.folder_name,
+                package_name=self.package_name,
+                spec_folder=self.spec_folder,
+                input_readme=self.input_readme
+            )
+
+            assert os.path.isfile(metadata_file_path) == True
+            assert os.path.isfile(manifest_file) == True
+            with open(manifest_file, "r") as f:
+                meta_line = "include _meta.json\n"
+                line = f.readline()
+                second_line = f.readline()
+                assert meta_line == line
+                assert meta_line != second_line