From 09c016bbfb291568d8d45ff6c20a2d63188df92a Mon Sep 17 00:00:00 2001 From: Santiago Quiroga <22756465+quirogas@users.noreply.github.com> Date: Fri, 14 Jul 2023 09:25:33 -0400 Subject: [PATCH] Feature/mp init blank template (#229) * Add template selector * Add Blank template adapter --- docs/get_started.md | 39 ++-- docs/references/mp-init.md | 4 +- .../adapter_template/adapter.py | 5 +- .../adapter_template/new_adapter_template.py | 173 ++++++++++++++++++ .../constant.py | 4 + .../mp_init.py | 54 +++++- 6 files changed, 252 insertions(+), 27 deletions(-) create mode 100644 vmware_aria_operations_integration_sdk/adapter_template/new_adapter_template.py diff --git a/docs/get_started.md b/docs/get_started.md index 7b77eacf..e981a390 100644 --- a/docs/get_started.md +++ b/docs/get_started.md @@ -27,7 +27,7 @@ In addition, at least one Cloud Proxy (also version 8.10 or later) must be set u go to [Docker's installation documentation](https://docs.docker.com/engine/install/), follow the instructions provided for your operating system. Finally, make sure the Docker default socket is enabled in the **Advanced** tab in **Settings** (version 4.18.0 and above). -* Container registry with read and write access (see [FAQ and Troubleshooting](troubleshooting_and_faq.md#why-do-i-need-a-container-registry?) for additional information). +* Container registry with read and write access (see [FAQ and Troubleshooting](troubleshooting_and_faq.md#why-do-i-need-a-container-registry) for additional information). * Python3 3.9.0 or later. Updating to the latest stable version is recommended. Python 3.8 and earlier (including Python2) are not supported. For instructions on installing Python, go to [Python's installation documentation](https://wiki.python.org/moin/BeginnersGuide/Download), and follow the instructions provided for your operating system. @@ -95,18 +95,29 @@ the creation of a new management pack project. PNG format and 256x256 pixels. An icon file can be added later by copying the icon to the root project directory and setting the value of the `"pak_icon"` key to the icon's file name in the `manifest.txt` file. -For complete documentation of the `mp-init` tool including an overview of its output, see the [MP Initialization Tool Documentation](references/mp-init.md). +8. `Select a template for your project` -### Template Project -Every new project creates a file system that has the basic project structure required to develop and build a Management Pack. + Both of the available options will generate a project structure that can be modified into an adapter: + + `Sample Adapter`: Creates a template adapter that collects several objects and metrics from the container the adapter is running. + The template adapter has comments throughout its code that explain what the code does and how to customize it for your adapter. + + `New Adapter`: Creates methods, minimal comments, and code necessary to implement test + connection, collection, adapter definition, and endpoints logic. + + For the purposes of this *Get Started* guide, select **Sample Adapter**. + +???+ info + + For complete documentation of the `mp-init` tool, including an overview of its output, + see the [MP Initialization Tool Documentation](references/mp-init.md). + +### Project Templates +Both templates create a file system that has the basic project structure required to develop and build a Management Pack. Each file and directory is discussed in depth in the [mp-init](references/mp-init.md) documentation. `app/adapter.py` is the adapter's entry point and the best starting point. -`adapter.py` is a template adapter that collects several objects and metrics from the -container in which the adapter is running. The template adapter has comments throughout its code that explain what the code does -and how to customize it for your adapter. - -The methods inside the adapter template are required, and should be modified to generate a custom +The methods inside `adapter.py` are required, and should be modified to generate a custom adapter. Each method fulfills a request from the VMware Aria Operations collector, and can be tested individually using `mp-test` (covered in [Testing a Management Pack](#testing-a-management-pack)). @@ -151,7 +162,7 @@ Each method is described below: when using advanced features of the `describe.xml` file that are not present in this method. -For further guidance on using the template project, consult the `Guides` section. +For further guidance on using the sample adapter, consult the `Guides` section. ### Testing a Management Pack @@ -171,7 +182,7 @@ For Windows: ```cmd venv\Scripts\activate.bat ``` -???+ note +???+ note To exit the virtual environment, run `deactivate` in the virtual environment. @@ -185,9 +196,9 @@ reads the `conf/describe.xml` file to find the connection parameters and credent prompts for each. This is similar to creating a new _Adapter Instance_ in the VMware Aria Operations UI. Connections are automatically saved per project, and can be reused when re-running the `mp-test` tool. -???+ note +???+ note - In the template project, the only connection parameter is `ID`, and because it connects to the container it is running on, + In the sample adapter, the only connection parameter is `ID`, and because it connects to the container it is running on, this parameter is not necessary; it is only there as an example, and can be set to any value. The template also implements an example Test Connection. If a Test Connection is run (see below), with the `ID` set to the text `bad`, then the Test Connection will fail. @@ -229,7 +240,7 @@ It should return successfully, then click `ADD`. By default, a collection will run every 5 minutes. The first collection should happen immediately. However, newly-created objects cannot have metrics, properties, and events added to them. After the second collection, approximately five -minutes later, the objects' metrics, properties, and events should appear. These can be checked by navigating to +minutes later, the objects' metrics, properties, and events should appear. These can be checked by navigating to **Environment → Object Browser → All Objects** and expanding the Adapter and associated object types and object. ![CPU Idle Time](images/test-adapter-cpu-idle-time.png) diff --git a/docs/references/mp-init.md b/docs/references/mp-init.md index 01f9d84b..0d05bb03 100644 --- a/docs/references/mp-init.md +++ b/docs/references/mp-init.md @@ -3,13 +3,13 @@ ## Purpose -The mp-init tool generates a template project that contains all the requirements to develop a Management Pack with a +The `mp-init` tool generates project templates that contain all the requirements to develop a Management Pack with a containerized adapter for VMware Aria Operations. To build a Management Pack, use the [build tool](mp-build.md). When calling `mp-init`, the user will be prompted with a series of questions. The script will use these questions to generate an initial project structure and create classifiers that other tools and VMware Aria Operations will use. ## Prerequisites -* The [VMware Aria Operations Integration SDK](../index.md#installation) is installed, with the virtual environment active. +* The [VMware Aria Operations Integration SDK](../get_started.md#installation) is installed, with the virtual environment active. ## Input diff --git a/vmware_aria_operations_integration_sdk/adapter_template/adapter.py b/vmware_aria_operations_integration_sdk/adapter_template/adapter.py index cf9050fc..e4a1532a 100644 --- a/vmware_aria_operations_integration_sdk/adapter_template/adapter.py +++ b/vmware_aria_operations_integration_sdk/adapter_template/adapter.py @@ -112,12 +112,11 @@ def test(adapter_instance: AdapterInstance) -> TestResult: result.with_error("The ID is bad") # otherwise, the test has passed except Exception as e: - # TODO: If any connections are still open, make sure they are closed before - # returning logger.error("Unexpected connection test error") logger.exception(e) result.with_error("Unexpected connection test error: " + repr(e)) finally: + # TODO: If any connections are still open, make sure they are closed before returning logger.debug(f"Returning test result: {result.get_json()}") return result @@ -186,11 +185,11 @@ def collect(adapter_instance: AdapterInstance) -> CollectResult: system.add_child(disk) system.add_child(cpu) except Exception as e: - # TODO: If any connections are still open, make sure they are closed before returning logger.error("Unexpected collection error") logger.exception(e) result.with_error("Unexpected collection error: " + repr(e)) finally: + # TODO: If any connections are still open, make sure they are closed before returning logger.debug(f"Returning collection result {result.get_json()}") return result diff --git a/vmware_aria_operations_integration_sdk/adapter_template/new_adapter_template.py b/vmware_aria_operations_integration_sdk/adapter_template/new_adapter_template.py new file mode 100644 index 00000000..59ce2b29 --- /dev/null +++ b/vmware_aria_operations_integration_sdk/adapter_template/new_adapter_template.py @@ -0,0 +1,173 @@ +# Copyright 2022-2023 VMware, Inc. +# SPDX-License-Identifier: Apache-2.0 +import sys +from typing import List + +import aria.ops.adapter_logging as logging +from aria.ops.adapter_instance import AdapterInstance +from aria.ops.definition.adapter_definition import AdapterDefinition +from aria.ops.result import CollectResult +from aria.ops.result import EndpointResult +from aria.ops.result import TestResult +from aria.ops.timer import Timer +from constants import ADAPTER_KIND +from constants import ADAPTER_NAME + +logger = logging.getLogger(__name__) + + +def get_adapter_definition() -> AdapterDefinition: + """ + The adapter definition defines the object types and attribute types (metric/property) that are present + in a collection. Setting these object types and attribute types helps VMware Aria Operations to + validate, process, and display the data correctly. + :return: AdapterDefinition + """ + with Timer(logger, "Get Adapter Definition"): + definition = AdapterDefinition(ADAPTER_KIND, ADAPTER_NAME) + + # TODO: Add parameters and credentials + + # The key 'container_memory_limit' is a special key read by the VMware Aria Operations + # collector to determine how much memory to allocate to the docker container running + # this adapter. It does not need to be read inside the adapter code. However, removing + # the definition from the object model will remove the ability to change the container + # memory limit during the adapter's configuration, and the VMware Aria Operations collector + # will give 1024 MB of memory to the container running the adapter instance. + definition.define_int_parameter( + "container_memory_limit", + label="Adapter Memory Limit (MB)", + description="Sets the maximum amount of memory VMware Aria Operations can " + "allocate to the container running this adapter instance.", + required=True, + advanced=True, + default=1024, + ) + + # TODO: Add object types, including identifiers, metrics, and properties + + logger.debug(f"Returning adapter definition: {definition.to_json()}") + return definition + + +def test(adapter_instance: AdapterInstance) -> TestResult: + with Timer(logger, "Test"): + result = TestResult() + try: + # A typical test connection will generally consist of: + # 1. Read identifier values from adapter_instance that are required to + # connect to the target(s) + # 2. Connect to the target(s), and retrieve some sample data + # 3. Disconnect cleanly from the target (ensure this happens even if an + # error occurs) + # 4. If any of the above failed, return an error, otherwise pass. + + # TODO: Add connection testing logic + pass # TODO: Remove pass statement + + except Exception as e: + logger.error("Unexpected connection test error") + logger.exception(e) + result.with_error("Unexpected connection test error: " + repr(e)) + finally: + # TODO: If any connections are still open, make sure they are closed before returning + logger.debug(f"Returning test result: {result.get_json()}") + return result + + +def collect(adapter_instance: AdapterInstance) -> CollectResult: + with Timer(logger, "Collection"): + result = CollectResult() + try: + # A typical collection will generally consist of: + # 1. Read identifier values from adapter_instance that are required to + # connect to the target(s) + # 2. Connect to the target(s), and retrieve data + # 3. Add the data into a CollectResult's objects, properties, metrics, etc + # 4. Disconnect cleanly from the target (ensure this happens even if an + # error occurs) + # 5. Return the CollectResult. + + # TODO: Add collection logic + pass # TODO: Remove pass statement + + except Exception as e: + logger.error("Unexpected collection error") + logger.exception(e) + result.with_error("Unexpected collection error: " + repr(e)) + finally: + # TODO: If any connections are still open, make sure they are closed before returning + logger.debug(f"Returning collection result {result.get_json()}") + return result + + +def get_endpoints(adapter_instance: AdapterInstance) -> EndpointResult: + with Timer(logger, "Get Endpoints"): + result = EndpointResult() + # In the case that an SSL Certificate is needed to communicate to the target, + # add each URL that the adapter uses here. Often this will be derived from a + # 'host' parameter in the adapter instance. In this Adapter we don't use any + # HTTPS connections, so we won't add any. If we did, we might do something like + # this: + # result.with_endpoint(adapter_instance.get_identifier_value("host")) + # + # Multiple endpoints can be returned, like this: + # result.with_endpoint(adapter_instance.get_identifier_value("primary_host")) + # result.with_endpoint(adapter_instance.get_identifier_value("secondary_host")) + # + # This 'get_endpoints' method will be run before the 'test' method, + # and VMware Aria Operations will use the results to extract a certificate from + # each URL. If the certificate is not trusted by the VMware Aria Operations + # Trust Store, the user will be prompted to either accept or reject the + # certificate. If it is accepted, the certificate will be added to the + # AdapterInstance object that is passed to the 'test' and 'collect' methods. + # Any certificate that is encountered in those methods should then be validated + # against the certificate(s) in the AdapterInstance. + + # TODO: Add any additional endpoints if any + + logger.debug(f"Returning endpoints: {result.get_json()}") + return result + + +# Main entry point of the adapter. You should not need to modify anything below this line. +def main(argv: List[str]) -> None: + logging.setup_logging("adapter.log") + # Start a new log file by calling 'rotate'. By default, the last five calls will be + # retained. If the logs are not manually rotated, the 'setup_logging' call should be + # invoked with the 'max_size' parameter set to a reasonable value, e.g., + # 10_489_760 (10MB). + logging.rotate() + logger.info(f"Running adapter code with arguments: {argv}") + if len(argv) != 3: + # `inputfile` and `outputfile` are always automatically appended to the + # argument list by the server + logger.error("Arguments must be ") + exit(1) + + method = argv[0] + try: + if method == "test": + test(AdapterInstance.from_input()).send_results() + elif method == "endpoint_urls": + get_endpoints(AdapterInstance.from_input()).send_results() + elif method == "collect": + collect(AdapterInstance.from_input()).send_results() + elif method == "adapter_definition": + result = get_adapter_definition() + if type(result) is AdapterDefinition: + result.send_results() + else: + logger.info( + "get_adapter_definition method did not return an AdapterDefinition" + ) + exit(1) + else: + logger.error(f"Command {method} not found") + exit(1) + finally: + logger.info(Timer.graph()) + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/vmware_aria_operations_integration_sdk/constant.py b/vmware_aria_operations_integration_sdk/constant.py index a8e7ea45..4d670332 100644 --- a/vmware_aria_operations_integration_sdk/constant.py +++ b/vmware_aria_operations_integration_sdk/constant.py @@ -47,3 +47,7 @@ ENDPOINTS_URLS_ENDPOINT = "endpointURLs" ADAPTER_DEFINITION_ENDPOINT = "adapterDefinition" API_VERSION_ENDPOINT = "apiVersion" + +# CLI options keys +SAMPLE_ADAPTER_OPTION_KEY = "sample_adapter" +NEW_ADAPTER_OPTION_KEY = "new_adapter" diff --git a/vmware_aria_operations_integration_sdk/mp_init.py b/vmware_aria_operations_integration_sdk/mp_init.py index 325bf372..69d9880b 100644 --- a/vmware_aria_operations_integration_sdk/mp_init.py +++ b/vmware_aria_operations_integration_sdk/mp_init.py @@ -1,4 +1,4 @@ -# Copyright 2022 VMware, Inc. +# Copyright 2022-2023 VMware, Inc. # SPDX-License-Identifier: Apache-2.0 import argparse import json @@ -20,7 +20,13 @@ from vmware_aria_operations_integration_sdk.constant import CONTAINER_BASE_NAME from vmware_aria_operations_integration_sdk.constant import CONTAINER_REGISTRY_HOST from vmware_aria_operations_integration_sdk.constant import CONTAINER_REGISTRY_PATH +from vmware_aria_operations_integration_sdk.constant import ( + NEW_ADAPTER_OPTION_KEY, +) from vmware_aria_operations_integration_sdk.constant import REPO_NAME +from vmware_aria_operations_integration_sdk.constant import ( + SAMPLE_ADAPTER_OPTION_KEY, +) from vmware_aria_operations_integration_sdk.constant import VERSION_FILE from vmware_aria_operations_integration_sdk.filesystem import mkdir from vmware_aria_operations_integration_sdk.filesystem import rmdir @@ -164,6 +170,7 @@ def create_project( eula_file: str, icon_file: str, language: str, + template_style: str, ) -> None: mkdir(path) @@ -204,7 +211,7 @@ def create_project( # create project structure executable_directory_path = build_project_structure( - path, adapter_key, name, language + path, adapter_key, name, language, template_style ) # create Dockerfile @@ -319,8 +326,24 @@ def main() -> None: # description="The language for the Management Pack determines the language for the template\n" # "source and build files.", # ) - # create project_directory + template_style = selection_prompt( + "Select a template for your project", + items=[ + ( + SAMPLE_ADAPTER_OPTION_KEY, + "Sample Adapter", + ), + ( + NEW_ADAPTER_OPTION_KEY, + "New Adapter", + ), + ], + description="- Sample Adapter: Generates a working adapter with comments throughout its code\n" + "- New Adapter: The minimum necessary code to start developing an adapter\n\n" + "For more information visit https://vmware.github.io/vmware-aria-operations-integration-sdk/get_started/#template-projects", + ) + # create project_directory with Spinner("Creating Project"): create_project( path, @@ -331,6 +354,7 @@ def main() -> None: eula_file, icon_file, language, + template_style, ) print("") print("") @@ -421,7 +445,7 @@ def create_commands_file( def build_project_structure( - path: str, adapter_kind: str, name: str, language: str + path: str, adapter_kind: str, name: str, language: str, template_style: str ) -> str: logger.debug("generating project structure") project_directory = "" # this is where all the source code will reside @@ -433,7 +457,8 @@ def build_project_structure( # create template requirements.txt requirements_file = os.path.join(path, "adapter_requirements.txt") with open(requirements_file, "w") as requirements: - requirements.write("psutil==5.9.4\n") + if template_style == SAMPLE_ADAPTER_OPTION_KEY: + requirements.write("psutil==5.9.4\n") requirements.write("vmware-aria-operations-integration-sdk-lib==0.7.*\n") # create development requirements file @@ -465,9 +490,22 @@ def build_project_structure( "Could not install sdk tools into the development virtual environment." ) - # copy adapter.py into app directory - with resources.path(adapter_template, "adapter.py") as src: - dest = os.path.join(path, project_directory) + namespace_package_indicator_file = os.path.join( + path, project_directory, "__init__.py" + ) + with open(namespace_package_indicator_file, "w"): + os.utime(namespace_package_indicator_file) + + # copy the template code into app/adapter.py file + template = ( + "adapter.py" + if template_style == SAMPLE_ADAPTER_OPTION_KEY + else "new_adapter_template.py" + ) + with resources.as_file( + resources.files(adapter_template).joinpath(template) + ) as src: + dest = os.path.join(path, project_directory, "adapter.py") copy(src, dest) with open(