Merge pull request #318 from nutanix/dsl-400-cut

CALM-DSL changes for v4.0.0

CalmVersion

@@ -1 +1 @@
-3.8.1
+4.0.0

Makefile

@@ -13,6 +13,15 @@ dev:
 	venv/bin/pip3 install --no-cache -r requirements.txt -r dev-requirements.txt
 	venv/bin/python3 setup.py develop
 
+windev:
+	# Setup our python3 based virtualenv in windows machine
+	# This step assumes python3 is installed on your dev machine
+	[ -f venv/Scripts/python3 ] || python -m venv venv
+	venv/Scripts/python -m pip install --upgrade pip
+	venv/Scripts/pip install setuptools --upgrade --ignore-installed
+	venv/Scripts/pip install --no-cache -r requirements.txt -r dev-requirements.txt
+	venv/Scripts/python setup.py develop
+
 test-bed: dev
 	venv/bin/python3 tests/testprep.py
 

README.md

@@ -2,9 +2,9 @@
 
 ![Build](https://github.com/nutanix/calm-dsl/workflows/Setup%20&%20build%20calm-dsl/badge.svg)
 
-`Latest release version: 3.8.1, Latest-release-tag: v3.8.1`
+`Latest release version: 4.0.0, Latest-release-tag: v4.0.0`
 
-`Latest Release Notes:` [read here](release-notes/3.8.1)
+`Latest Release Notes:` [read here](release-notes/4.0.0)
 
 
 # Nutanix Cloud Manager (NCM) Self Service (formerly Calm) DSL

calm/dsl/api/provider.py

@@ -1,12 +1,47 @@
+import json
+import os
+
+from calm.dsl.log import get_logging_handle
+
 from .resource import ResourceAPI
 from .connection import REQUEST
+from .util import strip_provider_secrets, patch_secrets, strip_uuids
+
+LOG = get_logging_handle(__name__)
 
 
 class ProviderAPI(ResourceAPI):
     def __init__(self, connection):
         super().__init__(connection, resource_type="providers", calm_api=True)
 
         self.CREATE = self.PREFIX
+        self.BULK_CREATE = self.CREATE + "/bulk"
+        self.BULK_UPDATE = self.ITEM + "/bulk"
+        self.BULK_READ = self.ITEM + "/bulk"
+        self.COMPILE = self.ITEM + "/compile"
+        self.TEST_PROVIDER_VERIFY = self.PREFIX + "/{}/actions/{}/test_run"
+        self.ABORT_RUN = self.ITEM + "/runlogs/{}/abort"
+        self.POLL_RUN = self.ITEM + "/runlogs/{}"
+        self.CHILD_RUNLOG_LIST = self.ITEM + "/runlogs/{}/children/list"
+        self.RUNLOG_OUTPUT = self.ITEM + "/runlogs/{}/children/{}/output"
+        self.RUNLOG_LIST = self.CREATE + "/runlogs/list"
+        self.CLONE = self.ITEM + "/clone"
+        self.IMPORT_JSON = self.PREFIX + "/import_json"
+        self.IMPORT_FILE = self.PREFIX + "/import_file"
+        self.EXPORT_JSON = self.ITEM + "/export_json"
+        self.EXPORT_FILE = self.ITEM + "/export_file"
+
+    def check_if_provider_already_exists(self, provider_name):
+        params = {"filter": "name=={};state!=DELETED".format(provider_name)}
+        res, err = self.list(params=params)
+        if err:
+            return None, err
+
+        response = res.json()
+        entities = response.get("entities", None)
+        if entities and len(entities) > 0:
+            return entities[0], None
+        return None, None
 
     def create(self, provider_payload):
         return self.connection._call(
@@ -16,3 +51,225 @@ def create(self, provider_payload):
             method=REQUEST.METHOD.POST,
             timeout=(5, 300),
         )
+
+    # This is equivalent to "compile" of whole Provider tree as present in Calm UI
+    def preview_validate(self, uuid):
+        return self.connection._call(
+            self.COMPILE.format(uuid), verify=False, method=REQUEST.METHOD.GET
+        )
+
+    def bulk_create(self, provider_payload):
+        res, err = self.connection._call(
+            self.BULK_CREATE,
+            verify=False,
+            request_json=provider_payload,
+            method=REQUEST.METHOD.POST,
+        )
+        if err:
+            return res, err
+
+        return self.preview_validate(res.json()["metadata"]["uuid"])
+
+    def bulk_read(self, id):
+        return self.connection._call(
+            self.BULK_READ.format(id), verify=False, method=REQUEST.METHOD.GET
+        )
+
+    def bulk_update(self, uuid, provider_payload):
+        res, err = self.connection._call(
+            self.BULK_UPDATE.format(uuid),
+            verify=False,
+            request_json=provider_payload,
+            method=REQUEST.METHOD.PUT,
+        )
+        if err:
+            return res, err
+
+        return self.preview_validate(uuid)
+
+    def run(self, uuid, action_uuid, payload):
+        return self.connection._call(
+            self.TEST_PROVIDER_VERIFY.format(uuid, action_uuid),
+            verify=False,
+            request_json=payload,
+            method=REQUEST.METHOD.POST,
+        )
+
+    def list_child_runlogs(self, provider_uuid, runlog_uuid):
+        return self.connection._call(
+            self.CHILD_RUNLOG_LIST.format(provider_uuid, runlog_uuid),
+            verify=False,
+            request_json={},
+            method=REQUEST.METHOD.POST,
+        )
+
+    def list_runlogs(self, payload=None):
+        return self.connection._call(
+            self.RUNLOG_LIST,
+            verify=False,
+            request_json=payload,
+            method=REQUEST.METHOD.POST,
+        )
+
+    def runlog_output(self, provider_uuid, runlog_uuid, child_runlog_uuid):
+        return self.connection._call(
+            self.RUNLOG_OUTPUT.format(provider_uuid, runlog_uuid, child_runlog_uuid),
+            verify=False,
+            method=REQUEST.METHOD.GET,
+        )
+
+    def poll_action_run(self, uuid, runlog_uuid, payload=None):
+        if payload:
+            return self.connection._call(
+                self.POLL_RUN.format(uuid, runlog_uuid),
+                request_json=payload,
+                verify=False,
+                method=REQUEST.METHOD.POST,
+            )
+        else:
+            return self.connection._call(
+                self.POLL_RUN.format(uuid, runlog_uuid),
+                verify=False,
+                method=REQUEST.METHOD.GET,
+            )
+
+    def abort(self, uuid, runlog_uuid):
+        return self.connection._call(
+            self.ABORT_RUN.format(uuid, runlog_uuid),
+            verify=False,
+            request_json={},
+            method=REQUEST.METHOD.POST,
+        )
+
+    def clone(self, uuid, clone_payload):
+        return self.connection._call(
+            self.CLONE.format(uuid),
+            verify=False,
+            request_json=clone_payload,
+            method=REQUEST.METHOD.POST,
+        )
+
+    def import_json(self, provider_json):
+        return self.connection._call(
+            self.IMPORT_JSON,
+            verify=False,
+            request_json=provider_json,
+            method=REQUEST.METHOD.POST,
+        )
+
+    def export_json(self, uuid):
+        return self.connection._call(
+            self.EXPORT_JSON.format(uuid), verify=False, method=REQUEST.METHOD.GET
+        )
+
+    def export_file(self, uuid, passphrase=None):
+        if passphrase:
+            return self.connection._call(
+                self.EXPORT_FILE.format(uuid),
+                verify=False,
+                method=REQUEST.METHOD.POST,
+                request_json={"passphrase": passphrase},
+                files=[],
+            )
+        return self.connection._call(
+            self.EXPORT_FILE.format(uuid), verify=False, method=REQUEST.METHOD.GET
+        )
+
+    def export_provider(self, uuid, passphrase=None):
+        current_path = os.path.dirname(os.path.realpath(__file__))
+        if passphrase:
+            res, err = self.connection._call(
+                self.EXPORT_FILE.format(uuid),
+                verify=False,
+                method=REQUEST.METHOD.POST,
+                request_json={"passphrase": passphrase},
+                files=[],
+            )
+        else:
+            res, err = self.connection._call(
+                self.EXPORT_FILE.format(uuid), verify=False, method=REQUEST.METHOD.GET
+            )
+
+        if err:
+            raise Exception("[{}] - {}".format(err["code"], err["error"]))
+
+        with open(current_path + "/" + uuid + ".json", "wb") as downloaded_file:
+            for chunk in res.iter_content(chunk_size=2048):
+                downloaded_file.write(chunk)
+
+        return current_path + "/" + uuid + ".json"
+
+    def upload_using_import_file(self, payload, files):
+        return self.connection._call(
+            self.IMPORT_FILE,
+            verify=False,
+            files=files,
+            request_json=payload,
+            method=REQUEST.METHOD.POST,
+        )
+
+    def upload_with_decompiled_secrets(
+        self,
+        provider_payload,
+        passphrase,
+        decompiled_secrets=[],
+    ):
+        """
+        Used to create a provider if it contains encrypted secrets from decompilation
+
+        Args:
+            provider_payload (dict): payload of the provider
+            passphrase (string): passphrase for creating provider with secrets (it should be same as the one provided while decompilation)
+            decompiled_secrets (list): contains all the secrets that were present in the decompiled provider
+        """
+        secret_map = {}
+        secret_variables = []
+        not_stripped_secrets = []
+        provider_name = provider_payload["spec"]["name"]
+        provider_resources = provider_payload["spec"]["resources"]
+        LOG.debug("provider_resources pre-stripping secrets")
+        LOG.debug(provider_resources)
+        strip_provider_secrets(
+            provider_name,
+            provider_resources,
+            secret_map,
+            secret_variables,
+            decompiled_secrets=decompiled_secrets,
+            not_stripped_secrets=not_stripped_secrets,
+        )
+        LOG.debug("provider_resources post-stripping secrets")
+        LOG.debug(provider_resources)
+
+        strip_uuids(provider_resources)
+        LOG.debug("provider_resources post-stripping UUIDs")
+        LOG.debug(provider_resources)
+        provider_payload["spec"]["resources"] = provider_resources
+        files = {"file": ("file", json.dumps(provider_payload))}
+        res, err = self.upload_using_import_file(
+            {"name": provider_name, "passphrase": passphrase}, files
+        )
+        if err:
+            return res, err
+
+        # Add secrets and update provider
+        provider = res.json()
+        del provider["status"]
+        LOG.info("Patching newly created/updated secrets")
+        for k in secret_map:
+            LOG.debug("[CREATED/MODIFIED] credential -> '{}'".format(k))
+        for s in secret_variables:
+            LOG.debug("[CREATED/MODIFIED] variable -> '{}' path: {}".format(s[2], s[0]))
+
+        patch_secrets(
+            provider["spec"]["resources"],
+            secret_map,
+            secret_variables,
+            not_stripped_secrets,
+        )
+        LOG.debug("Update provider payload:")
+        LOG.debug(provider)
+
+        # Update provider
+        update_payload = provider
+        uuid = provider["metadata"]["uuid"]
+        return self.bulk_update(uuid, update_payload)

calm/dsl/api/resource_type.py

@@ -6,9 +6,11 @@ class ResourceTypeAPI(ResourceAPI):
     def __init__(self, connection):
         super().__init__(connection, resource_type="resource_types", calm_api=True)
         self.CREATE = self.PREFIX
+        self.LIST = self.PREFIX + "/list"
         self.TEST_RUNBOOK = self.PREFIX + "/{}/test_runbook/{}/run"
         self.PLATFORM_LIST = self.PREFIX + "/platform_list"
         self.UPDATE = self.PREFIX + "/{}"
+        self.TEST_EXECUTE = self.PREFIX + "/{}/actions/{}/test_run"
 
     def create(self, resource_type_payload):
         return self.connection._call(
@@ -19,6 +21,16 @@ def create(self, resource_type_payload):
             timeout=(5, 300),
         )
 
+    def list(self, payload={}):
+        if not payload.get("length"):
+            payload["length"] = 20
+        return self.connection._call(
+            self.LIST,
+            verify=False,
+            request_json=payload,
+            method=REQUEST.METHOD.POST,
+        )
+
     def update(self, uuid, resource_type_payload):
         return self.connection._call(
             self.UPDATE.format(uuid),
@@ -27,6 +39,14 @@ def update(self, uuid, resource_type_payload):
             method=REQUEST.METHOD.PUT,
         )
 
+    def run(self, resource_type_uuid, action_uuid, payload):
+        return self.connection._call(
+            self.TEST_EXECUTE.format(resource_type_uuid, action_uuid),
+            request_json=payload,
+            verify=False,
+            method=REQUEST.METHOD.POST,
+        )
+
     def run_test_runbook(self, resource_type_id, action_id, payload):
         return self.connection._call(
             self.TEST_RUNBOOK.format(resource_type_id, action_id),

calm/dsl/api/runbook.py

@@ -1,13 +1,17 @@
 import os
+import sys
 from distutils.version import LooseVersion as LV
 
 
 from .resource import ResourceAPI
 from .connection import REQUEST
 from .util import strip_secrets, patch_secrets
 from calm.dsl.config import get_context
+from calm.dsl.log import get_logging_handle
 from .project import ProjectAPI
 
+LOG = get_logging_handle(__name__)
+
 
 class RunbookAPI(ResourceAPI):
     def __init__(self, connection):
@@ -34,6 +38,7 @@ def __init__(self, connection):
         self.MARKETPLACE_EXECUTE = self.PREFIX + "/marketplace_execute"
         self.MARKETPLACE_CLONE = self.PREFIX + "/marketplace_clone"
         self.VARIABLE_VALUES = self.ITEM + "/variables/{}/values"
+        self.CLONE = self.PREFIX + "/{}/clone"
 
     def upload(self, payload):
         return self.connection._call(
@@ -435,3 +440,21 @@ def variable_values(self, uuid, var_uuid, payload={}):
         return self.connection._call(
             url, verify=False, method=REQUEST.METHOD.POST, request_json=payload
         )
+
+    def clone(self, uuid, payload):
+        from calm.dsl.store.version import Version
+
+        calm_version = Version.get_version("Calm")
+
+        if LV(calm_version) < LV("4.0.0"):
+            LOG.error(
+                "Runbook clone is supported from Calm version 4.0.0. Please upgrade your Calm version to use this feature."
+            )
+            sys.exit("Runbook clone is supported from calm version 4.0.0 onwards")
+
+        return self.connection._call(
+            self.CLONE.format(uuid),
+            verify=False,
+            request_json=payload,
+            method=REQUEST.METHOD.POST,
+        )