Pipeline Property Substitution

api_app/_version.py

@@ -1 +1 @@
-__version__ = "0.3.14"
+__version__ = "0.3.15"

api_app/models/domain/resource.py

@@ -1,5 +1,5 @@
 from enum import Enum
-from typing import List, Optional
+from typing import List, Optional, Union
 from pydantic import Field
 from models.domain.azuretremodel import AzureTREModel
 from models.domain.request_action import RequestAction
@@ -70,4 +70,4 @@ def get_resource_request_message_payload(self, operation_id: str, step_id: str,
 
 class Output(AzureTREModel):
     Name: str = Field(title="", description="")
-    Value: str = Field(title="", description="")
+    Value: Union[list, dict, str] = Field(None, title="", description="")

api_app/models/domain/resource_template.py

@@ -1,4 +1,4 @@
-from typing import Dict, Any, List, Optional
+from typing import Dict, Any, List, Optional, Union
 
 from pydantic import Field
 
@@ -34,7 +34,9 @@ class CustomAction(AzureTREModel):
 class PipelineStepProperty(AzureTREModel):
     name: str = Field(title="name", description="name of the property to update")
     type: str = Field(title="type", description="data type of the property to update")
-    value: str = Field(title="value", description="value to use in substitution for the property to update")
+    value: Union[dict, str] = Field(None, title="value", description="value to use in substitution for the property to update")
+    arraySubstitutionAction: Optional[str] = Field("", title="Array Substitution Action", description="How to treat existing values of this property in an array [overwrite | append | replace | remove]")
+    arrayMatchField: Optional[str] = Field("", title="Array match field", description="Name of the field to use for finding an item in an array - to replace/remove it")
 
 
 class PipelineStep(AzureTREModel):

api_app/service_bus/deployment_status_update.py

@@ -112,7 +112,7 @@ def create_updated_resource_document(resource: dict, message: DeploymentStatusUp
 
     # although outputs are likely to be relevant when resources are moving to "deployed" status,
     # lets not limit when we update them and have the resource process make that decision.
-    output_dict = {output.Name: output.Value.strip("'").strip('"') for output in message.outputs}
+    output_dict = {output.Name: output.Value.strip("'").strip('"') if isinstance(output.Value, str) else output.Value for output in message.outputs}
     resource["properties"].update(output_dict)
 
     # if deleted - mark as isActive = False

api_app/service_bus/helpers.py

@@ -2,8 +2,8 @@
 from azure.servicebus import ServiceBusMessage
 from azure.servicebus.aio import ServiceBusClient
 from contextlib import asynccontextmanager
-
 from pydantic import parse_obj_as
+from service_bus.substitutions import substitute_properties
 from models.domain.resource_template import PipelineStep
 from models.domain.operation import OperationStep
 from models.domain.resource import Resource, ResourceType
@@ -52,7 +52,7 @@ async def send_deployment_message(content, correlation_id, session_id, action):
 
 
 def update_resource_for_step(operation_step: OperationStep, resource_repo: ResourceRepository, resource_template_repo: ResourceTemplateRepository, primary_resource_id: str, resource_to_update_id: str, primary_action: str, user: User) -> Resource:
-    # create properties dict - for now we create a basic, string only dict to use as a patch
+
     # get primary resource to use in substitutions
     primary_resource = resource_repo.get_resource_by_id(primary_resource_id)
 
@@ -77,20 +77,16 @@ def update_resource_for_step(operation_step: OperationStep, resource_repo: Resou
     if template_step is None:
         raise f"Cannot find step with id of {operation_step.stepId} in template {primary_resource.templateName} for action {primary_action}"
 
-    # TODO: actual substitution logic #1679
-    properties = {}
-    for prop in template_step.properties:
-        properties[prop.name] = prop.value
-
     if template_step.resourceAction == "upgrade":
         resource_to_send = try_upgrade_with_retries(
             num_retries=3,
             attempt_count=0,
             resource_repo=resource_repo,
             resource_template_repo=resource_template_repo,
-            properties=properties,
             user=user,
-            resource_to_update_id=resource_to_update_id
+            resource_to_update_id=resource_to_update_id,
+            template_step=template_step,
+            primary_resource=primary_resource
         )
 
         return resource_to_send
@@ -99,14 +95,15 @@ def update_resource_for_step(operation_step: OperationStep, resource_repo: Resou
         raise Exception("Only upgrade is currently supported for pipeline steps")
 
 
-def try_upgrade_with_retries(num_retries: int, attempt_count: int, resource_repo: ResourceRepository, resource_template_repo: ResourceTemplateRepository, properties: dict, user: User, resource_to_update_id: str) -> Resource:
+def try_upgrade_with_retries(num_retries: int, attempt_count: int, resource_repo: ResourceRepository, resource_template_repo: ResourceTemplateRepository, user: User, resource_to_update_id: str, template_step: PipelineStep, primary_resource: Resource) -> Resource:
     try:
         return try_upgrade(
             resource_repo=resource_repo,
             resource_template_repo=resource_template_repo,
-            properties=properties,
             user=user,
-            resource_to_update_id=resource_to_update_id
+            resource_to_update_id=resource_to_update_id,
+            template_step=template_step,
+            primary_resource=primary_resource
         )
     except CosmosAccessConditionFailedError as e:
         logging.warn(f"Etag mismatch for {resource_to_update_id}. Retrying.")
@@ -116,17 +113,21 @@ def try_upgrade_with_retries(num_retries: int, attempt_count: int, resource_repo
                 attempt_count=(attempt_count + 1),
                 resource_repo=resource_repo,
                 resource_template_repo=resource_template_repo,
-                properties=properties,
                 user=user,
-                resource_to_update_id=resource_to_update_id
+                resource_to_update_id=resource_to_update_id,
+                template_step=template_step,
+                primary_resource=primary_resource
             )
         else:
             raise e
 
 
-def try_upgrade(resource_repo: ResourceRepository, resource_template_repo: ResourceTemplateRepository, properties: dict, user: User, resource_to_update_id: str) -> Resource:
+def try_upgrade(resource_repo: ResourceRepository, resource_template_repo: ResourceTemplateRepository, user: User, resource_to_update_id: str, template_step: PipelineStep, primary_resource: Resource) -> Resource:
     resource_to_update = resource_repo.get_resource_by_id(resource_to_update_id)
 
+    # substitute values into new property bag for update
+    properties = substitute_properties(template_step, primary_resource, resource_to_update)
+
     # get the template for the resource to upgrade
     parent_service_name = ""
     if resource_to_update.resourceType == ResourceType.UserResource:

api_app/service_bus/substitutions.py

@@ -0,0 +1,103 @@
+from typing import Union
+from models.domain.resource_template import PipelineStep
+from models.domain.resource import Resource
+
+
+def substitute_properties(template_step: PipelineStep, primary_resource: Resource, resource_to_update: Resource) -> dict:
+    properties = {}
+    primary_resource_dict = primary_resource.dict()
+
+    for prop in template_step.properties:
+        val = prop.value
+        if isinstance(prop.value, dict):
+            val = recurse_object(prop.value, primary_resource_dict)
+
+            if prop.type == 'array':
+                if prop.name in resource_to_update.properties:
+                    existing_arr = resource_to_update.properties[prop.name]
+                else:
+                    existing_arr = []
+
+                if prop.arraySubstitutionAction == 'overwrite':
+                    existing_arr = [val]
+
+                if prop.arraySubstitutionAction == 'append':
+                    existing_arr.append(val)
+
+                if prop.arraySubstitutionAction == 'remove':
+                    item_index = find_item_index(existing_arr, prop.arrayMatchField, val)
+                    if item_index > -1:
+                        del existing_arr[item_index]
+
+                if prop.arraySubstitutionAction == 'replace':
+                    item_index = find_item_index(existing_arr, prop.arrayMatchField, val)
+                    if item_index > -1:
+                        existing_arr[item_index] = val
+                    else:
+                        existing_arr.append(val)
+
+                properties[prop.name] = existing_arr
+
+            else:
+                properties[prop.name] = val
+
+        else:
+            val = substitute_value(val, primary_resource_dict)
+            properties[prop.name] = val
+
+    return properties
+
+
+def find_item_index(array: list, arrayMatchField: str, val: dict) -> int:
+    for i in range(0, len(array)):
+        if array[i][arrayMatchField] == val[arrayMatchField]:
+            return i
+    return -1
+
+
+def recurse_object(obj: dict, primary_resource_dict: dict) -> dict:
+    for prop in obj:
+        if isinstance(obj[prop], list):
+            for i in range(0, len(obj[prop])):
+                obj[prop][i] = recurse_object(obj[prop][i], primary_resource_dict)
+        if isinstance(obj[prop], dict):
+            obj[prop] = recurse_object(obj[prop])
+        else:
+            obj[prop] = substitute_value(obj[prop], primary_resource_dict)
+
+    return obj
+
+
+def substitute_value(val: str, primary_resource_dict: dict) -> Union[dict, list, str]:
+    if "{{" not in val:
+        return val
+
+    val = val.replace("{{ ", "{{").replace(" }}", "}}")
+
+    # if the value being substituted in is a simple type, we can return it in the string, to allow for concatenation
+    # like "This was deployed by {{ resource.id }}"
+    # else if the value being injected in is a dict/list - we shouldn't try to concatenate that, we'll return the true value and drop any surrounding text
+
+    # extract the tokens to replace
+    tokens = []
+    parts = val.split("{{")
+    for p in parts:
+        if len(p) > 0 and "}}" in p:
+            t = p[0:p.index("}}")]
+            tokens.append(t)
+
+    for t in tokens:
+        # t = "resource.properties.prop_1"
+        p = t.split(".")
+        if p[0] == "resource":
+            prop_to_get = primary_resource_dict
+            for i in range(1, len(p)):
+                prop_to_get = prop_to_get[p[i]]
+
+            # if the value to inject is actually an object / list - just return it, else replace the value in the string
+            if isinstance(prop_to_get, dict) or isinstance(prop_to_get, list):
+                return prop_to_get
+            else:
+                val = val.replace("{{" + t + "}}", str(prop_to_get))
+
+    return val

api_app/tests_ma/conftest.py

@@ -1,5 +1,6 @@
 import uuid
 import pytest
+from models.domain.resource import Resource
 from models.domain.user_resource import UserResource
 from models.domain.shared_service import SharedService
 from tests_ma.test_api.test_routes.test_resource_helpers import FAKE_CREATE_TIMESTAMP
@@ -192,7 +193,7 @@ def user_resource_template_in_response(input_user_resource_template):
 
 
 @pytest.fixture
-def multi_step_resource_template(basic_shared_service_template):
+def multi_step_resource_template(basic_shared_service_template) -> ResourceTemplate:
     return ResourceTemplate(
         id="123",
         name="template1",
@@ -316,3 +317,89 @@ def multi_step_operation(test_user, basic_shared_service_template, basic_shared_
             )
         ]
     )
+
+
+@pytest.fixture
+def primary_resource() -> Resource:
+    return Resource(
+        id="123",
+        name="test resource",
+        isEnabled=True,
+        templateName="template name",
+        templateVersion="7",
+        resourceType="workspace",
+        _etag="",
+        properties={
+            "display_name": "test_resource name",
+            "address_prefix": ["172.0.0.1", "192.168.0.1"],
+            "fqdn": ["*pypi.org", "files.pythonhosted.org", "security.ubuntu.com"],
+            "my_protocol": "MyCoolProtocol"
+        },
+    )
+
+
+@pytest.fixture
+def resource_to_update() -> Resource:
+    return Resource(
+        id="123",
+        name="Firewall",
+        isEnabled=True,
+        templateName="template name",
+        templateVersion="7",
+        resourceType="workspace",
+        _etag="",
+        properties={},
+    )
+
+
+@pytest.fixture
+def pipeline_step() -> PipelineStep:
+    return PipelineStep(
+        properties=[
+            PipelineStepProperty(
+                name="rule_collections",
+                type="array",
+                arraySubstitutionAction="overwrite",
+                arrayMatchField="name",
+                value={
+                    "name": "arc-web_app_subnet_nexus_api",
+                    "action": "Allow",
+                    "rules": [
+                        {
+                            "name": "nexus-package-sources-api",
+                            "description": "Deployed by {{ resource.id }}",
+                            "protocols": [
+                                {"port": "443", "type": "Https"},
+                                {"port": "80", "type": "{{ resource.properties.my_protocol }}"},
+                            ],
+                            "target_fqdns": "{{ resource.properties.fqdn }}",
+                            "source_addresses": "{{ resource.properties.address_prefix }}",
+                        }
+                    ]
+                }
+            )
+        ]
+    )
+
+
+@pytest.fixture
+def simple_pipeline_step() -> PipelineStep:
+    return PipelineStep(
+        properties=[
+            PipelineStepProperty(
+                name="just_text",
+                type="string",
+                value="Updated by {{resource.id}}"
+            ),
+            PipelineStepProperty(
+                name="just_text_2",
+                type="string",
+                value="No substitution, just a fixed string here"
+            ),
+            PipelineStepProperty(
+                name="just_text_3",
+                type="string",
+                value="Multiple substitutions -> {{resource.id}} and {{resource.templateName}}"
+            )
+        ]
+    )