Convert KMS translator to a Lookup

stacker/config/translators/__init__.py

@@ -1,7 +1,5 @@
 import yaml
 
-from .vault import vault_constructor
 from .kms import kms_simple_constructor
 
-yaml.add_constructor('!vault', vault_constructor)
 yaml.add_constructor('!kms', kms_simple_constructor)

stacker/config/translators/kms.py

@@ -1,58 +1,7 @@
-import base64
-
-import botocore.session
-
-from .base import read_value_from_path
-
-
-def kms_simple_decrypt(value):
-    """Decrypt the specified value with a master key in KMS.
-
-    kmssimple field types should be in the following format:
-
-        [<region>@]<base64 encrypted value>
-
-    Note: The region is optional, and defaults to us-east-1 if not given.
-
-    For example:
-
-        # We use the aws cli to get the encrypted value for the string
-        # "PASSWORD" using the master key called "myStackerKey" in us-east-1
-        $ aws --region us-east-1 kms encrypt --key-id alias/myStackerKey \
-                --plaintext "PASSWORD" --output text --query CiphertextBlob
-
-        CiD6bC8t2Y<...encrypted blob...>
-
-        # In stacker we would reference the encrypted value like:
-        conf_key: !kms us-east-1@CiD6bC8t2Y<...encrypted blob...>
-
-        You can optionally store the encrypted value in a file, ie:
-
-        kms_value.txt
-        us-east-1@CiD6bC8t2Y<...encrypted blob...>
-
-        and reference it within stacker (NOTE: the path should be relative to
-        the stacker config file):
-
-        conf_key: !kms file://kms_value.txt
-
-        # Both of the above would resolve to
-        conf_key: PASSWORD
-
-    """
-    value = read_value_from_path(value)
-
-    region = "us-east-1"
-    if "@" in value:
-        region, value = value.split("@", 1)
-
-    s = botocore.session.get_session()
-    kms = s.create_client("kms", region_name=region)
-    decoded = base64.b64decode(value)
-    response = kms.decrypt(CiphertextBlob=decoded)
-    return response["Plaintext"]
+# NOTE: The translator is going to be deprecated in favor of the lookup
+from ...lookups.handlers.kms import handler
 
 
 def kms_simple_constructor(loader, node):
     value = loader.construct_scalar(node)
-    return kms_simple_decrypt(value)
+    return handler(value)

stacker/lookups/__init__.py

@@ -7,14 +7,14 @@
 from .registry import register_lookup_handler  # NOQA
 
 LOOKUP_REGEX = re.compile("""
-\$\{                                # opening brace for the lookup
-((?P<type>[._\-a-zA-Z0-9]*(?=\s))   # type of lookup, must be followed by a
-                                    # space to allow for defaulting to "output"
-                                    # type
-?\s*                                # any number of spaces separating the type
-                                    # from the input
-(?P<input>[,\._\-a-zA-Z0-9\:\s]+)   # the input value to the lookup
-)\}                                 # closing brace of the lookup
+\$\{                                   # opening brace for the lookup
+((?P<type>[._\-a-zA-Z0-9]*(?=\s))      # type of lookup, must be followed by a
+                                       # space to allow for defaulting to
+                                       # "output" type
+?\s*                                   # any number of spaces separating the
+                                       # type from the input
+(?P<input>[@\+\/,\._\-a-zA-Z0-9\:\s]+) # the input value to the lookup
+)\}                                    # closing brace of the lookup
 """, re.VERBOSE)
 
 Lookup = namedtuple("Lookup", ("type", "input", "raw"))

stacker/lookups/handlers/kms.py

@@ -0,0 +1,55 @@
+import base64
+
+import botocore.session
+
+from ...util import read_value_from_path
+
+TYPE_NAME = "kms"
+
+
+def handler(value, **kwargs):
+    """Decrypt the specified value with a master key in KMS.
+
+    kmssimple field types should be in the following format:
+
+        [<region>@]<base64 encrypted value>
+
+    Note: The region is optional, and defaults to us-east-1 if not given.
+
+    For example:
+
+        # We use the aws cli to get the encrypted value for the string
+        # "PASSWORD" using the master key called "myStackerKey" in us-east-1
+        $ aws --region us-east-1 kms encrypt --key-id alias/myStackerKey \
+                --plaintext "PASSWORD" --output text --query CiphertextBlob
+
+        CiD6bC8t2Y<...encrypted blob...>
+
+        # In stacker we would reference the encrypted value like:
+        conf_key: ${kms us-east-1@CiD6bC8t2Y<...encrypted blob...>}
+
+        You can optionally store the encrypted value in a file, ie:
+
+        kms_value.txt
+        us-east-1@CiD6bC8t2Y<...encrypted blob...>
+
+        and reference it within stacker (NOTE: the path should be relative to
+        the stacker config file):
+
+        conf_key: ${kms file://kms_value.txt}
+
+        # Both of the above would resolve to
+        conf_key: PASSWORD
+
+    """
+    value = read_value_from_path(value)
+
+    region = "us-east-1"
+    if "@" in value:
+        region, value = value.split("@", 1)
+
+    s = botocore.session.get_session()
+    kms = s.create_client("kms", region_name=region)
+    decoded = base64.b64decode(value)
+    response = kms.decrypt(CiphertextBlob=decoded)
+    return response["Plaintext"]

stacker/lookups/registry.py

@@ -2,6 +2,7 @@
 from ..util import load_object_from_string
 
 from .handlers import output
+from .handlers import kms
 from .handlers import xref
 
 LOOKUP_HANDLERS = {}
@@ -50,4 +51,5 @@ def resolve_lookups(lookups, context, provider):
     return resolved_lookups
 
 register_lookup_handler(output.TYPE_NAME, output.handler)
+register_lookup_handler(kms.TYPE_NAME, kms.handler)
 register_lookup_handler(xref.TYPE_NAME, xref.handler)

stacker/tests/lookups/handlers/test_kms.py

@@ -0,0 +1,28 @@
+import base64
+from mock import patch
+import unittest
+
+from stacker.lookups.handlers.kms import handler
+
+
+class TestKMSHandler(unittest.TestCase):
+
+    def setUp(self):
+        patcher = patch("botocore.session")
+        self.addCleanup(patcher.stop)
+        self.session = patcher.start()
+        self.kms = self.session.get_session().create_client()
+        self.input = base64.b64encode("encrypted test value")
+        self.value = {"Plaintext": "test value"}
+
+    def test_kms_handler(self):
+        self.kms.decrypt.return_value = self.value
+        decrypted = handler(self.input)
+        self.assertEqual(decrypted, self.value["Plaintext"])
+
+    def test_kms_handler_with_region(self):
+        handler("us-west-2@{}".format(self.input))
+        self.assertEqual(self.kms.decrypt.call_args[1]["CiphertextBlob"],
+                         "encrypted test value")
+        kwargs = self.session.get_session().create_client.call_args[1]
+        self.assertEqual(kwargs["region_name"], "us-west-2")

stacker/tests/test_config.py

@@ -25,18 +25,6 @@ def test_valid_env_substitution(self):
         c = parse_config("a: $a", {"a": "A"})
         self.assertEqual(c["a"], "A")
 
-    @patch("stacker.config.translators.vault.get_vaulted_value")
-    def test_custom_constructors(self, patched):
-        patched.return_value = "stub"
-        c = parse_config("a: $a", {"a": "!vault some_encrypted_value"})
-        self.assertEqual(c["a"], "stub")
-
-    @patch("stacker.config.translators.vault.subprocess")
-    def test_vault_constructor(self, patched):
-        patched.check_output.return_value = "secret\n"
-        c = parse_config("a: $a", {"a": "!vault secret/hello@value"})
-        self.assertEqual(c["a"], "secret")
-
     def test_blank_env_values(self):
         conf = """a: ${key1}"""
         e = parse_environment("""key1:""")

stacker/tests/test_lookups.py

@@ -51,3 +51,24 @@ def test_nested_lookups_string(self):
     def test_comma_delimited(self):
         lookups = extract_lookups("${noop val1,val2}")
         self.assertEqual(len(lookups), 1)
+
+    def test_kms_lookup(self):
+        lookups = extract_lookups("${kms CiADsGxJp1mCR21fjsVjVxr7RwuO2FE3ZJqC4iG0Lm+HkRKwAQEBAgB4A7BsSadZgkdtX47FY1ca+0cLjthRN2SaguIhtC5vh5EAAACHMIGEBgkqhkiG9w0BBwagdzB1AgEAMHAGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQM3IKyEoNEQVxN3BaaAgEQgEOpqa0rcl3WpHOmblAqL1rOPRyokO3YXcJAAB37h/WKLpZZRAWV2h9C67xjlsj3ebg+QIU91T/}")  # NOQA
+        self.assertEqual(len(lookups), 1)
+        lookup = list(lookups)[0]
+        self.assertEqual(lookup.type, "kms")
+        self.assertEqual(lookup.input, "CiADsGxJp1mCR21fjsVjVxr7RwuO2FE3ZJqC4iG0Lm+HkRKwAQEBAgB4A7BsSadZgkdtX47FY1ca+0cLjthRN2SaguIhtC5vh5EAAACHMIGEBgkqhkiG9w0BBwagdzB1AgEAMHAGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQM3IKyEoNEQVxN3BaaAgEQgEOpqa0rcl3WpHOmblAqL1rOPRyokO3YXcJAAB37h/WKLpZZRAWV2h9C67xjlsj3ebg+QIU91T/")  # NOQA
+
+    def test_kms_lookup_with_region(self):
+        lookups = extract_lookups("${kms us-west-2@CiADsGxJp1mCR21fjsVjVxr7RwuO2FE3ZJqC4iG0Lm+HkRKwAQEBAgB4A7BsSadZgkdtX47FY1ca+0cLjthRN2SaguIhtC5vh5EAAACHMIGEBgkqhkiG9w0BBwagdzB1AgEAMHAGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQM3IKyEoNEQVxN3BaaAgEQgEOpqa0rcl3WpHOmblAqL1rOPRyokO3YXcJAAB37h/WKLpZZRAWV2h9C67xjlsj3ebg+QIU91T/}")  # NOQA
+        self.assertEqual(len(lookups), 1)
+        lookup = list(lookups)[0]
+        self.assertEqual(lookup.type, "kms")
+        self.assertEqual(lookup.input, "us-west-2@CiADsGxJp1mCR21fjsVjVxr7RwuO2FE3ZJqC4iG0Lm+HkRKwAQEBAgB4A7BsSadZgkdtX47FY1ca+0cLjthRN2SaguIhtC5vh5EAAACHMIGEBgkqhkiG9w0BBwagdzB1AgEAMHAGCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQM3IKyEoNEQVxN3BaaAgEQgEOpqa0rcl3WpHOmblAqL1rOPRyokO3YXcJAAB37h/WKLpZZRAWV2h9C67xjlsj3ebg+QIU91T/")  # NOQA
+
+    def test_kms_file_lookup(self):
+        lookups = extract_lookups("${kms file://path/to/some/file.txt}")
+        self.assertEqual(len(lookups), 1)
+        lookup = list(lookups)[0]
+        self.assertEqual(lookup.type, "kms")
+        self.assertEqual(lookup.input, "file://path/to/some/file.txt")

stacker/util.py

@@ -2,6 +2,7 @@
 import uuid
 import importlib
 import logging
+import os
 import re
 import sys
 import time
@@ -330,3 +331,33 @@ def handle_hooks(stage, hooks, region, context):
                 sys.exit(1)
             logger.warning("Non-required hook %s failed. Return value: %s",
                            hook["path"], result)
+
+
+def get_config_directory():
+    """Return the directory the config file is located in.
+
+    This enables us to use relative paths in config values.
+
+    """
+    # avoid circular import
+    from ...commands.stacker import Stacker
+    command = Stacker()
+    namespace = command.parse_args()
+    return os.path.dirname(namespace.config.name)
+
+
+def read_value_from_path(value):
+    """Enables translators to read values from files.
+
+    The value can be referred to with the `file://` prefix. ie:
+
+        conf_key: ${kms file://kms_value.txt}
+
+    """
+    if value.startswith('file://'):
+        path = value.split('file://', 1)[1]
+        config_directory = get_config_directory()
+        relative_path = os.path.join(config_directory, path)
+        with open(relative_path) as read_file:
+            value = read_file.read()
+    return value