Added support for loading files as parameter values, including interpolation

docs/lookups.rst

@@ -139,6 +139,78 @@ For example::
 
   ConfVariable: ${xref fully-qualified-stack::SomeOutput}
 
+.. file:
+
+File Lookup
+-----------
+
+The ``file`` lookup type allows the loading of arbitrary data from files on
+disk. The lookup additionally supports using a ``codec`` to manipulate or
+wrap the file contents prior to injecting it. The parameterized-b64 ``codec``
+is particularly useful to allow the interpolation of CloudFormation parameters
+in a UserData attribute of an instance or launch configuration.
+
+Basic examples::
+
+  # We've written a file to /some/path:
+  $ echo "hello there" > /some/path
+
+  # In stacker we would reference the contents of this file with the following
+  conf_key: ${file plain:file://some/path}
+
+  # The above would resolve to
+  conf_key: hello there
+
+  # Or, if we used wanted a base64 encoded copy of the file data
+  conf_key: ${file base64:file://some/path}
+
+  # The above would resolve to
+  conf_key: aGVsbG8gdGhlcmUK
+
+Supported codecs:
+ - plain
+ - base64 - encode the plain text file at the given path with base64 prior
+   to returning it
+ - parameterized - the same as plain, but additionally supports
+   referencing CloudFormation parameters to create userdata that's
+   supplemented with information from the template, as is commonly needed
+   in EC2 UserData. For example, given a template parameter of BucketName,
+   the file could contain the following text::
+
+     #!/bin/sh
+     aws s3 sync s3://{{BucketName}}/somepath /somepath
+
+   and then you could use something like this in the YAML config file::
+
+     UserData: ${file parameterized:/path/to/file}
+
+   resulting in the UserData parameter being defined as::
+
+     { "Fn::Join" : ["", [
+       "#!/bin/sh\naws s3 sync s3://",
+       {"Ref" : "BucketName"},
+       "/somepath /somepath"
+     ]] }
+
+ - parameterized-b64 - the same as parameterized, with the results additionally
+   wrapped in { "Fn::Base64": ... } , which is what you actually need for
+   EC2 UserData
+
+When using parameterized-b64 for UserData, you should use a local_parameter defined
+as such::
+
+  from troposphere import AWSHelperFn
+
+  "UserData": {
+    "type": AWSHelperFn,
+    "description": "Instance user data",
+    "default": Ref("AWS::NoValue")
+  }
+
+and then assign UserData in a LaunchConfiguration or Instance to self.get_variables()["UserData"].
+Note that we use AWSHelperFn as the type because the parameterized-b64 codec returns either a
+Base64 or a GenericHelperFn troposphere object.
+
 Custom Lookups
 --------------
 

stacker/lookups/handlers/file.py

@@ -0,0 +1,116 @@
+import re
+import base64
+
+from ...util import read_value_from_path
+from troposphere import GenericHelperFn, Base64
+
+TYPE_NAME = "file"
+
+
+def handler(value, **kwargs):
+    """Translate a filename into the file contents, optionally encoding or interpolating the input
+
+    Fields should use the following format:
+
+        <codec>:<path>
+
+    For example:
+
+        # We've written a file to /some/path:
+        $ echo "hello there" > /some/path
+
+        # In stacker we would reference the contents of this file with the following
+        conf_key: ${file plain:file://some/path}
+
+        # The above would resolve to
+        conf_key: hello there
+
+        # Or, if we used wanted a base64 encoded copy of the file data
+        conf_key: ${file base64:file://some/path}
+
+        # The above would resolve to
+        conf_key: aGVsbG8gdGhlcmUK
+
+    Supported codecs:
+     - plain
+     - base64 - encode the plain text file at the given path with base64 prior
+       to returning it
+     - parameterized - the same as plain, but additionally supports
+       referencing template parameters to create userdata that's supplemented
+       with information from the template, as is commonly needed in EC2
+       UserData. For example, given a template parameter of BucketName, the
+       file could contain the following text:
+
+         #!/bin/sh
+         aws s3 sync s3://{{BucketName}}/somepath /somepath
+
+       and then you could use something like this in the YAML config file:
+
+         UserData: ${file parameterized:/path/to/file}
+
+       resulting in the UserData parameter being defined as:
+
+         { "Fn::Join" : ["", [
+           "#!/bin/sh\naws s3 sync s3://",
+           {"Ref" : "BucketName"},
+           "/somepath /somepath"
+         ]] }
+
+     - parameterized-b64 - the same as parameterized, with the results additionally
+       wrapped in { "Fn::Base64": ... } , which is what you actually need for
+       EC2 UserData
+
+    When using parameterized-b64 for UserData, you should use a variable defined
+    as such:
+
+    from troposphere import AWSHelperFn
+
+      "UserData": {
+        "type": AWSHelperFn,
+        "description": "Instance user data",
+        "default": Ref("AWS::NoValue")
+      }
+
+    and then assign UserData in a LaunchConfiguration or Instance to self.get_variables()["UserData"].
+    Note that we use AWSHelperFn as the type because the parameterized-b64 codec returns either a
+    Base64 or a GenericHelperFn troposphere object
+    """
+
+    try:
+        codec, path = value.split(":", 1)
+    except ValueError:
+        raise TypeError(
+            "File value must be of the format"
+            " \"<codec>:<path>\" (got %s)" % (value)
+        )
+
+    value = read_value_from_path(path)
+
+    return CODECS[codec](value)
+
+
+def parameterized_codec(raw, b64):
+    pattern = re.compile(r'{{(\w+)}}')
+
+    parts = []
+    s_index = 0
+
+    for match in pattern.finditer(raw):
+        parts.append(raw[s_index:match.start()])
+        parts.append({"Ref": match.group(1)})
+        s_index = match.end()
+
+    parts.append(raw[s_index:])
+    result = {"Fn::Join": ["", parts]}
+
+    # Note, since we want a raw JSON object (not a string) output in the template,
+    # we wrap the result in GenericHelperFn (not needed if we're using Base64)
+    return Base64(result) if b64 else GenericHelperFn(result)
+
+
+CODECS = {
+    "plain": lambda x: x,
+    "base64": base64.b64encode,
+    "parameterized": lambda x: parameterized_codec(x, False),
+    "parameterized-b64": lambda x: parameterized_codec(x, True)
+}

stacker/lookups/registry.py

@@ -4,6 +4,7 @@
 from .handlers import output
 from .handlers import kms
 from .handlers import xref
+from .handlers import file as file_handler
 
 LOOKUP_HANDLERS = {}
 DEFAULT_LOOKUP = output.TYPE_NAME
@@ -53,3 +54,4 @@ def resolve_lookups(lookups, context, provider):
 register_lookup_handler(output.TYPE_NAME, output.handler)
 register_lookup_handler(kms.TYPE_NAME, kms.handler)
 register_lookup_handler(xref.TYPE_NAME, xref.handler)
+register_lookup_handler(file_handler.TYPE_NAME, file_handler.handler)

stacker/tests/lookups/handlers/test_file.py

@@ -0,0 +1,54 @@
+import unittest
+import mock
+import base64
+import troposphere
+
+from stacker.lookups.handlers.file import parameterized_codec, handler
+
+
+class TestFileTranslator(unittest.TestCase):
+    def setUp(self):
+        pass
+
+    def test_parameterized_codec_b64(self):
+        expected = {'Fn::Base64': {'Fn::Join': ['', ['Test ', {'Ref': 'Interpolation'}, ' Here']]}}
+        self.assertEqual(expected, parameterized_codec('Test {{Interpolation}} Here', True).data)
+
+    def test_parameterized_codec_plain(self):
+        expected = {'Fn::Join': ['', ['Test ', {'Ref': 'Interpolation'}, ' Here']]}
+        self.assertEqual(expected, parameterized_codec('Test {{Interpolation}} Here', False).data)
+
+    def test_file_loaded(self):
+        with mock.patch('stacker.lookups.handlers.file.read_value_from_path', return_value='') as amock:
+            handler('plain:file://tmp/test')
+            amock.assert_called_with('file://tmp/test')
+
+    def test_handler_plain(self):
+        expected = 'Hello, world'
+        with mock.patch('stacker.lookups.handlers.file.read_value_from_path', return_value=expected):
+            out = handler('plain:file://tmp/test')
+            self.assertEqual(expected, out)
+
+    def test_handler_b64(self):
+        expected = 'Hello, world'
+        with mock.patch('stacker.lookups.handlers.file.read_value_from_path', return_value=expected):
+            out = handler('base64:file://tmp/test')
+            self.assertEqual(expected, base64.b64decode(out))
+
+    def test_handler_parameterized(self):
+        expected = 'Hello, world'
+        with mock.patch('stacker.lookups.handlers.file.read_value_from_path', return_value=expected):
+            out = handler('parameterized:file://tmp/test')
+            self.assertEqual(troposphere.GenericHelperFn, type(out))
+
+    def test_handler_parameterized_b64(self):
+        expected = 'Hello, world'
+        with mock.patch('stacker.lookups.handlers.file.read_value_from_path', return_value=expected):
+            out = handler('parameterized-b64:file://tmp/test')
+            self.assertEqual(troposphere.Base64, type(out))
+
+    def test_unknown_codec(self):
+        expected = 'Hello, world'
+        with mock.patch('stacker.lookups.handlers.file.read_value_from_path', return_value=expected):
+            with self.assertRaises(KeyError):
+                handler('bad:file://tmp/test')

stacker/util.py

@@ -340,7 +340,7 @@ def get_config_directory():
 
     """
     # avoid circular import
-    from ...commands.stacker import Stacker
+    from .commands.stacker import Stacker
     command = Stacker()
     namespace = command.parse_args()
     return os.path.dirname(namespace.config.name)