diff --git a/docs/lookups.rst b/docs/lookups.rst index eac6d1db5..9263df64a 100644 --- a/docs/lookups.rst +++ b/docs/lookups.rst @@ -139,6 +139,78 @@ For example:: ConfVariable: ${xref fully-qualified-stack::SomeOutput} +.. file: + +File Lookup +----------- + +The ``file`` lookup type allows the loading of arbitrary data from files on +disk. The lookup additionally supports using a ``codec`` to manipulate or +wrap the file contents prior to injecting it. The parameterized-b64 ``codec`` +is particularly useful to allow the interpolation of CloudFormation parameters +in a UserData attribute of an instance or launch configuration. + +Basic examples:: + + # We've written a file to /some/path: + $ echo "hello there" > /some/path + + # In stacker we would reference the contents of this file with the following + conf_key: ${file plain:file://some/path} + + # The above would resolve to + conf_key: hello there + + # Or, if we used wanted a base64 encoded copy of the file data + conf_key: ${file base64:file://some/path} + + # The above would resolve to + conf_key: aGVsbG8gdGhlcmUK + +Supported codecs: + - plain + - base64 - encode the plain text file at the given path with base64 prior + to returning it + - parameterized - the same as plain, but additionally supports + referencing CloudFormation parameters to create userdata that's + supplemented with information from the template, as is commonly needed + in EC2 UserData. For example, given a template parameter of BucketName, + the file could contain the following text:: + + #!/bin/sh + aws s3 sync s3://{{BucketName}}/somepath /somepath + + and then you could use something like this in the YAML config file:: + + UserData: ${file parameterized:/path/to/file} + + resulting in the UserData parameter being defined as:: + + { "Fn::Join" : ["", [ + "#!/bin/sh\naws s3 sync s3://", + {"Ref" : "BucketName"}, + "/somepath /somepath" + ]] } + + - parameterized-b64 - the same as parameterized, with the results additionally + wrapped in { "Fn::Base64": ... } , which is what you actually need for + EC2 UserData + +When using parameterized-b64 for UserData, you should use a local_parameter defined +as such:: + + from troposphere import AWSHelperFn + + "UserData": { + "type": AWSHelperFn, + "description": "Instance user data", + "default": Ref("AWS::NoValue") + } + +and then assign UserData in a LaunchConfiguration or Instance to self.get_variables()["UserData"]. +Note that we use AWSHelperFn as the type because the parameterized-b64 codec returns either a +Base64 or a GenericHelperFn troposphere object. + Custom Lookups -------------- diff --git a/stacker/lookups/handlers/file.py b/stacker/lookups/handlers/file.py new file mode 100644 index 000000000..faede6aba --- /dev/null +++ b/stacker/lookups/handlers/file.py @@ -0,0 +1,116 @@ +import re +import base64 + +from ...util import read_value_from_path +from troposphere import GenericHelperFn, Base64 + +TYPE_NAME = "file" + + +def handler(value, **kwargs): + """Translate a filename into the file contents, optionally encoding or interpolating the input + + Fields should use the following format: + + : + + For example: + + # We've written a file to /some/path: + $ echo "hello there" > /some/path + + # In stacker we would reference the contents of this file with the following + conf_key: ${file plain:file://some/path} + + # The above would resolve to + conf_key: hello there + + # Or, if we used wanted a base64 encoded copy of the file data + conf_key: ${file base64:file://some/path} + + # The above would resolve to + conf_key: aGVsbG8gdGhlcmUK + + Supported codecs: + - plain + - base64 - encode the plain text file at the given path with base64 prior + to returning it + - parameterized - the same as plain, but additionally supports + referencing template parameters to create userdata that's supplemented + with information from the template, as is commonly needed in EC2 + UserData. For example, given a template parameter of BucketName, the + file could contain the following text: + + #!/bin/sh + aws s3 sync s3://{{BucketName}}/somepath /somepath + + and then you could use something like this in the YAML config file: + + UserData: ${file parameterized:/path/to/file} + + resulting in the UserData parameter being defined as: + + { "Fn::Join" : ["", [ + "#!/bin/sh\naws s3 sync s3://", + {"Ref" : "BucketName"}, + "/somepath /somepath" + ]] } + + - parameterized-b64 - the same as parameterized, with the results additionally + wrapped in { "Fn::Base64": ... } , which is what you actually need for + EC2 UserData + + When using parameterized-b64 for UserData, you should use a variable defined + as such: + + from troposphere import AWSHelperFn + + "UserData": { + "type": AWSHelperFn, + "description": "Instance user data", + "default": Ref("AWS::NoValue") + } + + and then assign UserData in a LaunchConfiguration or Instance to self.get_variables()["UserData"]. + Note that we use AWSHelperFn as the type because the parameterized-b64 codec returns either a + Base64 or a GenericHelperFn troposphere object + """ + + try: + codec, path = value.split(":", 1) + except ValueError: + raise TypeError( + "File value must be of the format" + " \":\" (got %s)" % (value) + ) + + value = read_value_from_path(path) + + return CODECS[codec](value) + + +def parameterized_codec(raw, b64): + pattern = re.compile(r'{{(\w+)}}') + + parts = [] + s_index = 0 + + for match in pattern.finditer(raw): + parts.append(raw[s_index:match.start()]) + parts.append({"Ref": match.group(1)}) + s_index = match.end() + + parts.append(raw[s_index:]) + result = {"Fn::Join": ["", parts]} + + # Note, since we want a raw JSON object (not a string) output in the template, + # we wrap the result in GenericHelperFn (not needed if we're using Base64) + return Base64(result) if b64 else GenericHelperFn(result) + + +CODECS = { + "plain": lambda x: x, + "base64": base64.b64encode, + "parameterized": lambda x: parameterized_codec(x, False), + "parameterized-b64": lambda x: parameterized_codec(x, True) +} diff --git a/stacker/lookups/registry.py b/stacker/lookups/registry.py index 5efc5a90f..6aa60bc84 100644 --- a/stacker/lookups/registry.py +++ b/stacker/lookups/registry.py @@ -4,6 +4,7 @@ from .handlers import output from .handlers import kms from .handlers import xref +from .handlers import file as file_handler LOOKUP_HANDLERS = {} DEFAULT_LOOKUP = output.TYPE_NAME @@ -53,3 +54,4 @@ def resolve_lookups(lookups, context, provider): register_lookup_handler(output.TYPE_NAME, output.handler) register_lookup_handler(kms.TYPE_NAME, kms.handler) register_lookup_handler(xref.TYPE_NAME, xref.handler) +register_lookup_handler(file_handler.TYPE_NAME, file_handler.handler) diff --git a/stacker/tests/lookups/handlers/test_file.py b/stacker/tests/lookups/handlers/test_file.py new file mode 100644 index 000000000..af4df9446 --- /dev/null +++ b/stacker/tests/lookups/handlers/test_file.py @@ -0,0 +1,51 @@ +import unittest +import mock +import base64 +import troposphere + +from stacker.lookups.handlers.file import parameterized_codec, handler + + +class TestFileTranslator(unittest.TestCase): + def test_parameterized_codec_b64(self): + expected = {'Fn::Base64': {'Fn::Join': ['', ['Test ', {'Ref': 'Interpolation'}, ' Here']]}} + self.assertEqual(expected, parameterized_codec('Test {{Interpolation}} Here', True).data) + + def test_parameterized_codec_plain(self): + expected = {'Fn::Join': ['', ['Test ', {'Ref': 'Interpolation'}, ' Here']]} + self.assertEqual(expected, parameterized_codec('Test {{Interpolation}} Here', False).data) + + def test_file_loaded(self): + with mock.patch('stacker.lookups.handlers.file.read_value_from_path', return_value='') as amock: + handler('plain:file://tmp/test') + amock.assert_called_with('file://tmp/test') + + def test_handler_plain(self): + expected = 'Hello, world' + with mock.patch('stacker.lookups.handlers.file.read_value_from_path', return_value=expected): + out = handler('plain:file://tmp/test') + self.assertEqual(expected, out) + + def test_handler_b64(self): + expected = 'Hello, world' + with mock.patch('stacker.lookups.handlers.file.read_value_from_path', return_value=expected): + out = handler('base64:file://tmp/test') + self.assertEqual(expected, base64.b64decode(out)) + + def test_handler_parameterized(self): + expected = 'Hello, world' + with mock.patch('stacker.lookups.handlers.file.read_value_from_path', return_value=expected): + out = handler('parameterized:file://tmp/test') + self.assertEqual(troposphere.GenericHelperFn, type(out)) + + def test_handler_parameterized_b64(self): + expected = 'Hello, world' + with mock.patch('stacker.lookups.handlers.file.read_value_from_path', return_value=expected): + out = handler('parameterized-b64:file://tmp/test') + self.assertEqual(troposphere.Base64, type(out)) + + def test_unknown_codec(self): + expected = 'Hello, world' + with mock.patch('stacker.lookups.handlers.file.read_value_from_path', return_value=expected): + with self.assertRaises(KeyError): + handler('bad:file://tmp/test') diff --git a/stacker/util.py b/stacker/util.py index eef8b1584..8226c23bc 100644 --- a/stacker/util.py +++ b/stacker/util.py @@ -340,7 +340,7 @@ def get_config_directory(): """ # avoid circular import - from ...commands.stacker import Stacker + from .commands.stacker import Stacker command = Stacker() namespace = command.parse_args() return os.path.dirname(namespace.config.name)