Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Self parse user data #306

Merged
merged 14 commits into from
Jan 30, 2017
45 changes: 44 additions & 1 deletion stacker/blueprints/base.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
import copy
import hashlib
import logging
import re
import base64

from troposphere import (
Parameter,
Ref,
Template,
Template
)

from ..exceptions import (
Expand Down Expand Up @@ -221,6 +223,7 @@ def resolve_variable(var_name, var_def, provided_variable, blueprint_name):


class Blueprint(object):

"""Base implementation for rendering a troposphere template.

Args:
Expand Down Expand Up @@ -402,6 +405,46 @@ def render_template(self):
version = hashlib.md5(rendered).hexdigest()[:8]
return (version, rendered)

def parse_user_data(self, raw_user_data):
"""Translate a userdata file to into the file contents.

It supports referencing template variables to create userdata
that's supplemented with information from the data, as commonly
required when creating EC2 userdata files. Automatically, encodes
the data file to base64 after it is processed.

Args:
raw_user_data (str): the user data with the cloud-init info

Returns:
str: The parsed user data, with all the variables values and
refs replaced with their resolved values.

"""
pattern = re.compile(r'{{([::|\w]+)}}')
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

One issue w/ building your own templating engine is that there's no way to escape this with the current implementation, right? Python actually has a second string formatting/templating system that you can use perhaps? https://docs.python.org/3/library/string.html#format-string-syntax

res = ""
start_index = 0
variables = self.get_variables()

for match in pattern.finditer(raw_user_data):
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Lets go with the standard python string.Template method of templating: https://docs.python.org/3/library/string.html#template-strings

Also, please double check that the parameters aren't availabe in get_variables - seems strange, but worth double checking (@mhahn might know)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

  1. I decided to stick to just using a regex because the standard delimiter in the Template strings is $ which is very commonly used in UserData for other purposes. We can subclass the Template class and define our own custom rules for the delimiters, but I would argue that what we have right now is cleaner.

  2. Parameters are in get_variables, however, they are stored as CFNParameter objects. In my new commit I addressed this is a cleaner way.

res += raw_user_data[start_index:match.start()]

key = match.group(1)

if key in variables:
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I wouldn't usually check for the existence of a key - instead you should use try/except:

try:
    v = variables[key]
except KeyError:
    raise MissingVariable(self.name, key)

Then work with v for the rest of the logic, outside of the block.

if type(variables[key]) is CFNParameter:
res += variables[key].to_parameter_value()
else:
res += variables[key]
else:
raise MissingVariable(self.name, key)

start_index = match.end()

res += raw_user_data[start_index:]

return base64.b64encode(res)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It might be better to not encode this at all, letting the user choose whether to encode it either using base64 or the Base64 troposphere method. what do you think?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think that's fair. They could extend the script if they wanted to.


@property
def rendered(self):
if not self._rendered:
Expand Down
46 changes: 45 additions & 1 deletion stacker/tests/blueprints/test_base.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
import unittest
import base64

from mock import MagicMock
from troposphere import (
Base64,
Ref,
s3,
s3
)

from stacker.blueprints.base import (
Expand Down Expand Up @@ -44,6 +45,7 @@ def mock_lookup_handler(value, provider=None, context=None, fqn=False,


class TestBuildParameter(unittest.TestCase):

def test_base_parameter(self):
p = build_parameter("BasicParam", {"type": "String"})
p.validate()
Expand Down Expand Up @@ -72,6 +74,7 @@ class TestBlueprint(Blueprint):
}

class TestBlueprintSublcass(TestBlueprint):

def defined_variables(self):
variables = super(TestBlueprintSublcass,
self).defined_variables()
Expand Down Expand Up @@ -528,3 +531,44 @@ class TestBlueprint(Blueprint):

with self.assertRaises(AttributeError):
TestBlueprint(name="test", context=MagicMock())

def test_parse_user_data(self):
class TestBlueprint(Blueprint):
VARIABLES = {
"Param1": {"type": str}
}

blueprint = TestBlueprint(name="test", context=MagicMock())
variables = [Variable("Param1", "test1")]
blueprint.resolve_variables(variables)
userdata_raw = "Param1: {{Param1}}"
userdata = blueprint.parse_user_data(userdata_raw)
print(userdata)
self.assertEqual(userdata, base64.b64encode("Param1: test1"))

def test_parse_user_data_cfn_parameters(self):
class TestBlueprint(Blueprint):
VARIABLES = {
"Param2": {"type": CFNString},
}

blueprint = TestBlueprint(name="test", context=MagicMock())
variables = [Variable("Param1", "test1"), Variable("Param2", "test2")]
blueprint.resolve_variables(variables)
userdata_raw = "Param2: {{Param2}}!"
userdata = blueprint.parse_user_data(userdata_raw)
self.assertEqual(userdata, base64.b64encode("Param2: test2!"))

def test_parse_user_data_fails(self):
class TestBlueprint(Blueprint):
VARIABLES = {
"Param1": {"type": str},
"Param2": {"type": str},
}

blueprint = TestBlueprint(name="test", context=MagicMock())
variables = [Variable("Param1", "test1"), Variable("Param2", "test2")]
blueprint.resolve_variables(variables)
userdata_raw = "My name is {{Param1}} and {{Param3}}."
with self.assertRaises(MissingVariable):
blueprint.parse_user_data(userdata_raw)