diff --git a/docs/environments.rst b/docs/environments.rst index 420dd20a2..77ce1b87b 100644 --- a/docs/environments.rst +++ b/docs/environments.rst @@ -3,7 +3,15 @@ Environments ============ When running stacker, you can optionally provide an "environment" file. The -stacker config file will be interpolated as a `string.Template +environment file defines values, which can then be referred to by name from +your stack config file. The environment file is interpreted as YAML if it +ends in `.yaml` or `.yml`, otherwise it's interpreted as simple key/value +pairs. + +Key/Value environments +---------------------- + +The stacker config file will be interpolated as a `string.Template `_ using the key/value pairs from the environment file. The format of the file is a single key/value per line, separated by a colon (**:**), like this:: @@ -43,6 +51,58 @@ files in your config. For example:: variables: InstanceType: ${web_instance_type} +YAML environments +----------------- + +YAML environments allow for more complex environment configuration rather +than simple text substitution, and support YAML features like anchors and +references. To build on the example above, let's define a stack that's +a little more complex:: + + stacks: + - name: webservers + class_path: stacker_blueprints.asg.AutoscalingGroup + variables: + InstanceType: ${web_instance_type} + IngressCIDRsByPort: ${ingress_cidrs_by_port} + +We've defined a stack which expects a list of ingress CIDR's allowed access to +each port. Our environment files would look like this:: + + # in the file: stage.yml + web_instance_type: m3.medium + ingress_cidrs_by_port: + 80: + - 192.168.1.0/8 + 8080: + - 0.0.0.0/0 + + # in the file: prod.env + web_instance_type: c4.xlarge + ingress_cidrs_by_port: + 80: + - 192.168.1.0/8 + 443: + - 10.0.0.0/16 + - 10.1.0.0/16 + +The YAML format allows for specifying lists, maps, and supports all `pyyaml` +functionality allowed in `safe_load()` function. + +Variable substitution in the YAML case is a bit more complex than in the +`string.Template` case. Objects can only be substituted for variables in the +case where we perform a full substitution, such as this:: + + vpcID: ${vpc_variable} + +We can not substitute an object in a sub-string, such as this:: + + vpcID: prefix-${vpc_variable} + +It makes no sense to substitute a complex object in this case, and we will raise +an error if that happens. You can still perform this substitution with +primitives; numbers, strings, but not dicts or lists. + .. note:: Namespace defined in the environment file has been deprecated in favor of defining the namespace in the config and will be removed in a future release. diff --git a/stacker/commands/stacker/base.py b/stacker/commands/stacker/base.py index c3f2084d7..f49aa64b4 100644 --- a/stacker/commands/stacker/base.py +++ b/stacker/commands/stacker/base.py @@ -7,8 +7,13 @@ import signal from collections import Mapping import logging +import os.path -from ...environment import parse_environment +from ...environment import ( + DictWithSourceType, + parse_environment, + parse_yaml_environment +) logger = logging.getLogger(__name__) @@ -63,8 +68,14 @@ def key_value_arg(string): def environment_file(input_file): """Reads a stacker environment file and returns the resulting data.""" + + is_yaml = os.path.splitext(input_file)[1].lower() in ['.yaml', '.yml'] + with open(input_file) as fd: - return parse_environment(fd.read()) + if is_yaml: + return parse_yaml_environment(fd.read()) + else: + return parse_environment(fd.read()) class BaseCommand(object): @@ -158,12 +169,17 @@ def add_arguments(self, parser): "-v", "--verbose", action="count", default=0, help="Increase output verbosity. May be specified up to twice.") parser.add_argument( - "environment", type=environment_file, nargs='?', default={}, - help="Path to a simple `key: value` pair environment file. The " - "values in the environment file can be used in the stack " - "config as if it were a string.Template type: " + "environment", type=environment_file, nargs='?', + default=DictWithSourceType('simple'), + help="Path to an environment file. The file can be a simple " + "`key: value` pair environment file, or a YAML file ending in" + ".yaml or .yml. In the simple key:value case, values in the " + "environment file can be used in the stack config as if it " + "were a string.Template type: " "https://docs.python.org/2/library/" - "string.html#template-strings.") + "string.html#template-strings. In the YAML case, variable" + "references in the stack config are replaced with the objects" + "in the environment after parsing") parser.add_argument( "config", type=argparse.FileType(), help="The config file where stack configuration is located. Must " diff --git a/stacker/config/__init__.py b/stacker/config/__init__.py index 5fdde4162..4c2192b28 100644 --- a/stacker/config/__init__.py +++ b/stacker/config/__init__.py @@ -3,10 +3,12 @@ from __future__ import absolute_import from future import standard_library standard_library.install_aliases() +from past.types import basestring from builtins import str import copy import sys import logging +import re from string import Template from io import StringIO @@ -32,6 +34,7 @@ from ..lookups import register_lookup_handler from ..util import merge_map, yaml_to_ordered_dict, SourceProcessor from .. import exceptions +from ..environment import DictWithSourceType # register translators (yaml constructors) from .translators import * # NOQA @@ -83,33 +86,138 @@ def render(raw_config, environment=None): Args: raw_config (str): the raw stacker configuration string. - environment (dict, optional): any environment values that should be - passed to the config + environment (DictWithSourceType, optional): any environment values that + should be passed to the config Returns: str: the stacker configuration populated with any values passed from the environment """ - - t = Template(raw_config) - buff = StringIO() if not environment: environment = {} - try: - substituted = t.substitute(environment) - except KeyError as e: - raise exceptions.MissingEnvironment(e.args[0]) - except ValueError: - # Support "invalid" placeholders for lookup placeholders. - substituted = t.safe_substitute(environment) - - if not isinstance(substituted, str): - substituted = substituted.decode('utf-8') - - buff.write(substituted) - buff.seek(0) - return buff.read() + # If we have a naked dict, we got here through the old non-YAML path, so + # we can't have a YAML config file. + is_yaml = False + if type(environment) == DictWithSourceType: + is_yaml = environment.source_type == 'yaml' + + if is_yaml: + # First, read the config as yaml + config = yaml.safe_load(raw_config) + + # Next, we need to walk the yaml structure, and find all things which + # look like variable references. This regular expression is copied from + # string.template to match variable references identically as the + # simple configuration case below. We've got two cases of this pattern, + # since python 2.7 doesn't support re.fullmatch(), so we have to add + # the end of line anchor to the inner patterns. + idpattern = r'[_a-z][_a-z0-9]*' + pattern = r""" + %(delim)s(?: + (?P%(id)s) | # delimiter and a Python identifier + {(?P%(id)s)} # delimiter and a braced identifier + ) + """ % {'delim': re.escape('$'), + 'id': idpattern, + } + full_pattern = r""" + %(delim)s(?: + (?P%(id)s)$ | # delimiter and a Python identifier + {(?P%(id)s)}$ # delimiter and a braced identifier + ) + """ % {'delim': re.escape('$'), + 'id': idpattern, + } + exp = re.compile(pattern, re.IGNORECASE | re.VERBOSE) + full_exp = re.compile(full_pattern, re.IGNORECASE | re.VERBOSE) + new_config = substitute_references(config, environment, exp, full_exp) + # Now, re-encode the whole thing as YAML and return that. + return yaml.safe_dump(new_config) + else: + t = Template(raw_config) + buff = StringIO() + + try: + substituted = t.substitute(environment) + except KeyError as e: + raise exceptions.MissingEnvironment(e.args[0]) + except ValueError: + # Support "invalid" placeholders for lookup placeholders. + substituted = t.safe_substitute(environment) + + if not isinstance(substituted, str): + substituted = substituted.decode('utf-8') + + buff.write(substituted) + buff.seek(0) + return buff.read() + + +def substitute_references(root, environment, exp, full_exp): + # We need to check for something being a string in both python 2.7 and + # 3+. The aliases in the future package don't work for yaml sourced + # strings, so we have to spin our own. + def isstr(s): + try: + return isinstance(s, basestring) + except NameError: + return isinstance(s, str) + + if isinstance(root, list): + result = [] + for x in root: + result.append(substitute_references(x, environment, exp, full_exp)) + return result + elif isinstance(root, dict): + result = {} + for k, v in root.items(): + result[k] = substitute_references(v, environment, exp, full_exp) + return result + elif isstr(root): + # Strings are the special type where all substitutions happen. If we + # encounter a string object in the expression tree, we need to perform + # one of two different kinds of matches on it. First, if the entire + # string is a variable, we can replace it with an arbitrary object; + # dict, list, primitive. If the string contains variables within it, + # then we have to do string substitution. + match_obj = full_exp.match(root.strip()) + if match_obj: + matches = match_obj.groupdict() + var_name = matches['named'] or matches['braced'] + if var_name is not None: + value = environment.get(var_name) + if value is None: + raise exceptions.MissingEnvironment(var_name) + return value + + # Returns if an object is a basic type. Once again, the future package + # overrides don't work for string here, so we have to special case it + def is_basic_type(o): + if isstr(o): + return True + basic_types = [int, bool, float] + for t in basic_types: + if isinstance(o, t): + return True + return False + + # If we got here, then we didn't have any full matches, now perform + # partial substitutions within a string. + def replace(mo): + name = mo.groupdict()['braced'] or mo.groupdict()['named'] + if not name: + return root[mo.start():mo.end()] + val = environment.get(name) + if val is None: + raise exceptions.MissingEnvironment(name) + if not is_basic_type(val): + raise exceptions.WrongEnvironmentType(name) + return str(val) + value = exp.sub(replace, root) + return value + # In all other unhandled cases, return a copy of the input + return copy.copy(root) def parse(raw_config): diff --git a/stacker/environment.py b/stacker/environment.py index 4ac753611..e4a2be174 100644 --- a/stacker/environment.py +++ b/stacker/environment.py @@ -2,9 +2,27 @@ from __future__ import division from __future__ import absolute_import +import yaml + + +class DictWithSourceType(dict): + """An environment dict which keeps track of its source. + + Environment files may be loaded from simple key/value files, or from + structured YAML files, and we need to render them using a different + strategy based on their source. This class adds a source_type property + to a dict which keeps track of whether the source for the dict is + yaml or simple. + """ + def __init__(self, source_type, *args): + dict.__init__(self, args) + if source_type not in ['yaml', 'simple']: + raise ValueError('source_type must be yaml or simple') + self.source_type = source_type + def parse_environment(raw_environment): - environment = {} + environment = DictWithSourceType('simple') for line in raw_environment.split('\n'): line = line.strip() if not line: @@ -20,3 +38,13 @@ def parse_environment(raw_environment): environment[key] = value.strip() return environment + + +def parse_yaml_environment(raw_environment): + environment = DictWithSourceType('yaml') + parsed_env = yaml.safe_load(raw_environment) + + if type(parsed_env) != dict: + raise ValueError('Environment must be valid YAML') + environment.update(parsed_env) + return environment diff --git a/stacker/exceptions.py b/stacker/exceptions.py index e1ae8339f..e4b5f7939 100644 --- a/stacker/exceptions.py +++ b/stacker/exceptions.py @@ -158,6 +158,14 @@ def __init__(self, key, *args, **kwargs): super(MissingEnvironment, self).__init__(message, *args, **kwargs) +class WrongEnvironmentType(Exception): + + def __init__(self, key, *args, **kwargs): + self.key = key + message = "Environment key %s can't be merged into a string" % (key,) + super(WrongEnvironmentType, self).__init__(message, *args, **kwargs) + + class ImproperlyConfigured(Exception): def __init__(self, cls, error, *args, **kwargs): diff --git a/stacker/hooks/iam.py b/stacker/hooks/iam.py index 009888157..f04b51f28 100644 --- a/stacker/hooks/iam.py +++ b/stacker/hooks/iam.py @@ -46,6 +46,7 @@ def create_ecs_service_role(provider, context, **kwargs): raise policy = Policy( + Version='2012-10-17', Statement=[ Statement( Effect=Allow, diff --git a/stacker/tests/test_config.py b/stacker/tests/test_config.py index 87876c0d2..9795784a2 100644 --- a/stacker/tests/test_config.py +++ b/stacker/tests/test_config.py @@ -4,6 +4,7 @@ from builtins import next import sys import unittest +import yaml from stacker.config import ( render_parse_load, @@ -14,7 +15,10 @@ process_remote_sources ) from stacker.config import Config, Stack -from stacker.environment import parse_environment +from stacker.environment import ( + parse_environment, + parse_yaml_environment +) from stacker import exceptions from stacker.lookups.registry import LOOKUP_HANDLERS @@ -49,6 +53,84 @@ def test_render_blank_env_values(self): c = render(conf, e) self.assertEqual("namespace: !!str", c) + def test_render_yaml(self): + conf = """ + namespace: ${namespace} + list_var: ${env_list} + dict_var: ${env_dict} + str_var: ${env_str} + nested_list: + - ${list_1} + - ${dict_1} + - ${str_1} + nested_dict: + a: ${list_1} + b: ${dict_1} + c: ${str_1} + empty: ${empty_string} + substr: prefix-${str_1}-suffix + multiple: ${str_1}-${str_2} + dont_match_this: ${output something} + """ + env = """ + namespace: test + env_list: &listAnchor + - a + - b + - c + env_dict: &dictAnchor + a: 1 + b: 2 + c: 3 + env_str: Hello World! + list_1: *listAnchor + dict_1: *dictAnchor + str_1: another str + str_2: hello + empty_string: "" + """ + e = parse_yaml_environment(env) + c = render(conf, e) + + # Parse the YAML again, so that we can check structure + pc = yaml.safe_load(c) + + exp_dict = {'a': 1, 'b': 2, 'c': 3} + exp_list = ['a', 'b', 'c'] + + self.assertEquals(pc['namespace'], 'test') + self.assertEquals(pc['list_var'], exp_list) + self.assertEquals(pc['dict_var'], exp_dict) + self.assertEquals(pc['str_var'], 'Hello World!') + self.assertEquals(pc['nested_list'][0], exp_list) + self.assertEquals(pc['nested_list'][1], exp_dict) + self.assertEquals(pc['nested_list'][2], 'another str') + self.assertEquals(pc['nested_dict']['a'], exp_list) + self.assertEquals(pc['nested_dict']['b'], exp_dict) + self.assertEquals(pc['nested_dict']['c'], 'another str') + self.assertEquals(pc['empty'], '') + self.assertEquals(pc['substr'], 'prefix-another str-suffix') + self.assertEquals(pc['multiple'], 'another str-hello') + self.assertEquals(pc['dont_match_this'], '${output something}') + + def test_render_yaml_errors(self): + # We shouldn't be able to substitute an object into a string + conf = "something: prefix-${var_name}" + env = """ + var_name: + foo: bar + """ + e = parse_yaml_environment(env) + with self.assertRaises(exceptions.WrongEnvironmentType): + render(conf, e) + + # Missing keys need to raise errors too + conf = "something: ${variable}" + env = "some_other_variable: 5" + e = parse_yaml_environment(env) + with self.assertRaises(exceptions.MissingEnvironment): + render(conf, e) + def test_config_validate_missing_stack_source(self): config = Config({ "namespace": "prod", diff --git a/stacker/tests/test_environment.py b/stacker/tests/test_environment.py index 1d8acd979..bed424333 100644 --- a/stacker/tests/test_environment.py +++ b/stacker/tests/test_environment.py @@ -3,7 +3,10 @@ from __future__ import absolute_import import unittest -from stacker.environment import parse_environment +from stacker.environment import ( + DictWithSourceType, + parse_environment +) test_env = """key1: value1 # some: comment @@ -31,7 +34,7 @@ class TestEnvironment(unittest.TestCase): def test_simple_key_value_parsing(self): parsed_env = parse_environment(test_env) - self.assertTrue(isinstance(parsed_env, dict)) + self.assertTrue(isinstance(parsed_env, DictWithSourceType)) self.assertEqual(parsed_env["key1"], "value1") self.assertEqual(parsed_env["key2"], "value2") self.assertEqual(parsed_env["key3"], "some:complex::value")