Merge pull request cloudtools#523 from remind101/dag-1.1

Generate a DAG internally

CHANGELOG.md

@@ -1,6 +1,7 @@
 ## Upcoming/Master
 
 - assertRenderedBlueprint always dumps current results [GH-528]
+- stacker now builds a DAG internally [GH-523]
 
 ## 1.1.4 (2018-01-26)
 

stacker/actions/base.py

@@ -1,10 +1,9 @@
-import copy
 import logging
-import sys
+
+from ..plan2 import Step, build_plan
 
 import botocore.exceptions
 from stacker.session_cache import get_session
-from stacker.exceptions import PlanFailed
 
 from stacker.util import (
     ensure_s3_bucket,
@@ -14,6 +13,40 @@
 logger = logging.getLogger(__name__)
 
 
+def plan(description, action, stacks,
+         targets=None, tail=None,
+         reverse=False):
+    """A simple helper that builds a graph based plan from a set of stacks.
+
+    Args:
+        description (str): a description of the plan.
+        action (func): a function to call for each stack.
+        stacks (list): a list of :class:`stacker.stack.Stack` objects to build.
+        targets (list): an optional list of targets to filter the graph to.
+        tail (func): an optional function to call to tail the stack progress.
+        reverse (bool): if True, execute the graph in reverse (useful for
+            destroy actions).
+
+    Returns:
+        :class:`plan.Plan`: The resulting plan object
+    """
+
+    steps = [
+        Step(stack, fn=action, watch_func=tail)
+        for stack in stacks]
+
+    plan = build_plan(
+        description=description,
+        steps=steps,
+        targets=targets,
+        reverse=reverse)
+
+    for step in steps:
+        step.status_changed_func = plan._check_point
+
+    return plan
+
+
 def stack_template_key_name(blueprint):
     """Given a blueprint, produce an appropriate key name.
 
@@ -126,13 +159,9 @@ def s3_stack_push(self, blueprint, force=False):
         return template_url
 
     def execute(self, *args, **kwargs):
-        try:
-            self.pre_run(*args, **kwargs)
-            self.run(*args, **kwargs)
-            self.post_run(*args, **kwargs)
-        except PlanFailed as e:
-            logger.error(e.message)
-            sys.exit(1)
+        self.pre_run(*args, **kwargs)
+        self.run(*args, **kwargs)
+        self.post_run(*args, **kwargs)
 
     def pre_run(self, *args, **kwargs):
         pass
@@ -142,60 +171,3 @@ def run(self, *args, **kwargs):
 
     def post_run(self, *args, **kwargs):
         pass
-
-    def _get_all_stack_names(self, dependencies):
-        """Get all stack names specified in dependencies.
-
-        Args:
-            - dependencies (dict): a dictionary where each key should be the
-                fully qualified name of a stack whose value is an array of
-                fully qualified stack names that the stack depends on.
-
-        Returns:
-            set: set of all stack names
-
-        """
-        return set(
-            dependencies.keys() +
-            [item for items in dependencies.values() for item in items]
-        )
-
-    def get_stack_execution_order(self, dependencies):
-        """Return the order in which the stacks should be executed.
-
-        Args:
-            - dependencies (dict): a dictionary where each key should be the
-                fully qualified name of a stack whose value is an array of
-                fully qualified stack names that the stack depends on. This is
-                used to generate the order in which the stacks should be
-                executed.
-
-        Returns:
-            array: An array of stack names in the order which they should be
-                executed.
-
-        """
-        # copy the dependencies since we pop items out of it to get the
-        # execution order, we don't want to mutate the one passed in
-        dependencies = copy.deepcopy(dependencies)
-        pending_steps = []
-        executed_steps = []
-        stack_names = self._get_all_stack_names(dependencies)
-        for stack_name in stack_names:
-            requirements = dependencies.get(stack_name, None)
-            if not requirements:
-                dependencies.pop(stack_name, None)
-                pending_steps.append(stack_name)
-
-        while dependencies:
-            for step in pending_steps:
-                for stack_name, requirements in dependencies.items():
-                    if step in requirements:
-                        requirements.remove(step)
-
-                    if not requirements:
-                        dependencies.pop(stack_name)
-                        pending_steps.append(stack_name)
-                pending_steps.remove(step)
-                executed_steps.append(step)
-        return executed_steps + pending_steps

stacker/actions/build.py

@@ -1,6 +1,7 @@
 import logging
+import sys
 
-from .base import BaseAction
+from .base import BaseAction, plan
 
 from ..providers.base import Template
 from .. import util
@@ -10,7 +11,6 @@
     StackDoesNotExist,
 )
 
-from ..plan import Plan
 from ..status import (
     NotSubmittedStatus,
     NotUpdatedStatus,
@@ -315,31 +315,12 @@ def _template(self, blueprint):
             return Template(body=blueprint.rendered)
 
     def _generate_plan(self, tail=False):
-        plan_kwargs = {}
-        if tail:
-            plan_kwargs["watch_func"] = self.provider.tail_stack
-
-        plan = Plan(description="Create/Update stacks",
-                    logger_type=self.context.logger_type, **plan_kwargs)
-        stacks = self.context.get_stacks_dict()
-        dependencies = self._get_dependencies()
-        for stack_name in self.get_stack_execution_order(dependencies):
-            try:
-                stack = stacks[stack_name]
-            except KeyError:
-                raise StackDoesNotExist(stack_name)
-            plan.add(
-                stack,
-                run_func=self._launch_stack,
-                requires=dependencies.get(stack_name),
-            )
-        return plan
-
-    def _get_dependencies(self):
-        dependencies = {}
-        for stack in self.context.get_stacks():
-            dependencies[stack.fqn] = stack.requires
-        return dependencies
+        return plan(
+            description="Create/Update stacks",
+            action=self._launch_stack,
+            tail=self.provider.tail_stack if tail else None,
+            stacks=self.context.get_stacks(),
+            targets=self.context.stack_names)
 
     def pre_run(self, outline=False, dump=False, *args, **kwargs):
         """Any steps that need to be taken prior to running the action."""
@@ -363,7 +344,8 @@ def run(self, outline=False, tail=False, dump=False, *args, **kwargs):
         if not outline and not dump:
             plan.outline(logging.DEBUG)
             logger.debug("Launching stacks: %s", ", ".join(plan.keys()))
-            plan.execute()
+            if not plan.execute():
+                sys.exit(1)
         else:
             if outline:
                 plan.outline()

stacker/actions/destroy.py

@@ -1,14 +1,14 @@
 import logging
+import sys
 
-from .base import BaseAction
+from .base import BaseAction, plan
 from ..exceptions import StackDoesNotExist
 from .. import util
 from ..status import (
     CompleteStatus,
     SubmittedStatus,
     SUBMITTED,
 )
-from ..plan import Plan
 
 from ..status import StackDoesNotExist as StackDoesNotExistStatus
 
@@ -31,33 +31,14 @@ class Action(BaseAction):
 
     """
 
-    def _get_dependencies(self, stacks_dict):
-        dependencies = {}
-        for stack_name, stack in stacks_dict.iteritems():
-            required_stacks = stack.requires
-            if not required_stacks:
-                if stack_name not in dependencies:
-                    dependencies[stack_name] = required_stacks
-                continue
-
-            for requirement in required_stacks:
-                dependencies.setdefault(requirement, set()).add(stack_name)
-        return dependencies
-
     def _generate_plan(self, tail=False):
-        plan_kwargs = {}
-        if tail:
-            plan_kwargs["watch_func"] = self.provider.tail_stack
-        plan = Plan(description="Destroy stacks", **plan_kwargs)
-        stacks_dict = self.context.get_stacks_dict()
-        dependencies = self._get_dependencies(stacks_dict)
-        for stack_name in self.get_stack_execution_order(dependencies):
-            plan.add(
-                stacks_dict[stack_name],
-                run_func=self._destroy_stack,
-                requires=dependencies.get(stack_name),
-            )
-        return plan
+        return plan(
+            description="Destroy stacks",
+            action=self._destroy_stack,
+            tail=self.provider.tail_stack if tail else None,
+            stacks=self.context.get_stacks(),
+            targets=self.context.stack_names,
+            reverse=True)
 
     def _destroy_stack(self, stack, **kwargs):
         try:
@@ -101,9 +82,9 @@ def run(self, force, tail=False, *args, **kwargs):
         if force:
             # need to generate a new plan to log since the outline sets the
             # steps to COMPLETE in order to log them
-            debug_plan = self._generate_plan()
-            debug_plan.outline(logging.DEBUG)
-            plan.execute()
+            plan.outline(logging.DEBUG)
+            if not plan.execute():
+                sys.exit(1)
         else:
             plan.outline(message="To execute this plan, run with \"--force\" "
                                  "flag.")

stacker/actions/diff.py

@@ -1,12 +1,13 @@
 import difflib
 import json
 import logging
+import sys
 from operator import attrgetter
 
+from .base import plan
 from . import build
 from .. import exceptions
-from ..plan import COMPLETE, Plan
-from ..status import NotSubmittedStatus, NotUpdatedStatus
+from ..status import NotSubmittedStatus, NotUpdatedStatus, COMPLETE
 
 logger = logging.getLogger(__name__)
 
@@ -230,23 +231,18 @@ def _diff_stack(self, stack, **kwargs):
         return COMPLETE
 
     def _generate_plan(self):
-        plan = Plan(description="Diff stacks")
-        stacks = self.context.get_stacks_dict()
-        dependencies = self._get_dependencies()
-        for stack_name in self.get_stack_execution_order(dependencies):
-            plan.add(
-                stacks[stack_name],
-                run_func=self._diff_stack,
-                requires=dependencies.get(stack_name),
-            )
-        return plan
+        return plan(
+            description="Diff stacks",
+            action=self._diff_stack,
+            stacks=self.context.get_stacks(),
+            targets=self.context.stack_names)
 
     def run(self, *args, **kwargs):
         plan = self._generate_plan()
-        debug_plan = self._generate_plan()
-        debug_plan.outline(logging.DEBUG)
+        plan.outline(logging.DEBUG)
         logger.info("Diffing stacks: %s", ", ".join(plan.keys()))
-        plan.execute()
+        if not plan.execute():
+            sys.exit(1)
 
     """Don't ever do anything for pre_run or post_run"""
     def pre_run(self, *args, **kwargs):

stacker/commands/stacker/build.py

@@ -27,10 +27,10 @@ def add_arguments(self, parser):
                                  "the config.")
         parser.add_argument("--stacks", action="append",
                             metavar="STACKNAME", type=str,
-                            help="Only work on the stacks given. Can be "
-                                 "specified more than once. If not specified "
-                                 "then stacker will work on all stacks in the "
-                                 "config file.")
+                            help="Only work on the stacks given, and their "
+                                 "dependencies. Can be specified more than "
+                                 "once. If not specified then stacker will "
+                                 "work on all stacks in the config file.")
         parser.add_argument("-t", "--tail", action="store_true",
                             help="Tail the CloudFormation logs while working "
                                  "with stacks")

stacker/context.py

@@ -119,10 +119,7 @@ def mappings(self):
         return self.config.mappings or {}
 
     def _get_stack_definitions(self):
-        if not self.stack_names:
-            return self.config.stacks
-        return [s for s in self.config.stacks if s.name in
-                self.stack_names]
+        return self.config.stacks
 
     def get_stacks(self):
         """Get the stacks for the current action.