8000 DAG by ejholmes · Pull Request #280 · cloudtools/stacker · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

DAG #280

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 12 commits into from
Closed

DAG #280

Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
.PHONY: build
.PHONY: build test lint

build:
docker build -t remind101/stacker .

test:
lint:
flake8 --exclude stacker/tests/ stacker
flake8 --ignore N802 stacker/tests # ignore setUp naming

test: lint
python setup.py test
57 changes: 16 additions & 41 deletions stacker/actions/base.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,28 @@
import copy
import logging
import sys
import threading
import signal

import boto3
import botocore.exceptions

logger = logging.getLogger(__name__)


def cancel():
"""Returns a threading.Event() that will get set when SIGTERM, or
SIGINT are triggered. This can be used to cancel execution of threads.
"""
cancel = threading.Event()

def cancel_execution(signum, frame):
cancel.set()

signal.signal(signal.SIGINT, cancel_execution)
signal.signal(signal.SIGTERM, cancel_execution)
return cancel


def stack_template_key_name(blueprint):
"""Given a blueprint, produce an appropriate key name.

Expand Down Expand Up @@ -150,43 +165,3 @@ def _get_all_stack_names(self, dependencies):
dependencies.keys() +
[item for items in dependencies.values() for item in items]
)

def get_stack_execution_order(self, dependencies):
"""Return the order in which the stacks should be executed.

Args:
- dependencies (dict): a dictionary where each key should be the
fully qualified name of a stack whose value is an array of
fully qualified stack names that the stack depends on. This is
used to generate the order in which the stacks should be
executed.

Returns:
array: An array of stack names in the order which they should be
executed.

"""
# copy the dependencies since we pop items out of it to get the
# execution order, we don't want to mutate the one passed in
dependencies = copy.deepcopy(dependencies)
pending_steps = []
executed_steps = []
stack_names = self._get_all_stack_names(dependencies)
for stack_name in stack_names:
requirements = dependencies.get(stack_name, None)
if not requirements:
dependencies.pop(stack_name, None)
pending_steps.append(stack_name)

while dependencies:
for step in pending_steps:
for stack_name, requirements in dependencies.items():
if step in requirements:
requirements.remove(step)

if not requirements:
dependencies.pop(stack_name)
pending_steps.append(stack_name)
pending_steps.remove(step)
executed_steps.append(step)
return executed_steps + pending_steps
27 changes: 7 additions & 20 deletions stacker/actions/build.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import logging

from .base import BaseAction
from .base import BaseAction, cancel
from .. import exceptions, util
from ..exceptions import StackDidNotChange
from ..plan import Plan
Expand Down Expand Up @@ -271,26 +271,10 @@ def _handle_missing_parameters(self, params, required_params,
return params.items()

def _generate_plan(self, tail=False):
plan_kwargs = {}
if tail:
plan_kwargs["watch_func"] = self.provider.tail_stack
plan = Plan(description="Create/Update stacks", **plan_kwargs)
stacks = self.context.get_stacks_dict()
dependencies = self._get_dependencies()
for stack_name in self.get_stack_execution_order(dependencies):
plan.add(
stacks[stack_name],
run_func=self._launch_stack,
requires=dependencies.get(stack_name),
)
plan = Plan(description="Create/Update stacks")
plan.build(self.context.get_stacks())
return plan
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This gets removed in this PR, which may not be correct. This seems to tail the events, but I'm not sure I've ever actually seen stacker do this.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah - this is used by some people with --tail. Does it cause issues if you leave it in?


def _get_dependencies(self):
dependencies = {}
for stack in self.context.get_stacks():
dependencies[stack.fqn] = stack.requires
return dependencies

def pre_run(self, outline=False, *args, **kwargs):
"""Any steps that need to be taken prior to running the action."""
pre_build = self.context.config.get("pre_build")
Expand All @@ -308,7 +292,10 @@ def run(self, outline=False, tail=False, dump=False, *args, **kwargs):
if not outline and not dump:
plan.outline(logging.DEBUG)
logger.debug("Launching stacks: %s", ", ".join(plan.keys()))
plan.execute()
plan.execute(
self._launch_stack,
parallel=self.provider.supports_parallel,
cancel=cancel())
else:
if outline:
plan.outline()
Expand Down
34 changes: 7 additions & 27 deletions stacker/actions/destroy.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import logging

from .base import BaseAction
from .base import BaseAction, cancel
from ..exceptions import StackDoesNotExist
from .. import util
from ..status import (
Expand Down Expand Up @@ -31,32 +31,9 @@ class Action(BaseAction):

"""

def _get_dependencies(self, stacks_dict):
dependencies = {}
for stack_name, stack in stacks_dict.iteritems():
required_stacks = stack.requires
if not required_stacks:
if stack_name not in dependencies:
dependencies[stack_name] = required_stacks
continue

for requirement in required_stacks:
dependencies.setdefault(requirement, set()).add(stack_name)
return dependencies

def _generate_plan(self, tail=False):
plan_kwargs = {}
if tail:
plan_kwargs["watch_func"] = self.provider.tail_stack
plan = Plan(description="Destroy stacks", **plan_kwargs)
stacks_dict = self.context.get_stacks_dict()
dependencies = self._get_dependencies(stacks_dict)
for stack_name in self.get_stack_execution_order(dependencies):
plan.add(
stacks_dict[stack_name],
run_func=self._destroy_stack,
requires=dependencies.get(stack_name),
)
plan = Plan(description="Destroy stacks", reverse=True)
plan.build(self.context.get_stacks())
return plan

def _destroy_stack(self, stack, **kwargs):
Expand Down Expand Up @@ -100,7 +77,10 @@ def run(self, force, tail=False, *args, **kwargs):
# steps to COMPLETE in order to log them
debug_plan = self._generate_plan()
debug_plan.outline(logging.DEBUG)
plan.execute()
plan.execute(
self._destroy_stack,
parallel=self.provider.supports_parallel,
cancel=cancel())
else:
plan.outline(message="To execute this plan, run with \"--force\" "
"flag.")
Expand Down
Loading
0