From 9e48512311f4644cd7f6acb13d000aefecd0daed Mon Sep 17 00:00:00 2001 From: Pierre-Louis Bonicoli Date: Thu, 8 Feb 2018 10:51:22 +0100 Subject: [PATCH 1/7] Enable validate-config tests --- test/run_tests.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/run_tests.yml b/test/run_tests.yml index 2086e787..72d2423e 100644 --- a/test/run_tests.yml +++ b/test/run_tests.yml @@ -117,4 +117,8 @@ include_role: name: validate-rebuild-features tags: rebuild + - name: "Validate config" + include_role: + name: validate-config + tags: config From bc16f8d844fd6ff1bb45436955f8f4f030e94bb8 Mon Sep 17 00:00:00 2001 From: Pierre-Louis Bonicoli Date: Thu, 8 Feb 2018 10:52:49 +0100 Subject: [PATCH 2/7] Fix TypeError exception (Python 3 compatibility) Error message: ______ TestAnsibleContainerConfig.test_should_also_replace_pwd_in_volumes ______ self = def test_should_also_replace_pwd_in_volumes(self): # test that ${PWD} gets resolved self.config.cli_vars_files = self.vars_files self.config.set_env('prod') container.ENV = 'conductor' container.utils.DataLoader = DataLoader container.utils.VariableManager = VariableManager container.utils.RoleInclude = RoleInclude > conductor_config = AnsibleContainerConductorConfig(self.config._config) ansible-container/test/tests/validate_config.py:145: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ ansible-container/container/__init__.py:19: in __wrapped__ return fn(*args, **kwargs) ansible-container/container/config.py:357: in __init__ self._process_defaults() ansible-container/container/config.py:390: in _process_defaults callback=lambda processed: self._templar.set_available_variables( ansible-container/container/config.py:375: in _process_section yaml.round_trip_dump(value, buffer) [...] venv/3.6/lib/python3.6/site-packages/ruamel/yaml/emitter.py:422: in expect_flow_sequence self.write_indicator(u' ' * ind + u'[', True, whitespace=True) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = indicator = '[', need_whitespace = True, whitespace = True, indention = False def write_indicator(self, indicator, need_whitespace, whitespace=False, indention=False): # type: (Any, Any, bool, bool) -> None if self.whitespace or not need_whitespace: data = indicator else: data = u' ' + indicator self.whitespace = whitespace self.indention = self.indention and indention self.column += len(data) self.open_ended = False if bool(self.encoding): data = data.encode(self.encoding) > self.stream.write(data) E TypeError: a bytes-like object is required, not 'str' venv/3.6/lib/python3.6/site-packages/ruamel/yaml/emitter.py:1059: TypeError --- container/config.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/container/config.py b/container/config.py index 83dec053..42765aed 100644 --- a/container/config.py +++ b/container/config.py @@ -6,13 +6,12 @@ logger = getLogger(__name__) from abc import ABCMeta, abstractproperty, abstractmethod -from io import BytesIO import os from os import path import copy import json import re -from six import add_metaclass, iteritems, PY2, string_types, text_type +from six import add_metaclass, iteritems, PY2, string_types, text_type, StringIO from collections import Mapping from .utils.ordereddict import ordereddict @@ -371,7 +370,7 @@ def _process_section(self, section_value, callback=None, templar=None): elif isinstance(value, (list, dict)): # if it's a dimensional structure, it's cheaper just to serialize # it, treat it like a template, and then deserialize it again - buffer = BytesIO() # use bytes explicitly, not unicode + buffer = StringIO() yaml.round_trip_dump(value, buffer) processed[key] = yaml.round_trip_load( templar.template(buffer.getvalue()) From 11ddabf582c0b099ba598669d23cdffc29cef978 Mon Sep 17 00:00:00 2001 From: Pierre-Louis Bonicoli Date: Thu, 8 Feb 2018 11:00:22 +0100 Subject: [PATCH 3/7] Don't redefine built-in 'buffer' --- container/config.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/container/config.py b/container/config.py index 42765aed..0d729e73 100644 --- a/container/config.py +++ b/container/config.py @@ -370,10 +370,10 @@ def _process_section(self, section_value, callback=None, templar=None): elif isinstance(value, (list, dict)): # if it's a dimensional structure, it's cheaper just to serialize # it, treat it like a template, and then deserialize it again - buffer = StringIO() - yaml.round_trip_dump(value, buffer) + stream = StringIO() + yaml.round_trip_dump(value, stream) processed[key] = yaml.round_trip_load( - templar.template(buffer.getvalue()) + templar.template(stream.getvalue()) ) else: # ints, booleans, etc. From c0369d70830e6ee8de9ae75127bea83ec9cf7872 Mon Sep 17 00:00:00 2001 From: Pierre-Louis Bonicoli Date: Thu, 8 Feb 2018 11:01:28 +0100 Subject: [PATCH 4/7] validate-config test: add required 'settings' key --- test/roles/validate-config/files/container.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/roles/validate-config/files/container.yml b/test/roles/validate-config/files/container.yml index f158ae28..0cb31cc1 100644 --- a/test/roles/validate-config/files/container.yml +++ b/test/roles/validate-config/files/container.yml @@ -1,4 +1,7 @@ version: '2' +settings: + conductor: + base: "centos:7" defaults: web_image: "centos:7" web_ports: ['8080:80'] From 9c1b52c2ca1fdd3c8dcad91fd2090227c6b0d2ce Mon Sep 17 00:00:00 2001 From: Pierre-Louis Bonicoli Date: Thu, 8 Feb 2018 11:02:47 +0100 Subject: [PATCH 5/7] validate-config test: fix assertions --- test/roles/validate-config/tasks/main.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/test/roles/validate-config/tasks/main.yml b/test/roles/validate-config/tasks/main.yml index 8f70dab9..3533e927 100644 --- a/test/roles/validate-config/tasks/main.yml +++ b/test/roles/validate-config/tasks/main.yml @@ -51,12 +51,13 @@ - include: includes/show-output.yml output_file=./task.output registered_output="{{ output }}" +# Expected output: +# Playbook generated: [{'hosts': u'web', 'roles': [ordereddict([('role', 'test-variables')])], 'vars': {u'debug': 0, u'web_image': u'centos:7', u'foo': u'bar', u'project_path': u'/foo', u'web_ports': [u'8080:80']}}] [container.utils] - name: Test build output for container.yml defaults assert: that: - - "'web_image=centos:7' in build_output.stdout" + - "build_output.stdout is search(\"'web_image': u'centos:7'\")" - "'8080:80' in build_output.stdout" - - "'debug=0' in build_output.stdout" - - "'foo=bar' in build_output.stdout" - - "'project_path=/foo' in build_output.stdout" - + - "build_output.stdout is search(\"'debug': 0\")" + - "build_output.stdout is search(\"'foo': u'bar'\")" + - "build_output.stdout is search(\"'project_path': u'/foo'\")" From bd1c048fca99f9a80ea4551b918734f2bc625492 Mon Sep 17 00:00:00 2001 From: Pierre-Louis Bonicoli Date: Thu, 8 Feb 2018 11:04:47 +0100 Subject: [PATCH 6/7] validate-config test: add required parameter --- test/tests/validate_config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/tests/validate_config.py b/test/tests/validate_config.py index b734ee17..b588fd84 100644 --- a/test/tests/validate_config.py +++ b/test/tests/validate_config.py @@ -31,7 +31,7 @@ def setUp(self): container.ENV = 'host' container.config.Templar = Templar container.config.AnsibleUnsafeText = AnsibleUnsafeText - self.config = get_config(self.project_path, vars_files=None, engine_name='docker') + self.config = get_config(self.project_path, vars_files=None, engine_name='docker', config_file='container.yml') def tearDown(self): pass @@ -50,7 +50,7 @@ def test_should_have_project_name_equal_basename(self): self.assertEqual(self.config.project_name, os.path.basename(self.project_path)) def test_should_have_project_name_equal_cli(self): - config = get_config(self.project_path, vars_files=None, engine_name='docker', project_name='foo') + config = get_config(self.project_path, vars_files=None, engine_name='docker', project_name='foo', config_file='container.yml') self.assertEqual(config.project_name, 'foo') def test_should_have_project_name_equal_settings(self): From 3aba711900a2564a0f59de1d14ada199670f5588 Mon Sep 17 00:00:00 2001 From: Pierre-Louis Bonicoli Date: Thu, 8 Feb 2018 11:07:08 +0100 Subject: [PATCH 7/7] add test for #882 --- container/utils/__init__.py | 18 ++++++++++-------- test/tests/validate_config.py | 9 ++++++++- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/container/utils/__init__.py b/container/utils/__init__.py index cc603d49..5db685aa 100644 --- a/container/utils/__init__.py +++ b/container/utils/__init__.py @@ -252,6 +252,16 @@ def generate_playbook_for_role(service_name, vars, role): logger.debug('Playbook generated: %s', playbook) return playbook + +def get_dependencies_for_role(role_path): + meta_main_path = os.path.join(role_path, 'meta', 'main.yml') + if os.path.exists(meta_main_path): + meta_main = yaml.safe_load(open(meta_main_path)) + if meta_main: + for dependency in meta_main.get('dependencies', []): + yield dependency.get('role', None) + + @container.conductor_only def get_role_fingerprint(role, service_name, config_vars): """ @@ -307,14 +317,6 @@ def hash_role(hash_obj, role_path): else: hash_dir(hash_obj, src) - def get_dependencies_for_role(role_path): - meta_main_path = os.path.join(role_path, 'meta', 'main.yml') - if os.path.exists(meta_main_path): - meta_main = yaml.safe_load(open(meta_main_path)) - if meta_main: - for dependency in meta_main.get('dependencies', []): - yield dependency.get('role', None) - hash_obj = hashlib.sha256() # Account for variables passed to the role by including the invocation string hash_obj.update((json.dumps(role) if not isinstance(role, string_types) else role) + '::') diff --git a/test/tests/validate_config.py b/test/tests/validate_config.py index b588fd84..3aed1e8c 100644 --- a/test/tests/validate_config.py +++ b/test/tests/validate_config.py @@ -3,9 +3,11 @@ import json import container -from container.utils import get_config from container.config import AnsibleContainerConductorConfig from container.exceptions import AnsibleContainerConfigException +from container.utils import get_dependencies_for_role, get_config +from ansible.compat.tests.mock import mock_open, patch +from ansible.module_utils.six.moves import builtins from ansible.playbook.role.include import RoleInclude try: @@ -171,3 +173,8 @@ def test_should_use_dev_overrides(self): # self.assertEqual(self.config._config['services']['web']['environment'][2], 'VERSION={0}'.format(__version__)) + @patch('os.path.exists', lambda x: True) + def test_get_dependencies_for_role(self): + with patch.object(builtins, 'open', mock_open(read_data=b'---\n')): + for x in get_dependencies_for_role('dummy'): + pass