From cb096d611a518e6c2abe0c4d31324ad1de3ea5a0 Mon Sep 17 00:00:00 2001 From: artsyjian <63004951+artsyjian@users.noreply.github.com> Date: Mon, 3 Feb 2025 08:58:31 -0800 Subject: [PATCH 1/3] feat: fallback to "docker compose" when docker-compose not found (#434) --- hokusai/cli/base.py | 4 +- hokusai/commands/check.py | 14 +-- hokusai/commands/development.py | 147 +++++++++++++------------- hokusai/commands/test.py | 52 +++++---- hokusai/lib/docker_compose_helpers.py | 58 +++++++++- hokusai/services/docker.py | 31 ++---- 6 files changed, 183 insertions(+), 123 deletions(-) diff --git a/hokusai/cli/base.py b/hokusai/cli/base.py index bd634446..d1493260 100644 --- a/hokusai/cli/base.py +++ b/hokusai/cli/base.py @@ -97,8 +97,10 @@ def test(build, cleanup, filename, service_name, verbose): @base.command(context_settings=CONTEXT_SETTINGS) -def check(): +@click.option('-v', '--verbose', type=click.BOOL, is_flag=True, help='Verbose output') +def check(verbose): """Check Hokusai dependencies and configuration""" + set_verbosity(verbose) command( hokusai.check ) diff --git a/hokusai/commands/check.py b/hokusai/commands/check.py index e7f29f7f..6618d822 100644 --- a/hokusai/commands/check.py +++ b/hokusai/commands/check.py @@ -3,8 +3,9 @@ import os from hokusai import CWD -from hokusai.lib.common import get_region_name, print_red, print_green, shout +from hokusai.lib.common import get_region_name, print_green, print_red, shout from hokusai.lib.config import HOKUSAI_CONFIG_DIR, BUILD_YAML_FILE, TEST_YML_FILE, DEVELOPMENT_YML_FILE, config +from hokusai.lib.docker_compose_helpers import detect_compose_command from hokusai.lib.exceptions import CalledProcessError, HokusaiError from hokusai.lib.template_selector import TemplateSelector from hokusai.services.ecr import ECR @@ -32,12 +33,11 @@ def check_err(check_item): check_err('docker') return_code += 1 - try: - shout('which docker-compose') - check_ok('docker-compose') - except CalledProcessError: - check_err('docker-compose') - return_code += 1 + compose_command = detect_compose_command() + print_green( + '\u2714 ' + + f'Will use "{compose_command}" command for Docker Compose' + ) try: shout('which kubectl') diff --git a/hokusai/commands/development.py b/hokusai/commands/development.py index 968075ad..2827a744 100644 --- a/hokusai/commands/development.py +++ b/hokusai/commands/development.py @@ -1,107 +1,108 @@ import os import signal -from hokusai import CWD -from hokusai.lib.config import HOKUSAI_CONFIG_DIR, DEVELOPMENT_YML_FILE, config -from hokusai.lib.common import print_green, shout, EXIT_SIGNALS -from hokusai.lib.exceptions import HokusaiError +from hokusai.lib.common import EXIT_SIGNALS, print_green, shout +from hokusai.lib.config import DEVELOPMENT_YML_FILE, config +from hokusai.lib.docker_compose_helpers import generate_compose_command, get_yaml_template from hokusai.services.docker import Docker -from hokusai.lib.template_selector import TemplateSelector -from hokusai.lib.docker_compose_helpers import follow_extends -from hokusai.services.yaml_spec import YamlSpec -def dev_start(build, detach, filename): - if filename is None: - yaml_template = TemplateSelector().get(os.path.join(CWD, HOKUSAI_CONFIG_DIR, DEVELOPMENT_YML_FILE)) - else: - yaml_template = TemplateSelector().get(filename) - - docker_compose_yml = YamlSpec(yaml_template).to_file() - follow_extends(docker_compose_yml) +def dev_start(build, detach, filename): + compose_command = generate_compose_command( + filename, + default_yaml_file=DEVELOPMENT_YML_FILE + ) def cleanup(*args): - shout("COMPOSE_COMPATIBILITY=true docker-compose -f %s -p hokusai stop" % docker_compose_yml, print_output=True) + shout( + f'{compose_command} -p hokusai stop', + print_output=True + ) for sig in EXIT_SIGNALS: signal.signal(sig, cleanup) - opts = '' if build: + yaml_template = get_yaml_template( + filename, + default_yaml_file=DEVELOPMENT_YML_FILE + ) Docker().build(filename=yaml_template) if detach: opts += ' -d' - if not detach: - print_green("Starting development environment... Press Ctrl+C to stop.") - - shout("COMPOSE_COMPATIBILITY=true docker-compose -f %s -p hokusai up%s" % (docker_compose_yml, opts), print_output=True) - + print_green( + "Starting development environment... Press Ctrl+C to stop." + ) + shout( + f'{compose_command} -p hokusai up{opts}', + print_output=True + ) if detach: - print_green("Run `hokousai dev stop` to shut down, or `hokusai dev logs --follow` to tail output.") + print_green( + "Run `hokousai dev stop` to shut down, or `hokusai dev logs --follow` to tail output." + ) def dev_stop(filename): - if filename is None: - yaml_template = TemplateSelector().get(os.path.join(CWD, HOKUSAI_CONFIG_DIR, DEVELOPMENT_YML_FILE)) - else: - yaml_template = TemplateSelector().get(filename) - - docker_compose_yml = YamlSpec(yaml_template).to_file() - follow_extends(docker_compose_yml) - - shout("COMPOSE_COMPATIBILITY=true docker-compose -f %s -p hokusai stop" % docker_compose_yml, print_output=True) + compose_command = generate_compose_command( + filename, + default_yaml_file=DEVELOPMENT_YML_FILE + ) + shout( + f'{compose_command} -p hokusai stop', + print_output=True + ) def dev_status(filename): - if filename is None: - yaml_template = TemplateSelector().get(os.path.join(CWD, HOKUSAI_CONFIG_DIR, DEVELOPMENT_YML_FILE)) - else: - yaml_template = TemplateSelector().get(filename) - - docker_compose_yml = YamlSpec(yaml_template).to_file() - follow_extends(docker_compose_yml) - - shout("COMPOSE_COMPATIBILITY=true docker-compose -f %s -p hokusai ps" % docker_compose_yml, print_output=True) + compose_command = generate_compose_command( + filename, + default_yaml_file=DEVELOPMENT_YML_FILE + ) + shout( + f'{compose_command} -p hokusai ps', + print_output=True + ) def dev_logs(follow, tail, filename): - if filename is None: - yaml_template = TemplateSelector().get(os.path.join(CWD, HOKUSAI_CONFIG_DIR, DEVELOPMENT_YML_FILE)) - else: - yaml_template = TemplateSelector().get(filename) - - docker_compose_yml = YamlSpec(yaml_template).to_file() - follow_extends(docker_compose_yml) - + compose_command = generate_compose_command( + filename, + default_yaml_file=DEVELOPMENT_YML_FILE + ) opts = '' if follow: opts += ' --follow' if tail: opts += " --tail=%i" % tail - - shout("COMPOSE_COMPATIBILITY=true docker-compose -f %s -p hokusai logs%s" % (docker_compose_yml, opts), print_output=True) + shout( + f'{compose_command} -p hokusai logs{opts}', + print_output=True + ) def dev_run(container_command, service_name, stop, filename): - if filename is None: - yaml_template = TemplateSelector().get(os.path.join(CWD, HOKUSAI_CONFIG_DIR, DEVELOPMENT_YML_FILE)) - else: - yaml_template = TemplateSelector().get(filename) - - docker_compose_yml = YamlSpec(yaml_template).to_file() - follow_extends(docker_compose_yml) - + compose_command = generate_compose_command( + filename, + default_yaml_file=DEVELOPMENT_YML_FILE + ) if service_name is None: service_name = config.project_name - - shout("COMPOSE_COMPATIBILITY=true docker-compose -f %s -p hokusai run %s %s" % (docker_compose_yml, service_name, container_command), print_output=True) - + shout( + f'{compose_command} -p hokusai run {service_name} {container_command}', + print_output=True + ) if stop: - shout("COMPOSE_COMPATIBILITY=true docker-compose -f %s -p hokusai stop" % docker_compose_yml, print_output=True) + shout( + f'{compose_command} -p hokusai stop', + print_output=True + ) def dev_clean(filename): - if filename is None: - yaml_template = TemplateSelector().get(os.path.join(CWD, HOKUSAI_CONFIG_DIR, DEVELOPMENT_YML_FILE)) - else: - yaml_template = TemplateSelector().get(filename) - - docker_compose_yml = YamlSpec(yaml_template).to_file() - follow_extends(docker_compose_yml) - - shout("COMPOSE_COMPATIBILITY=true docker-compose -f %s -p hokusai stop" % docker_compose_yml, print_output=True) - shout("COMPOSE_COMPATIBILITY=true docker-compose -f %s -p hokusai rm --force" % docker_compose_yml, print_output=True) + compose_command = generate_compose_command( + filename, + default_yaml_file=DEVELOPMENT_YML_FILE + ) + shout( + f'{compose_command} -p hokusai stop', + print_output=True + ) + shout( + f'{compose_command} -p hokusai rm --force', + print_output=True + ) diff --git a/hokusai/commands/test.py b/hokusai/commands/test.py index eae35523..d568bc94 100644 --- a/hokusai/commands/test.py +++ b/hokusai/commands/test.py @@ -1,50 +1,62 @@ import os import signal -from hokusai import CWD -from hokusai.lib.config import HOKUSAI_CONFIG_DIR, TEST_YML_FILE, config -from hokusai.lib.common import print_green, print_red, shout, EXIT_SIGNALS +from hokusai.lib.common import EXIT_SIGNALS, print_green, print_red, shout +from hokusai.lib.config import TEST_YML_FILE, config +from hokusai.lib.docker_compose_helpers import generate_compose_command, get_yaml_template from hokusai.lib.exceptions import CalledProcessError, HokusaiError from hokusai.services.docker import Docker -from hokusai.lib.template_selector import TemplateSelector -from hokusai.lib.docker_compose_helpers import follow_extends -from hokusai.services.yaml_spec import YamlSpec -def test(build, cleanup, filename, service_name): - if filename is None: - yaml_template = TemplateSelector().get(os.path.join(CWD, HOKUSAI_CONFIG_DIR, TEST_YML_FILE)) - else: - yaml_template = TemplateSelector().get(filename) - docker_compose_yml = YamlSpec(yaml_template).to_file() - follow_extends(docker_compose_yml) +def test(build, cleanup, filename, service_name): + compose_command = generate_compose_command( + filename, + default_yaml_file=TEST_YML_FILE + ) def on_cleanup(*args): - shout("COMPOSE_COMPATIBILITY=true docker-compose -f %s -p hokusai stop" % docker_compose_yml) - shout("COMPOSE_COMPATIBILITY=true docker-compose -f %s -p hokusai rm --force" % docker_compose_yml) - + shout( + f'{compose_command} -p hokusai stop', + ) + shout( + f'{compose_command} -p hokusai rm --force', + ) if cleanup: for sig in EXIT_SIGNALS: signal.signal(sig, on_cleanup) if build: + yaml_template = get_yaml_template( + filename, + default_yaml_file=TEST_YML_FILE + ) Docker().build(filename=yaml_template) if service_name is None: service_name = config.project_name opts = " --abort-on-container-exit --exit-code-from %s" % service_name - - print_green("Starting test environment... Press Ctrl+C to stop.", newline_after=True) + print_green( + "Starting test environment... Press Ctrl+C to stop.", + newline_after=True + ) try: - return_code = int(shout("COMPOSE_COMPATIBILITY=true docker-compose -f %s -p hokusai up%s" % (docker_compose_yml, opts), print_output=True)) + return_code = int( + shout( + f'{compose_command} -p hokusai up{opts}', + print_output=True + ) + ) except CalledProcessError as e: print_red("CalledProcessError return code: %s" % e.returncode) if cleanup: on_cleanup() raise HokusaiError('Tests Failed') if return_code: - raise HokusaiError('Tests Failed - Exit Code: %s\n' % return_code, return_code=return_code) + raise HokusaiError( + 'Tests Failed - Exit Code: %s\n' % return_code, + return_code=return_code + ) else: print_green("Tests Passed") diff --git a/hokusai/lib/docker_compose_helpers.py b/hokusai/lib/docker_compose_helpers.py index 24965f5e..2aca0b26 100644 --- a/hokusai/lib/docker_compose_helpers.py +++ b/hokusai/lib/docker_compose_helpers.py @@ -3,11 +3,30 @@ import yaml from hokusai import CWD +from hokusai.lib.common import get_verbosity, print_green, print_yellow, shout from hokusai.lib.config import HOKUSAI_CONFIG_DIR -from hokusai.lib.exceptions import HokusaiError +from hokusai.lib.exceptions import CalledProcessError from hokusai.lib.template_selector import TemplateSelector from hokusai.services.yaml_spec import YamlSpec + +def detect_compose_command(): + ''' decide what command to use for Docker Compose ''' + command_to_use = '' + try: + shout('which docker-compose') + if get_verbosity(): + print_green('Found docker-compose.') + command_to_use = 'docker-compose' + except CalledProcessError: + if get_verbosity(): + print_yellow( + 'docker-compose command not found. ' + + 'Will use "docker compose" assuming it exists.' + ) + command_to_use = 'docker compose' + return command_to_use + def follow_extends(docker_compose_yml): with open(docker_compose_yml, 'r') as f: rendered_templates = [] @@ -22,3 +41,40 @@ def follow_extends(docker_compose_yml): extended_template = TemplateSelector().get(extended_template_path) rendered_templates.append(YamlSpec(extended_template).to_file()) return rendered_templates + +def generate_compose_command(filename, default_yaml_file): + ''' return Docker Compose command ''' + docker_compose_yml = render_docker_compose_yml( + filename, + default_yaml_file + ) + # docker-compose v2 switched to using '-' as separator in image name, + # resulting in 'hokusai-' + # COMPOSE_COMPATIBILITY=true forces v2 to use '_', + # resulting in 'hokusai_', matching v1 + return ( + 'COMPOSE_COMPATIBILITY=true ' + + detect_compose_command() + + f' -f {docker_compose_yml}' + ) + +def get_yaml_template(filename, default_yaml_file): + ''' return yaml template ''' + if filename is None: + yaml_template = TemplateSelector().get( + os.path.join( + CWD, + HOKUSAI_CONFIG_DIR, + default_yaml_file + ) + ) + else: + yaml_template = TemplateSelector().get(filename) + return yaml_template + +def render_docker_compose_yml(filename, default_yaml_file): + ''' return path of rendered Docker Compose yaml ''' + yaml_template = get_yaml_template(filename, default_yaml_file) + docker_compose_yml = YamlSpec(yaml_template).to_file() + follow_extends(docker_compose_yml) + return docker_compose_yml diff --git a/hokusai/services/docker.py b/hokusai/services/docker.py index d9469484..e855fd0c 100644 --- a/hokusai/services/docker.py +++ b/hokusai/services/docker.py @@ -1,32 +1,21 @@ import os -from hokusai import CWD -from hokusai.lib.config import HOKUSAI_CONFIG_DIR, BUILD_YAML_FILE, config from hokusai.lib.common import shout -from hokusai.lib.template_selector import TemplateSelector -from hokusai.services.yaml_spec import YamlSpec +from hokusai.lib.config import BUILD_YAML_FILE, config +from hokusai.lib.docker_compose_helpers import generate_compose_command + class Docker: def build(self, filename=None): - if filename is None: - yaml_template = TemplateSelector().get(os.path.join(CWD, HOKUSAI_CONFIG_DIR, BUILD_YAML_FILE)) - else: - yaml_template = TemplateSelector().get(filename) - - docker_compose_yml = YamlSpec(yaml_template).to_file() - - # docker-compose v2 switched to using '-' as separator in image name, resulting in 'hokusai-' - # COMPOSE_COMPATIBILITY=true forces v2 to use '_', resulting in 'hokusai_', matching v1 - env_vars = "DOCKER_DEFAULT_PLATFORM=linux/amd64 COMPOSE_COMPATIBILITY=true" - - compose_command = f"docker-compose -f {docker_compose_yml} -p hokusai build" - compose_options = "--progress plain" - build_command = f"{env_vars} {compose_command} {compose_options}" - + env_vars = "DOCKER_DEFAULT_PLATFORM=linux/amd64" + compose_command = generate_compose_command( + filename, + default_yaml_file=BUILD_YAML_FILE + ) + opts = "--progress plain" + build_command = f'{env_vars} {compose_command} -p hokusai build {opts}' if config.pre_build: build_command = "%s && %s" % (config.pre_build, build_command) - if config.post_build: build_command = "%s && %s" % (build_command, config.post_build) - shout(build_command, print_output=True) From 09c07ca5ed09ec1f353786b4be00cd6fcb34701f Mon Sep 17 00:00:00 2001 From: artsyjian <63004951+artsyjian@users.noreply.github.com> Date: Thu, 6 Feb 2025 02:00:04 -0800 Subject: [PATCH 2/3] prepare v3.1.0 (#435) --- CHANGELOG.md | 8 ++++++++ hokusai/RELEASE_VERSION | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fba2e3e5..8f7135e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +## v3.1.0 + +[Documentation](https://github.com/artsy/hokusai/blob/v3.1.0/README.md) + +### Notable changes since v3.0.0 + +* cb096d6 feat: fallback to "docker compose" when docker-compose not found (#434) + ## v3.0.0 [Documentation](https://github.com/artsy/hokusai/blob/v3.0.0/README.md) diff --git a/hokusai/RELEASE_VERSION b/hokusai/RELEASE_VERSION index 4a36342f..fd2a0186 100644 --- a/hokusai/RELEASE_VERSION +++ b/hokusai/RELEASE_VERSION @@ -1 +1 @@ -3.0.0 +3.1.0 From adbc92991cc6245ddf468e30777145b2d0d0285f Mon Sep 17 00:00:00 2001 From: Jian Date: Wed, 12 Feb 2025 11:29:06 -0800 Subject: [PATCH 3/3] bump golang --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 7094c743..84b45467 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -266,7 +266,7 @@ jobs: release_github: docker: - - image: golang:1.19 + - image: golang:1.23 steps: - checkout - run: apt-get -qq update