From 3c451b5503901c9aebb849bebe0197542238a39d Mon Sep 17 00:00:00 2001 From: Luc Peterson Date: Tue, 24 Aug 2021 15:50:40 -0700 Subject: [PATCH 1/6] Modifying task id to include directory --- merlin/common/tasks.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/merlin/common/tasks.py b/merlin/common/tasks.py index f55395fcd..8d27e0e2c 100644 --- a/merlin/common/tasks.py +++ b/merlin/common/tasks.py @@ -31,6 +31,8 @@ """Test tasks.""" from __future__ import absolute_import, unicode_literals +from datetime import datetime + import logging import os @@ -70,6 +72,11 @@ STOP_COUNTDOWN = 60 +def log_result(result, step_dir, result_file): + now = datetime.now().strftime("%c") + cmd = f"flock --timeout 60 {result_file} echo '{now},{step_dir},{result}' >> {result_file}" + _ = os.system(cmd) + @shared_task( bind=True, autoretry_for=retry_exceptions, @@ -104,6 +111,7 @@ def merlin_step(self, *args, **kwargs): self.max_retries = step.max_retries step_name = step.name() step_dir = step.get_workspace() + LOG.error(self.request.id) LOG.debug(f"merlin_step: step_name '{step_name}' step_dir '{step_dir}'") finished_filename = os.path.join(step_dir, "MERLIN_FINISHED") # if we've already finished this task, skip it @@ -112,6 +120,7 @@ def merlin_step(self, *args, **kwargs): result = ReturnCode.OK else: result = step.execute(config) + log_result(result, step_dir, f'{step_dir}/../results_log.txt') if result == ReturnCode.OK: LOG.info(f"Step '{step_name}' in '{step_dir}' finished successfully.") # touch a file indicating we're done with this step @@ -296,6 +305,7 @@ def add_merlin_expanded_chain_to_chord( adapter_config=adapter_config, ) new_step.set(queue=step.get_task_queue()) + new_step.set(task_id=os.path.join(workspace, relative_paths[sample_id])) new_chain.append(new_step) all_chains.append(new_chain) @@ -355,7 +365,8 @@ def add_simple_chain_to_chord(self, task_type, chain_, adapter_config): new_steps = [ task_type.s(step, adapter_config=adapter_config).set( - queue=step.get_task_queue() + queue=step.get_task_queue(), + task_id=step.get_workspace(), ) ] all_chains.append(new_steps) From a371a4fc2c10577653e8241aca90aae42ee377f5 Mon Sep 17 00:00:00 2001 From: Luc Peterson Date: Thu, 6 Jun 2024 17:00:26 -0700 Subject: [PATCH 2/6] Update changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ca60ba17..3c422f868 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to Merlin will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] +### Changed +- Task id is now the path to the directory + ## [1.12.2b1] ### Added - Conflict handler option to the `dict_deep_merge` function in `utils.py` From 91ad91be4093dd0275a04a15d2fc18dd3d4f7ce6 Mon Sep 17 00:00:00 2001 From: Luc Peterson Date: Thu, 6 Jun 2024 17:03:05 -0700 Subject: [PATCH 3/6] fix style --- merlin/common/tasks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/merlin/common/tasks.py b/merlin/common/tasks.py index 6c3e54c08..143d3bf12 100644 --- a/merlin/common/tasks.py +++ b/merlin/common/tasks.py @@ -77,6 +77,7 @@ # R0914: too many local variables # R0915: too many statements + @shared_task( # noqa: C901 bind=True, autoretry_for=retry_exceptions, From bbaa61bfdfa6ff313f198adc4e06184d371e0412 Mon Sep 17 00:00:00 2001 From: Luc Peterson Date: Mon, 10 Jun 2024 21:15:14 -0700 Subject: [PATCH 4/6] Update CHANGELOG.md --- CHANGELOG.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c422f868..07ef76622 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,10 +4,6 @@ All notable changes to Merlin will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] -### Changed -- Task id is now the path to the directory - ## [1.12.2b1] ### Added - Conflict handler option to the `dict_deep_merge` function in `utils.py` @@ -30,6 +26,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Applying filters for `merlin detailed-status` will now log debug statements instead of warnings - Modified the unit tests for the `merlin status` command to use pytest rather than unittest - Added fixtures for `merlin status` tests that copy the workspace to a temporary directory so you can see exactly what's run in a test +- Task id is now the path to the directory ### Fixed - Bugfix for output of `merlin example openfoam_wf_singularity` From 2f632846de2bc12d4c0e9e273aa14426da8fbc78 Mon Sep 17 00:00:00 2001 From: Luc Peterson Date: Wed, 12 Jun 2024 10:09:47 -0700 Subject: [PATCH 5/6] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 07ef76622..567d91a31 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Applying filters for `merlin detailed-status` will now log debug statements instead of warnings - Modified the unit tests for the `merlin status` command to use pytest rather than unittest - Added fixtures for `merlin status` tests that copy the workspace to a temporary directory so you can see exactly what's run in a test + - Task id is now the path to the directory ### Fixed From fd6c44c1b6a2bbf18492ceaa3b4391a194838f96 Mon Sep 17 00:00:00 2001 From: Luc Peterson Date: Wed, 12 Jun 2024 12:37:33 -0700 Subject: [PATCH 6/6] update CHANGELOG --- CHANGELOG.md | 1 - 1 file changed, 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a6da02eac..21b4427b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,7 +27,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Modified the unit tests for the `merlin status` command to use pytest rather than unittest - Added fixtures for `merlin status` tests that copy the workspace to a temporary directory so you can see exactly what's run in a test - Batch block and workers now allow for variables to be used in node settings - - Task id is now the path to the directory ### Fixed