From 8fca0a55ca876696724bcd2d6209265ec626df52 Mon Sep 17 00:00:00 2001 From: ismail simsek <6005685+ismailsimsek@users.noreply.github.com> Date: Sun, 2 Mar 2025 16:16:26 +0100 Subject: [PATCH 1/9] Add workflow_dispatch to github test action --- .github/workflows/tests-dbt-version.yml | 1 + opendbt/__init__.py | 9 +- opendbt/airflow/plugin.py | 1 - opendbt/dbt/__init__.py | 54 ++++---- opendbt/dbt/shared/cli/main.py | 4 + opendbt/dbt/v17/adapters/factory.py | 4 + opendbt/dbt/v17/config/runtime.py | 4 + opendbt/dbt/v17/task/docs/generate.py | 3 + opendbt/dbt/v17/task/run.py | 3 + opendbt/dbt/v18/adapters/factory.py | 4 + opendbt/dbt/v18/config/runtime.py | 7 +- opendbt/dbt/v18/task/docs/generate.py | 3 + opendbt/dbt/v18/task/run.py | 3 + opendbt/runtime_patcher.py | 161 ++++++++++++++++++++++++ 14 files changed, 219 insertions(+), 42 deletions(-) create mode 100644 opendbt/runtime_patcher.py diff --git a/.github/workflows/tests-dbt-version.yml b/.github/workflows/tests-dbt-version.yml index 93f529c..c060795 100644 --- a/.github/workflows/tests-dbt-version.yml +++ b/.github/workflows/tests-dbt-version.yml @@ -28,6 +28,7 @@ jobs: # FIX for protobuf issue: https://github.com/dbt-labs/dbt-core/issues/9759 pip install -q "apache-airflow" "protobuf>=4.25.3,<5.0.0" "opentelemetry-proto<1.28.0" --prefer-binary pip install -q .[test] --prefer-binary + pip install -q dbt-core==${{ inputs.dbt-version }}.* dbt-duckdb==${{ inputs.dbt-version }}.* --force-reinstall --upgrade python --version python -c "from dbt.version import get_installed_version as get_dbt_version;print(f'dbt version={get_dbt_version()}')" python -m compileall -f opendbt diff --git a/opendbt/__init__.py b/opendbt/__init__.py index 1f30cfc..494776d 100644 --- a/opendbt/__init__.py +++ b/opendbt/__init__.py @@ -1,15 +1,12 @@ -import logging import os import sys from pathlib import Path -###################### -from opendbt.dbt import patch_dbt - -patch_dbt() +# IMPORTANT! this will import the overrides, and activates the patches +# IMPORTANT! `opendbt.dbt` import needs to happen before any `dbt` import +from opendbt.dbt import * from opendbt.logger import OpenDbtLogger from opendbt.utils import Utils -###################### from dbt.cli.main import dbtRunner as DbtCliRunner from dbt.cli.main import dbtRunnerResult diff --git a/opendbt/airflow/plugin.py b/opendbt/airflow/plugin.py index e3ab9d5..72f993f 100644 --- a/opendbt/airflow/plugin.py +++ b/opendbt/airflow/plugin.py @@ -52,7 +52,6 @@ def manifest(self): # static_url_path='/dbtdocsview' ) - class AirflowDbtDocsPlugin(AirflowPlugin): name = "DBT Docs Plugin" flask_blueprints = [bp] diff --git a/opendbt/dbt/__init__.py b/opendbt/dbt/__init__.py index 43bef51..e39aa6f 100644 --- a/opendbt/dbt/__init__.py +++ b/opendbt/dbt/__init__.py @@ -1,37 +1,27 @@ -import dbt from dbt import version from packaging.version import Version +import opendbt.dbt.shared.cli.main +from opendbt.runtime_patcher import RuntimePatcher -def patch_dbt(): - # ================================================================================================================ - # Monkey Patching! Override dbt lib code with new one - # ================================================================================================================ - dbt_version = Version(version.get_installed_version().to_version_string(skip_matcher=True)) - if Version("1.6.0") <= dbt_version < Version("1.8.0"): - from opendbt.dbt.v17.config.runtime import OpenDbtRuntimeConfig - dbt.config.RuntimeConfig = OpenDbtRuntimeConfig - from opendbt.dbt.v17.task.docs.generate import OpenDbtGenerateTask - dbt.task.generate.GenerateTask = OpenDbtGenerateTask - from opendbt.dbt.v17.adapters.factory import OpenDbtAdapterContainer - dbt.adapters.factory.FACTORY = OpenDbtAdapterContainer() - from opendbt.dbt.v17.task.run import ModelRunner - dbt.task.run.ModelRunner = ModelRunner - elif Version("1.8.0") <= dbt_version < Version("1.10.0"): - from opendbt.dbt.v18.config.runtime import OpenDbtRuntimeConfig - dbt.config.RuntimeConfig = OpenDbtRuntimeConfig - from opendbt.dbt.v18.task.docs.generate import OpenDbtGenerateTask - dbt.task.docs.generate.GenerateTask = OpenDbtGenerateTask - from opendbt.dbt.v18.adapters.factory import OpenDbtAdapterContainer - dbt.adapters.factory.FACTORY = OpenDbtAdapterContainer() - from opendbt.dbt.v18.task.run import ModelRunner - dbt.task.run.ModelRunner = ModelRunner - else: - raise Exception( - f"Unsupported dbt version {dbt_version}, please make sure dbt version is supported/integrated by opendbt") +dbt_version = Version(version.get_installed_version().to_version_string(skip_matcher=True)) +if Version("1.6.0") <= dbt_version < Version("1.8.0"): + from opendbt.dbt.v17.adapters.factory import OpenDbtAdapterContainer + from opendbt.dbt.v17.config.runtime import OpenDbtRuntimeConfig + from opendbt.dbt.v17.task.docs.generate import OpenDbtGenerateTask + from opendbt.dbt.v17.task.run import ModelRunner +elif Version("1.8.0") <= dbt_version < Version("1.10.0"): + from opendbt.dbt.v18.adapters.factory import OpenDbtAdapterContainer + from opendbt.dbt.v18.config.runtime import OpenDbtRuntimeConfig + from opendbt.dbt.v18.task.docs.generate import OpenDbtGenerateTask + from opendbt.dbt.v18.task.run import ModelRunner +else: + raise Exception( + f"Unsupported dbt version {dbt_version}, please make sure dbt version is supported/integrated by opendbt") - # shared code patches - import opendbt.dbt.shared.cli.main - dbt.cli.main.sqlfluff = opendbt.dbt.shared.cli.main.sqlfluff - dbt.cli.main.sqlfluff_lint = opendbt.dbt.shared.cli.main.sqlfluff_lint - dbt.cli.main.sqlfluff_fix = opendbt.dbt.shared.cli.main.sqlfluff_fix +RuntimePatcher(module_name="dbt.adapters.factory").patch_attribute(attribute_name="FACTORY", + new_value=OpenDbtAdapterContainer()) +# shared code patches +from opendbt.dbt.shared.cli.main import sqlfluff +from opendbt.dbt.shared.cli.main import sqlfluff_lint +from opendbt.dbt.shared.cli.main import sqlfluff_fix diff --git a/opendbt/dbt/shared/cli/main.py b/opendbt/dbt/shared/cli/main.py index a5c1bef..2d86c16 100644 --- a/opendbt/dbt/shared/cli/main.py +++ b/opendbt/dbt/shared/cli/main.py @@ -3,12 +3,14 @@ from dbt.cli.main import global_flags, cli from opendbt.dbt.shared.task.sqlfluff import SqlFluffTasks +from opendbt.runtime_patcher import PatchFunction # dbt docs @cli.group() @click.pass_context @global_flags +@PatchFunction(module_name="dbt.cli.main", target_name="sqlfluff") def sqlfluff(ctx, **kwargs): """Generate or serve the documentation website for your project""" @@ -45,6 +47,7 @@ def sqlfluff(ctx, **kwargs): @requires.project @requires.runtime_config @requires.manifest(write=False) +@PatchFunction(module_name="dbt.cli.main", target_name="sqlfluff_lint") def sqlfluff_lint(ctx, **kwargs): """Generate the documentation website for your project""" task = SqlFluffTasks( @@ -90,6 +93,7 @@ def sqlfluff_lint(ctx, **kwargs): @requires.project @requires.runtime_config @requires.manifest(write=False) +@PatchFunction(module_name="dbt.cli.main", target_name="sqlfluff_lint") def sqlfluff_fix(ctx, **kwargs): """Generate the documentation website for your project""" task = SqlFluffTasks( diff --git a/opendbt/dbt/v17/adapters/factory.py b/opendbt/dbt/v17/adapters/factory.py index ac67251..fea3402 100644 --- a/opendbt/dbt/v17/adapters/factory.py +++ b/opendbt/dbt/v17/adapters/factory.py @@ -7,9 +7,13 @@ from dbt.events.types import AdapterRegistered from dbt.semver import VersionSpecifier +from opendbt.runtime_patcher import PatchClass + +@PatchClass(module_name="dbt.adapters.factory", target_name="AdapterContainer") class OpenDbtAdapterContainer(factory.AdapterContainer): DBT_CUSTOM_ADAPTER_VAR = 'dbt_custom_adapter' + def register_adapter(self, config: 'AdapterRequiredConfig') -> None: # ==== CUSTOM CODE ==== # ==== END CUSTOM CODE ==== diff --git a/opendbt/dbt/v17/config/runtime.py b/opendbt/dbt/v17/config/runtime.py index 6e7921c..f7af0f1 100644 --- a/opendbt/dbt/v17/config/runtime.py +++ b/opendbt/dbt/v17/config/runtime.py @@ -10,6 +10,8 @@ ) from typing_extensions import override +from opendbt.runtime_patcher import PatchClass + def load_yml_dict(file_path): ret = {} @@ -17,8 +19,10 @@ def load_yml_dict(file_path): ret = _load_yaml(file_path) or {} return ret + # pylint: disable=too-many-ancestors @dataclass +@PatchClass(module_name="dbt.config", target_name="RuntimeConfig") class OpenDbtRuntimeConfig(RuntimeConfig): def load_dependence_projects(self): dependencies_yml_dict = load_yml_dict(f"{self.project_root}/{DEPENDENCIES_FILE_NAME}") diff --git a/opendbt/dbt/v17/task/docs/generate.py b/opendbt/dbt/v17/task/docs/generate.py index 79f0020..b66b7be 100644 --- a/opendbt/dbt/v17/task/docs/generate.py +++ b/opendbt/dbt/v17/task/docs/generate.py @@ -4,7 +4,10 @@ import click from dbt.task.generate import GenerateTask +from opendbt.runtime_patcher import PatchClass + +@PatchClass(module_name="dbt.task.generate", target_name="GenerateTask") class OpenDbtGenerateTask(GenerateTask): def deploy_user_index_html(self): diff --git a/opendbt/dbt/v17/task/run.py b/opendbt/dbt/v17/task/run.py index b0b5cf1..61d94bc 100644 --- a/opendbt/dbt/v17/task/run.py +++ b/opendbt/dbt/v17/task/run.py @@ -6,7 +6,10 @@ ) from dbt.task import run +from opendbt.runtime_patcher import PatchClass + +@PatchClass(module_name="dbt.task.run", target_name="ModelRunner") class ModelRunner(run.ModelRunner): def print_result_adapter_response(self, result): diff --git a/opendbt/dbt/v18/adapters/factory.py b/opendbt/dbt/v18/adapters/factory.py index d6bada2..1d8a4df 100644 --- a/opendbt/dbt/v18/adapters/factory.py +++ b/opendbt/dbt/v18/adapters/factory.py @@ -11,9 +11,13 @@ from dbt_common.events.base_types import EventLevel from dbt_common.events.functions import fire_event +from opendbt.runtime_patcher import PatchClass + +@PatchClass(module_name="dbt.adapters.factory", target_name="AdapterContainer") class OpenDbtAdapterContainer(factory.AdapterContainer): DBT_CUSTOM_ADAPTER_VAR = 'dbt_custom_adapter' + def register_adapter( self, config: 'AdapterRequiredConfig', diff --git a/opendbt/dbt/v18/config/runtime.py b/opendbt/dbt/v18/config/runtime.py index 3b48cf8..a84be1b 100644 --- a/opendbt/dbt/v18/config/runtime.py +++ b/opendbt/dbt/v18/config/runtime.py @@ -5,14 +5,15 @@ from dbt.config import RuntimeConfig from dbt.config.project import load_yml_dict from dbt.constants import DEPENDENCIES_FILE_NAME -from dbt.exceptions import ( - DbtProjectError, NonUniquePackageNameError, -) +from dbt.exceptions import DbtProjectError, NonUniquePackageNameError from typing_extensions import override +from opendbt.runtime_patcher import PatchClass # pylint: disable=too-many-ancestors @dataclass +@PatchClass(module_name="dbt.config", target_name="RuntimeConfig") +@PatchClass(module_name="dbt.cli.requires", target_name="RuntimeConfig") class OpenDbtRuntimeConfig(RuntimeConfig): def load_dependence_projects(self): dependencies_yml_dict = load_yml_dict(f"{self.project_root}/{DEPENDENCIES_FILE_NAME}") diff --git a/opendbt/dbt/v18/task/docs/generate.py b/opendbt/dbt/v18/task/docs/generate.py index 0b02efa..fb3b7fb 100644 --- a/opendbt/dbt/v18/task/docs/generate.py +++ b/opendbt/dbt/v18/task/docs/generate.py @@ -4,7 +4,10 @@ import click from dbt.task.docs.generate import GenerateTask +from opendbt.runtime_patcher import PatchClass + +@PatchClass(module_name="dbt.task.docs.generate", target_name="GenerateTask") class OpenDbtGenerateTask(GenerateTask): def deploy_user_index_html(self): diff --git a/opendbt/dbt/v18/task/run.py b/opendbt/dbt/v18/task/run.py index 80755a9..3b19ab3 100644 --- a/opendbt/dbt/v18/task/run.py +++ b/opendbt/dbt/v18/task/run.py @@ -6,7 +6,10 @@ from dbt_common.events.base_types import EventLevel from dbt_common.events.functions import fire_event +from opendbt.runtime_patcher import PatchClass + +@PatchClass(module_name="dbt.task.run", target_name="ModelRunner") class ModelRunner(run.ModelRunner): def print_result_adapter_response(self, result): diff --git a/opendbt/runtime_patcher.py b/opendbt/runtime_patcher.py new file mode 100644 index 0000000..3543b2c --- /dev/null +++ b/opendbt/runtime_patcher.py @@ -0,0 +1,161 @@ +import importlib +from typing import Callable, Type, Any +from opendbt.logger import OpenDbtLogger + + +class RuntimePatcher(OpenDbtLogger): + """ + A utility class for patching modules and classes at runtime. + + This class provides a simplified way to replace existing functions, + classes, or attributes within modules with custom implementations. + """ + + def __init__(self, module_name: str): + """ + Initializes the RuntimePatcher for a specific module. + + Args: + module_name: The name of the module to patch (e.g., "dbt.config"). + """ + self.module_name = module_name + self.module = importlib.import_module(module_name) + + def patch_function(self, function_name: str, new_function: Callable): + """ + Patches a function within the module. + + Args: + function_name: The name of the function to patch. + new_function: The new function to use as a replacement. + """ + setattr(self.module, function_name, new_function) + self.log.debug(f"Patched function: {self.module_name}.{function_name}") + + def patch_class(self, class_name: str, new_class: Type): + """ + Patches a class within the module. + + Args: + class_name: The name of the class to patch. + new_class: The new class to use as a replacement. + """ + setattr(self.module, class_name, new_class) + self.log.debug(f"Patched class: {self.module_name}.{class_name}") + + def patch_attribute(self, attribute_name: str, new_value: Any): + """ + Patches an attribute within the module. + + Args: + attribute_name: The name of the attribute to patch. + new_value: The new value to assign to the attribute. + """ + setattr(self.module, attribute_name, new_value) + self.log.debug(f"Patched attribute: {self.module_name}.{attribute_name}") + + def patch_class_method(self, class_name: str, method_name: str, new_method: Callable): + """ + Patches a class method within the module. + + Args: + class_name: The name of the class containing the method. + method_name: The name of the method to patch. + new_method: The new method to use as a replacement. + """ + target_class = getattr(self.module, class_name) + setattr(target_class, method_name, new_method) + self.log.debug(f"Patched class method: {self.module_name}.{class_name}.{method_name}") + + +class _PatchDecorator: + """ + Base class for patch decorators + """ + + def __init__(self, module_name: str, target_name: str): + self.module_name = module_name + self.target_name = target_name + self.patcher = RuntimePatcher(self.module_name) + + +class PatchClass(_PatchDecorator): + """ + A decorator for patching classes at runtime. + """ + + def __call__(self, target: Type): + self.patcher.patch_class(self.target_name, target) + return target + + +class PatchFunction(_PatchDecorator): + """ + A decorator for patching functions at runtime. + """ + + def __call__(self, target: Callable): + self.patcher.patch_function(self.target_name, target) + return target + + +class PatchAttribute(_PatchDecorator): + """ + A decorator for patching attributes at runtime. + """ + + def __call__(self, target: Any): + # if it is callable, call it to get the value + if callable(target): + target = target() + self.patcher.patch_attribute(self.target_name, target) + return target + + +class PatchClassMethod(_PatchDecorator): + """ + A decorator for patching class methods at runtime. + """ + + def __init__(self, module_name: str, class_name: str, method_name: str): + super().__init__(module_name, class_name) + self.method_name = method_name + + def __call__(self, target: Callable): + self.patcher.patch_class_method(self.target_name, self.method_name, target) + return target + +# Example Usage: + +# Example to use PatchClass for override the ModelRunner class +# @PatchClass(module_name="dbt.task.run", target_name="ModelRunner") +# class CustomModelRunner: +# def __init__(self, *args, **kwargs): +# print("Custom ModelRunner initialized!") +# +# +# # Example to use PatchClass for override the RuntimeConfig class +# @PatchClass(module_name="dbt.config", target_name="RuntimeConfig") +# class CustomRuntimeConfig: +# def __init__(self, *args, **kwargs): +# print("Custom RuntimeConfig initialized!") +# +# # Example to use PatchAttribute for override the FACTORY attribute +# @PatchAttribute(module_name="dbt.adapters.factory", target_name="FACTORY") +# def get_custom_open_dbt_adapter_container(): +# class CustomOpenDbtAdapterContainer: +# def __init__(self, *args, **kwargs): +# print("Custom OpenDbtAdapterContainer initialized!") +# return CustomOpenDbtAdapterContainer +# +# +# # Example to use PatchFunction for override the sqlfluff_lint function +# @PatchFunction(module_name="dbt.cli.main", target_name="sqlfluff_lint") +# def custom_sqlfluff_lint(): +# print("Custom sqlfluff_lint called!") + +# Example to patch class method +# @PatchClassMethod(module_name="dbt.adapters.factory", class_name="AdapterContainer", method_name="get_adapter") +# def custom_get_adapter(self, *args, **kwargs): +# print("Custom get_adapter method called!") +# return "Custom Adapter" From e2715c5b58db1e5978d4e6a5778f12b9cc4d4823 Mon Sep 17 00:00:00 2001 From: ismail simsek <6005685+ismailsimsek@users.noreply.github.com> Date: Sun, 2 Mar 2025 21:20:56 +0100 Subject: [PATCH 2/9] Add workflow_dispatch to github test action --- tests/base_dbt_test.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/base_dbt_test.py b/tests/base_dbt_test.py index 534a04e..305b6ab 100644 --- a/tests/base_dbt_test.py +++ b/tests/base_dbt_test.py @@ -7,13 +7,18 @@ class BaseDbtTest(TestCase): - RESOURCES_DIR = Path(__file__).parent.joinpath("resources") + TESTS_ROOT = Path(__file__).parent + PROJECT_ROOT = TESTS_ROOT.parent + RESOURCES_DIR = TESTS_ROOT.joinpath("resources") DBTCORE_DIR = RESOURCES_DIR.joinpath("dbtcore") DBTFINANCE_DIR = RESOURCES_DIR.joinpath("dbtfinance") DBT_VERSION = get_dbt_version() @classmethod def setUpClass(cls): + BaseDbtTest.PROJECT_ROOT.joinpath("dev.duckdb").unlink(missing_ok=True) + BaseDbtTest.RESOURCES_DIR.joinpath("dev.duckdb").unlink(missing_ok=True) + dpf = OpenDbtCli(project_dir=BaseDbtTest.DBTFINANCE_DIR, profiles_dir=BaseDbtTest.DBTFINANCE_DIR) dpc = OpenDbtCli(project_dir=BaseDbtTest.DBTCORE_DIR, profiles_dir=BaseDbtTest.DBTCORE_DIR) dpf.invoke(args=["clean"]) From 8219cdc4e6a1270314cb3f574208a40d8b01d9e7 Mon Sep 17 00:00:00 2001 From: ismail simsek <6005685+ismailsimsek@users.noreply.github.com> Date: Sun, 2 Mar 2025 21:23:25 +0100 Subject: [PATCH 3/9] Add workflow_dispatch to github test action --- tests/base_dbt_test.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/base_dbt_test.py b/tests/base_dbt_test.py index 305b6ab..a66664e 100644 --- a/tests/base_dbt_test.py +++ b/tests/base_dbt_test.py @@ -23,3 +23,8 @@ def setUpClass(cls): dpc = OpenDbtCli(project_dir=BaseDbtTest.DBTCORE_DIR, profiles_dir=BaseDbtTest.DBTCORE_DIR) dpf.invoke(args=["clean"]) dpc.invoke(args=["clean"]) + + def setUp(self): + # Setup actions to be performed before each test + BaseDbtTest.PROJECT_ROOT.joinpath("dev.duckdb").unlink(missing_ok=True) + BaseDbtTest.RESOURCES_DIR.joinpath("dev.duckdb").unlink(missing_ok=True) \ No newline at end of file From 17b4b79a815eaddff7e583106e95f954ee240261 Mon Sep 17 00:00:00 2001 From: ismail simsek <6005685+ismailsimsek@users.noreply.github.com> Date: Sun, 2 Mar 2025 21:36:44 +0100 Subject: [PATCH 4/9] Add workflow_dispatch to github test action --- opendbt/examples.py | 19 ++++++++----------- tests/test_opendbt_cli.py | 4 ++-- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/opendbt/examples.py b/opendbt/examples.py index 4b5034e..539a2bf 100644 --- a/opendbt/examples.py +++ b/opendbt/examples.py @@ -97,14 +97,11 @@ def email_dbt_test_callback(event: "EventMsg"): --------------- full data --------------- dbt data: {event.data} """ - try: - # send email alert using airflow - from airflow.utils.email import send_email - send_email( - subject=email_subject, - to="my-slack-notification-channel@slack.com", - html_content=email_html_content - ) - except Exception as _: - # Used by unittest, expecting airflow error - logging.getLogger('dbtcallbacks').error("Airflow send_email failed! this is expected for unit testing!") + # @TODO send email alert using airflow + # from airflow.utils.email import send_email + # send_email( + # subject=email_subject, + # to="my-slack-notification-channel@slack.com", + # html_content=email_html_content + # ) + logging.getLogger('dbtcallbacks').error("Callback email sent!") diff --git a/tests/test_opendbt_cli.py b/tests/test_opendbt_cli.py index a031487..8e8caff 100644 --- a/tests/test_opendbt_cli.py +++ b/tests/test_opendbt_cli.py @@ -30,7 +30,7 @@ def test_cli_callbacks(self): dp.invoke(args=["test", '--select', 'my_first_dbt_model', "--profiles-dir", dp.project_dir.as_posix()]) self.assertIn('DBT callback `email_dbt_test_callback` called', str(cm.output)) - self.assertIn('Airflow send_email failed! this is expected for unit testing!', str(cm.output)) + self.assertIn('Callback email sent', str(cm.output)) # self.assertIn('dbt test', str(cm.output)) def test_cli_run_models(self): @@ -41,5 +41,5 @@ def test_cli_run_models(self): def test_cli_run_cross_project_ref_models(self): dpf = OpenDbtCli(project_dir=self.DBTFINANCE_DIR) dpc = OpenDbtCli(project_dir=self.DBTCORE_DIR) - dpf.invoke(args=['run', '--select', 'my_cross_project_ref_model', "--profiles-dir", dpf.project_dir.as_posix()]) dpc.invoke(args=['run', '--select', 'my_core_table1', "--profiles-dir", dpc.project_dir.as_posix()]) + dpf.invoke(args=['run', '--select', 'my_cross_project_ref_model', "--profiles-dir", dpf.project_dir.as_posix()]) From 9845c58cf2eb4770cfa34335191803c20640d5da Mon Sep 17 00:00:00 2001 From: ismail simsek <6005685+ismailsimsek@users.noreply.github.com> Date: Sun, 2 Mar 2025 21:38:44 +0100 Subject: [PATCH 5/9] Add workflow_dispatch to github test action --- tests/test_opendbt_cli.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_opendbt_cli.py b/tests/test_opendbt_cli.py index 8e8caff..50ce058 100644 --- a/tests/test_opendbt_cli.py +++ b/tests/test_opendbt_cli.py @@ -27,7 +27,10 @@ def test_cli_callbacks(self): self.assertIn(email_dbt_test_callback, dp.project_callbacks) with self.assertLogs('dbtcallbacks', level='INFO') as cm: - dp.invoke(args=["test", '--select', 'my_first_dbt_model', "--profiles-dir", dp.project_dir.as_posix()]) + try: + dp.invoke(args=["test", '--select', 'my_first_dbt_model', "--profiles-dir", dp.project_dir.as_posix()]) + except: + pass self.assertIn('DBT callback `email_dbt_test_callback` called', str(cm.output)) self.assertIn('Callback email sent', str(cm.output)) From ff47ad36e6ef475631dc2a662291000b9d5ae2a8 Mon Sep 17 00:00:00 2001 From: ismail simsek <6005685+ismailsimsek@users.noreply.github.com> Date: Sun, 2 Mar 2025 21:41:19 +0100 Subject: [PATCH 6/9] Add workflow_dispatch to github test action --- opendbt/__init__.py | 8 -------- opendbt/dbt/__init__.py | 9 +++++++++ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/opendbt/__init__.py b/opendbt/__init__.py index 494776d..67c8a5f 100644 --- a/opendbt/__init__.py +++ b/opendbt/__init__.py @@ -8,14 +8,6 @@ from opendbt.logger import OpenDbtLogger from opendbt.utils import Utils -from dbt.cli.main import dbtRunner as DbtCliRunner -from dbt.cli.main import dbtRunnerResult -from dbt.config import PartialProject -from dbt.contracts.graph.manifest import Manifest -from dbt.contracts.results import RunResult -from dbt.exceptions import DbtRuntimeError -from dbt.task.base import get_nearest_project_dir - class OpenDbtCli: def __init__(self, project_dir: Path, profiles_dir: Path = None, callbacks: list = None): diff --git a/opendbt/dbt/__init__.py b/opendbt/dbt/__init__.py index e39aa6f..411ad53 100644 --- a/opendbt/dbt/__init__.py +++ b/opendbt/dbt/__init__.py @@ -25,3 +25,12 @@ from opendbt.dbt.shared.cli.main import sqlfluff from opendbt.dbt.shared.cli.main import sqlfluff_lint from opendbt.dbt.shared.cli.main import sqlfluff_fix + +# dbt imports +from dbt.cli.main import dbtRunner as DbtCliRunner +from dbt.cli.main import dbtRunnerResult +from dbt.config import PartialProject +from dbt.contracts.graph.manifest import Manifest +from dbt.contracts.results import RunResult +from dbt.exceptions import DbtRuntimeError +from dbt.task.base import get_nearest_project_dir \ No newline at end of file From 7e8b73b1a92920740b78b4a771ce048df367eea6 Mon Sep 17 00:00:00 2001 From: ismail simsek <6005685+ismailsimsek@users.noreply.github.com> Date: Sun, 2 Mar 2025 21:44:14 +0100 Subject: [PATCH 7/9] Add workflow_dispatch to github test action --- opendbt/dbt/__init__.py | 57 +++++++++++++++++++++-------------------- 1 file changed, 29 insertions(+), 28 deletions(-) diff --git a/opendbt/dbt/__init__.py b/opendbt/dbt/__init__.py index 411ad53..c749ee2 100644 --- a/opendbt/dbt/__init__.py +++ b/opendbt/dbt/__init__.py @@ -4,33 +4,34 @@ import opendbt.dbt.shared.cli.main from opendbt.runtime_patcher import RuntimePatcher -dbt_version = Version(version.get_installed_version().to_version_string(skip_matcher=True)) -if Version("1.6.0") <= dbt_version < Version("1.8.0"): - from opendbt.dbt.v17.adapters.factory import OpenDbtAdapterContainer - from opendbt.dbt.v17.config.runtime import OpenDbtRuntimeConfig - from opendbt.dbt.v17.task.docs.generate import OpenDbtGenerateTask - from opendbt.dbt.v17.task.run import ModelRunner -elif Version("1.8.0") <= dbt_version < Version("1.10.0"): - from opendbt.dbt.v18.adapters.factory import OpenDbtAdapterContainer - from opendbt.dbt.v18.config.runtime import OpenDbtRuntimeConfig - from opendbt.dbt.v18.task.docs.generate import OpenDbtGenerateTask - from opendbt.dbt.v18.task.run import ModelRunner -else: - raise Exception( - f"Unsupported dbt version {dbt_version}, please make sure dbt version is supported/integrated by opendbt") +try: + dbt_version = Version(version.get_installed_version().to_version_string(skip_matcher=True)) + if Version("1.6.0") <= dbt_version < Version("1.8.0"): + from opendbt.dbt.v17.adapters.factory import OpenDbtAdapterContainer + from opendbt.dbt.v17.config.runtime import OpenDbtRuntimeConfig + from opendbt.dbt.v17.task.docs.generate import OpenDbtGenerateTask + from opendbt.dbt.v17.task.run import ModelRunner + elif Version("1.8.0") <= dbt_version < Version("1.10.0"): + from opendbt.dbt.v18.adapters.factory import OpenDbtAdapterContainer + from opendbt.dbt.v18.config.runtime import OpenDbtRuntimeConfig + from opendbt.dbt.v18.task.docs.generate import OpenDbtGenerateTask + from opendbt.dbt.v18.task.run import ModelRunner + else: + raise Exception( + f"Unsupported dbt version {dbt_version}, please make sure dbt version is supported/integrated by opendbt") -RuntimePatcher(module_name="dbt.adapters.factory").patch_attribute(attribute_name="FACTORY", - new_value=OpenDbtAdapterContainer()) -# shared code patches -from opendbt.dbt.shared.cli.main import sqlfluff -from opendbt.dbt.shared.cli.main import sqlfluff_lint -from opendbt.dbt.shared.cli.main import sqlfluff_fix + RuntimePatcher(module_name="dbt.adapters.factory").patch_attribute(attribute_name="FACTORY", + new_value=OpenDbtAdapterContainer()) + # shared code patches + from opendbt.dbt.shared.cli.main import sqlfluff + from opendbt.dbt.shared.cli.main import sqlfluff_lint + from opendbt.dbt.shared.cli.main import sqlfluff_fix -# dbt imports -from dbt.cli.main import dbtRunner as DbtCliRunner -from dbt.cli.main import dbtRunnerResult -from dbt.config import PartialProject -from dbt.contracts.graph.manifest import Manifest -from dbt.contracts.results import RunResult -from dbt.exceptions import DbtRuntimeError -from dbt.task.base import get_nearest_project_dir \ No newline at end of file + # dbt imports + from dbt.cli.main import dbtRunner as DbtCliRunner + from dbt.cli.main import dbtRunnerResult + from dbt.config import PartialProject + from dbt.contracts.graph.manifest import Manifest + from dbt.contracts.results import RunResult + from dbt.exceptions import DbtRuntimeError + from dbt.task.base import get_nearest_project_dir \ No newline at end of file From 6c23ddc89d0aeaf4c75591d43e90ba496f01d294 Mon Sep 17 00:00:00 2001 From: ismail simsek <6005685+ismailsimsek@users.noreply.github.com> Date: Sun, 2 Mar 2025 21:48:25 +0100 Subject: [PATCH 8/9] Add workflow_dispatch to github test action --- opendbt/dbt/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/opendbt/dbt/__init__.py b/opendbt/dbt/__init__.py index c749ee2..fb9286b 100644 --- a/opendbt/dbt/__init__.py +++ b/opendbt/dbt/__init__.py @@ -34,4 +34,6 @@ from dbt.contracts.graph.manifest import Manifest from dbt.contracts.results import RunResult from dbt.exceptions import DbtRuntimeError - from dbt.task.base import get_nearest_project_dir \ No newline at end of file + from dbt.task.base import get_nearest_project_dir +except: + raise \ No newline at end of file From 3498f54ef356667fc58ff285a1a3a4a542003aa1 Mon Sep 17 00:00:00 2001 From: ismail simsek <6005685+ismailsimsek@users.noreply.github.com> Date: Sun, 2 Mar 2025 21:58:46 +0100 Subject: [PATCH 9/9] Add workflow_dispatch to github test action --- opendbt/dbt/v17/config/runtime.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/opendbt/dbt/v17/config/runtime.py b/opendbt/dbt/v17/config/runtime.py index f7af0f1..985848b 100644 --- a/opendbt/dbt/v17/config/runtime.py +++ b/opendbt/dbt/v17/config/runtime.py @@ -5,24 +5,21 @@ from dbt.config import RuntimeConfig from dbt.config.project import path_exists, _load_yaml from dbt.constants import DEPENDENCIES_FILE_NAME -from dbt.exceptions import ( - DbtProjectError, NonUniquePackageNameError, -) +from dbt.exceptions import DbtProjectError, NonUniquePackageNameError from typing_extensions import override from opendbt.runtime_patcher import PatchClass - def load_yml_dict(file_path): ret = {} if path_exists(file_path): ret = _load_yaml(file_path) or {} return ret - # pylint: disable=too-many-ancestors @dataclass @PatchClass(module_name="dbt.config", target_name="RuntimeConfig") +@PatchClass(module_name="dbt.cli.requires", target_name="RuntimeConfig") class OpenDbtRuntimeConfig(RuntimeConfig): def load_dependence_projects(self): dependencies_yml_dict = load_yml_dict(f"{self.project_root}/{DEPENDENCIES_FILE_NAME}")