From 9224c0e7a955394a98247671e3cd8c94c7dbbb62 Mon Sep 17 00:00:00 2001 From: Michael Grupp Date: Fri, 18 Aug 2023 01:54:10 +0200 Subject: [PATCH 1/7] Fix compatibility with cythonized tasks. (#3246) `types.GeneratorType` is not compatible with the generator type of Cython. This leads to `_run_get_new_deps()` returning directly when the task is from a cythonized module and [yields dynamic dependencies](https://luigi.readthedocs.io/en/stable/tasks.html?highlight=yield#dynamic-dependencies). --- luigi/worker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/luigi/worker.py b/luigi/worker.py index 18cd565b8e..01a2772a68 100644 --- a/luigi/worker.py +++ b/luigi/worker.py @@ -29,6 +29,7 @@ """ import collections +import collections.abc import datetime import getpass import importlib @@ -47,7 +48,6 @@ import threading import time import traceback -import types from luigi import notifications from luigi.event import Event @@ -137,7 +137,7 @@ def __init__(self, task, worker_id, result_queue, status_reporter, def _run_get_new_deps(self): task_gen = self.task.run() - if not isinstance(task_gen, types.GeneratorType): + if not isinstance(task_gen, collections.abc.Generator): return None next_send = None From 902dd4989742d77bb17a8ba62c5cc6b780d5280b Mon Sep 17 00:00:00 2001 From: Dillon Stadther Date: Thu, 17 Aug 2023 20:32:05 -0400 Subject: [PATCH 2/7] fix flake8 error in master --- test/worker_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/worker_test.py b/test/worker_test.py index 39d3e466f8..eaad63b17b 100644 --- a/test/worker_test.py +++ b/test/worker_test.py @@ -1359,8 +1359,8 @@ def complete(self): worker = Worker(scheduler) a = A() - with mock.patch.object(worker._scheduler, 'announce_scheduling_failure', side_effect=Exception('Unexpected')),\ - self.assertRaises(Exception): + with mock.patch.object(worker._scheduler, 'announce_scheduling_failure', + side_effect=Exception('Unexpected')), self.assertRaises(Exception): worker.add(a) self.assertTrue(len(emails) == 2) # One for `complete` error, one for exception in announcing. self.assertTrue('Luigi: Framework error while scheduling' in emails[1]) From 9c719b0b74b6243ca6c5776a93ca24f73b9a7c48 Mon Sep 17 00:00:00 2001 From: ewallace-RI <85639928+ewallace-RI@users.noreply.github.com> Date: Wed, 6 Sep 2023 17:53:48 -0700 Subject: [PATCH 3/7] Do not escape
tags in task tooltips, so that the line breaks are formatted correctly. (#3250) --- luigi/static/visualiser/js/graph.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/luigi/static/visualiser/js/graph.js b/luigi/static/visualiser/js/graph.js index e646e4e025..f996526bd7 100644 --- a/luigi/static/visualiser/js/graph.js +++ b/luigi/static/visualiser/js/graph.js @@ -277,14 +277,14 @@ Graph = (function() { .appendTo(g); var titleText = node.name; - var content = $.map(node.params, function (value, name) { return name + ": " + value; }).join("
"); + var content = $.map(node.params, function (value, name) { return escapeHtml(name + ": " + value); }).join("
"); g.attr("title", titleText) .popover({ trigger: 'hover', container: 'body', html: true, placement: 'top', - content: escapeHtml(content) + content: content }); }); From f78a067b952a97241d0ab6bace724e3bd6ce530b Mon Sep 17 00:00:00 2001 From: Adrian Stachlewski Date: Thu, 7 Sep 2023 03:03:05 +0200 Subject: [PATCH 4/7] Silence false positive UnconsumedParameterWarning (#3235) * Silence false positive UnconsumedParameterWarning Do not emit UnconsumedParameterWarning when parameter name is written in camelCase or with dashes. * Silence warnings for not consumed autoload_range and no_configure_logging parameters in core config. --------- Co-authored-by: Dillon Stadther --- luigi/configuration/cfg_parser.py | 1 + luigi/interface.py | 4 +++ luigi/task.py | 4 ++- test/task_test.py | 59 +++++++++++++++++++++++++++++++ 4 files changed, 67 insertions(+), 1 deletion(-) diff --git a/luigi/configuration/cfg_parser.py b/luigi/configuration/cfg_parser.py index 734aedcea3..abca6d713a 100644 --- a/luigi/configuration/cfg_parser.py +++ b/luigi/configuration/cfg_parser.py @@ -117,6 +117,7 @@ def before_write(self, parser, section, option, value): class LuigiConfigParser(BaseParser, ConfigParser): NO_DEFAULT = object() enabled = True + optionxform = str _instance = None _config_paths = [ '/etc/luigi/client.cfg', # Deprecated old-style global luigi config diff --git a/luigi/interface.py b/luigi/interface.py index dbfbac683b..867463c159 100644 --- a/luigi/interface.py +++ b/luigi/interface.py @@ -49,6 +49,10 @@ class core(task.Config): This is arguably a bit of a hack. ''' use_cmdline_section = False + ignore_unconsumed = { + 'autoload_range', + 'no_configure_logging', + } local_scheduler = parameter.BoolParameter( default=False, diff --git a/luigi/task.py b/luigi/task.py index 59c8558192..f5a108dc23 100644 --- a/luigi/task.py +++ b/luigi/task.py @@ -437,9 +437,11 @@ def list_to_tuple(x): if not hasattr(cls, "_unconsumed_params"): cls._unconsumed_params = set() if task_family in conf.sections(): + ignore_unconsumed = getattr(cls, 'ignore_unconsumed', set()) for key, value in conf[task_family].items(): + key = key.replace('-', '_') composite_key = f"{task_family}_{key}" - if key not in result and composite_key not in cls._unconsumed_params: + if key not in result and key not in ignore_unconsumed and composite_key not in cls._unconsumed_params: warnings.warn( "The configuration contains the parameter " f"'{key}' with value '{value}' that is not consumed by the task " diff --git a/test/task_test.py b/test/task_test.py index bb5578bfe2..e2da34796c 100644 --- a/test/task_test.py +++ b/test/task_test.py @@ -218,6 +218,65 @@ class TaskB(luigi.Task): f"the task '{task_name}'." ) + @with_config( + { + "TaskEdgeCase": { + "camelParam": "camelCase", + "underscore_param": "underscore", + "dash-param": "dash", + }, + } + ) + def test_unconsumed_params_edge_cases(self): + class TaskEdgeCase(luigi.Task): + camelParam = luigi.Parameter() + underscore_param = luigi.Parameter() + dash_param = luigi.Parameter() + + with warnings.catch_warnings(record=True) as w: + warnings.filterwarnings( + action="ignore", + category=Warning, + ) + warnings.simplefilter( + action="always", + category=luigi.parameter.UnconsumedParameterWarning, + ) + + task = TaskEdgeCase() + assert len(w) == 0 + assert task.camelParam == "camelCase" + assert task.underscore_param == "underscore" + assert task.dash_param == "dash" + + @with_config( + { + "TaskIgnoreUnconsumed": { + "a": "a", + "b": "b", + "c": "c", + }, + } + ) + def test_unconsumed_params_ignore_unconsumed(self): + class TaskIgnoreUnconsumed(luigi.Task): + ignore_unconsumed = {"b", "d"} + + a = luigi.Parameter() + + with warnings.catch_warnings(record=True) as w: + warnings.filterwarnings( + action="ignore", + category=Warning, + ) + warnings.simplefilter( + action="always", + category=luigi.parameter.UnconsumedParameterWarning, + ) + + TaskIgnoreUnconsumed() + assert len(w) == 1 + class TaskFlattenOutputTest(unittest.TestCase): def test_single_task(self): From ef9952a66ec0b8f5bd15f658fb9834dacfd7787f Mon Sep 17 00:00:00 2001 From: Gregory Borodin Date: Fri, 22 Sep 2023 23:07:11 +0200 Subject: [PATCH 5/7] Fix pymssql import (#3252) * `import pymssql` * Revert "`import pymssql`" This reverts commit 7881915b125286994c7bfaa4bd54b0909f57f7ca. * Fix exception name --- luigi/contrib/mssqldb.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/luigi/contrib/mssqldb.py b/luigi/contrib/mssqldb.py index 90e30bd2e2..57c0570673 100644 --- a/luigi/contrib/mssqldb.py +++ b/luigi/contrib/mssqldb.py @@ -22,7 +22,7 @@ logger = logging.getLogger('luigi-interface') try: - import _mssql + from pymssql import _mssql except ImportError: logger.warning("Loading MSSQL module without the python package pymssql. \ This will crash at runtime if SQL Server functionality is used.") @@ -107,7 +107,7 @@ def exists(self, connection=None): WHERE update_id = %s """.format(marker_table=self.marker_table), (self.update_id,)) - except _mssql.MSSQLDatabaseException as e: + except _mssql.MssqlDatabaseException as e: # Error number for table doesn't exist if e.number == 208: row = None @@ -145,7 +145,7 @@ def create_marker_table(self): """ .format(marker_table=self.marker_table) ) - except _mssql.MSSQLDatabaseException as e: + except _mssql.MssqlDatabaseException as e: # Table already exists code if e.number == 2714: pass From f74f1f13fc372804dcee91fd517e6b0990176d07 Mon Sep 17 00:00:00 2001 From: Jorge <46056498+jorgectf@users.noreply.github.com> Date: Mon, 25 Sep 2023 14:04:01 +0200 Subject: [PATCH 6/7] Add CodeQL workflow (#3244) Co-authored-by: sonjaer --- .github/workflows/codeql.yml | 65 ++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 .github/workflows/codeql.yml diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 0000000000..12be2b33aa --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,65 @@ +name: "CodeQL" + +on: + push: + branches: [ 'master' ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ 'master' ] + schedule: + - cron: '29 18 * * 0' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ 'python', 'javascript' ] + # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] + # Use only 'java' to analyze code written in Java, Kotlin or both + # Use only 'javascript' to analyze code written in JavaScript, TypeScript or both + # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + # Initializes the CodeQL tools for scanning. + - name: Initialize CodeQL + uses: github/codeql-action/init@v2 + with: + languages: ${{ matrix.language }} + # If you wish to specify custom queries, you can do so here or in a config file. + # By default, queries listed here will override any specified in a config file. + # Prefix the list here with "+" to use these queries and those in the config file. + + # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # queries: security-extended,security-and-quality + + + # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift). + # If this step fails, then you should remove it and run the build manually (see below) + - name: Autobuild + uses: github/codeql-action/autobuild@v2 + + # ℹī¸ Command-line programs to run using the OS shell. + # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun + + # If the Autobuild fails above, remove it and uncomment the following three lines. + # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. + + # - run: | + # echo "Run, Build Application using script" + # ./location_of_script_within_repo/buildscript.sh + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v2 + with: + category: "/language:${{matrix.language}}" From 319ce20df8c861abc01a88ad7523814ba6c1a286 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9s=20G=C3=B3mez?= Date: Thu, 5 Oct 2023 13:06:13 +0200 Subject: [PATCH 7/7] Version 3.4.0 (#3256) --- luigi/__meta__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/luigi/__meta__.py b/luigi/__meta__.py index 01166d9064..b3adee7a35 100644 --- a/luigi/__meta__.py +++ b/luigi/__meta__.py @@ -7,5 +7,5 @@ __author__ = 'The Luigi Authors' __contact__ = 'https://github.com/spotify/luigi' __license__ = 'Apache License 2.0' -__version__ = '3.3.0' +__version__ = '3.4.0' __status__ = 'Production'