From d547e6389052d902b661dc33f592f511dbd59f79 Mon Sep 17 00:00:00 2001 From: Eric Carmichael Date: Sat, 16 Jul 2022 12:56:22 -0700 Subject: [PATCH 1/9] add slack logger --- README.md | 53 +++++++++++++++++++++++++++ ckc/logging.py | 98 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 151 insertions(+) create mode 100644 ckc/logging.py diff --git a/README.md b/README.md index dc59b93..723f004 100644 --- a/README.md +++ b/README.md @@ -107,6 +107,59 @@ class WhateverTest(TestCase): ``` +#### Slack logging + +Get a Slack webhook URL and set `SLACK_WEBHOOK_URL` env var. You can also set `DJANGO_SLACK_LOG_LEVEL` +with info, warning, etc. + +Modify your Celery settings: +```py +# Let our slack logger handle celery stuff +CELERY_WORKER_HIJACK_ROOT_LOGGER = False +``` + +Example `LOGGING` configuration that turns on Slack logging if `SLACK_WEBHOOK_URL` env var is found: +```py +LOGGING = { + 'version': 1, + 'disable_existing_loggers': False, + 'formatters': { + 'colored': { + '()': 'colorlog.ColoredFormatter', + 'format': "%(log_color)s%(levelname)-8s%(reset)s %(white)s%(message)s", + } + }, + 'handlers': { + 'console': { + 'class': 'logging.StreamHandler', + 'formatter': 'colored', + }, + }, + 'loggers': { + '': { + 'handlers': ['console'], + 'level': os.getenv('DJANGO_LOG_LEVEL', 'INFO'), + }, + 'django': { + 'handlers': ['console'], + 'level': os.getenv('DJANGO_LOG_LEVEL', 'INFO'), + 'propagate': False, + } + }, +} + +SLACK_WEBHOOK_URL = os.getenv('SLACK_WEBHOOK_URL', '') +if SLACK_WEBHOOK_URL: + LOGGING['handlers']['slack'] = { + 'class': 'ckc.logging.CkcSlackHandler', + 'level': os.getenv('DJANGO_SLACK_LOG_LEVEL', 'ERROR'), + } + + LOGGING['loggers']['django']['handlers'] = ['console', 'slack'] + LOGGING['loggers']['']['handlers'] = ['console', 'slack'] +``` + + #### `./manage.py` commands | command | description| diff --git a/ckc/logging.py b/ckc/logging.py new file mode 100644 index 0000000..e50987f --- /dev/null +++ b/ckc/logging.py @@ -0,0 +1,98 @@ +""" +Most of this Slack logging is pulled from here: + https://github.com/junhwi/python-slack-logger/ +""" +import os +import logging +import json + +from logging import LogRecord +from urllib.parse import urlparse +from logging.handlers import HTTPHandler + + +class SlackHandler(HTTPHandler): + def __init__(self, url, username=None, icon_url=None, icon_emoji=None, channel=None, mention=None): + o = urlparse(url) + is_secure = o.scheme == 'https' + HTTPHandler.__init__(self, o.netloc, o.path, method="POST", secure=is_secure) + self.username = username + self.icon_url = icon_url + self.icon_emoji = icon_emoji + self.channel = channel + self.mention = mention and mention.lstrip('@') + + def mapLogRecord(self, record): + text = self.format(record) + + if isinstance(self.formatter, SlackFormatter): + payload = { + 'attachments': [ + text, + ], + } + if self.mention: + payload['text'] = '<@{0}>'.format(self.mention) + else: + if self.mention: + text = '<@{0}> {1}'.format(self.mention, text) + payload = { + 'text': text, + } + + if self.username: + payload['username'] = self.username + if self.icon_url: + payload['icon_url'] = self.icon_url + if self.icon_emoji: + payload['icon_emoji'] = self.icon_emoji + if self.channel: + payload['channel'] = self.channel + + ret = { + 'payload': json.dumps(payload), + } + return ret + + +class SlackFormatter(logging.Formatter): + def format(self, record): + ret = {} + if record.levelname == 'INFO': + ret['color'] = 'good' + elif record.levelname == 'WARNING': + ret['color'] = 'warning' + elif record.levelname == 'ERROR': + ret['color'] = '#E91E63' + elif record.levelname == 'CRITICAL': + ret['color'] = 'danger' + + ret['author_name'] = record.levelname + ret['title'] = record.name + ret['ts'] = record.created + ret['text'] = super(SlackFormatter, self).format(record) + return ret + + +class SlackLogFilter(logging.Filter): + """ + Logging filter to decide when logging to Slack is requested, using + the `extra` kwargs: + `logger.info("...", extra={'notify_slack': True})` + """ + + def filter(self, record): + return getattr(record, 'notify_slack', False) + + +class CkcSlackHandler(SlackHandler): + """ + Override the default handler to insert our own URL + """ + def __init__(self, **kwargs): + url = os.getenv('SLACK_WEBHOOK_URL') + super().__init__(url, **kwargs) + + def format(self, record: LogRecord) -> str: + """Surround our log message in a "code block" for styling.""" + return f"```{super().format(record)}```" From f421f0b8be1eb412f335465cfd2ddb9087be7447 Mon Sep 17 00:00:00 2001 From: Eric Carmichael Date: Sat, 16 Jul 2022 13:02:29 -0700 Subject: [PATCH 2/9] bump version to 0.0.8 --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 095e3f8..4f388b6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -39,7 +39,7 @@ name = django-ckc author = Eric Carmichael author_email = eric@ckcollab.com description = tools, utilities, etc. we use across projects @ ckc -version = 0.0.7 +version = 0.0.8 url = https://github.com/ckcollab/django-ckc keywords = django From bb5c69b9c788792fb5a701ca159f5945de4639fc Mon Sep 17 00:00:00 2001 From: Eric Carmichael Date: Sat, 16 Jul 2022 13:03:34 -0700 Subject: [PATCH 3/9] check against more modern versions of python --- .github/workflows/pythonpackage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 21bb94f..73f8434 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.6, 3.7, 3.8] + python-version: [3.8, 3.9, 3.10] django-version: ['<3', '>=3'] steps: From 259ea57a49cb8d760b9447f8b2eec056f792366e Mon Sep 17 00:00:00 2001 From: Eric Carmichael Date: Sat, 16 Jul 2022 13:04:44 -0700 Subject: [PATCH 4/9] also change django versions to test against --- .github/workflows/pythonpackage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index 73f8434..a9425a7 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -11,7 +11,7 @@ jobs: strategy: matrix: python-version: [3.8, 3.9, 3.10] - django-version: ['<3', '>=3'] + django-version: ['<4', '>=4'] steps: - uses: actions/checkout@v2 From 1745773a4a417c5af587452319fad82e3abde8b9 Mon Sep 17 00:00:00 2001 From: Eric Carmichael Date: Sat, 16 Jul 2022 13:14:55 -0700 Subject: [PATCH 5/9] tests passing locally --- Dockerfile | 10 ++++++++-- ckc/fields.py | 2 ++ requirements.txt | 4 ++-- setup.cfg | 3 +-- 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index a92ab5a..f26a70f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9.2 +FROM python:3.10.5 # Spatial packages and such RUN apt-get update && apt-get install -y libgdal-dev libsqlite3-mod-spatialite @@ -8,8 +8,14 @@ ADD requirements.txt . RUN pip install -r requirements.txt # add all of our source + config +#ADD ckc/ /src/ckc/ +#ADD testproject/ /src/testproject +#ADD tests/ /src/tests +#ADD setup.cfg /src +#WORKDIR /src + +ADD testproject/ /src/ ADD ckc/ /src/ckc/ -ADD testproject/ /src/testproject ADD tests/ /src/tests ADD setup.cfg /src WORKDIR /src diff --git a/ckc/fields.py b/ckc/fields.py index 87b7766..84fd0b5 100644 --- a/ckc/fields.py +++ b/ckc/fields.py @@ -1,3 +1,5 @@ +from collections import OrderedDict + from rest_framework import serializers diff --git a/requirements.txt b/requirements.txt index 68f4717..4b210f4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,7 +11,7 @@ djangorestframework==3.12.4 factory-boy==3.2.0 # tests -pytest==5.4.1 -pytest-django==3.9.0 +pytest<7 +pytest-django==4.5.2 pytest-pythonpath==0.7.3 flake8==3.7.9 diff --git a/setup.cfg b/setup.cfg index 4f388b6..5cada64 100644 --- a/setup.cfg +++ b/setup.cfg @@ -26,8 +26,7 @@ per-file-ignores = [tool:pytest] addopts = --reuse-db -DJANGO_SETTINGS_MODULE = testproject.settings -python_paths = testproject +DJANGO_SETTINGS_MODULE = settings python_files = tests/integration/*.py tests/functional/*.py From 63f914935e8f8d1514f9b606eb8c2aeafb4ba4e8 Mon Sep 17 00:00:00 2001 From: Eric Carmichael Date: Sat, 16 Jul 2022 13:16:29 -0700 Subject: [PATCH 6/9] bump versions --- requirements.txt | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/requirements.txt b/requirements.txt index 4b210f4..534e081 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,17 +1,16 @@ # These requirements are for local development and testing of the module # python packaging -twine==3.1.1 +twine==4.0.1 # django stuff -Django==3.1.12 -djangorestframework==3.12.4 +Django==4.0.6 +djangorestframework==3.13.1 # factories -factory-boy==3.2.0 +factory-boy==3.2.1 # tests -pytest<7 +pytest==7.1.2 pytest-django==4.5.2 -pytest-pythonpath==0.7.3 -flake8==3.7.9 +flake8==4.0.1 From b5322778f79f9cbf43a85b17d591e49d8681c71d Mon Sep 17 00:00:00 2001 From: Eric Carmichael Date: Sat, 16 Jul 2022 13:18:34 -0700 Subject: [PATCH 7/9] fix test paths --- Dockerfile | 8 +------- setup.cfg | 5 ++--- testproject/settings.py | 2 ++ 3 files changed, 5 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index f26a70f..edbbbeb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,14 +8,8 @@ ADD requirements.txt . RUN pip install -r requirements.txt # add all of our source + config -#ADD ckc/ /src/ckc/ -#ADD testproject/ /src/testproject -#ADD tests/ /src/tests -#ADD setup.cfg /src -#WORKDIR /src - -ADD testproject/ /src/ ADD ckc/ /src/ckc/ +ADD testproject/ /src/testproject ADD tests/ /src/tests ADD setup.cfg /src WORKDIR /src diff --git a/setup.cfg b/setup.cfg index 5cada64..1c98221 100644 --- a/setup.cfg +++ b/setup.cfg @@ -26,12 +26,11 @@ per-file-ignores = [tool:pytest] addopts = --reuse-db -DJANGO_SETTINGS_MODULE = settings +DJANGO_SETTINGS_MODULE = testproject.settings +pythonpath = . testproject python_files = tests/integration/*.py tests/functional/*.py -test_paths = - tests/ [metadata] name = django-ckc diff --git a/testproject/settings.py b/testproject/settings.py index fbcf852..ce23d56 100644 --- a/testproject/settings.py +++ b/testproject/settings.py @@ -3,6 +3,8 @@ DEBUG = True +USE_TZ = True + BASE_DIR = os.path.dirname(__file__) # NOTE: We're using Geospatial sqlite jazz From 63f7c73a95c178f06532cf31a72a4e1280c41d92 Mon Sep 17 00:00:00 2001 From: Eric Carmichael Date: Sat, 16 Jul 2022 13:19:42 -0700 Subject: [PATCH 8/9] fix py3.1 version in github tests ?? 3.10.5 to be explicit --- .github/workflows/pythonpackage.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml index a9425a7..7341188 100644 --- a/.github/workflows/pythonpackage.yml +++ b/.github/workflows/pythonpackage.yml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.8, 3.9, 3.10] + python-version: [3.8, 3.9, 3.10.5] django-version: ['<4', '>=4'] steps: From ac2879c0f143d5565434e3705fc91fe32e78b737 Mon Sep 17 00:00:00 2001 From: Eric Carmichael Date: Sat, 16 Jul 2022 13:24:05 -0700 Subject: [PATCH 9/9] add pytz since it was removed in django 4 --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 534e081..e957c80 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,6 +6,7 @@ twine==4.0.1 # django stuff Django==4.0.6 djangorestframework==3.13.1 +pytz==2022.1 # factories factory-boy==3.2.1