diff --git a/.github/workflows/testrunner.yaml b/.github/workflows/testrunner.yaml new file mode 100644 index 0000000..63eac8c --- /dev/null +++ b/.github/workflows/testrunner.yaml @@ -0,0 +1,25 @@ +name: Testrunner + +on: + pull_request: + +jobs: + lint: + uses: zeitonline/gh-action-workflows/.github/workflows/lefthook.yaml@a7aaf1e4ad49e558e917f8d283ee751c38f30c16 # 4.5.0 + with: + python-version: "3.14" + setup-command: bin/build lefthook + baseproject-name: pypi-oss + baseproject-environment: production + + test: + needs: + - lint + runs-on: zon-ubuntu-general-dind + permissions: + contents: read + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7 + - run: bin/test diff --git a/.lefthook.yaml b/.lefthook.yaml new file mode 100644 index 0000000..2649a10 --- /dev/null +++ b/.lefthook.yaml @@ -0,0 +1,11 @@ +pre-commit: + jobs: + - name: ruff + glob: "*.py" + group: + piped: true + jobs: + - name: format + run: .venv/bin/ruff format --force-exclude --exit-non-zero-on-format {staged_files} + - name: check + run: .venv/bin/ruff check --force-exclude --fix --exit-non-zero-on-fix {staged_files} diff --git a/bin/build b/bin/build new file mode 100755 index 0000000..6f60634 --- /dev/null +++ b/bin/build @@ -0,0 +1,31 @@ +#!/bin/bash + +set -e +if [[ -n "$CI" ]]; then set -x; fi +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + + +function main() { + cd "$DIR/.." + + local command=$1 + case $command in + lefthook) + build_lefthook + ;; + *) + build_lefthook + ;; + esac +} + +function build_lefthook() { + uv sync --frozen --only-group lint + + if [[ -z "$CI" && ! -f .git/hooks/pre-commit ]]; then + lefthook install + fi +} + + +main "$@" diff --git a/pyproject.toml b/pyproject.toml index 1f33ef2..76ea375 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,7 @@ prometheus = "celery_redis_prometheus.exporter:main" [dependency-groups] dev = ["pytest"] +lint = ["lefthook", "ruff"] release = ["towncrier"] @@ -51,3 +52,18 @@ underlines = ["", ""] directory = "change" name = "" showcontent = true + + +[tool.ruff] +line-length = 100 + +[tool.ruff.lint] +select = ["E", "F", "I"] +fixable = ["ALL"] + +[tool.ruff.format] +quote-style = "single" + +[tool.ruff.lint.isort] +from-first = true +lines-after-imports = 2 diff --git a/src/celery_redis_prometheus/exporter.py b/src/celery_redis_prometheus/exporter.py index 3532728..3120aa3 100644 --- a/src/celery_redis_prometheus/exporter.py +++ b/src/celery_redis_prometheus/exporter.py @@ -1,14 +1,15 @@ from functools import wraps -import celery.bin.base -import click +import _thread import collections import json import logging -import prometheus_client -import _thread import threading import time +import celery.bin.base +import click +import prometheus_client + log = logging.getLogger(__name__) @@ -21,14 +22,14 @@ STATS = { - 'tasks': prometheus_client.Counter( - 'celery_tasks_total', 'Number of tasks', ['queue', 'state']), + 'tasks': prometheus_client.Counter('celery_tasks_total', 'Number of tasks', ['queue', 'state']), 'queuetime': prometheus_client.Histogram( - 'celery_task_queuetime_seconds', 'Task queue wait time', ['queue']), + 'celery_task_queuetime_seconds', 'Task queue wait time', ['queue'] + ), 'runtime': prometheus_client.Histogram( - 'celery_task_runtime_seconds', 'Task runtime', ['queue']), - 'queues': prometheus_client.Gauge( - 'celery_queue_length', 'Queue length', ['queue']) + 'celery_task_runtime_seconds', 'Task runtime', ['queue'] + ), + 'queues': prometheus_client.Gauge('celery_queue_length', 'Queue length', ['queue']), } @@ -36,8 +37,8 @@ @click.option('--host', default='0.0.0.0', help='Listen host') @click.option('--port', default=9691, help='Listen port') @click.option( - '--queuelength-interval', default=0, - help='Check queue lengths every x seconds (0=disabled)') + '--queuelength-interval', default=0, help='Check queue lengths every x seconds (0=disabled)' +) @click.option('--verbose', is_flag=True, help='Enable debug logging') @click.pass_context def main(ctx, host, port, queuelength_interval, verbose): @@ -68,7 +69,10 @@ def main(ctx, host, port, queuelength_interval, verbose): except Exception as e: log.error( 'Failed to capture events: "%s", trying again in %s seconds.', - e, try_interval, exc_info=True) + e, + try_interval, + exc_info=True, + ) time.sleep(try_interval) @@ -78,11 +82,11 @@ def wrapper(self, event): self.state.event(event) task = self.state.tasks.get(event['uuid']) return fn(self, event, task) + return wrapper class CeleryEventReceiver: - def __init__(self, app): self.app = app @@ -91,8 +95,7 @@ def on_task_started(self, event, task): log.debug('Started %s', task) STATS['tasks'].labels(task.routing_key, 'started').inc() if task.sent: - STATS['queuetime'].labels(task.routing_key).observe( - time.time() - task.sent) + STATS['queuetime'].labels(task.routing_key).observe(time.time() - task.sent) @task_handler def on_task_succeeded(self, event, task): @@ -125,18 +128,20 @@ def __call__(self, *args, **kw): kw.setdefault('wakeup', False) with self.app.connection() as connection: - recv = self.app.events.Receiver(connection, handlers={ - 'task-started': self.on_task_started, - 'task-succeeded': self.on_task_succeeded, - 'task-failed': self.on_task_failed, - 'task-retried': self.on_task_retried, - '*': self.state.event, - }) + recv = self.app.events.Receiver( + connection, + handlers={ + 'task-started': self.on_task_started, + 'task-succeeded': self.on_task_succeeded, + 'task-failed': self.on_task_failed, + 'task-retried': self.on_task_retried, + '*': self.state.event, + }, + ) recv.capture(*args, **kw) class QueueLengthMonitor(threading.Thread): - def __init__(self, app, interval): super().__init__() self.app = app @@ -149,8 +154,7 @@ def run(self): lengths = collections.Counter() with self.app.connection() as connection: - pipe = connection.channel().client.pipeline( - transaction=False) + pipe = connection.channel().client.pipeline(transaction=False) for queue in self.app.conf['task_queues']: # Not claimed by any worker yet pipe.llen(queue.name) @@ -173,9 +177,7 @@ def run(self): time.sleep(self.interval) except Exception: - log.error( - 'Uncaught exception, preventing thread from crashing.', - exc_info=True) + log.error('Uncaught exception, preventing thread from crashing.', exc_info=True) def stop(self): self.running = False diff --git a/src/celery_redis_prometheus/tests/test_exporter.py b/src/celery_redis_prometheus/tests/test_exporter.py index 8e0bc0a..cb05d58 100644 --- a/src/celery_redis_prometheus/tests/test_exporter.py +++ b/src/celery_redis_prometheus/tests/test_exporter.py @@ -1,26 +1,26 @@ -from . import conftest -import celery_redis_prometheus.exporter -import pytest import threading +import pytest + +import celery_redis_prometheus.exporter + +from . import conftest + def test_collects_task_events(celery_worker): - receiver = celery_redis_prometheus.exporter.CeleryEventReceiver( - conftest.CELERY) + receiver = celery_redis_prometheus.exporter.CeleryEventReceiver(conftest.CELERY) # 3 = recived + started + succeeded thread = threading.Thread(target=lambda: receiver(limit=3)) thread.start() conftest.celery_ping.delay().get() thread.join() data = celery_redis_prometheus.exporter.STATS['tasks'].collect() - item = [x for x in data[0].samples - if x.labels.get('state', '') == 'succeeded'] + item = [x for x in data[0].samples if x.labels.get('state', '') == 'succeeded'] assert item[0].value == 1 def test_sets_separate_state_for_retry_failed(celery_worker): - receiver = celery_redis_prometheus.exporter.CeleryEventReceiver( - conftest.CELERY) + receiver = celery_redis_prometheus.exporter.CeleryEventReceiver(conftest.CELERY) # 6 = received + started + retry + received + started + failed thread = threading.Thread(target=lambda: receiver(limit=6)) thread.start() @@ -28,6 +28,5 @@ def test_sets_separate_state_for_retry_failed(celery_worker): conftest.provoke_retry.delay().get() thread.join() data = celery_redis_prometheus.exporter.STATS['tasks'].collect() - item = [x for x in data[0].samples - if x.labels.get('state', '') == 'retries-exceeded'] + item = [x for x in data[0].samples if x.labels.get('state', '') == 'retries-exceeded'] assert item[0].value == 1 diff --git a/uv.lock b/uv.lock index 39f4bb8..d96ac94 100644 --- a/uv.lock +++ b/uv.lock @@ -205,6 +205,10 @@ dev = [ { name = "pytest", version = "8.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, { name = "pytest", version = "9.1.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, ] +lint = [ + { name = "lefthook" }, + { name = "ruff" }, +] release = [ { name = "towncrier", version = "23.6.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.8'" }, { name = "towncrier", version = "24.8.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.8.*'" }, @@ -220,6 +224,10 @@ requires-dist = [ [package.metadata.requires-dev] dev = [{ name = "pytest" }] +lint = [ + { name = "lefthook" }, + { name = "ruff" }, +] release = [{ name = "towncrier" }] [[package]] @@ -557,6 +565,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fb/0f/834427d8c03ff1d7e867d3db3d176470c64871753252b21b4f4897d1fa45/kombu-5.6.2-py3-none-any.whl", hash = "sha256:efcfc559da324d41d61ca311b0c64965ea35b4c55cc04ee36e55386145dace93", size = 214219, upload-time = "2025-12-29T20:30:05.74Z" }, ] +[[package]] +name = "lefthook" +version = "2.1.17" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/30/ab/1d5364eb167e0bbdb7894d453fe8f25973087b3c680837d2cc5639fb71d5/lefthook-2.1.17-py3-none-macosx_10_15_x86_64.whl", hash = "sha256:53dc47b923127867010e68c324e09bbd455463954f7629c88102faa01dbdf61a", size = 5446224, upload-time = "2026-10-05T07:48:37.068Z" }, + { url = "https://files.pythonhosted.org/packages/a2/22/02c06debb039dbfd3f7e5cfb0b3f81efa49b6271a2cf5887e717ac34b165/lefthook-2.1.17-py3-none-macosx_11_0_arm64.whl", hash = "sha256:8faa20d78977bb3c5176d04c151c19060280a47448f339e19c0a61be32c9f143", size = 4965211, upload-time = "2026-10-05T07:48:38.642Z" }, + { url = "https://files.pythonhosted.org/packages/0c/5d/0ffb1d67b25c625797dfac31fe269a9bc14c8719b07772b8772b4a7735dc/lefthook-2.1.17-py3-none-manylinux_2_17_aarch64.whl", hash = "sha256:05e913e5bad044a50b105de6bfa077f082cd0c5e78a7de758840362f8cf4c698", size = 4794103, upload-time = "2026-10-05T07:48:40.223Z" }, + { url = "https://files.pythonhosted.org/packages/06/ae/fc4c0d2bbcd386ea4a2bc6c8296747786b54ebce004f1935fa137a61d606/lefthook-2.1.17-py3-none-manylinux_2_17_x86_64.whl", hash = "sha256:e5b4ad680d6d141abd7be5e1cf29dde285e2c5fc60d7170b6bcde9874dd1cfb4", size = 5376403, upload-time = "2026-10-05T07:48:41.773Z" }, + { url = "https://files.pythonhosted.org/packages/3f/ac/006681926a87602b43fe34698d3a04bd50f750388bafee7bc1711a72cd64/lefthook-2.1.17-py3-none-win_amd64.whl", hash = "sha256:fbf76ad507b6d77af021a818e897c9c8071a77566081def5e2242f68f55b8da8", size = 5526254, upload-time = "2026-10-05T07:48:43.229Z" }, + { url = "https://files.pythonhosted.org/packages/7d/af/8c4c93349d1ced1bb51f08fbb3d3d5714abbcc3dc20d322311cf9abdd471/lefthook-2.1.17-py3-none-win_arm64.whl", hash = "sha256:83035a1e902c76295c4e2b80ceb38a4a1928e1d888ed81d3e7b0dede1397a3e7", size = 4875677, upload-time = "2026-10-05T07:48:44.824Z" }, +] + [[package]] name = "markupsafe" version = "2.1.5" @@ -1064,6 +1085,31 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4f/ef/c66110d46fb800dda0bf33164182dfadabe26a90e4476844d502a23dca8e/pytz-2026.5-py2.py3-none-any.whl", hash = "sha256:e658af3757f9e26a9d25dd2aff38335acd92bc9104f890a894b2c1ba28311b03", size = 506342, upload-time = "2026-10-04T02:37:56.814Z" }, ] +[[package]] +name = "ruff" +version = "0.16.10" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c4/49/23802c45f093eb14bde54b141d2b2f058edfa63a06db7beed047308cc08f/ruff-0.16.10.tar.gz", hash = "sha256:eff4728c4eaae93f0955cd264d24b2ab348e74bf59986ccf282ba6dc16b3b017", size = 4958724, upload-time = "2026-10-01T18:03:21.697Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2f/21/ebce22e1d90cdb2cd691026b9c6e9bec6499f0b396089481755b5d49efff/ruff-0.16.10-py3-none-linux_armv6l.whl", hash = "sha256:488b0fe3f3574210e5cf80d9f59b9e3ab17a127a8155de3f307b392589cfb511", size = 10095557, upload-time = "2026-10-01T18:02:36.072Z" }, + { url = "https://files.pythonhosted.org/packages/cb/98/a54de85876a8b2612bfa0d84c7b9abfb39c6a3354aee7800b09c1649b9e3/ruff-0.16.10-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:e748ff95c934c4e978783b8e687bc174e7bd84e8ad24e3243e1ecfcda5e0282d", size = 10340577, upload-time = "2026-10-01T18:02:39.258Z" }, + { url = "https://files.pythonhosted.org/packages/9f/16/1a5a4a2657effe4806110f2b907313802f1367fcdbb29e8122766e407fab/ruff-0.16.10-py3-none-macosx_11_0_arm64.whl", hash = "sha256:3031a4a2e8e7b8a46f70be45f198c35a11ece509a94b80334d8d397a33c67550", size = 9774282, upload-time = "2026-10-01T18:02:41.79Z" }, + { url = "https://files.pythonhosted.org/packages/6e/fb/470085af734da396e68cd80fb0a3e7c109a459716ae59588c0f6fab8a17d/ruff-0.16.10-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:494401c86df4c4c25f69b9419605d944467ee98c42fb6ad405ef4fa40b8fb67d", size = 9920895, upload-time = "2026-10-01T18:02:44.458Z" }, + { url = "https://files.pythonhosted.org/packages/57/de/f10cffe4f88a37ea6615ec460f01bf76f0bc1477737a35d9ee61a630a78b/ruff-0.16.10-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d203abc0ff2b773ee33d00ab8df0bb67046fbc7c07b119332f08b9b345cf8221", size = 9902707, upload-time = "2026-10-01T18:02:47.041Z" }, + { url = "https://files.pythonhosted.org/packages/ff/44/3fdcedf83ae60ef837dd239170e480dce606a9142cfa2db911afdf855fd9/ruff-0.16.10-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bd83d1235a5258d318477bc5b576303974cbdef5df0c01a1bff14efcc23bd12a", size = 10619287, upload-time = "2026-10-01T18:02:49.485Z" }, + { url = "https://files.pythonhosted.org/packages/c1/62/02e76a5574002153618eafbb70e159468addc72a5d2c488e65cbb4639d2d/ruff-0.16.10-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bc2610fb269fa56dd8a68669ae470fa6272902668c0fc2ebc3aa112b2633d5b8", size = 11339942, upload-time = "2026-10-01T18:02:52.008Z" }, + { url = "https://files.pythonhosted.org/packages/6b/c4/cde27d47ad8d4126c587e608c67c46e7feac11763f606d261a1bc8a489e6/ruff-0.16.10-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3e70175e29cc94c26ea296c80e470180b744b7419026898e58f520c6ab32578e", size = 10934316, upload-time = "2026-10-01T18:02:54.811Z" }, + { url = "https://files.pythonhosted.org/packages/e4/03/17234145f302a645a123e8c3bb2411ecf4669fbc350de3b0d803230a1729/ruff-0.16.10-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f33f43a864a8483eebd160e713336c8bab02c934feaff0a33cf5ccb41546d09a", size = 10387968, upload-time = "2026-10-01T18:02:57.497Z" }, + { url = "https://files.pythonhosted.org/packages/71/29/2493af60240ee7644b38a4b821f5f9c3b5a4fa3178770fe1d0c685217395/ruff-0.16.10-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:1dfc6f0088149fb6a362c1c446bcbb3fd2157b3852fe2fa68409276eab9ad9b3", size = 10537906, upload-time = "2026-10-01T18:03:00.006Z" }, + { url = "https://files.pythonhosted.org/packages/2e/9a/f56b28f3b143bb9e518c34fb89ab191b626bca8b70dbf8af0d2e2d473572/ruff-0.16.10-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:6553498afc35f580f036030795810b9e6bcea31604b0fd9e8d352795473042e3", size = 10012938, upload-time = "2026-10-01T18:03:03.006Z" }, + { url = "https://files.pythonhosted.org/packages/c2/c4/fca37362848ea4d4d80712df13632e645e7c7cfb1bdedf140699ac7a090b/ruff-0.16.10-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:a3b8471dea115d37f123882be852bed13403746d3a76c11de4a19ec5f9ff5a03", size = 9897945, upload-time = "2026-10-01T18:03:05.818Z" }, + { url = "https://files.pythonhosted.org/packages/7d/c7/e0bc57664d6e0c61fd22f620af260fe9662d7e1ddb320ea7f160e76ef165/ruff-0.16.10-py3-none-musllinux_1_2_i686.whl", hash = "sha256:92e59a70bcbd9d3a5483656da906ec28edfdacfce00afd99edb8b4e9d15644be", size = 10333134, upload-time = "2026-10-01T18:03:08.25Z" }, + { url = "https://files.pythonhosted.org/packages/21/aa/5c9f3b68737c0e4a33a1db5dfab44f7b786d96ca91a7233112ddab1e6dc2/ruff-0.16.10-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:7ae7375f803b5520dc9f546bed7e3a0acb70b91812e9bb4b19927de22f25b77d", size = 10741702, upload-time = "2026-10-01T18:03:10.775Z" }, + { url = "https://files.pythonhosted.org/packages/78/fa/0f9c2020dc316c53d983be011720b8157cc052b97e3f4dfa7db6d0f880a6/ruff-0.16.10-py3-none-win32.whl", hash = "sha256:2a12e01cb9156c10c466f63b46eaae5ecea28dfbd21b5836353ae498e7d1349a", size = 10139176, upload-time = "2026-10-01T18:03:13.267Z" }, + { url = "https://files.pythonhosted.org/packages/99/29/cfb0df9448d4d4ad48c2de029ada9ebd71baa6da983a6c77ee6c6cd0fe82/ruff-0.16.10-py3-none-win_amd64.whl", hash = "sha256:97f2015c92aa97105b0eab19eb5d224884399281cfc5da86a92db4ab5e7fb2ca", size = 10584734, upload-time = "2026-10-01T18:03:16.006Z" }, + { url = "https://files.pythonhosted.org/packages/fc/05/c16957eb287c3fc062e032619a25868d93d408a844b725bcf514f7a378ff/ruff-0.16.10-py3-none-win_arm64.whl", hash = "sha256:25a65fe998c4e6861ec079ada5826a2fc605e6cbccbe9dcd7fac1f54e791621b", size = 10366440, upload-time = "2026-10-01T18:03:19.04Z" }, +] + [[package]] name = "six" version = "1.17.0"