Make CLI test failures verbose by not catching exceptions in the test runner

flexmeasures/cli/tests/test_utils.py

@@ -145,3 +145,25 @@ def cmd(name):
     assert result.exit_code == 0
     assert "Option '--old-name' will be replaced by '--name'." in result.output
     assert result.output.endswith("foo\n")
+
+
+@pytest.mark.xfail(
+    strict=True,
+    raises=RuntimeError,
+    reason="CustomFlaskCliRunner lets exceptions propagate instead of catching them",
+)
+def test_custom_cli_runner_raises_exceptions(app):
+    """Verify that the custom CLI runner does not catch exceptions.
+
+    This test is expected to fail: CustomFlaskCliRunner propagates exceptions
+    raised inside a CLI command instead of swallowing them (as the default runner does).
+    If this test unexpectedly passes, it means exceptions are being caught again,
+    which would make failing CLI tests much harder to debug.
+    """
+
+    @click.command()
+    def failing_command():
+        raise RuntimeError("This exception should propagate out of the CLI runner")
+
+    runner = app.test_cli_runner()
+    runner.invoke(failing_command)

flexmeasures/conftest.py

@@ -12,10 +12,10 @@
 import pandas as pd
 import numpy as np
 from flask import request, jsonify, Flask
+from flask.testing import FlaskCliRunner
 from flask_sqlalchemy import SQLAlchemy
 from flask_security import roles_accepted
 from timely_beliefs.sensors.func_store.knowledge_horizons import x_days_ago_at_y_oclock
-
 from werkzeug.exceptions import (
     InternalServerError,
     BadRequest,
@@ -64,10 +64,23 @@
 """
 
 
+class CustomFlaskCliRunner(FlaskCliRunner):
+    """A CLI test runner that lets exceptions propagate instead of catching them.
+
+    This makes test failures more informative: instead of seeing only a non-zero
+    exit code, the full exception with traceback is shown directly.
+    """
+
+    def invoke(self, *args, **kwargs):
+        kwargs.setdefault("catch_exceptions", False)
+        return super().invoke(*args, **kwargs)
+
+
 @pytest.fixture(scope="session")
 def app():
     print("APP FIXTURE")
     test_app = create_app(env="testing")
+    test_app.test_cli_runner_class = CustomFlaskCliRunner
 
     with test_app.app_context():
         yield test_app