diff --git a/validation/rules/python-rules.yaml b/validation/rules/python-rules.yaml index 4df7c02b..abc9843f 100644 --- a/validation/rules/python-rules.yaml +++ b/validation/rules/python-rules.yaml @@ -59,25 +59,26 @@ # P-006: check-test-files-exist # -# Severity is a function of API maturity and release type: -# - default (alpha, wip, any non-rc release): hint -# - initial (0.x) + rc/public release: warn -# - stable (>=1.x) + rc/public release: error +# Severity ramps on target_api_status (the canonical content-ramp field; see +# severity-assignment-guide.md ยง2), anchored on the public/stable obligation +# with overrides only demoting: +# - default (stable @ rc/public): error +# - draft/alpha (either maturity): hint +# - initial (0.x) @ rc/public: warn - id: P-006 engine: python engine_rule: check-test-files-exist short_title: "Missing test definition file for API" documentation_url: "https://github.com/camaraproject/tooling/blob/main/documentation/validation/faq.md#p-006-missing-test-files" conditional_level: - default: hint + default: error overrides: - condition: - target_api_maturity: [stable] - target_release_type: [pre-release-rc, public-release] - level: error + target_api_status: [draft, alpha] + level: hint - condition: target_api_maturity: [initial] - target_release_type: [pre-release-rc, public-release] + target_api_status: [rc, public] level: warn suggestion: >- Test files are optional for alpha releases but expected before the @@ -90,29 +91,37 @@ # Parses the Feature line of .feature files to extract the version # (e.g. "Feature: CAMARA API, vwip - Operation foo") and compares # against the expected version from info.version. +# +# Severity ramps on target_api_status, anchored on the public/stable +# obligation (warn); initial (0.x) APIs stay at hint throughout. - id: P-007 engine: python engine_rule: check-test-file-version short_title: "Test file: feature-line version mismatch" conditional_level: - default: hint + default: warn overrides: - condition: - target_api_maturity: [stable] - target_release_type: [pre-release-rc, public-release] - level: warn + target_api_status: [draft, alpha] + level: hint + - condition: + target_api_maturity: [initial] + level: hint # P-008: check-test-directory-exists +# +# Severity ramps on target_api_status, anchored on the public/stable +# obligation (warn); no maturity split. - id: P-008 engine: python engine_rule: check-test-directory-exists short_title: "Test_definitions directory is missing" conditional_level: - default: hint + default: warn overrides: - condition: - target_release_type: [pre-release-rc, public-release] - level: warn + target_api_status: [draft, alpha] + level: hint # P-009: check-release-plan-semantics - id: P-009 diff --git a/validation/tests/test_postfilter_levels.py b/validation/tests/test_postfilter_levels.py index 4bc4f3d5..49657427 100644 --- a/validation/tests/test_postfilter_levels.py +++ b/validation/tests/test_postfilter_levels.py @@ -268,3 +268,84 @@ def test_monotonic_non_decreasing(self, rule_id): for s in ("draft", "alpha", "rc", "public") ] assert levels == sorted(levels), f"{rule_id} severity decreases: {levels}" + + +# --------------------------------------------------------------------------- +# Regression: P-006/P-007/P-008 (test-file rules) ramp on target_api_status, +# not target_release_type, anchored on the public/stable obligation with +# overrides only demoting. See validation-rules/017. +# --------------------------------------------------------------------------- + + +class TestTestFileRulesStatusRamp: + """P-006 splits by maturity; P-007/P-008 grade the same for both.""" + + @pytest.mark.parametrize("status", ["draft", "alpha"]) + def test_p006_hint_pre_alpha_both_maturities(self, status): + rule = _rule_by_id("P-006") + ctx = _make_context() + for maturity in ("stable", "initial"): + api = _make_api(target_api_maturity=maturity, target_api_status=status) + assert resolve_level(rule, ctx, api) == "hint" + + @pytest.mark.parametrize("status", ["rc", "public"]) + def test_p006_stable_error_initial_warn(self, status): + rule = _rule_by_id("P-006") + ctx = _make_context() + stable = _make_api(target_api_maturity="stable", target_api_status=status) + initial = _make_api(target_api_maturity="initial", target_api_status=status) + assert resolve_level(rule, ctx, stable) == "error" + assert resolve_level(rule, ctx, initial) == "warn" + + def test_p006_monotonic_both_maturities(self): + rule = _rule_by_id("P-006") + ctx = _make_context() + for maturity in ("stable", "initial"): + levels = [ + _SEVERITY_ORDER[ + resolve_level( + rule, + ctx, + _make_api(target_api_maturity=maturity, target_api_status=s), + ) + ] + for s in ("draft", "alpha", "rc", "public") + ] + assert levels == sorted(levels), f"P-006 ({maturity}) decreases: {levels}" + + @pytest.mark.parametrize("status", ["draft", "alpha"]) + def test_p007_hint_pre_rc_stable(self, status): + rule = _rule_by_id("P-007") + ctx = _make_context() + api = _make_api(target_api_maturity="stable", target_api_status=status) + assert resolve_level(rule, ctx, api) == "hint" + + @pytest.mark.parametrize("status", ["rc", "public"]) + def test_p007_warn_at_rc_public_stable(self, status): + rule = _rule_by_id("P-007") + ctx = _make_context() + api = _make_api(target_api_maturity="stable", target_api_status=status) + assert resolve_level(rule, ctx, api) == "warn" + + @pytest.mark.parametrize("status", ["draft", "alpha", "rc", "public"]) + def test_p007_initial_stays_hint(self, status): + rule = _rule_by_id("P-007") + ctx = _make_context() + api = _make_api(target_api_maturity="initial", target_api_status=status) + assert resolve_level(rule, ctx, api) == "hint" + + @pytest.mark.parametrize("status", ["draft", "alpha"]) + def test_p008_hint_pre_rc(self, status): + rule = _rule_by_id("P-008") + ctx = _make_context() + for maturity in ("stable", "initial"): + api = _make_api(target_api_maturity=maturity, target_api_status=status) + assert resolve_level(rule, ctx, api) == "hint" + + @pytest.mark.parametrize("status", ["rc", "public"]) + def test_p008_warn_at_rc_public(self, status): + rule = _rule_by_id("P-008") + ctx = _make_context() + for maturity in ("stable", "initial"): + api = _make_api(target_api_maturity=maturity, target_api_status=status) + assert resolve_level(rule, ctx, api) == "warn"