Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 25 additions & 16 deletions validation/rules/python-rules.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -59,25 +59,26 @@

# P-006: check-test-files-exist
#
# Severity is a function of API maturity and release type:
# - default (alpha, wip, any non-rc release): hint
# - initial (0.x) + rc/public release: warn
# - stable (>=1.x) + rc/public release: error
# Severity ramps on target_api_status (the canonical content-ramp field; see
# severity-assignment-guide.md §2), anchored on the public/stable obligation
# with overrides only demoting:
# - default (stable @ rc/public): error
# - draft/alpha (either maturity): hint
# - initial (0.x) @ rc/public: warn
- id: P-006
engine: python
engine_rule: check-test-files-exist
short_title: "Missing test definition file for API"
documentation_url: "https://github.com/camaraproject/tooling/blob/main/documentation/validation/faq.md#p-006-missing-test-files"
conditional_level:
default: hint
default: error
overrides:
- condition:
target_api_maturity: [stable]
target_release_type: [pre-release-rc, public-release]
level: error
target_api_status: [draft, alpha]
level: hint
- condition:
target_api_maturity: [initial]
target_release_type: [pre-release-rc, public-release]
target_api_status: [rc, public]
level: warn
suggestion: >-
Test files are optional for alpha releases but expected before the
Expand All @@ -90,29 +91,37 @@
# Parses the Feature line of .feature files to extract the version
# (e.g. "Feature: CAMARA API, vwip - Operation foo") and compares
# against the expected version from info.version.
#
# Severity ramps on target_api_status, anchored on the public/stable
# obligation (warn); initial (0.x) APIs stay at hint throughout.
- id: P-007
engine: python
engine_rule: check-test-file-version
short_title: "Test file: feature-line version mismatch"
conditional_level:
default: hint
default: warn
overrides:
- condition:
target_api_maturity: [stable]
target_release_type: [pre-release-rc, public-release]
level: warn
target_api_status: [draft, alpha]
level: hint
- condition:
target_api_maturity: [initial]
level: hint

# P-008: check-test-directory-exists
#
# Severity ramps on target_api_status, anchored on the public/stable
# obligation (warn); no maturity split.
- id: P-008
engine: python
engine_rule: check-test-directory-exists
short_title: "Test_definitions directory is missing"
conditional_level:
default: hint
default: warn
overrides:
- condition:
target_release_type: [pre-release-rc, public-release]
level: warn
target_api_status: [draft, alpha]
level: hint

# P-009: check-release-plan-semantics
- id: P-009
Expand Down
81 changes: 81 additions & 0 deletions validation/tests/test_postfilter_levels.py
Original file line number Diff line number Diff line change
Expand Up @@ -268,3 +268,84 @@ def test_monotonic_non_decreasing(self, rule_id):
for s in ("draft", "alpha", "rc", "public")
]
assert levels == sorted(levels), f"{rule_id} severity decreases: {levels}"


# ---------------------------------------------------------------------------
# Regression: P-006/P-007/P-008 (test-file rules) ramp on target_api_status,
# not target_release_type, anchored on the public/stable obligation with
# overrides only demoting. See validation-rules/017.
# ---------------------------------------------------------------------------


class TestTestFileRulesStatusRamp:
"""P-006 splits by maturity; P-007/P-008 grade the same for both."""

@pytest.mark.parametrize("status", ["draft", "alpha"])
def test_p006_hint_pre_alpha_both_maturities(self, status):
rule = _rule_by_id("P-006")
ctx = _make_context()
for maturity in ("stable", "initial"):
api = _make_api(target_api_maturity=maturity, target_api_status=status)
assert resolve_level(rule, ctx, api) == "hint"

@pytest.mark.parametrize("status", ["rc", "public"])
def test_p006_stable_error_initial_warn(self, status):
rule = _rule_by_id("P-006")
ctx = _make_context()
stable = _make_api(target_api_maturity="stable", target_api_status=status)
initial = _make_api(target_api_maturity="initial", target_api_status=status)
assert resolve_level(rule, ctx, stable) == "error"
assert resolve_level(rule, ctx, initial) == "warn"

def test_p006_monotonic_both_maturities(self):
rule = _rule_by_id("P-006")
ctx = _make_context()
for maturity in ("stable", "initial"):
levels = [
_SEVERITY_ORDER[
resolve_level(
rule,
ctx,
_make_api(target_api_maturity=maturity, target_api_status=s),
)
]
for s in ("draft", "alpha", "rc", "public")
]
assert levels == sorted(levels), f"P-006 ({maturity}) decreases: {levels}"

@pytest.mark.parametrize("status", ["draft", "alpha"])
def test_p007_hint_pre_rc_stable(self, status):
rule = _rule_by_id("P-007")
ctx = _make_context()
api = _make_api(target_api_maturity="stable", target_api_status=status)
assert resolve_level(rule, ctx, api) == "hint"

@pytest.mark.parametrize("status", ["rc", "public"])
def test_p007_warn_at_rc_public_stable(self, status):
rule = _rule_by_id("P-007")
ctx = _make_context()
api = _make_api(target_api_maturity="stable", target_api_status=status)
assert resolve_level(rule, ctx, api) == "warn"

@pytest.mark.parametrize("status", ["draft", "alpha", "rc", "public"])
def test_p007_initial_stays_hint(self, status):
rule = _rule_by_id("P-007")
ctx = _make_context()
api = _make_api(target_api_maturity="initial", target_api_status=status)
assert resolve_level(rule, ctx, api) == "hint"

@pytest.mark.parametrize("status", ["draft", "alpha"])
def test_p008_hint_pre_rc(self, status):
rule = _rule_by_id("P-008")
ctx = _make_context()
for maturity in ("stable", "initial"):
api = _make_api(target_api_maturity=maturity, target_api_status=status)
assert resolve_level(rule, ctx, api) == "hint"

@pytest.mark.parametrize("status", ["rc", "public"])
def test_p008_warn_at_rc_public(self, status):
rule = _rule_by_id("P-008")
ctx = _make_context()
for maturity in ("stable", "initial"):
api = _make_api(target_api_maturity=maturity, target_api_status=status)
assert resolve_level(rule, ctx, api) == "warn"