diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml new file mode 100644 index 00000000000..8801916b92a --- /dev/null +++ b/.github/workflows/tests.yaml @@ -0,0 +1,31 @@ +name: Tests +on: + pull_request: + branches: + - main +permissions: + contents: read + +jobs: + tests: + concurrency: + group: tests-${{ github.event_name == 'push' && github.run_number || github.ref }} + cancel-in-progress: true + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Setup Python + uses: actions/setup-python@v4 + with: + python-version: "3.11" + + - name: Install dependencies + run: pip install .[test] + + - name: Show versions + run: pip freeze + + - name: Run pytest + run: pytest tests/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 970404b81f9..6b007dcdf20 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -41,19 +41,8 @@ repos: types_or: [python, rst, markdown] additional_dependencies: [tomli] - - repo: local - hooks: - - id: mypy - # note: assumes python env is setup and activated - name: mypy - entry: mypy - language: system - pass_filenames: false - types: [python] - stages: [manual] - - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.11.1 + rev: v2.0.0 hooks: - id: mypy require_serial: true diff --git a/ci/make_issues.py b/ci/make_issues.py index 33c13525165..ce983a90fdd 100644 --- a/ci/make_issues.py +++ b/ci/make_issues.py @@ -53,9 +53,9 @@ def time_to_str(x: float) -> str: is_negative = x < 0.0 if x >= 1.0: result = f"{x:0.3f}s" - elif x >= 0.001: # noqa: PLR2004 + elif x >= 0.001: result = f"{x * 1000:0.3f}ms" - elif x >= 0.000001: # noqa: PLR2004 + elif x >= 0.000001: result = f"{x * (1000 ** 2):0.3f}us" else: result = f"{x * (1000 ** 3):0.3f}ns" diff --git a/pyproject.toml b/pyproject.toml index ca0d04bd6c6..65f0a611847 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -74,4 +74,7 @@ select = [ "W", # pycodestyle - warning "YTT", # flake8-2020 ] -ignore = ["A002"] +ignore = [ + "A002", + "PLR2004", # Magic value used in comparison +] diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_find_commit_to_run.py b/tests/test_find_commit_to_run.py new file mode 100644 index 00000000000..1b31e3594b5 --- /dev/null +++ b/tests/test_find_commit_to_run.py @@ -0,0 +1,105 @@ +from __future__ import annotations + +import subprocess +from collections.abc import Callable +from pathlib import Path +from typing import Any + +import pytest + +from ci.find_commit_to_run import run + + +class _FakeCompleted: + def __init__(self, stdout: bytes) -> None: + self.stdout = stdout + + +def _fake_subprocess_run(shas: list[str]) -> Callable[..., _FakeCompleted]: + out = "\n".join(f"{sha} commit message {sha}" for sha in shas).encode() + + def _run(cmd: Any, **kwargs: Any) -> _FakeCompleted: + return _FakeCompleted(out) + + return _run + + +def test_run_no_existing_shas_picks_first( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + repo = tmp_path / "repo" + repo.mkdir() + monkeypatch.setattr(subprocess, "run", _fake_subprocess_run(["sha1111", "sha2222"])) + + run(input_path=tmp_path, repo_path=repo) + + assert capsys.readouterr().out.strip() == "sha1111" + + +def test_run_skips_existing_shas( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + (tmp_path / "shas.txt").write_text("sha1111\nsha2222\n") + repo = tmp_path / "repo" + repo.mkdir() + monkeypatch.setattr( + subprocess, + "run", + _fake_subprocess_run(["sha1111", "sha2222", "sha3333"]), + ) + + run(input_path=tmp_path, repo_path=repo) + + assert capsys.readouterr().out.strip() == "sha3333" + + +def test_run_all_existing_prints_none( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + (tmp_path / "shas.txt").write_text("sha1111\nsha2222\n") + repo = tmp_path / "repo" + repo.mkdir() + monkeypatch.setattr(subprocess, "run", _fake_subprocess_run(["sha1111", "sha2222"])) + + run(input_path=tmp_path, repo_path=repo) + + assert capsys.readouterr().out.strip() == "NONE" + + +def test_run_accepts_string_paths( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + repo = tmp_path / "repo" + repo.mkdir() + monkeypatch.setattr(subprocess, "run", _fake_subprocess_run(["abcdef0"])) + + run(input_path=str(tmp_path), repo_path=str(repo)) + + assert capsys.readouterr().out.strip() == "abcdef0" + + +def test_run_ignores_blank_lines_in_shas_file( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + (tmp_path / "shas.txt").write_text("sha1111\n\nsha2222\n") + repo = tmp_path / "repo" + repo.mkdir() + monkeypatch.setattr( + subprocess, + "run", + _fake_subprocess_run(["sha1111", "sha2222", "sha3333"]), + ) + + run(input_path=tmp_path, repo_path=repo) + + assert capsys.readouterr().out.strip() == "sha3333" diff --git a/tests/test_make_issues.py b/tests/test_make_issues.py new file mode 100644 index 00000000000..297b9f4e6dc --- /dev/null +++ b/tests/test_make_issues.py @@ -0,0 +1,165 @@ +from __future__ import annotations + +import pandas as pd +import pytest + +from ci.make_issues import ( + escape_ansi, + get_commit_range, + make_body, + time_to_str, +) + + +@pytest.mark.parametrize( + "value,expected", + [ + (1.5, "1.500s"), + (1.0, "1.000s"), + (0.5, "500.000ms"), + (0.001, "1.000ms"), + (0.0005, "500.000us"), + (0.000001, "1.000us"), + (0.0000005, "500.000ns"), + ], +) +def test_time_to_str_positive(value: float, expected: str) -> None: + assert time_to_str(value) == expected + + +def test_escape_ansi_passes_plain_text() -> None: + assert escape_ansi("hello world") == "hello world" + + +def test_escape_ansi_strips_color_codes() -> None: + colored = "\x1b[31mhello\x1b[0m world" + assert escape_ansi(colored) == "hello world" + + +def _benchmarks_with_shas(shas: list[str], dates: list[str]) -> pd.DataFrame: + return pd.DataFrame({"sha": shas, "date": pd.to_datetime(dates)}) + + +def test_get_commit_range_returns_prev_sha_to_sha() -> None: + df = _benchmarks_with_shas( + ["a", "b", "c"], ["2024-01-01", "2024-01-02", "2024-01-03"] + ) + assert get_commit_range(benchmarks=df, sha="c") == "b...c" + assert get_commit_range(benchmarks=df, sha="b") == "a...b" + + +def test_get_commit_range_orders_by_date_not_input_order() -> None: + # Rows are not in date order; the function should still pick the previous + # sha by date. + df = _benchmarks_with_shas( + ["c", "a", "b"], ["2024-01-03", "2024-01-01", "2024-01-02"] + ) + assert get_commit_range(benchmarks=df, sha="c") == "b...c" + + +def _regression_frame() -> pd.DataFrame: + return pd.DataFrame( + { + "sha": ["abc123", "abc123", "abc123", "def456"], + "is_regression": [True, True, True, True], + "name": ["bench.foo", "bench.foo", "bench.bar", "bench.foo"], + "params": ["x=1", "x=2", "", "x=1"], + "abs_change": [0.5, 0.0005, 1.5, 0.1], + "pct_change": [0.10, 0.20, 0.30, 0.40], + } + ) + + +def test_make_body_includes_commit_range_link() -> None: + body = make_body( + base_url="https://github.com/pandas-dev/pandas/compare/", + commit_range="aaa...bbb", + benchmarks=_regression_frame(), + sha="abc123", + ) + assert ( + "[Commit Range](https://github.com/pandas-dev/pandas/compare/aaa...bbb)" in body + ) + + +def test_make_body_only_includes_target_sha_regressions() -> None: + body = make_body( + base_url="https://github.com/pandas-dev/pandas/compare/", + commit_range="aaa...bbb", + benchmarks=_regression_frame(), + sha="abc123", + ) + # Both benchmarks for abc123 should appear; def456 should not. + assert "bench.foo" in body + assert "bench.bar" in body + assert "def456" not in body + + +def test_make_body_renders_param_sublist_for_nonempty_params() -> None: + body = make_body( + base_url="https://github.com/pandas-dev/pandas/compare/", + commit_range="aaa...bbb", + benchmarks=_regression_frame(), + sha="abc123", + ) + # Non-empty params should produce indented sub-bullets with severity. + assert " - [ ] [x=1]" in body + assert " - [ ] [x=2]" in body + assert "10.000% (500.000ms)" in body + assert "20.000% (500.000us)" in body + + +def test_make_body_inlines_severity_when_params_empty() -> None: + body = make_body( + base_url="https://github.com/pandas-dev/pandas/compare/", + commit_range="aaa...bbb", + benchmarks=_regression_frame(), + sha="abc123", + ) + # bench.bar has empty params; severity should be on the benchmark line. + msg = ( + " - [ ] [bench.bar](https://pandas-dev.github.io/asv-runner/#bench.bar)" + " - 30.000% (1.500s)" + ) + assert msg in body + + +def test_make_body_shorten_collapses_param_sublist() -> None: + full = make_body( + base_url="https://github.com/pandas-dev/pandas/compare/", + commit_range="aaa...bbb", + benchmarks=_regression_frame(), + sha="abc123", + ) + short = make_body( + base_url="https://github.com/pandas-dev/pandas/compare/", + commit_range="aaa...bbb", + benchmarks=_regression_frame(), + sha="abc123", + shorten=True, + ) + assert len(short) < len(full) + # In shorten mode, no indented sub-bullets are emitted. + assert " - [ ]" not in short + assert "10.000% (500.000ms)" in short + + +def test_make_body_excludes_non_regression_rows() -> None: + df = pd.DataFrame( + { + "sha": ["abc", "abc"], + "is_regression": [True, False], + "name": ["bench.foo", "bench.bar"], + "params": ["", ""], + "abs_change": [0.5, 0.5], + "pct_change": [0.10, 0.10], + } + ) + body = make_body( + base_url="https://example.com/", + commit_range="x...y", + benchmarks=df, + sha="abc", + ) + assert "bench.foo" in body + assert "bench.bar" not in body diff --git a/tests/test_process_results.py b/tests/test_process_results.py new file mode 100644 index 00000000000..7b06e48e8a9 --- /dev/null +++ b/tests/test_process_results.py @@ -0,0 +1,300 @@ +from __future__ import annotations + +import datetime as dt +import json +from pathlib import Path +from typing import Any + +import pandas as pd +import pytest + +from ci.process_results import ( + BASE_COLUMNS, + DERIVED_COLUMNS, + PARQUET_DIRNAME, + build_new_rows, + detect_regression, + load_existing, + run, +) + + +def _write_benchmarks(input_path: Path, benchmarks: dict[str, Any]) -> None: + results_dir = input_path / "results" + results_dir.mkdir(parents=True, exist_ok=True) + (results_dir / "benchmarks.json").write_text(json.dumps(benchmarks)) + + +def _write_result( + input_path: Path, + sha: str, + when: dt.datetime, + results: dict[str, Any], + result_columns: list[str] | None = None, +) -> None: + asvrunner_dir = input_path / "results" / "asvrunner" + asvrunner_dir.mkdir(parents=True, exist_ok=True) + payload = { + "commit_hash": sha, + "date": int(when.timestamp() * 1000), + "result_columns": result_columns or ["result", "params"], + "results": results, + } + (asvrunner_dir / f"{sha}.json").write_text(json.dumps(payload)) + + +def test_load_existing_returns_none_when_missing(tmp_path: Path) -> None: + assert load_existing(tmp_path / "does-not-exist.parquet") is None + + +def test_load_existing_normalizes_added_date_dtype(tmp_path: Path) -> None: + parquet_path = tmp_path / "results.parquet" + df = pd.DataFrame( + { + "sha": pd.array(["a", "b"], dtype="string[pyarrow]"), + "result": pd.array([1.0, 2.0], dtype="float64[pyarrow]"), + "added_date": ["2024-01-01", "2024-01-02"], + } + ) + df.to_parquet( + parquet_path, + index=False, + partition_cols=["added_date"], + basename_template="part-{i}.parquet", + ) + + loaded = load_existing(parquet_path) + + assert loaded is not None + assert str(loaded["added_date"].dtype) == "string" + + +def test_build_new_rows_produces_expected_rows(tmp_path: Path) -> None: + _write_benchmarks( + tmp_path, + { + "version": "1.0", + "bench.foo": {"param_names": ["a", "b"]}, + }, + ) + _write_result( + tmp_path, + "deadbeef", + dt.datetime(2024, 1, 1), + { + "bench.foo": [ + [1.0, 2.0], + [["1", "2"], ["3"]], + ], + }, + ) + + df = build_new_rows(tmp_path, skip_shas=set(), added_date="2024-01-02") + + assert len(df) == 2 + assert set(df["sha"]) == {"deadbeef"} + assert set(df["params"]) == {"a=1, b=3", "a=2, b=3"} + assert set(df["name"]) == {"bench.foo"} + assert sorted(df["result"].tolist()) == [1.0, 2.0] + assert set(df["added_date"]) == {"2024-01-02"} + + +def test_build_new_rows_skips_machine_json(tmp_path: Path) -> None: + _write_benchmarks( + tmp_path, + { + "version": "1.0", + "bench.foo": {"param_names": ["a"]}, + }, + ) + _write_result( + tmp_path, + "deadbeef", + dt.datetime(2024, 1, 1), + {"bench.foo": [[1.0], [["1"]]]}, + ) + # machine.json must be ignored even though it lives alongside results. + (tmp_path / "results" / "asvrunner" / "machine.json").write_text( + json.dumps({"unrelated": "data"}) + ) + + df = build_new_rows(tmp_path, skip_shas=set(), added_date="2024-01-02") + + assert len(df) == 1 + + +def test_build_new_rows_respects_skip_shas(tmp_path: Path) -> None: + _write_benchmarks( + tmp_path, + { + "version": "1.0", + "bench.foo": {"param_names": ["a"]}, + }, + ) + _write_result( + tmp_path, + "skipme", + dt.datetime(2024, 1, 1), + {"bench.foo": [[1.0], [["1"]]]}, + ) + _write_result( + tmp_path, + "keepme", + dt.datetime(2024, 1, 2), + {"bench.foo": [[2.0], [["1"]]]}, + ) + + df = build_new_rows(tmp_path, skip_shas={"skipme"}, added_date="2024-01-02") + + assert set(df["sha"]) == {"keepme"} + + +def test_build_new_rows_empty_when_no_result_files(tmp_path: Path) -> None: + _write_benchmarks( + tmp_path, + { + "version": "1.0", + "bench.foo": {"param_names": ["a"]}, + }, + ) + (tmp_path / "results" / "asvrunner").mkdir(parents=True) + + df = build_new_rows(tmp_path, skip_shas=set(), added_date="2024-01-02") + + assert len(df) == 0 + assert "name" in df.columns + assert "added_date" in df.columns + + +def _stable_frame(n: int, value: float = 1.0) -> pd.DataFrame: + dates = pd.date_range("2024-01-01", periods=n, freq="D") + return pd.DataFrame( + { + "name": ["bench.foo"] * n, + "params": ["a=1"] * n, + "date": dates, + "result": [value] * n, + } + ) + + +def test_detect_regression_adds_derived_columns() -> None: + df = _stable_frame(30) + + out = detect_regression(df, window_size=5) + + for col in DERIVED_COLUMNS: + assert col in out.columns + + +def test_detect_regression_no_regressions_on_constant_series() -> None: + df = _stable_frame(30, value=1.0) + + out = detect_regression(df, window_size=5) + + assert not out["is_regression"].any() + + +def test_detect_regression_flags_step_change() -> None: + # 30 points stable, then a sharp slowdown that holds — established_worst + # from window_size ago is well under 0.95 * established_best now. + n = 60 + dates = pd.date_range("2024-01-01", periods=n, freq="D") + values = [1.0] * (n // 2) + [10.0] * (n // 2) + df = pd.DataFrame( + { + "name": ["bench.foo"] * n, + "params": ["a=1"] * n, + "date": dates, + "result": values, + } + ) + + out = detect_regression(df, window_size=5) + + assert out["is_regression"].sum() >= 1 + + +def test_detect_regression_drops_null_result_rows() -> None: + df = _stable_frame(10) + df.loc[0, "result"] = None + + out = detect_regression(df, window_size=5) + + assert len(out) == 9 + + +def test_run_writes_parquet_with_expected_columns(tmp_path: Path) -> None: + input_path = tmp_path / "input" + output_path = tmp_path / "output" + output_path.mkdir() + + _write_benchmarks( + input_path, + { + "version": "1.0", + "bench.foo": {"param_names": ["a"]}, + }, + ) + base = dt.datetime(2024, 1, 1) + for i in range(30): + sha = f"sha{i:03d}" + _write_result( + input_path, + sha, + base + dt.timedelta(days=i), + {"bench.foo": [[1.0], [["1"]]]}, + ) + + run(input_path, output_path) + + parquet_path = output_path / PARQUET_DIRNAME + assert parquet_path.exists() + df = pd.read_parquet(parquet_path) + for col in BASE_COLUMNS: + assert col in df.columns + for col in DERIVED_COLUMNS: + assert col in df.columns + assert len(df) == 30 + + +def test_run_appends_to_existing_results(tmp_path: Path) -> None: + input_path = tmp_path / "input" + output_path = tmp_path / "output" + output_path.mkdir() + + _write_benchmarks( + input_path, + { + "version": "1.0", + "bench.foo": {"param_names": ["a"]}, + }, + ) + base = dt.datetime(2024, 1, 1) + for i in range(15): + _write_result( + input_path, + f"sha{i:03d}", + base + dt.timedelta(days=i), + {"bench.foo": [[1.0], [["1"]]]}, + ) + run(input_path, output_path) + + # Add new result files for additional shas; existing ones should be skipped + # and the new ones appended. + for i in range(15, 25): + _write_result( + input_path, + f"sha{i:03d}", + base + dt.timedelta(days=i), + {"bench.foo": [[1.0], [["1"]]]}, + ) + run(input_path, output_path) + + df = pd.read_parquet(output_path / PARQUET_DIRNAME) + assert len(df) == 25 + assert df["sha"].nunique() == 25 + + +if __name__ == "__main__": + pytest.main([__file__])