refactor/docs: improve compile_pip_requirements error message and docs (#2792)

Resolution failure is the most common error from pip-compile, so we
should make sure the error message is as clean as it can be. Previously,
the output was cluttered with the exception traceback, which makes the
actual error hard to see (several nested traceback).

The new output shortens it with a nicer message:

```
Checking _main/requirements_lock.txt
  ERROR: Cannot install requests<2.24 and requests~=2.25.1 because these package versions have conflicting dependencies.
ResolutionImpossible: for help visit https://pip.pypa.io/en/latest/topics/dependency-resolution/#dealing-with-dependency-conflicts
```

Fixes #2763

---------

Co-authored-by: Richard Levasseur <[email protected]>

Author: brandonchinn178

docs/pypi-dependencies.md

@@ -5,8 +5,40 @@
 
 Using PyPI packages (aka "pip install") involves two main steps.
 
-1. [Installing third party packages](#installing-third-party-packages)
-2. [Using third party packages as dependencies](#using-third-party-packages)
+1. [Generating requirements file](#generating-requirements-file)
+2. [Installing third party packages](#installing-third-party-packages)
+3. [Using third party packages as dependencies](#using-third-party-packages)
+
+{#generating-requirements-file}
+## Generating requirements file
+
+Generally, when working on a Python project, you'll have some dependencies that themselves have other dependencies. You might also specify dependency bounds instead of specific versions. So you'll need to generate a full list of all transitive dependencies and pinned versions for every dependency.
+
+Typically, you'd have your dependencies specified in `pyproject.toml` or `requirements.in` and generate the full pinned list of dependencies in `requirements_lock.txt`, which you can manage with the `compile_pip_requirements` Bazel rule:
+
+```starlark
+load("@rules_python//python:pip.bzl", "compile_pip_requirements")
+
+compile_pip_requirements(
+    name = "requirements",
+    src = "requirements.in",
+    requirements_txt = "requirements_lock.txt",
+)
+```
+
+This rule generates two targets:
+- `bazel run [name].update` will regenerate the `requirements_txt` file
+- `bazel test [name]_test` will test that the `requirements_txt` file is up to date
+
+For more documentation, see the API docs under {obj}`@rules_python//python:pip.bzl`.
+
+Once you generate this fully specified list of requirements, you can install the requirements with the instructions in [Installing third party packages](#installing-third-party-packages).
+
+:::{warning}
+If you're specifying dependencies in `pyproject.toml`, make sure to include the `[build-system]` configuration, with pinned dependencies. `compile_pip_requirements` will use the build system specified to read your project's metadata, and you might see non-hermetic behavior if you don't pin the build system.
+
+Not specifying `[build-system]` at all will result in using a default `[build-system]` configuration, which uses unpinned versions ([ref](https://peps.python.org/pep-0518/#build-system-table)).
+:::
 
 {#installing-third-party-packages}
 ## Installing third party packages
@@ -27,8 +59,7 @@ pip.parse(
 )
 use_repo(pip, "my_deps")
 ```
-For more documentation, including how the rules can update/create a requirements
-file, see the bzlmod examples under the {gh-path}`examples` folder or the documentation
+For more documentation, see the bzlmod examples under the {gh-path}`examples` folder or the documentation
 for the {obj}`@rules_python//python/extensions:pip.bzl` extension.
 
 ```{note}

python/private/pypi/dependency_resolver/dependency_resolver.py

@@ -15,14 +15,17 @@
 "Set defaults for the pip-compile command to run it under Bazel"
 
 import atexit
+import functools
 import os
 import shutil
 import sys
 from pathlib import Path
-from typing import Optional, Tuple
+from typing import List, Optional, Tuple
 
 import click
 import piptools.writer as piptools_writer
+from pip._internal.exceptions import DistributionNotFound
+from pip._vendor.resolvelib.resolvers import ResolutionImpossible
 from piptools.scripts.compile import cli
 
 from python.runfiles import runfiles
@@ -82,15 +85,15 @@ def _locate(bazel_runfiles, file):
 @click.command(context_settings={"ignore_unknown_options": True})
 @click.option("--src", "srcs", multiple=True, required=True)
 @click.argument("requirements_txt")
-@click.argument("update_target_label")
+@click.argument("target_label_prefix")
 @click.option("--requirements-linux")
 @click.option("--requirements-darwin")
 @click.option("--requirements-windows")
 @click.argument("extra_args", nargs=-1, type=click.UNPROCESSED)
 def main(
     srcs: Tuple[str, ...],
     requirements_txt: str,
-    update_target_label: str,
+    target_label_prefix: str,
     requirements_linux: Optional[str],
     requirements_darwin: Optional[str],
     requirements_windows: Optional[str],
@@ -152,9 +155,10 @@ def main(
         # or shutil.copyfile, as they will fail with OSError: [Errno 18] Invalid cross-device link.
         shutil.copy(resolved_requirements_file, requirements_out)
 
-    update_command = os.getenv("CUSTOM_COMPILE_COMMAND") or "bazel run %s" % (
-        update_target_label,
+    update_command = (
+        os.getenv("CUSTOM_COMPILE_COMMAND") or f"bazel run {target_label_prefix}.update"
     )
+    test_command = f"bazel test {target_label_prefix}_test"
 
     os.environ["CUSTOM_COMPILE_COMMAND"] = update_command
     os.environ["PIP_CONFIG_FILE"] = os.getenv("PIP_CONFIG_FILE") or os.devnull
@@ -168,6 +172,12 @@ def main(
     )
     argv.extend(extra_args)
 
+    _run_pip_compile = functools.partial(
+        run_pip_compile,
+        argv,
+        srcs_relative=srcs_relative,
+    )
+
     if UPDATE:
         print("Updating " + requirements_file_relative)
 
@@ -187,49 +197,66 @@ def main(
                 atexit.register(
                     lambda: shutil.copy(absolute_output_file, requirements_file_tree)
                 )
-        cli(argv, standalone_mode=False)
+        _run_pip_compile(verbose_command=f"{update_command} -- --verbose")
         requirements_file_relative_path = Path(requirements_file_relative)
         content = requirements_file_relative_path.read_text()
         content = content.replace(absolute_path_prefix, "")
         requirements_file_relative_path.write_text(content)
     else:
-        # cli will exit(0) on success
-        try:
-            print("Checking " + requirements_file)
-            cli(argv)
-            print("cli() should exit", file=sys.stderr)
+        print("Checking " + requirements_file)
+        sys.stdout.flush()
+        _run_pip_compile(verbose_command=f"{test_command} --test_arg=--verbose")
+        golden = open(_locate(bazel_runfiles, requirements_file)).readlines()
+        out = open(requirements_out).readlines()
+        out = [line.replace(absolute_path_prefix, "") for line in out]
+        if golden != out:
+            import difflib
+
+            print("".join(difflib.unified_diff(golden, out)), file=sys.stderr)
+            print(
+                f"Lock file out of date. Run '{update_command}' to update.",
+                file=sys.stderr,
+            )
+            sys.exit(1)
+
+
+def run_pip_compile(
+    args: List[str],
+    *,
+    srcs_relative: List[str],
+    verbose_command: str,
+) -> None:
+    try:
+        cli(args, standalone_mode=False)
+    except DistributionNotFound as e:
+        if isinstance(e.__cause__, ResolutionImpossible):
+            # pip logs an informative error to stderr already
+            # just render the error and exit
+            print(e)
+            sys.exit(1)
+        else:
+            raise
+    except SystemExit as e:
+        if e.code == 0:
+            return  # shouldn't happen, but just in case
+        elif e.code == 2:
+            print(
+                "pip-compile exited with code 2. This means that pip-compile found "
+                "incompatible requirements or could not find a version that matches "
+                f"the install requirement in one of {srcs_relative}.\n"
+                "Try re-running with verbose:\n"
+                f"    {verbose_command}",
+                file=sys.stderr,
+            )
+            sys.exit(1)
+        else:
+            print(
+                f"pip-compile unexpectedly exited with code {e.code}.\n"
+                "Try re-running with verbose:\n"
+                f"    {verbose_command}",
+                file=sys.stderr,
+            )
             sys.exit(1)
-        except SystemExit as e:
-            if e.code == 2:
-                print(
-                    "pip-compile exited with code 2. This means that pip-compile found "
-                    "incompatible requirements or could not find a version that matches "
-                    f"the install requirement in one of {srcs_relative}.",
-                    file=sys.stderr,
-                )
-                sys.exit(1)
-            elif e.code == 0:
-                golden = open(_locate(bazel_runfiles, requirements_file)).readlines()
-                out = open(requirements_out).readlines()
-                out = [line.replace(absolute_path_prefix, "") for line in out]
-                if golden != out:
-                    import difflib
-
-                    print("".join(difflib.unified_diff(golden, out)), file=sys.stderr)
-                    print(
-                        "Lock file out of date. Run '"
-                        + update_command
-                        + "' to update.",
-                        file=sys.stderr,
-                    )
-                    sys.exit(1)
-                sys.exit(0)
-            else:
-                print(
-                    f"pip-compile unexpectedly exited with code {e.code}.",
-                    file=sys.stderr,
-                )
-                sys.exit(1)
 
 
 if __name__ == "__main__":

python/private/pypi/pip_compile.bzl

@@ -110,7 +110,7 @@ def pip_compile(
 
     args = ["--src=%s" % loc.format(src) for src in srcs] + [
         loc.format(requirements_txt),
-        "//%s:%s.update" % (native.package_name(), name),
+        "//%s:%s" % (native.package_name(), name),
         "--resolver=backtracking",
         "--allow-unsafe",
     ]