Run cudf-polars wheels unit tests with more than 1 process (#20124)

Follow up to #19980

Authors:
  - Matthew Roeschke (https://github.com/mroeschke)

Approvers:
  - Bradley Dice (https://github.com/bdice)
  - Vyas Ramasubramani (https://github.com/vyasr)

URL: #20124

Author: mroeschke

ci/test_python_other.sh

@@ -43,7 +43,7 @@ rapids-logger "pytest custreamz"
 rapids-logger "pytest cudf-polars"
 ./ci/run_cudf_polars_pytests.sh \
   --junitxml="${RAPIDS_TESTS_DIR}/junit-cudf-polars.xml" \
-  --numprocesses=6 \
+  --numprocesses=8 \
   --dist=worksteal \
   --cov-config=./pyproject.toml \
   --cov=cudf_polars \

ci/test_wheel_cudf_polars.sh

@@ -71,6 +71,8 @@ for version in "${VERSIONS[@]}"; do
 
     ./ci/run_cudf_polars_pytests.sh \
         "${COVERAGE_ARGS[@]}" \
+        --numprocesses=8 \
+        --dist=worksteal \
         --junitxml="${RAPIDS_TESTS_DIR}/junit-cudf-polars-${version}.xml"
 
     if [ $? -ne 0 ]; then

python/cudf_polars/tests/experimental/conftest.py

@@ -7,6 +7,8 @@
 import pytest
 
 
+# scope="session" is important to not cause singificant slowdowns in CI
+# https://github.com/rapidsai/cudf/pull/20137
 @pytest.fixture(autouse=True, scope="session")
 def dask_cluster(pytestconfig, worker_id):
     if (