Unpin urllib3 (#810)

* Unpin urllib3

* Fix rq instrumentation, change tox deps

* Swap in mocket for httpretty

* Pin elasticsearch until we can verify compat

* Artifact naming, download update

* Set up trusted publishing

Author: lancetarn

.github/workflows/main.yml

@@ -19,7 +19,6 @@ jobs:
       fail-fast: false
       matrix:
         python-version:
-        - "3.8"
         - "3.9"
         - "3.10"
         - "3.11"
@@ -50,57 +49,8 @@ jobs:
       REDIS_URL: redis://localhost:6379/0
 
     steps:
-    - uses: actions/checkout@v3
-    - uses: actions/setup-python@v4
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Upgrade packaging tools
-      run: python -m pip install --upgrade pip setuptools importlib_metadata virtualenv
-    - name: Install dependencies
-      run: python -m pip install --upgrade tox
-    - name: Run tox targets for ${{ matrix.python-version }}
-      run: |
-        ENV_PREFIX=$(tr -C -d "0-9" <<< "${{ matrix.python-version }}" | cut -c -3)
-        TOXENV=$(tox --listenvs | grep "^py$ENV_PREFIX" | tr '\n' ',') python -m tox
-
-  tests-ubuntu-20:
-    name: Python ${{ matrix.python-version }} Ubuntu 20
-    runs-on: ubuntu-20.04
-
-    strategy:
-      fail-fast: false
-      matrix:
-        python-version:
-        - "3.8"
-        - "3.9"
-
-    services:
-      elasticsearch:
-        image: elasticsearch:7.10.1
-        ports:
-        - 9200:9200
-        env:
-          discovery.type: single-node
-          xpack.security.enabled: false
-
-      mongodb:
-        image: mongo:4
-        ports:
-        - 27017:27017
-
-      redis:
-        image: redis:6
-        ports:
-        - 6379:6379
-
-    env:
-      ELASTICSEARCH_URL: http://localhost:9200/
-      MONGODB_URL: mongodb://localhost:27017/
-      REDIS_URL: redis://localhost:6379/0
-
-    steps:
-    - uses: actions/checkout@v3
-    - uses: actions/setup-python@v4
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
     - name: Upgrade packaging tools

.github/workflows/release.yml

@@ -14,15 +14,15 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ ubuntu-20.04 ]
+        os: [ ubuntu-22.04 ]
         python: ['3.10']
         manylinux_image: [ manylinux2014, manylinux_2_28 ]
         # Disable for platforms where pure Python wheels would be generated
-        cibw_skip: [ "pp38-* pp39-* pp310-* pp311-*" ]
+        cibw_skip: [ "pp38-* pp39-* pp310-* pp311-* pp312-* pp313-*" ]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         name: Install Python
         with:
           python-version: ${{ matrix.python }}
@@ -47,17 +47,18 @@ jobs:
           CIBW_ARCHS_LINUX: "auto aarch64"
         run: python -m cibuildwheel
 
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
+          name: ${{ matrix.manylinux_image }}-${{ matrix.python }}-${{ matrix.os }}
           path: ./wheelhouse/*.whl
 
   build_macos_wheels:
     name: Build macos wheels (cross-compiles arm64)
     runs-on: macos-13
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         name: Install Python
         with:
           python-version: "3.10"
@@ -72,23 +73,24 @@ jobs:
           CIBW_ARCHS_MACOS: "x86_64 arm64"
         run: python -m cibuildwheel
 
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
+          name: macos
           path: ./wheelhouse/*.whl
 
   build_pure_wheels:
     name: Build pure python wheels
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     strategy:
       matrix:
-        python: [3.9]
+        python: [3.8, 3.9]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
-      - uses: actions/setup-python@v4
+      - uses: actions/setup-python@v5
         name: Install Python
         with:
-          python-version: ${{ matrix.python-version }}
+          python-version: ${{ matrix.python }}
       - name: Install packaging tools
         run: |
           python -m pip install --upgrade pip setuptools importlib_metadata wheel
@@ -98,49 +100,52 @@ jobs:
           SCOUT_DISABLE_EXTENSIONS: "1"
         run: python setup.py bdist_wheel
 
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
+          name: pp${{ matrix.python }}
           path: dist/*.whl
 
   build_sdist:
     name: Build source distribution
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
-      - uses: actions/setup-python@v3
+      - uses: actions/setup-python@v5
         name: Install Python
         with:
           python-version: 3.9
 
       - name: Build sdist
         run: python setup.py sdist
 
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
+          name: sdist
           path: dist/*.tar.gz
 
   upload_pypi:
     needs: [cibuildwheel_py38plus, build_pure_wheels, build_sdist, build_macos_wheels]
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
+    permissions:
+      id-token: write
     steps:
       - name: Download distributions for publishing.
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
-          name: artifact
           path: dist
+          merge-multiple: true
 
       - name: Publish distributions to Test PyPI
         uses: pypa/gh-action-pypi-publish@release/v1
         with:
           user: __token__
           password: ${{ secrets.TEST_TWINE_PASSWORD }}
-          repository_url: https://test.pypi.org/legacy/
+          repository-url: https://test.pypi.org/legacy/
           skip_existing: true
 
       - name: Publish distributions to PyPI
         if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v')
         uses: pypa/gh-action-pypi-publish@release/v1
         with:
           user: __token__
-          password: ${{ secrets.TWINE_PASSWORD }}

setup.py

@@ -60,7 +60,7 @@
     install_requires=[
         "asgiref",
         "psutil>=5",
-        "urllib3~=2.2.0",
+        "urllib3",
         "certifi",
         "wrapt>=1.10,<2.0",
     ],

src/scout_apm/rq.py

@@ -73,6 +73,8 @@ def wrap_perform(wrapped, instance, args, kwargs):
     # assumption here.
     if instance.enqueued_at.tzinfo is None:
         queued_at = instance.enqueued_at.replace(tzinfo=dt.timezone.utc)
+    else:
+        queued_at = instance.enqueued_at
     queue_time = (dt.datetime.now(dt.timezone.utc) - queued_at).total_seconds()
     tracked_request.tag("queue_time", queue_time)
     operation = "Job/{}".format(instance.func_name)

tests/integration/core/test_error_service.py

@@ -6,8 +6,9 @@
 from datetime import datetime, timezone
 from time import sleep
 
-import httpretty
 import pytest
+from mocket import Mocketizer
+from mocket.plugins.httpretty import httpretty
 
 from scout_apm.core.config import scout_config
 from scout_apm.core.error_service import ErrorServiceThread
@@ -32,6 +33,7 @@ def error_monitor_errors(error_monitor_errors):
     return error_monitor_errors
 
 
+@pytest.mark.filterwarnings("ignore::pytest.PytestUnraisableExceptionWarning")
 @pytest.mark.parametrize(
     "config, decoded_body, expected_headers, expected_uri",
     [
@@ -73,7 +75,7 @@ def test_send(
     scout_config.set(**config)
 
     try:
-        with httpretty.enabled(allow_net_connect=False):
+        with Mocketizer():
             httpretty.register_uri(
                 httpretty.POST,
                 expected_uri,
@@ -82,16 +84,18 @@ def test_send(
             ErrorServiceThread.send(error={"foo": "BØØM!"})
             ErrorServiceThread.wait_until_drained()
 
-            request = httpretty.last_request()
-            assert (
-                json.loads(gzip_decompress(request.body).decode("utf-8"))
-                == decoded_body
-            )
-            assert request.headers.get("X-Error-Count") == "1"
+            # TODO - this is a hack to get the request body
+            request = httpretty.last_request
+            _ = request.event
+            raw_body = httpretty.last_request._parser.next_event().data
+
+            assert json.loads(gzip_decompress(raw_body).decode("utf-8")) == decoded_body
+            assert request.headers.get("x-error-count") == "1"
     finally:
         scout_config.reset_all()
 
 
+@pytest.mark.filterwarnings("ignore::pytest.PytestUnraisableExceptionWarning")
 def test_send_batch(error_service_thread):
     decompressed_body = {
         "notifier": "scout_apm_python",
@@ -100,7 +104,7 @@ def test_send_batch(error_service_thread):
         "problems": [{"foo": 0}, {"foo": 1}, {"foo": 2}, {"foo": 3}, {"foo": 4}],
     }
     try:
-        with httpretty.enabled(allow_net_connect=False):
+        with Mocketizer():
             httpretty.register_uri(
                 httpretty.POST,
                 "https://errors.scoutapm.com/apps/error.scout",
@@ -110,19 +114,23 @@ def test_send_batch(error_service_thread):
                 ErrorServiceThread.send(error={"foo": i})
             ErrorServiceThread.wait_until_drained()
 
-            request = httpretty.last_request()
+            # TODO - this is a hack to get the request body
+            request = httpretty.last_request
+            _ = request.event
+            raw_body = httpretty.last_request._parser.next_event().data
             assert (
-                json.loads(gzip_decompress(request.body).decode("utf-8"))
+                json.loads(gzip_decompress(raw_body).decode("utf-8"))
                 == decompressed_body
             )
-            assert request.headers.get("X-Error-Count") == "5"
+            assert request.headers.get("x-error-count") == "5"
     finally:
         scout_config.reset_all()
 
 
+@pytest.mark.filterwarnings("ignore::pytest.PytestUnraisableExceptionWarning")
 def test_send_api_error(error_service_thread, caplog):
     try:
-        with httpretty.enabled(allow_net_connect=False):
+        with Mocketizer():
             httpretty.register_uri(
                 httpretty.POST,
                 "https://errors.scoutapm.com/apps/error.scout",
@@ -140,8 +148,9 @@ def test_send_api_error(error_service_thread, caplog):
     )
 
 
+@pytest.mark.filterwarnings("ignore::pytest.PytestUnraisableExceptionWarning")
 def test_send_unserializable_data(error_service_thread, caplog):
-    with httpretty.enabled(allow_net_connect=False):
+    with Mocketizer():
         ErrorServiceThread.send(error={"value": datetime.now(tz=timezone.utc)})
         ErrorServiceThread.wait_until_drained()
 

tests/integration/instruments/test_urllib3.py

@@ -2,8 +2,10 @@
 
 import logging
 
-import httpretty
 import pytest
+from mocket import Mocketizer
+from mocket.mockhttp import Entry
+from mocket.plugins.httpretty import httprettified, httpretty
 
 from scout_apm.compat import urllib3_cert_pool_manager
 from scout_apm.instruments.urllib3 import ensure_installed
@@ -70,13 +72,12 @@ def test_install_fail_no_urlopen_attribute(caplog):
     )
 
 
+@pytest.mark.filterwarnings("ignore::pytest.PytestUnraisableExceptionWarning")
 def test_request(tracked_request):
     ensure_installed()
-    with httpretty.enabled(allow_net_connect=False):
-        httpretty.register_uri(
-            httpretty.GET, "https://example.com/", body="Hello World!"
-        )
+    Entry.single_register(Entry.GET, "https://example.com/", body="Hello World!")
 
+    with Mocketizer(strict_mode=True):
         http = urllib3_cert_pool_manager()
         response = http.request("GET", "https://example.com")
 
@@ -101,15 +102,16 @@ def test_request_type_error(tracked_request):
     assert span.tags["url"] == "https://example.com:443/"
 
 
+@pytest.mark.filterwarnings("ignore::pytest.PytestUnraisableExceptionWarning")
+@httprettified
 def test_request_ignore_errors_host(tracked_request):
     ensure_installed()
-    with httpretty.enabled(allow_net_connect=False):
-        httpretty.register_uri(
-            httpretty.POST, "https://errors.scoutapm.com", body="Hello World!"
-        )
+    httpretty.register_uri(
+        httpretty.POST, "https://errors.scoutapm.com", body="Hello World!"
+    )
 
-        http = urllib3_cert_pool_manager()
-        response = http.request("POST", "https://errors.scoutapm.com")
+    http = urllib3_cert_pool_manager()
+    response = http.request("POST", "https://errors.scoutapm.com")
 
     assert response.status == 200
     assert response.data == b"Hello World!"