perf(security): skip non-text subresources + fetch via per-page cache in guards

Route guards now abort image/media/font requests (never needed for inner_text)
and fetch the rest through the per-page _PinnedSessionCache (DNS + keep-alive)
instead of building a fresh session per subresource. Per-hop re-validation and
IP-pinning unchanged. Route-guard tests migrated to the cache seam; skip tests
added.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: FlyM1ss

Main/backend/datascraper/ssrf_guard.py

@@ -409,6 +409,17 @@ def _should_proxy(route) -> bool:
     return urlparse(request.url).scheme in _ALLOWED_SCHEMES
 
 
+def _should_skip_resource(route) -> bool:
+    """True for in-browser subresource types we never need for text extraction
+    (image/media/font by default — see ``_SKIP_RESOURCE_TYPES``). Aborting them
+    cuts DNS+TLS work and egress with no effect on ``page.inner_text``; aborting
+    more requests is also strictly less egress, so there is no SSRF downside."""
+    try:
+        return route.request.resource_type in _SKIP_RESOURCE_TYPES
+    except Exception:
+        return False
+
+
 def _forward_headers(route) -> dict:
     """The browser request's headers to replay through ``safe_get``, minus the
     framing/encoding headers requests must control (see
@@ -426,20 +437,20 @@ def _forward_headers(route) -> dict:
     }
 
 
-def _proxied_response_kwargs(request_url: str, headers: dict):
-    """Fetch an intercepted in-browser GET through the IP-pinned, byte-capped
-    ``safe_get`` and return the kwargs to fulfill the Playwright route with, or
-    ``None`` to fail closed (abort). Blocking — the async guard dispatches it via
-    ``asyncio.to_thread``. Both route guards share this one fail-closed policy
-    (which exceptions abort, what gets logged, which headers fulfill) so the sync
-    and async paths cannot drift. ``headers`` are the browser's own request
-    headers (UA/Accept/...) so the IP-pinned fetch stays indistinguishable to the
-    origin."""
+def _proxied_response_kwargs(cache: "_PinnedSessionCache", request_url: str, headers: dict):
+    """Fetch an intercepted in-browser GET through the per-page IP-pinned,
+    byte-capped ``cache`` and return the kwargs to fulfill the Playwright route
+    with, or ``None`` to fail closed (abort). Blocking — the async guard
+    dispatches it via ``asyncio.to_thread``. Both route guards share this one
+    fail-closed policy (which exceptions abort, what gets logged, which headers
+    fulfill) so the sync and async paths cannot drift. ``headers`` are the
+    browser's own request headers (UA/Accept/...) so the IP-pinned fetch stays
+    indistinguishable to the origin."""
     try:
-        # safe_get validates, pins to the resolved IP, follows redirects
-        # re-validating each, and byte-caps the body — raising UnsafeURLError on
-        # any violation.
-        response = safe_get(request_url, headers=headers)
+        # cache.fetch validates, pins to the resolved IP (reusing a per-host
+        # keep-alive session), follows redirects re-validating each, and byte-caps
+        # the body — raising UnsafeURLError on any violation.
+        response = cache.fetch(request_url, headers=headers)
     except UnsafeURLError as exc:
         logger.warning(
             "[ssrf_guard] aborting in-browser request to %s: %s",
@@ -464,24 +475,30 @@ def _proxied_response_kwargs(request_url: str, headers: dict):
 async def install_route_guard(page) -> None:
     """Register an async Playwright route handler on ALL URLs that fetches each
     intercepted in-browser request (top-level navigation OR subresource) through
-    the IP-pinned, byte-capped ``safe_get`` and fulfills Chromium with the
-    buffered response. For these HTTP(S) requests Chromium therefore never opens
-    its own socket and cannot be redirected to a private address by a
-    DNS-rebinding answer.
-
-    MUST be called BEFORE the first ``page.goto`` in every Playwright entrypoint
-    so EVERY navigation/subresource is pinned, not just the seed URL. Non-GET and
-    non-http(s) requests fail closed (aborted). WebSocket/WebRTC are not routed
-    through ``page.route`` and are out of scope (see module docstring)."""
+    a per-page IP-pinned, byte-capped, keep-alive cache and fulfills Chromium with
+    the buffered response. For these HTTP(S) requests Chromium therefore never
+    opens its own socket and cannot be redirected to a private address by a
+    DNS-rebinding answer. image/media/font requests are aborted outright (they
+    never feed text extraction); non-GET and non-http(s) requests fail closed.
+
+    MUST be called BEFORE the first ``page.goto`` so EVERY navigation/subresource
+    is pinned, not just the seed URL. Installed centrally by the PlaywrightBrowser
+    factory. WebSocket/WebRTC are not routed through ``page.route`` and are out of
+    scope (see module docstring)."""
+    cache = _PinnedSessionCache()
+    page.on("close", lambda *_: cache.close())
 
     async def _handler(route):
+        if _should_skip_resource(route):
+            await route.abort()
+            return
         if not _should_proxy(route):
             await route.abort()
             return
-        # _proxied_response_kwargs is blocking (it calls safe_get); run it off the
-        # event loop. route.request.* is read here on the loop before dispatch.
+        # _proxied_response_kwargs is blocking (it calls cache.fetch); run it off
+        # the event loop. route.request.* is read here on the loop before dispatch.
         kwargs = await asyncio.to_thread(
-            _proxied_response_kwargs, route.request.url, _forward_headers(route)
+            _proxied_response_kwargs, cache, route.request.url, _forward_headers(route)
         )
         if kwargs is None:  # fetch was unsafe or failed — fail closed.
             await route.abort()
@@ -493,17 +510,23 @@ async def _handler(route):
 
 def install_route_guard_sync(page) -> None:
     """Synchronous twin of :func:`install_route_guard` for the sync Playwright
-    fallback (``url_tools.scrape_with_playwright``). Same guarantee: every
-    in-browser GET is fulfilled from the IP-pinned ``safe_get`` so Chromium never
-    re-resolves DNS on its own socket; non-GET / non-http(s) fail closed.
+    fallback (``url_tools.scrape_with_playwright``). Same guarantees: every
+    in-browser GET is fulfilled from a per-page IP-pinned, keep-alive cache so
+    Chromium never re-resolves DNS on its own socket; image/media/font are
+    aborted outright; non-GET / non-http(s) fail closed.
 
     MUST be called BEFORE the first ``page.goto``."""
+    cache = _PinnedSessionCache()
+    page.on("close", lambda *_: cache.close())
 
     def _handler(route):
+        if _should_skip_resource(route):
+            route.abort()
+            return
         if not _should_proxy(route):
             route.abort()
             return
-        kwargs = _proxied_response_kwargs(route.request.url, _forward_headers(route))
+        kwargs = _proxied_response_kwargs(cache, route.request.url, _forward_headers(route))
         if kwargs is None:  # fetch was unsafe or failed — fail closed.
             route.abort()
             return

Main/backend/tests/test_ssrf_guard.py

@@ -189,10 +189,11 @@ async def fake_route(pattern, handler):
         "Host": "example.com",
     }
 
-    def _route(self, url, method="GET"):
+    def _route(self, url, method="GET", resource_type="document"):
         route = MagicMock()
         route.request.url = url
         route.request.method = method
+        route.request.resource_type = resource_type
         route.request.headers = dict(self._BROWSER_HEADERS)
         route.abort = AsyncMock()
         route.fulfill = AsyncMock()
@@ -209,8 +210,9 @@ async def run():
 
         asyncio.run(run())
 
-    @patch("datascraper.ssrf_guard.safe_get")
-    def test_route_guard_aborts_blocked_request(self, m_get):
+    @patch("datascraper.ssrf_guard._PinnedSessionCache")
+    def test_route_guard_aborts_blocked_request(self, m_cache):
+        m_get = m_cache.return_value.fetch
         m_get.side_effect = UnsafeURLError("blocked")
         route = self._route("http://evil.example.test/x")
         self._drive(route)
@@ -219,8 +221,9 @@ def test_route_guard_aborts_blocked_request(self, m_get):
         # Never delegate the fetch back to Chromium (would re-resolve DNS).
         route.continue_.assert_not_awaited()
 
-    @patch("datascraper.ssrf_guard.safe_get")
-    def test_route_guard_fulfills_public_request_from_pinned_fetch(self, m_get):
+    @patch("datascraper.ssrf_guard._PinnedSessionCache")
+    def test_route_guard_fulfills_public_request_from_pinned_fetch(self, m_cache):
+        m_get = m_cache.return_value.fetch
         resp = _FakeResp(
             status_code=200,
             headers={"Content-Type": "text/html", "Content-Encoding": "gzip"},
@@ -243,8 +246,9 @@ def test_route_guard_fulfills_public_request_from_pinned_fetch(self, m_get):
         m_get.assert_called_once()
         self.assertEqual(m_get.call_args.args[0], "http://example.com/x")
 
-    @patch("datascraper.ssrf_guard.safe_get")
-    def test_route_guard_forwards_browser_headers_to_pinned_fetch(self, m_get):
+    @patch("datascraper.ssrf_guard._PinnedSessionCache")
+    def test_route_guard_forwards_browser_headers_to_pinned_fetch(self, m_cache):
+        m_get = m_cache.return_value.fetch
         # The pinned fetch must present the browser's own User-Agent (the
         # context deliberately sets a Chrome UA to avoid bot-gating); SSRF
         # safety comes from IP-pinning, not from hiding the UA. Framing/encoding
@@ -264,15 +268,26 @@ def test_route_guard_forwards_browser_headers_to_pinned_fetch(self, m_get):
         self.assertNotIn("host", lowered)
         self.assertNotIn("accept-encoding", lowered)
 
-    @patch("datascraper.ssrf_guard.safe_get")
-    def test_route_guard_aborts_non_get(self, m_get):
-        # safe_get is GET-only; non-GET in-browser requests fail closed.
+    @patch("datascraper.ssrf_guard._PinnedSessionCache")
+    def test_route_guard_aborts_non_get(self, m_cache):
+        m_get = m_cache.return_value.fetch
+        # cache.fetch is GET-only; non-GET in-browser requests fail closed.
         route = self._route("http://example.com/api", method="POST")
         self._drive(route)
         route.abort.assert_awaited_once()
         route.fulfill.assert_not_awaited()
         m_get.assert_not_called()
 
+    @patch("datascraper.ssrf_guard._PinnedSessionCache")
+    def test_route_guard_aborts_skipped_resource(self, m_cache):
+        # image/media/font are aborted before any fetch — they don't feed
+        # inner_text, so we never spend DNS+TLS or egress on them.
+        route = self._route("http://example.com/logo.png", resource_type="image")
+        self._drive(route)
+        route.abort.assert_awaited_once()
+        route.fulfill.assert_not_awaited()
+        m_cache.return_value.fetch.assert_not_called()
+
 
 class SyncRouteGuardTests(SimpleTestCase):
     """install_route_guard_sync mirrors the async guard for the sync
@@ -287,10 +302,11 @@ def route(pattern, handler):
         page.route = route
         return page
 
-    def _route(self, url, method="GET"):
+    def _route(self, url, method="GET", resource_type="document"):
         route = MagicMock()
         route.request.url = url
         route.request.method = method
+        route.request.resource_type = resource_type
         route.request.headers = {
             "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) Chrome/120.0.0.0",
             "Accept-Encoding": "gzip, deflate, br",
@@ -304,16 +320,18 @@ def _drive(self, route):
         ssrf_guard.install_route_guard_sync(page)
         captured["handler"](route)
 
-    @patch("datascraper.ssrf_guard.safe_get")
-    def test_sync_guard_aborts_blocked_request(self, m_get):
+    @patch("datascraper.ssrf_guard._PinnedSessionCache")
+    def test_sync_guard_aborts_blocked_request(self, m_cache):
+        m_get = m_cache.return_value.fetch
         m_get.side_effect = UnsafeURLError("blocked")
         route = self._route("http://evil.example.test/x")
         self._drive(route)
         route.abort.assert_called_once()
         route.fulfill.assert_not_called()
 
-    @patch("datascraper.ssrf_guard.safe_get")
-    def test_sync_guard_fulfills_public_request(self, m_get):
+    @patch("datascraper.ssrf_guard._PinnedSessionCache")
+    def test_sync_guard_fulfills_public_request(self, m_cache):
+        m_get = m_cache.return_value.fetch
         resp = _FakeResp(status_code=200, headers={"Content-Type": "text/html"})
         resp._content = b"hi"
         m_get.return_value = resp
@@ -323,8 +341,9 @@ def test_sync_guard_fulfills_public_request(self, m_get):
         self.assertEqual(route.fulfill.call_args.kwargs["body"], b"hi")
         route.abort.assert_not_called()
 
-    @patch("datascraper.ssrf_guard.safe_get")
-    def test_sync_guard_forwards_browser_user_agent(self, m_get):
+    @patch("datascraper.ssrf_guard._PinnedSessionCache")
+    def test_sync_guard_forwards_browser_user_agent(self, m_cache):
+        m_get = m_cache.return_value.fetch
         resp = _FakeResp(status_code=200, headers={"Content-Type": "text/html"})
         resp._content = b"hi"
         m_get.return_value = resp
@@ -337,13 +356,22 @@ def test_sync_guard_forwards_browser_user_agent(self, m_get):
         self.assertNotIn("host", lowered)
         self.assertNotIn("accept-encoding", lowered)
 
-    @patch("datascraper.ssrf_guard.safe_get")
-    def test_sync_guard_aborts_non_get(self, m_get):
+    @patch("datascraper.ssrf_guard._PinnedSessionCache")
+    def test_sync_guard_aborts_non_get(self, m_cache):
+        m_get = m_cache.return_value.fetch
         route = self._route("http://example.com/api", method="POST")
         self._drive(route)
         route.abort.assert_called_once()
         m_get.assert_not_called()
 
+    @patch("datascraper.ssrf_guard._PinnedSessionCache")
+    def test_sync_guard_aborts_skipped_resource(self, m_cache):
+        route = self._route("http://example.com/font.woff2", resource_type="font")
+        self._drive(route)
+        route.abort.assert_called_once()
+        route.fulfill.assert_not_called()
+        m_cache.return_value.fetch.assert_not_called()
+
 
 class PinnedSessionCacheTests(SimpleTestCase):
     """The per-page cache reuses validated DNS + keep-alive sessions per host,