resolve

Author: Mantisus

CHANGELOG.md

@@ -2,6 +2,21 @@
 
 All notable changes to this project will be documented in this file.
 
+## [0.6.5](https://github.com/apify/crawlee-python/releases/tag/v0.6.5) (2025-03-13)
+
+### 🐛 Bug Fixes
+
+- Update to `browserforge` workaround ([#1075](https://github.com/apify/crawlee-python/pull/1075)) ([2378cf8](https://github.com/apify/crawlee-python/commit/2378cf84ab1ed06473049a9ddfca2ba6f166306d)) by [@Pijukatel](https://github.com/Pijukatel)
+
+
+## [0.6.4](https://github.com/apify/crawlee-python/releases/tag/v0.6.4) (2025-03-12)
+
+### 🐛 Bug Fixes
+
+- Add a check thread before set `add_signal_handler` ([#1068](https://github.com/apify/crawlee-python/pull/1068)) ([6983bda](https://github.com/apify/crawlee-python/commit/6983bda2dbc202b3ecbf7db62b11deee007b4b5f)) by [@Mantisus](https://github.com/Mantisus)
+- Temporary workaround for `browserforge` import time code execution ([#1073](https://github.com/apify/crawlee-python/pull/1073)) ([17d914f](https://github.com/apify/crawlee-python/commit/17d914f78242078f88c07d686a567d1091255eb1)) by [@Pijukatel](https://github.com/Pijukatel)
+
+
 ## [0.6.3](https://github.com/apify/crawlee-python/releases/tag/v0.6.3) (2025-03-07)
 
 ### 🚀 Features

Makefile

@@ -11,7 +11,6 @@ install-dev:
 	uv sync --all-extras
 	uv run pre-commit install
 	uv run playwright install
-	uv run python -m browserforge update
 
 build:
 	uv build --verbose

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "crawlee"
-version = "0.6.3"
+version = "0.6.5"
 description = "Crawlee for Python"
 authors = [{ name = "Apify Technologies s.r.o.", email = "[email protected]" }]
 license = { file = "LICENSE" }
@@ -33,19 +33,19 @@ keywords = [
     "scraping",
 ]
 dependencies = [
+    "apify_fingerprint_datapoints>=0.0.2",
     "browserforge>=1.2.3",
     "cachetools>=5.5.0",
     "colorama>=0.4.0",
-    "docutils>=0.21.0",
     "eval-type-backport>=0.2.0",
     "httpx[brotli,http2,zstd]>=0.27.0",
     "more-itertools>=10.2.0",
     "psutil>=6.0.0",
     "pydantic-settings>=2.2.0,<2.7.0",
     "pydantic>=2.8.0,!=2.10.0,!=2.10.1,!=2.10.2",
     "pyee>=9.0.0",
-    "rich>=13.9.0",
     "sortedcollections>=2.1.0",
+    "sortedcontainers>=2.4.0",
     "tldextract>=5.1.0",
     "typing-extensions>=4.1.0",
     "yarl>=1.18.0",
@@ -65,7 +65,7 @@ all = [
     "playwright>=1.27.0",
     "scikit-learn==1.5.2; python_version == '3.9'",
     "scikit-learn>=1.6.0; python_version >= '3.10'",
-    'typer>=0.12.0'
+    'typer>=0.12.0',
 ]
 adaptive-crawler = [
     "jaro-winkler>=2.0.3",
@@ -74,7 +74,7 @@ adaptive-crawler = [
     "scikit-learn>=1.6.0; python_version >= '3.10'",
 ]
 beautifulsoup = ["beautifulsoup4[lxml]>=4.12.0", "html5lib>=1.0"]
-cli = ["cookiecutter>=2.6.0", "inquirer>=3.3.0", "typer>=0.12.0"]
+cli = ["cookiecutter>=2.6.0", "inquirer>=3.3.0", "rich>=13.9.0", "typer>=0.12.0"]
 curl-impersonate = ["curl-cffi>=0.9.0"]
 parsel = ["parsel>=1.10.0"]
 playwright = ["playwright>=1.27.0"]
@@ -93,7 +93,7 @@ crawlee = "crawlee._cli:cli"
 [dependency-groups]
 dev = [
     "build~=1.2.0",
-    "filelock~=3.17.0",
+    "filelock~=3.18.0",
     "ipdb~=0.13.0",
     "mypy~=1.15.0",
     "pre-commit~=4.1.0",
@@ -106,7 +106,7 @@ dev = [
     "pytest-xdist~=3.6.0",
     "pytest~=8.3.0",
     "respx~=0.22.0",
-    "ruff~=0.9.0",
+    "ruff~=0.11.0",
     "setuptools~=76.0.0",                    # setuptools are used by pytest, but not explicitly required
     "sortedcontainers-stubs~=2.4.0",
     "types-beautifulsoup4~=4.12.0.20240229",
@@ -226,12 +226,13 @@ warn_unused_ignores = true
 [[tool.mypy.overrides]]
 # Example codes are sometimes showing integration of crawlee with external tool, which is not dependency of crawlee.
 module = [
-    "apify",                # Example code shows integration of apify and crawlee.
-    "camoufox",             # Example code shows integration of camoufox and crawlee.
-    "flask",                # Example code shows deploy on Google Cloud.
-    "functions_framework",  # Example code shows deploy on Google Cloud.
-    "jaro",                 # Untyped and stubs not available
-    "sklearn.linear_model", # Untyped and stubs not available
+    "apify",                        # Example code shows integration of apify and crawlee.
+    "apify_fingerprint_datapoints", # Untyped and stubs not available
+    "camoufox",                     # Example code shows integration of camoufox and crawlee.
+    "flask",                        # Example code shows deploy on Google Cloud.
+    "functions_framework",          # Example code shows deploy on Google Cloud.
+    "jaro",                         # Untyped and stubs not available
+    "sklearn.linear_model",         # Untyped and stubs not available
 ]
 ignore_missing_imports = true
 

src/crawlee/_autoscaling/snapshotter.py

@@ -186,7 +186,7 @@ def get_memory_sample(self, duration: timedelta | None = None) -> list[Snapshot]
         Returns:
             A sample of memory snapshots.
         """
-        snapshots = cast(list[Snapshot], self._memory_snapshots)
+        snapshots = cast('list[Snapshot]', self._memory_snapshots)
         return self._get_sample(snapshots, duration)
 
     @ensure_context
@@ -199,7 +199,7 @@ def get_event_loop_sample(self, duration: timedelta | None = None) -> list[Snaps
         Returns:
             A sample of event loop snapshots.
         """
-        snapshots = cast(list[Snapshot], self._event_loop_snapshots)
+        snapshots = cast('list[Snapshot]', self._event_loop_snapshots)
         return self._get_sample(snapshots, duration)
 
     @ensure_context
@@ -212,7 +212,7 @@ def get_cpu_sample(self, duration: timedelta | None = None) -> list[Snapshot]:
         Returns:
             A sample of CPU snapshots.
         """
-        snapshots = cast(list[Snapshot], self._cpu_snapshots)
+        snapshots = cast('list[Snapshot]', self._cpu_snapshots)
         return self._get_sample(snapshots, duration)
 
     @ensure_context
@@ -225,7 +225,7 @@ def get_client_sample(self, duration: timedelta | None = None) -> list[Snapshot]
         Returns:
             A sample of client snapshots.
         """
-        snapshots = cast(list[Snapshot], self._client_snapshots)
+        snapshots = cast('list[Snapshot]', self._client_snapshots)
         return self._get_sample(snapshots, duration)
 
     @staticmethod
@@ -255,7 +255,7 @@ def _snapshot_cpu(self, event_data: EventSystemInfoData) -> None:
             created_at=event_data.cpu_info.created_at,
         )
 
-        snapshots = cast(list[Snapshot], self._cpu_snapshots)
+        snapshots = cast('list[Snapshot]', self._cpu_snapshots)
         self._prune_snapshots(snapshots, event_data.cpu_info.created_at)
         self._cpu_snapshots.add(snapshot)
 
@@ -275,7 +275,7 @@ def _snapshot_memory(self, event_data: EventSystemInfoData) -> None:
             created_at=event_data.memory_info.created_at,
         )
 
-        snapshots = cast(list[Snapshot], self._memory_snapshots)
+        snapshots = cast('list[Snapshot]', self._memory_snapshots)
         self._prune_snapshots(snapshots, snapshot.created_at)
         self._memory_snapshots.add(snapshot)
         self._evaluate_memory_load(event_data.memory_info.current_size, event_data.memory_info.created_at)
@@ -295,7 +295,7 @@ def _snapshot_event_loop(self) -> None:
             event_loop_delay = snapshot.created_at - previous_snapshot.created_at - self._EVENT_LOOP_SNAPSHOT_INTERVAL
             snapshot.delay = event_loop_delay
 
-        snapshots = cast(list[Snapshot], self._event_loop_snapshots)
+        snapshots = cast('list[Snapshot]', self._event_loop_snapshots)
         self._prune_snapshots(snapshots, snapshot.created_at)
         self._event_loop_snapshots.add(snapshot)
 
@@ -312,7 +312,7 @@ def _snapshot_client(self) -> None:
         error_count = rate_limit_errors.get(self._CLIENT_RATE_LIMIT_ERROR_RETRY_COUNT, 0)
         snapshot = ClientSnapshot(error_count=error_count, max_error_count=self._max_client_errors)
 
-        snapshots = cast(list[Snapshot], self._client_snapshots)
+        snapshots = cast('list[Snapshot]', self._client_snapshots)
         self._prune_snapshots(snapshots, snapshot.created_at)
         self._client_snapshots.add(snapshot)
 

src/crawlee/_browserforge_workaround.py

@@ -0,0 +1,42 @@
+# ruff: noqa: N802
+
+
+def patch_browserforge() -> None:
+    """Patches `browserforge` to use data from `apify_fingerprint_datapoints`.
+
+    This avoids import time or runtime file downloads.
+    """
+    # Temporary fix until https://github.com/daijro/browserforge/pull/29 is merged
+    from pathlib import Path
+
+    import apify_fingerprint_datapoints
+    from browserforge import download
+
+    download.DATA_DIRS: dict[str, Path] = {  # type:ignore[misc]
+        'headers': apify_fingerprint_datapoints.get_header_network().parent,
+        'fingerprints': apify_fingerprint_datapoints.get_fingerprint_network().parent,
+    }
+
+    def DownloadIfNotExists(**flags: bool) -> None:
+        pass
+
+    download.DownloadIfNotExists = DownloadIfNotExists
+
+    import browserforge.bayesian_network
+
+    class BayesianNetwork(browserforge.bayesian_network.BayesianNetwork):
+        def __init__(self, path: Path) -> None:
+            """Inverted mapping as browserforge expects somewhat renamed file names."""
+            if path.name in download.DATA_FILES['headers']:
+                path = download.DATA_DIRS['headers'] / download.DATA_FILES['headers'][path.name]
+            else:
+                path = download.DATA_DIRS['fingerprints'] / download.DATA_FILES['fingerprints'][path.name]
+            super().__init__(path)
+
+    browserforge.bayesian_network.BayesianNetwork = BayesianNetwork  # type:ignore[misc]
+    import browserforge.headers.generator
+
+    browserforge.headers.generator.DATA_DIR = download.DATA_DIRS['headers']
+    import browserforge.fingerprints.generator
+
+    browserforge.headers.generator.DATA_DIR = download.DATA_DIRS['fingerprints']

src/crawlee/_cli.py

@@ -78,7 +78,7 @@ def _prompt_for_project_name(initial_project_name: str | None) -> str:
 
 def _prompt_text(message: str, default: str) -> str:
     return cast(
-        str,
+        'str',
         ConsoleRender().render(
             inquirer.Text(
                 name='text',
@@ -93,7 +93,7 @@ def _prompt_text(message: str, default: str) -> str:
 def _prompt_choice(message: str, choices: list[str]) -> str:
     """Prompt the user to pick one from a list of choices."""
     return cast(
-        str,
+        'str',
         ConsoleRender().render(
             inquirer.List(
                 name='choice',
@@ -106,7 +106,7 @@ def _prompt_choice(message: str, choices: list[str]) -> str:
 
 def _prompt_bool(message: str, *, default: bool) -> bool:
     return cast(
-        bool,
+        'bool',
         ConsoleRender().render(
             inquirer.Confirm(
                 name='confirm',

src/crawlee/_request.py

@@ -306,12 +306,12 @@ def get_query_param_from_url(self, param: str, *, default: str | None = None) ->
     @property
     def label(self) -> str | None:
         """A string used to differentiate between arbitrary request types."""
-        return cast(UserData, self.user_data).label
+        return cast('UserData', self.user_data).label
 
     @property
     def crawlee_data(self) -> CrawleeRequestData:
         """Crawlee-specific configuration stored in the `user_data`."""
-        user_data = cast(UserData, self.user_data)
+        user_data = cast('UserData', self.user_data)
         if user_data.crawlee_data is None:
             user_data.crawlee_data = CrawleeRequestData()
 

src/crawlee/_types.py

@@ -244,7 +244,7 @@ async def get_value(self, key: str, default_value: T | None = None) -> T | None:
 
     async def get_value(self, key: str, default_value: T | None = None) -> T | None:
         if key in self.updates:
-            return cast(T, self.updates[key].content)
+            return cast('T', self.updates[key].content)
 
         return await self._actual_key_value_store.get_value(key, default_value)
 

src/crawlee/_utils/console.py

@@ -0,0 +1,67 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+BORDER = {'TL': '┌', 'TR': '┐', 'BL': '└', 'BR': '┘', 'H': '─', 'V': '│', 'TM': '┬', 'BM': '┴'}
+
+
+def make_table(rows: Sequence[Sequence[str]], width: int = 100) -> str:
+    """Creates a text table using Unicode characters.
+
+    Args:
+        rows: A list of tuples/lists to be displayed in the table.
+        width: Maximum width of the table.
+    """
+    if not rows:
+        return ''
+
+    num_cols = max(len(row) for row in rows)
+
+    if num_cols == 0:
+        return ''
+
+    # Normalize the row size by filling missing columns with empty values
+    normalized_rows = [list(row) + [''] * (num_cols - len(row)) for row in rows]
+    col_widths = [max(len(str(row[i])) for row in normalized_rows) for i in range(num_cols)]
+    total_width = sum(col_widths) + (3 * num_cols) + 1
+
+    # If the table size is larger than `width`, set all columns to the same length
+    col_widths = col_widths if total_width <= width else [max(3, (width - (3 * num_cols) - 1) // num_cols)] * num_cols
+
+    # Initialize borders
+    top_parts, bottom_parts = [BORDER['TL']], [BORDER['BL']]
+
+    for i in range(num_cols):
+        h_border = BORDER['H'] * (col_widths[i] + 2)
+        top_parts.append(h_border)
+        bottom_parts.append(h_border)
+
+        if i < num_cols - 1:
+            top_parts.append(BORDER['TM'])
+            bottom_parts.append(BORDER['BM'])
+        else:
+            top_parts.append(BORDER['TR'])
+            bottom_parts.append(BORDER['BR'])
+
+    top_border, bottom_border = ''.join(top_parts), ''.join(bottom_parts)
+
+    result = [top_border]
+
+    for row in normalized_rows:
+        cells = []
+
+        for i, cell in enumerate(row):
+            # Trim the content if the length exceeds the widths of the column
+            norm_cell = f'{cell[: col_widths[i] - 3]}...' if len(cell) > col_widths[i] else cell.ljust(col_widths[i])
+            cells.append(norm_cell)
+
+        # row: │ cell1 │ cell2 │ ...
+        row_str = BORDER['V'] + ''.join(f' {cell} {BORDER["V"]}' for cell in cells)
+        result.append(row_str)
+
+    result.append(bottom_border)
+
+    return '\n'.join(result)

src/crawlee/_utils/models.py

@@ -14,7 +14,7 @@ def _timedelta_to_ms(td: timedelta | None) -> float | None:
         return float('inf')
     if td is None:
         return td
-    return int(round(td.total_seconds() * 1000))
+    return round(td.total_seconds() * 1000)
 
 
 def _timedelta_to_secs(td: timedelta | None) -> float | None:

src/crawlee/browsers/__init__.py

@@ -1,8 +1,15 @@
+# ruff: noqa: E402, TID252
+
 from crawlee._utils.try_import import install_import_hook as _install_import_hook
 from crawlee._utils.try_import import try_import as _try_import
 
 _install_import_hook(__name__)
 
+# Due to patch_browserforge
+from .._browserforge_workaround import patch_browserforge
+
+patch_browserforge()
+
 # The following imports are wrapped in try_import to handle optional dependencies,
 # ensuring the module can still function even if these dependencies are missing.
 with _try_import(__name__, 'BrowserPool'):

src/crawlee/browsers/_playwright_browser_controller.py

@@ -11,13 +11,13 @@
 
 from crawlee._utils.docs import docs_group
 from crawlee.browsers._browser_controller import BrowserController
-from crawlee.browsers._types import BrowserType
 from crawlee.fingerprint_suite import HeaderGenerator
 
 if TYPE_CHECKING:
     from collections.abc import Mapping
 
     from crawlee.browsers._playwright_browser import PlaywrightPersistentBrowser
+    from crawlee.browsers._types import BrowserType
     from crawlee.fingerprint_suite import FingerprintGenerator
     from crawlee.proxy_configuration import ProxyInfo
 
@@ -107,7 +107,7 @@ def is_browser_connected(self) -> bool:
     @property
     @override
     def browser_type(self) -> BrowserType:
-        return cast(BrowserType, self._browser.browser_type.name)
+        return cast('BrowserType', self._browser.browser_type.name)
 
     @override
     async def new_page(