|
| 1 | +from __future__ import annotations |
| 2 | + |
| 3 | +import asyncio |
| 4 | +from logging import getLogger |
| 5 | +from typing import TYPE_CHECKING, Any, ClassVar |
| 6 | + |
| 7 | +from apify_client import ApifyClientAsync |
| 8 | +from typing_extensions import override |
| 9 | + |
| 10 | +from crawlee.storage_clients._base import DatasetClient |
| 11 | +from crawlee.storage_clients.models import DatasetItemsListPage, DatasetMetadata |
| 12 | + |
| 13 | +if TYPE_CHECKING: |
| 14 | + from collections.abc import AsyncIterator |
| 15 | + from datetime import datetime |
| 16 | + |
| 17 | + from apify_client.clients import DatasetClientAsync |
| 18 | + |
| 19 | + from crawlee.configuration import Configuration |
| 20 | + |
| 21 | +logger = getLogger(__name__) |
| 22 | + |
| 23 | + |
| 24 | +class ApifyDatasetClient(DatasetClient): |
| 25 | + """An Apify platform implementation of the dataset client.""" |
| 26 | + |
| 27 | + _cache_by_name: ClassVar[dict[str, ApifyDatasetClient]] = {} |
| 28 | + """A dictionary to cache clients by their names.""" |
| 29 | + |
| 30 | + def __init__( |
| 31 | + self, |
| 32 | + *, |
| 33 | + id: str, |
| 34 | + name: str, |
| 35 | + created_at: datetime, |
| 36 | + accessed_at: datetime, |
| 37 | + modified_at: datetime, |
| 38 | + item_count: int, |
| 39 | + api_client: DatasetClientAsync, |
| 40 | + ) -> None: |
| 41 | + """Initialize a new instance. |
| 42 | +
|
| 43 | + Preferably use the `ApifyDatasetClient.open` class method to create a new instance. |
| 44 | + """ |
| 45 | + self._metadata = DatasetMetadata( |
| 46 | + id=id, |
| 47 | + name=name, |
| 48 | + created_at=created_at, |
| 49 | + accessed_at=accessed_at, |
| 50 | + modified_at=modified_at, |
| 51 | + item_count=item_count, |
| 52 | + ) |
| 53 | + |
| 54 | + self._api_client = api_client |
| 55 | + """The Apify dataset client for API operations.""" |
| 56 | + |
| 57 | + self._lock = asyncio.Lock() |
| 58 | + """A lock to ensure that only one operation is performed at a time.""" |
| 59 | + |
| 60 | + @override |
| 61 | + @property |
| 62 | + def metadata(self) -> DatasetMetadata: |
| 63 | + return self._metadata |
| 64 | + |
| 65 | + @override |
| 66 | + @classmethod |
| 67 | + async def open( |
| 68 | + cls, |
| 69 | + *, |
| 70 | + id: str | None, |
| 71 | + name: str | None, |
| 72 | + configuration: Configuration, |
| 73 | + ) -> ApifyDatasetClient: |
| 74 | + default_name = configuration.default_dataset_id |
| 75 | + token = 'configuration.apify_token' # TODO: use the real value |
| 76 | + api_url = 'configuration.apify_api_url' # TODO: use the real value |
| 77 | + |
| 78 | + name = name or default_name |
| 79 | + |
| 80 | + # Check if the client is already cached by name. |
| 81 | + if name in cls._cache_by_name: |
| 82 | + client = cls._cache_by_name[name] |
| 83 | + await client._update_metadata() # noqa: SLF001 |
| 84 | + return client |
| 85 | + |
| 86 | + # Otherwise, create a new one. |
| 87 | + apify_client_async = ApifyClientAsync( |
| 88 | + token=token, |
| 89 | + api_url=api_url, |
| 90 | + max_retries=8, |
| 91 | + min_delay_between_retries_millis=500, |
| 92 | + timeout_secs=360, |
| 93 | + ) |
| 94 | + |
| 95 | + apify_datasets_client = apify_client_async.datasets() |
| 96 | + |
| 97 | + metadata = DatasetMetadata.model_validate( |
| 98 | + await apify_datasets_client.get_or_create(name=id if id is not None else name), |
| 99 | + ) |
| 100 | + |
| 101 | + apify_dataset_client = apify_client_async.dataset(dataset_id=metadata.id) |
| 102 | + |
| 103 | + client = cls( |
| 104 | + id=metadata.id, |
| 105 | + name=metadata.name, |
| 106 | + created_at=metadata.created_at, |
| 107 | + accessed_at=metadata.accessed_at, |
| 108 | + modified_at=metadata.modified_at, |
| 109 | + item_count=metadata.item_count, |
| 110 | + api_client=apify_dataset_client, |
| 111 | + ) |
| 112 | + |
| 113 | + # Cache the client by name. |
| 114 | + cls._cache_by_name[name] = client |
| 115 | + |
| 116 | + return client |
| 117 | + |
| 118 | + @override |
| 119 | + async def drop(self) -> None: |
| 120 | + async with self._lock: |
| 121 | + await self._api_client.delete() |
| 122 | + |
| 123 | + # Remove the client from the cache. |
| 124 | + if self.metadata.name in self.__class__._cache_by_name: # noqa: SLF001 |
| 125 | + del self.__class__._cache_by_name[self.metadata.name] # noqa: SLF001 |
| 126 | + |
| 127 | + @override |
| 128 | + async def push_data(self, data: list[Any] | dict[str, Any]) -> None: |
| 129 | + async with self._lock: |
| 130 | + await self._api_client.push_items(items=data) |
| 131 | + await self._update_metadata() |
| 132 | + |
| 133 | + @override |
| 134 | + async def get_data( |
| 135 | + self, |
| 136 | + *, |
| 137 | + offset: int = 0, |
| 138 | + limit: int | None = 999_999_999_999, |
| 139 | + clean: bool = False, |
| 140 | + desc: bool = False, |
| 141 | + fields: list[str] | None = None, |
| 142 | + omit: list[str] | None = None, |
| 143 | + unwind: str | None = None, |
| 144 | + skip_empty: bool = False, |
| 145 | + skip_hidden: bool = False, |
| 146 | + flatten: list[str] | None = None, |
| 147 | + view: str | None = None, |
| 148 | + ) -> DatasetItemsListPage: |
| 149 | + response = await self._api_client.list_items( |
| 150 | + offset=offset, |
| 151 | + limit=limit, |
| 152 | + clean=clean, |
| 153 | + desc=desc, |
| 154 | + fields=fields, |
| 155 | + omit=omit, |
| 156 | + unwind=unwind, |
| 157 | + skip_empty=skip_empty, |
| 158 | + skip_hidden=skip_hidden, |
| 159 | + flatten=flatten, |
| 160 | + view=view, |
| 161 | + ) |
| 162 | + result = DatasetItemsListPage.model_validate(vars(response)) |
| 163 | + await self._update_metadata() |
| 164 | + return result |
| 165 | + |
| 166 | + @override |
| 167 | + async def iterate_items( |
| 168 | + self, |
| 169 | + *, |
| 170 | + offset: int = 0, |
| 171 | + limit: int | None = None, |
| 172 | + clean: bool = False, |
| 173 | + desc: bool = False, |
| 174 | + fields: list[str] | None = None, |
| 175 | + omit: list[str] | None = None, |
| 176 | + unwind: str | None = None, |
| 177 | + skip_empty: bool = False, |
| 178 | + skip_hidden: bool = False, |
| 179 | + ) -> AsyncIterator[dict]: |
| 180 | + async for item in self._api_client.iterate_items( |
| 181 | + offset=offset, |
| 182 | + limit=limit, |
| 183 | + clean=clean, |
| 184 | + desc=desc, |
| 185 | + fields=fields, |
| 186 | + omit=omit, |
| 187 | + unwind=unwind, |
| 188 | + skip_empty=skip_empty, |
| 189 | + skip_hidden=skip_hidden, |
| 190 | + ): |
| 191 | + yield item |
| 192 | + |
| 193 | + await self._update_metadata() |
| 194 | + |
| 195 | + async def _update_metadata(self) -> None: |
| 196 | + """Update the dataset metadata file with current information.""" |
| 197 | + metadata = await self._api_client.get() |
| 198 | + self._metadata = DatasetMetadata.model_validate(metadata) |
0 commit comments