Update KVS and its clients

Author: vdusek

Diff for: src/crawlee/_utils/file.py

@@ -2,7 +2,6 @@
 
 import asyncio
 import contextlib
-import io
 import json
 import mimetypes
 import os
@@ -83,21 +82,6 @@ def determine_file_extension(content_type: str) -> str | None:
     return ext[1:] if ext is not None else ext
 
 
-def is_file_or_bytes(value: Any) -> bool:
-    """Determine if the input value is a file-like object or bytes.
-
-    This function checks whether the provided value is an instance of bytes, bytearray, or io.IOBase (file-like).
-    The method is simplified for common use cases and may not cover all edge cases.
-
-    Args:
-        value: The value to be checked.
-
-    Returns:
-        True if the value is either a file-like object or bytes, False otherwise.
-    """
-    return isinstance(value, (bytes, bytearray, io.IOBase))
-
-
 async def json_dumps(obj: Any) -> str:
     """Serialize an object to a JSON-formatted string with specific settings.
 

Diff for: src/crawlee/storage_clients/_base/_dataset_client.py

@@ -56,6 +56,7 @@ def item_count(self) -> int:
     @abstractmethod
     async def open(
         cls,
+        *,
         id: str | None,
         name: str | None,
         storage_dir: Path,
@@ -82,7 +83,7 @@ async def drop(self) -> None:
         """
 
     @abstractmethod
-    async def push_data(self, data: list[Any] | dict[str, Any]) -> None:
+    async def push_data(self, *, data: list[Any] | dict[str, Any]) -> None:
         """Push data to the dataset.
 
         The backend method for the `Dataset.push_data` call.

Diff for: src/crawlee/storage_clients/_base/_key_value_store_client.py

@@ -6,111 +6,136 @@
 from crawlee._utils.docs import docs_group
 
 if TYPE_CHECKING:
-    from contextlib import AbstractAsyncContextManager
-
-    from httpx import Response
-
-    from crawlee.storage_clients.models import KeyValueStoreListKeysPage, KeyValueStoreMetadata, KeyValueStoreRecord
+    from collections.abc import AsyncIterator
+    from datetime import datetime
+    from pathlib import Path
+
+    from crawlee.storage_clients.models import (
+        KeyValueStoreRecord,
+        KeyValueStoreRecordMetadata,
+    )
+
+# Properties:
+# - id
+# - name
+# - created_at
+# - accessed_at
+# - modified_at
+
+# Methods:
+# - open
+# - drop
+# - get_value
+# - set_value
+# - delete_value
+# - iterate_keys
+# - get_public_url
 
 
 @docs_group('Abstract classes')
 class KeyValueStoreClient(ABC):
-    """An abstract class for key-value store resource clients.
+    """An abstract class for key-value store (KVS) resource clients.
 
     These clients are specific to the type of resource they manage and operate under a designated storage
     client, like a memory storage client.
     """
 
+    @property
     @abstractmethod
-    async def get(self) -> KeyValueStoreMetadata | None:
-        """Get metadata about the key-value store being managed by this client.
+    def id(self) -> str:
+        """The ID of the key-value store."""
 
-        Returns:
-            An object containing the key-value store's details, or None if the key-value store does not exist.
-        """
+    @property
+    @abstractmethod
+    def name(self) -> str | None:
+        """The name of the key-value store."""
 
+    @property
     @abstractmethod
-    async def delete(self) -> None:
-        """Permanently delete the key-value store managed by this client."""
+    def created_at(self) -> datetime:
+        """The time at which the key-value store was created."""
 
+    @property
     @abstractmethod
-    async def list_keys(
-        self,
+    def accessed_at(self) -> datetime:
+        """The time at which the key-value store was last accessed."""
+
+    @property
+    @abstractmethod
+    def modified_at(self) -> datetime:
+        """The time at which the key-value store was last modified."""
+
+    @classmethod
+    @abstractmethod
+    async def open(
+        cls,
         *,
-        limit: int = 1000,
-        exclusive_start_key: str | None = None,
-    ) -> KeyValueStoreListKeysPage:
-        """List the keys in the key-value store.
+        id: str | None,
+        name: str | None,
+        storage_dir: Path,
+    ) -> KeyValueStoreClient:
+        """Open existing or create a new key-value store client.
+
+        If a key-value store with the given name already exists, the appropriate key-value store client is returned.
+        Otherwise, a new key-value store is created and client for it is returned.
 
         Args:
-            limit: Number of keys to be returned. Maximum value is 1000.
-            exclusive_start_key: All keys up to this one (including) are skipped from the result.
+            id: The ID of the key-value store.
+            name: The name of the key-value store.
+            storage_dir: The path to the storage directory. If the client persists data, it should use this directory.
 
         Returns:
-            The list of keys in the key-value store matching the given arguments.
+            A key-value store client.
         """
 
     @abstractmethod
-    async def get_record(self, key: str) -> KeyValueStoreRecord | None:
-        """Retrieve the given record from the key-value store.
-
-        Args:
-            key: Key of the record to retrieve.
+    async def drop(self) -> None:
+        """Drop the whole key-value store and remove all its values.
 
-        Returns:
-            The requested record, or None, if the record does not exist
+        The backend method for the `KeyValueStore.drop` call.
         """
 
     @abstractmethod
-    async def get_record_as_bytes(self, key: str) -> KeyValueStoreRecord[bytes] | None:
-        """Retrieve the given record from the key-value store, without parsing it.
-
-        Args:
-            key: Key of the record to retrieve.
+    async def get_value(self, *, key: str) -> KeyValueStoreRecord | None:
+        """Retrieve the given record from the key-value store.
 
-        Returns:
-            The requested record, or None, if the record does not exist
+        The backend method for the `KeyValueStore.get_value` call.
         """
 
     @abstractmethod
-    async def stream_record(self, key: str) -> AbstractAsyncContextManager[KeyValueStoreRecord[Response] | None]:
-        """Retrieve the given record from the key-value store, as a stream.
+    async def set_value(self, *, key: str, value: Any, content_type: str | None = None) -> None:
+        """Set a value in the key-value store by its key.
 
-        Args:
-            key: Key of the record to retrieve.
-
-        Returns:
-            The requested record as a context-managed streaming Response, or None, if the record does not exist
+        The backend method for the `KeyValueStore.set_value` call.
         """
 
     @abstractmethod
-    async def set_record(self, key: str, value: Any, content_type: str | None = None) -> None:
-        """Set a value to the given record in the key-value store.
+    async def delete_value(self, *, key: str) -> None:
+        """Delete a value from the key-value store by its key.
 
-        Args:
-            key: The key of the record to save the value to.
-            value: The value to save into the record.
-            content_type: The content type of the saved value.
+        The backend method for the `KeyValueStore.delete_value` call.
         """
 
     @abstractmethod
-    async def delete_record(self, key: str) -> None:
-        """Delete the specified record from the key-value store.
+    async def iterate_keys(
+        self,
+        *,
+        exclusive_start_key: str | None = None,
+        limit: int | None = None,
+    ) -> AsyncIterator[KeyValueStoreRecordMetadata]:
+        """Iterate over all the existing keys in the key-value store.
 
-        Args:
-            key: The key of the record which to delete.
+        The backend method for the `KeyValueStore.iterate_keys` call.
         """
+        # This syntax is to make mypy properly work with abstract AsyncIterator.
+        # https://mypy.readthedocs.io/en/stable/more_types.html#asynchronous-iterators
+        raise NotImplementedError
+        if False:  # type: ignore[unreachable]
+            yield 0
 
     @abstractmethod
-    async def get_public_url(self, key: str) -> str:
+    async def get_public_url(self, *, key: str) -> str:
         """Get the public URL for the given key.
 
-        Args:
-            key: Key of the record for which URL is required.
-
-        Returns:
-            The public URL for the given key.
-
-        Raises:
-            ValueError: If the key does not exist.
+        The backend method for the `KeyValueStore.get_public_url` call.
         """

Diff for: src/crawlee/storage_clients/_file_system/_dataset_client.py

@@ -10,12 +10,11 @@
 from pydantic import ValidationError
 from typing_extensions import override
 
-from crawlee._consts import METADATA_FILENAME
 from crawlee._utils.crypto import crypto_random_object_id
 from crawlee.storage_clients._base import DatasetClient
 from crawlee.storage_clients.models import DatasetItemsListPage, DatasetMetadata
 
-from ._utils import json_dumps
+from ._utils import METADATA_FILENAME, json_dumps
 
 if TYPE_CHECKING:
     from collections.abc import AsyncIterator
@@ -129,6 +128,11 @@ async def open(
         Returns:
            A new instance of the file system dataset client.
         """
+        if id:
+            raise ValueError(
+                'Opening a dataset by "id" is not supported for file system storage client, use "name" instead.'
+            )
+
         name = name or cls._DEFAULT_NAME
         dataset_path = storage_dir / cls._STORAGE_SUBDIR / name
         metadata_path = dataset_path / METADATA_FILENAME