Replace scrapy.utils.defer.deferred_from_coro (Fix crawl getting stuck on Windows with Scrapy>=2.13)

pylintrc

@@ -20,3 +20,6 @@ disable=
 [FORMAT]
 expected-line-ending-format=LF
 max-line-length=99
+
+[MASTER]
+ignored-classes=twisted.internet.reactor

scrapy_playwright/_utils.py

@@ -10,6 +10,7 @@
 from scrapy.settings import Settings
 from scrapy.utils.python import to_unicode
 from twisted.internet.defer import Deferred
+from twisted.python import failure
 from w3lib.encoding import html_body_declared_encoding, http_content_type_encoding
 
 
@@ -115,24 +116,28 @@ class _ThreadedLoopAdapter:
     _stop_events: Dict[int, asyncio.Event] = {}
 
     @classmethod
-    async def _handle_coro(cls, coro, future) -> None:
+    async def _handle_coro(cls, coro: Awaitable, dfd: Deferred) -> None:
+        from twisted.internet import reactor
+
         try:
-            future.set_result(await coro)
+            result = await coro
         except Exception as exc:
-            future.set_exception(exc)
+            reactor.callFromThread(dfd.errback, failure.Failure(exc))
+        else:
+            reactor.callFromThread(dfd.callback, result)
 
     @classmethod
     async def _process_queue(cls) -> None:
         while any(not ev.is_set() for ev in cls._stop_events.values()):
-            coro, future = await cls._coro_queue.get()
-            asyncio.create_task(cls._handle_coro(coro, future))
+            coro, dfd = await cls._coro_queue.get()
+            asyncio.create_task(cls._handle_coro(coro, dfd))
             cls._coro_queue.task_done()
 
     @classmethod
     def _deferred_from_coro(cls, coro) -> Deferred:
-        future: asyncio.Future = asyncio.Future()
-        asyncio.run_coroutine_threadsafe(cls._coro_queue.put((coro, future)), cls._loop)
-        return scrapy.utils.defer.deferred_from_coro(future)
+        dfd: Deferred = Deferred()
+        asyncio.run_coroutine_threadsafe(cls._coro_queue.put((coro, dfd)), cls._loop)
+        return dfd
 
     @classmethod
     def start(cls, caller_id: int) -> None: