[Security] Remove mandatory ai-cloudscraper dependency

README.md

@@ -72,6 +72,9 @@
 # PYPI
 pip install --upgrade translators
 
+# Include asynchronous translation support
+pip install --upgrade 'translators[async]'
+
 # Conda (Not recommended)
 conda install conda-forge::translators
 
@@ -96,7 +99,7 @@ print(ts.translators_pool)
 print(ts.translate_text(q_text))
 print(ts.translate_html(q_html, translator='alibaba'))
 
-# async
+# async (requires translators[async])
 import asyncio
 print(asyncio.run(ts.translate_text(q_text, http_client='aiohttp', if_use_async=True)))
 

requirements.txt

@@ -6,7 +6,7 @@ httpx
 aiohttp
 requests
 niquests
-ai-cloudscraper
+cloudscraper
 cryptography
 brotlicffi
 brotli

setup.py

@@ -67,14 +67,16 @@
         'lxml>=5.4.0',
         'tqdm>=4.67.1',
         'pathos>=0.3.4',
-        # 'cloudscraper>=1.2.71',
-        'ai-cloudscraper>=3.7.6',
+        'cloudscraper>=1.2.71',
         'cryptography>=42.0.4',
         'brotlicffi>=1.2.0.0',
         'brotli>=1.2.0',
     ],
     python_requires='>=3.8',
-    extras_require={'pypi': ['build>=1.4.0', 'twine>=6.2.0', 'setuptools>=75.3.0']},
+    extras_require={
+        'async': ['aiohttp>=3.10.11'],
+        'pypi': ['build>=1.4.0', 'twine>=6.2.0', 'setuptools>=75.3.0'],
+    },
     zip_safe=False,
     entry_points={
         'console_scripts':

test/test_security_dependency.py

@@ -0,0 +1,90 @@
+import inspect
+import subprocess
+import sys
+import unittest
+
+import aiohttp
+import httpx
+
+import translators as ts
+from translators.server_async import Reverso, TranslatorError, Tse, _resolve_http_client
+
+
+class PackageLoadingTest(unittest.TestCase):
+    def test_sync_import_does_not_load_async_module(self):
+        result = subprocess.run(
+            [
+                sys.executable,
+                '-c',
+                "import sys; import translators; assert 'translators.server_async' not in sys.modules",
+            ],
+            check=False,
+            capture_output=True,
+            text=True,
+        )
+        self.assertEqual(result.returncode, 0, result.stderr)
+
+    def test_async_compatibility_names_are_coroutine_functions(self):
+        self.assertTrue(inspect.iscoroutinefunction(ts.translate_text_with_async))
+        self.assertTrue(inspect.iscoroutinefunction(ts.translate_html_with_async))
+        self.assertTrue(inspect.iscoroutinefunction(ts.preaccelerate_and_speedtest_with_async))
+        self.assertTrue(inspect.iscoroutinefunction(ts.close_with_async))
+
+
+class AsyncClientTest(unittest.IsolatedAsyncioTestCase):
+    async def test_session_type_selects_http_client(self):
+        aiohttp_session = aiohttp.ClientSession()
+        httpx_session = httpx.AsyncClient()
+        try:
+            self.assertEqual(_resolve_http_client(None, None), 'aiohttp')
+            self.assertEqual(_resolve_http_client(None, 'httpx'), 'httpx')
+            self.assertEqual(_resolve_http_client(aiohttp_session, None), 'aiohttp')
+            self.assertEqual(_resolve_http_client(httpx_session, None), 'httpx')
+            with self.assertRaisesRegex(TranslatorError, 'does not match'):
+                _resolve_http_client(aiohttp_session, 'httpx')
+            with self.assertRaisesRegex(TranslatorError, 'does not match'):
+                _resolve_http_client(httpx_session, 'aiohttp')
+        finally:
+            await aiohttp_session.close()
+            await httpx_session.aclose()
+
+    async def test_aiohttp_proxy_support_is_checked_before_session_creation(self):
+        proxy_url = 'http://127.0.0.1:9'
+        if 'proxy' not in inspect.signature(aiohttp.ClientSession).parameters:
+            with self.assertRaisesRegex(TranslatorError, 'use httpx'):
+                Tse.get_client_session('aiohttp', {'https': proxy_url})
+            return
+
+        session = Tse.get_client_session('aiohttp', {'https': proxy_url})
+        await session.close()
+
+    async def test_session_factories_close_cleanly(self):
+        aiohttp_session = Tse.get_client_session('aiohttp')
+        httpx_session = Tse.get_client_session('httpx', {'https': 'http://127.0.0.1:9'})
+        await aiohttp_session.close()
+        await httpx_session.aclose()
+
+    async def test_reverso_rejects_switching_a_live_cached_session(self):
+        sessions = (
+            (httpx.AsyncClient(), 'aiohttp'),
+            (aiohttp.ClientSession(), 'httpx'),
+        )
+        for session, requested_http_client in sessions:
+            reverso = Reverso()
+            reverso.session = session
+            try:
+                with self.assertRaisesRegex(TranslatorError, 'does not match'):
+                    await reverso.reverso_api(
+                        'hello',
+                        http_client=requested_http_client,
+                        if_print_warning=False,
+                    )
+            finally:
+                if isinstance(session, aiohttp.ClientSession):
+                    await session.close()
+                else:
+                    await session.aclose()
+
+
+if __name__ == '__main__':
+    unittest.main()

translators/__init__.py

@@ -1,6 +1,8 @@
 __version__ = "6.0.4"
 __author__ = "UlionTse"
 
+from importlib import import_module
+
 from translators.server import (
     translators_pool,
     get_languages,
@@ -11,12 +13,31 @@
     translate_html as translate_html_with_sync,
     preaccelerate_and_speedtest as preaccelerate_and_speedtest_with_sync,
 )
-from translators.server_async import (
-    translate_text as translate_text_with_async,
-    translate_html as translate_html_with_async,
-    preaccelerate_and_speedtest as preaccelerate_and_speedtest_with_async,
-    close as close_with_async,
-)
+
+
+def _get_async_function(function_name):
+    try:
+        server_async = import_module('translators.server_async')
+    except ModuleNotFoundError as exc:
+        if exc.name == 'aiohttp':
+            raise ImportError(
+                "Async translation requires the 'async' extra: "
+                "pip install 'translators[async]'"
+            ) from exc
+        raise
+    return getattr(server_async, function_name)
+
+
+async def translate_text_with_async(*args, **kwargs):
+    return await _get_async_function('translate_text')(*args, **kwargs)
+
+
+async def translate_html_with_async(*args, **kwargs):
+    return await _get_async_function('translate_html')(*args, **kwargs)
+
+
+async def preaccelerate_and_speedtest_with_async(*args, **kwargs):
+    return await _get_async_function('preaccelerate_and_speedtest')(*args, **kwargs)
 
 
 def translate_text(*args, **kwargs):
@@ -37,6 +58,10 @@ def preaccelerate_and_speedtest(*args, **kwargs):
     return preaccelerate_and_speedtest_with_sync(*args, **kwargs)
 
 
+async def close_with_async(*args, **kwargs):
+    return await _get_async_function('close')(*args, **kwargs)
+
+
 __all__ = (
     "__version__",
     "__author__",

translators/server_async.py

@@ -39,6 +39,7 @@
 import datetime
 import warnings
 import functools
+import inspect
 import urllib.parse
 from typing import Optional, Union, Tuple, List
 
@@ -47,7 +48,6 @@
 import httpx
 import aiohttp
 import requests
-import cloudscraper
 import lxml.etree as lxml_etree
 import cryptography.hazmat.primitives.ciphers as cry_ciphers
 import cryptography.hazmat.primitives.padding as cry_padding
@@ -94,6 +94,26 @@ class TranslatorError(Exception):
     pass
 
 
+def _resolve_http_client(
+    session: Optional[SessionType],
+    requested_http_client: Optional[str],
+    default: str = 'aiohttp',
+) -> str:
+    if session is None:
+        return requested_http_client or default
+
+    if isinstance(session, aiohttp.ClientSession):
+        session_http_client = 'aiohttp'
+    elif isinstance(session, httpx.AsyncClient):
+        session_http_client = 'httpx'
+    else:
+        raise TranslatorError('session must be aiohttp.ClientSession or httpx.AsyncClient.')
+
+    if requested_http_client and requested_http_client != session_http_client:
+        raise TranslatorError('http_client does not match the provided session type.')
+    return session_http_client
+
+
 class Tse:
     def __init__(self):
         self.author = 'UlionTse'
@@ -327,13 +347,19 @@ def get_client_session(http_client: str = 'aiohttp', proxies: Optional[dict] = N
 
         if http_client == 'aiohttp':
             proxy_url = proxies.get('http') or proxies.get('https')
-            session = aiohttp.ClientSession(
-                trust_env=True,
-                proxy=proxy_url,
-                requote_redirect_url=True,
-                raise_for_status=True,
-                connector=aiohttp.TCPConnector(force_close=True, enable_cleanup_closed=True),
-            )
+            if proxy_url and 'proxy' not in inspect.signature(aiohttp.ClientSession).parameters:
+                raise TranslatorError(
+                    'This aiohttp version does not support session-level proxies; use httpx instead.'
+                )
+            session_kwargs = {
+                'trust_env': True,
+                'requote_redirect_url': True,
+                'raise_for_status': True,
+                'connector': aiohttp.TCPConnector(force_close=True, enable_cleanup_closed=True),
+            }
+            if proxy_url:
+                session_kwargs['proxy'] = proxy_url
+            session = aiohttp.ClientSession(**session_kwargs)
 
         else:
             proxy_url = proxies.get('http') or proxies.get('https')
@@ -3174,7 +3200,7 @@ async def reverso_api(self, query_text: str, from_language: str = 'auto', to_lan
                 :param proxies: Optional[dict], default None.
                 :param sleep_seconds: float, default 0.
                 :param is_detail_result: bool, default False.
-                :param http_client: str, default 'aiohttp'.
+                :param http_client: str, default 'aiohttp'. Union['aiohttp', 'httpx']
                 :param if_ignore_limit_of_length: bool, default False.
                 :param limit_of_length: int, default 20000.
                 :param if_ignore_empty_query: bool, default False.
@@ -3189,7 +3215,15 @@ async def reverso_api(self, query_text: str, from_language: str = 'auto', to_lan
         timeout = kwargs.get('timeout', None)
         proxies = kwargs.get('proxies', None)
         sleep_seconds = kwargs.get('sleep_seconds', 0)
-        http_client = 'aiohttp'  # kwargs.get('http_client', 'aiohttp')  # ai-cloudscraper's aiohttp
+        provided_session = kwargs.get('session', None)
+        requested_http_client = kwargs.get('http_client', None)
+        if provided_session is not None and not Tse.if_session_exists(provided_session):
+            raise TranslatorError('The provided session is already closed.')
+        cached_session = self.session if Tse.if_session_exists(self.session) else None
+        if provided_session is not None and cached_session is not None and provided_session is not cached_session:
+            raise TranslatorError('Close the existing Reverso session before replacing it.')
+        active_session = provided_session or cached_session
+        http_client = _resolve_http_client(active_session, requested_http_client)
         if_print_warning = kwargs.get('if_print_warning', True)
         is_detail_result = kwargs.get('is_detail_result', False)
         update_session_after_freq = kwargs.get('update_session_after_freq', self.default_session_freq)
@@ -3200,9 +3234,7 @@ async def reverso_api(self, query_text: str, from_language: str = 'auto', to_lan
         not_update_cond_time = 1 if time.time() - self.begin_time < update_session_after_seconds else 0
         if not (Tse.if_session_exists(self.session) and self.language_map and not_update_cond_freq and not_update_cond_time and self.decrypt_language_map):
             self.begin_time = time.time()
-            self.session = kwargs.get('session', None) or Tse.get_client_session(http_client, proxies)
-            self.session = cloudscraper.create_async_scraper()
-            self.session.proxies = proxies
+            self.session = provided_session or Tse.get_client_session(http_client, proxies)
             # _ = await self.session.get(self.host_url, headers=self.host_headers, timeout=timeout)
 
             # self.language_url = re.compile(self.language_pattern).search(host_html).group()