Merge pull request #324 from protokoul/fix/splashrequest-url-required

Change url from required to optional parameter for SplashRequest

Author: wRAR

scrapy_splash/request.py

@@ -31,7 +31,7 @@ class SplashRequest(scrapy.Request):
     It requires SplashMiddleware to work.
     """
     def __init__(self,
-                 url,
+                 url=None,
                  callback=None,
                  method='GET',
                  endpoint='render.html',
@@ -48,6 +48,8 @@ def __init__(self,
                  meta=None,
                  **kwargs):
 
+        if url is None:
+            url = 'about:blank'
         url = to_unicode(url)
 
         meta = copy.deepcopy(meta) or {}

scrapy_splash/utils.py

@@ -5,12 +5,7 @@
 import six
 
 from scrapy.http import Headers
-import scrapy
-if scrapy.version_info >= (2, ):
-    from scrapy.utils.python import to_unicode
-else:
-    from scrapy.utils.python import to_native_str as to_unicode
-from scrapy.utils.python import to_bytes
+from scrapy.utils.python import to_unicode, to_bytes
 
 
 def dict_hash(obj, start=''):

setup.py

@@ -30,5 +30,5 @@
         'Topic :: Software Development :: Libraries :: Application Frameworks',
         'Topic :: Software Development :: Libraries :: Python Modules',
     ],
-    install_requires=['scrapy', 'six'],
+    install_requires=['scrapy>=2.4', 'six'],
 )

tests/test_fingerprints.py

@@ -142,9 +142,9 @@ def requests():
         dict(url=url2, args={'wait': 0.5}),     # 5
         dict(url=url3),                         # 6
         dict(url=url2, method='POST'),          # 7
-        dict(url=url3, args={'wait': 0.5}),     # 8
-        dict(url=url3, args={'wait': 0.5}),     # 9
-        dict(url=url3, args={'wait': 0.7}),     # 10
+        dict(args={'wait': 0.5}),               # 8
+        dict(args={'wait': 0.5}),               # 9
+        dict(args={'wait': 0.7}),               # 10
         dict(url=url4),                         # 11
     ]
     splash_requests = [SplashRequest(**kwargs) for kwargs in request_kwargs]

tests/test_middleware.py

@@ -630,6 +630,21 @@ def test_cache_args():
     assert mw._remote_keys == {}
 
 
+def test_splash_request_no_url():
+    mw = _get_mw()
+    lua_source = "function main(splash) return {result='ok'} end"
+    req1 = SplashRequest(meta={'splash': {
+        'args': {'lua_source': lua_source},
+        'endpoint': 'execute',
+    }})
+    req = mw.process_request(req1, None)
+    assert req.url == 'http://127.0.0.1:8050/execute'
+    assert json.loads(to_unicode(req.body)) == {
+        'url': 'about:blank',
+        'lua_source': lua_source
+    }
+
+
 def test_post_request():
     mw = _get_mw()
     for body in [b'', b'foo=bar']: