Add the script parameter (#29)

Add the 'script' parameter to the SeleniumRequest

Author: j4n7

README.md

@@ -52,7 +52,7 @@ def parse_result(self, response):
 ```
 
 ### Additional arguments
-The `scrapy_selenium.SeleniumRequest` accept 3 additional arguments:
+The `scrapy_selenium.SeleniumRequest` accept 4 additional arguments:
 
 #### `wait_time` / `wait_until`
 
@@ -80,6 +80,15 @@ yield SeleniumRequest(
 
 def parse_result(self, response):
     with open('image.png', 'wb') as image_file:
-        image_file.write(response.meta['screenshot])
+        image_file.write(response.meta['screenshot'])
 ```
 
+#### `script`
+When used, selenium will execute custom JavaScript code.
+```python
+yield SeleniumRequest(
+    url,
+    self.parse_result,
+    script='window.scrollTo(0, document.body.scrollHeight);',
+)
+```

scrapy_selenium/http.py

@@ -6,7 +6,7 @@
 class SeleniumRequest(Request):
     """Scrapy ``Request`` subclass providing additional arguments"""
 
-    def __init__(self, wait_time=None, wait_until=None, screenshot=False, *args, **kwargs):
+    def __init__(self, wait_time=None, wait_until=None, screenshot=False, script=None, *args, **kwargs):
         """Initialize a new selenium request
 
         Parameters
@@ -19,11 +19,14 @@ def __init__(self, wait_time=None, wait_until=None, screenshot=False, *args, **k
         screenshot: bool
             If True, a screenshot of the page will be taken and the data of the screenshot
             will be returned in the response "meta" attribute.
+        script: str
+            JavaScript code to execute.
 
         """
 
         self.wait_time = wait_time
         self.wait_until = wait_until
         self.screenshot = screenshot
+        self.script = script
 
         super().__init__(*args, **kwargs)

scrapy_selenium/middlewares.py

@@ -99,6 +99,9 @@ def process_request(self, request, spider):
         if request.screenshot:
             request.meta['screenshot'] = self.driver.get_screenshot_as_png()
 
+        if request.script:
+            self.driver.execute_script(request.script)
+
         body = str.encode(self.driver.page_source)
 
         # Expose the driver via the "meta" attribute

tests/test_middlewares.py

@@ -117,3 +117,21 @@ def test_process_request_should_return_a_screenshot_if_screenshot_option(self):
         )
 
         self.assertIsNotNone(html_response.meta['screenshot'])
+
+    def test_process_request_should_execute_script_if_script_option(self):
+        """Test that the ``process_request`` should execute the script and return a response"""
+
+        selenium_request = SeleniumRequest(
+            url='http://www.python.org',
+            script='document.title = "scrapy_selenium";'
+        )
+
+        html_response = self.selenium_middleware.process_request(
+            request=selenium_request,
+            spider=None
+        )
+
+        self.assertEqual(
+            html_response.selector.xpath('//title/text()').extract_first(),
+            'scrapy_selenium'
+        )