From 7d797a52e7e6eaacd817adfaff8a1f432f5bdcdd Mon Sep 17 00:00:00 2001 From: Lucy Wang Date: Tue, 15 Oct 2019 07:40:00 +0800 Subject: [PATCH 1/5] dockerfile: enable chrome DevTools Protocol (CDP) in QtWebEngine --- Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Dockerfile b/Dockerfile index cfe31cce8..a9c8a0a57 100644 --- a/Dockerfile +++ b/Dockerfile @@ -115,6 +115,9 @@ RUN pip3 install /app ENV PYTHONPATH $PYTHONPATH:/app #ENV QT_DEBUG_PLUGINS=1 +# Enable Chrome DevTools Protocol (CDP) in QtWebEngine +ENV QTWEBENGINE_REMOTE_DEBUGGING '127.0.0.1:9050' + # Chromium refuses to start under root RUN groupadd -r splash && useradd --no-log-init --create-home -r -g splash splash RUN chown -R splash:splash /app From 1a22fc28c4ede2b999be34299f8ac55b9f98edd2 Mon Sep 17 00:00:00 2001 From: Lucy Wang Date: Tue, 15 Oct 2019 07:40:15 +0800 Subject: [PATCH 2/5] allow render_all in chromium engine --- splash/engines/chromium/browser_tab.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/splash/engines/chromium/browser_tab.py b/splash/engines/chromium/browser_tab.py index 01f716e5c..f585a6839 100644 --- a/splash/engines/chromium/browser_tab.py +++ b/splash/engines/chromium/browser_tab.py @@ -145,8 +145,8 @@ def png(self, width=None, height=None, b64=False, render_all=False, "Getting PNG: width=%s, height=%s, " "render_all=%s, scale_method=%s, region=%s" % (width, height, render_all, scale_method, region), min_level=2) - if render_all: - raise ValueError("render_all=True is not supported yet") + # if render_all: + # raise ValueError("render_all=True is not supported yet") image = self._get_image('PNG', width, height, render_all, scale_method, region=region) @@ -165,8 +165,8 @@ def jpeg(self, width=None, height=None, b64=False, render_all=False, "render_all=%s, scale_method=%s, quality=%s, region=%s" % (width, height, render_all, scale_method, quality, region), min_level=2) - if render_all: - raise ValueError("render_all=True is not supported yet") + # if render_all: + # raise ValueError("render_all=True is not supported yet") image = self._get_image('JPEG', width, height, render_all, scale_method, region=region) @@ -178,6 +178,10 @@ def jpeg(self, width=None, height=None, b64=False, render_all=False, def _get_image(self, image_format, width, height, render_all, scale_method, region): + if render_all: + self.logger.log("Rendering whole page contents (RENDER_ALL)", + min_level=2) + self.set_viewport('full') renderer = QtChromiumScreenshotRenderer( self.web_page, self.logger, image_format, width=width, height=height, scale_method=scale_method, From 752b83fa5de6a71678acaf491559177e0326b2d5 Mon Sep 17 00:00:00 2001 From: Lucy Wang Date: Wed, 16 Oct 2019 12:33:53 +0800 Subject: [PATCH 3/5] replace qt5reactor with asyncqt eventloop (for websockets) --- Dockerfile | 2 +- dockerfiles/splash/install-python-splash-deps.sh | 4 ++-- requirements.txt | 2 +- setup.py | 2 +- splash/server.py | 16 ++++++++++++---- 5 files changed, 17 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index a9c8a0a57..1bc557962 100644 --- a/Dockerfile +++ b/Dockerfile @@ -116,7 +116,7 @@ ENV PYTHONPATH $PYTHONPATH:/app #ENV QT_DEBUG_PLUGINS=1 # Enable Chrome DevTools Protocol (CDP) in QtWebEngine -ENV QTWEBENGINE_REMOTE_DEBUGGING '127.0.0.1:9050' +ENV QTWEBENGINE_REMOTE_DEBUGGING "127.0.0.1:9050" # Chromium refuses to start under root RUN groupadd -r splash && useradd --no-log-init --create-home -r -g splash splash diff --git a/dockerfiles/splash/install-python-splash-deps.sh b/dockerfiles/splash/install-python-splash-deps.sh index a9b535d2f..19298f8bf 100755 --- a/dockerfiles/splash/install-python-splash-deps.sh +++ b/dockerfiles/splash/install-python-splash-deps.sh @@ -5,7 +5,7 @@ install_python_deps () { # Install python-level dependencies. ${_PYTHON} -m pip install -U pip setuptools six && \ ${_PYTHON} -m pip install \ - qt5reactor==0.5 \ + asyncqt==0.7.0 \ psutil==5.0.0 \ "Twisted[http2]==19.7.0" \ adblockparser==0.7 \ @@ -17,4 +17,4 @@ install_python_deps () { ${_PYTHON} -m pip install https://github.com/sunu/pyre2/archive/c610be52c3b5379b257d56fc0669d022fd70082a.zip#egg=re2 } -install_python_deps \ No newline at end of file +install_python_deps diff --git a/requirements.txt b/requirements.txt index 8fe7531f4..086a03615 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ # install PyQt5 (Splash is tested on PyQT 5.13) # and the following packages: twisted[http2] == 19.7.0 -qt5reactor +asyncqt == 0.7.0 psutil adblockparser >= 0.5 https://github.com/sunu/pyre2/archive/c610be52c3b5379b257d56fc0669d022fd70082a.zip#egg=re2 diff --git a/setup.py b/setup.py index 06608f6c3..2d0d6206a 100644 --- a/setup.py +++ b/setup.py @@ -60,7 +60,7 @@ def get_version(): 'zip_safe': False, 'install_requires': [ 'Twisted[http2] >= 19.7.0', - 'qt5reactor', + 'asyncqt', 'psutil', 'adblockparser', 'xvfbwrapper', diff --git a/splash/server.py b/splash/server.py index 7680810dc..57c22cc84 100644 --- a/splash/server.py +++ b/splash/server.py @@ -1,3 +1,4 @@ +import asyncio import os import sys import optparse @@ -7,6 +8,9 @@ import functools import faulthandler +from asyncqt import QEventLoop +from twisted.internet import asyncioreactor + from splash import defaults, __version__ from splash import xvfb from splash.qtutils import init_qt_app @@ -15,10 +19,14 @@ def install_qtreactor(verbose): - init_qt_app(verbose) - import qt5reactor - qt5reactor.install() - + """ + Install asyncqt eventloop for asyncio, and then install the + asyncioreactor for twisted. + """ + qtapp = init_qt_app(verbose) + loop = QEventLoop(qtapp) + asyncio.set_event_loop(loop) + asyncioreactor.install(loop) def parse_opts(jupyter=False, argv=None): if argv is None: From 0e69dbd389bed9ac8122a4daca2f9c0f581648c6 Mon Sep 17 00:00:00 2001 From: Lucy Wang Date: Wed, 16 Oct 2019 15:35:09 +0800 Subject: [PATCH 4/5] Revert "allow render_all in chromium engine" This reverts commit 28633969e3b8e52404edc2292f77007940af88a2. --- splash/engines/chromium/browser_tab.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/splash/engines/chromium/browser_tab.py b/splash/engines/chromium/browser_tab.py index f585a6839..01f716e5c 100644 --- a/splash/engines/chromium/browser_tab.py +++ b/splash/engines/chromium/browser_tab.py @@ -145,8 +145,8 @@ def png(self, width=None, height=None, b64=False, render_all=False, "Getting PNG: width=%s, height=%s, " "render_all=%s, scale_method=%s, region=%s" % (width, height, render_all, scale_method, region), min_level=2) - # if render_all: - # raise ValueError("render_all=True is not supported yet") + if render_all: + raise ValueError("render_all=True is not supported yet") image = self._get_image('PNG', width, height, render_all, scale_method, region=region) @@ -165,8 +165,8 @@ def jpeg(self, width=None, height=None, b64=False, render_all=False, "render_all=%s, scale_method=%s, quality=%s, region=%s" % (width, height, render_all, scale_method, quality, region), min_level=2) - # if render_all: - # raise ValueError("render_all=True is not supported yet") + if render_all: + raise ValueError("render_all=True is not supported yet") image = self._get_image('JPEG', width, height, render_all, scale_method, region=region) @@ -178,10 +178,6 @@ def jpeg(self, width=None, height=None, b64=False, render_all=False, def _get_image(self, image_format, width, height, render_all, scale_method, region): - if render_all: - self.logger.log("Rendering whole page contents (RENDER_ALL)", - min_level=2) - self.set_viewport('full') renderer = QtChromiumScreenshotRenderer( self.web_page, self.logger, image_format, width=width, height=height, scale_method=scale_method, From 014e37c80a154f8ee5dab3bf573fb810e1790b96 Mon Sep 17 00:00:00 2001 From: Lucy Wang Date: Fri, 18 Oct 2019 12:11:49 +0800 Subject: [PATCH 5/5] fix splash server exit when testing --- splash/qtutils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/splash/qtutils.py b/splash/qtutils.py index 759033dc2..538e9b771 100644 --- a/splash/qtutils.py +++ b/splash/qtutils.py @@ -139,6 +139,9 @@ def awake(self): diff = time.time() - self.blockedAt log.msg("awake; block time: %0.4f" % diff, system="QAbstractEventDispatcher") _qtapp = QApp(sys.argv) + # we have to set this, because otherwise the program may quit when + # the last web view window is closed. + _qtapp.setQuitOnLastWindowClosed(False) return _qtapp