Add py3.11 support

.github/workflows/tests.yml

@@ -6,7 +6,7 @@ jobs:
   unittest:
     strategy:
       matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10"]
+        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
     runs-on: macos-latest
     name: Python ${{ matrix.python-version }} tests
     steps:
@@ -31,7 +31,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v1
         with:
-          python-version: "3.8"
+          python-version: "3.10"
       - name: Install dependency
         run: |
           pip install pip -U

Dockerfile

@@ -5,7 +5,7 @@ WORKDIR /usr/src/frontend
 RUN if [[ -n "$USE_MIRROR" ]] ; then npm --registry https://registry.npmmirror.com install ; else npm install ; fi \
   && npm run build
 
-FROM python:3.8-alpine as pybuilder
+FROM python:3.10-alpine as pybuilder
 ARG USE_MIRROR 
 ENV PYTHONUNBUFFERED 1
 COPY . /usr/src

lyrebird/base_server.py

@@ -15,6 +15,8 @@ def __init__(self):
         self.running = False
         self.name = None
         self.event_thread = None
+        self.args = []
+        self.kwargs = {}
 
     def run(self, queue, config, *args, **kwargs):
         '''
@@ -41,15 +43,15 @@ def run(self, queue, config, *args, **kwargs):
         '''
         pass
 
-    def start(self, *args, **kwargs):
+    def start(self):
         if self.running:
             return
 
         global service_msg_queue
         config = application.config.raw()
         self.server_process = Process(group=None, target=self.run,
-                                      args=[service_msg_queue, config, *args],
-                                      kwargs=kwargs,
+                                      args=[service_msg_queue, config, self.args],
+                                      kwargs=self.kwargs,
                                       daemon=True)
         self.server_process.start()
         self.running = True

lyrebird/manager.py

@@ -24,6 +24,7 @@
 from lyrebird.notice_center import NoticeCenter
 from lyrebird.plugins import PluginManager
 from lyrebird.mitm.proxy_server import LyrebirdProxyServer
+from lyrebird.mitm import init_mitm
 from lyrebird.task import BackgroundTaskServer
 from lyrebird.base_server import MultiProcessServerMessageDispatcher
 from lyrebird import utils

lyrebird/mitm/__init__.py

@@ -0,0 +1 @@
+from lyrebird.mitm.mitm_installer import init_mitm

lyrebird/mitm/mitm_installer.py

@@ -0,0 +1,107 @@
+import sys
+import click
+import shutil
+import requests
+import tarfile
+import tempfile
+from pathlib import Path
+from lyrebird import log
+from lyrebird import application
+
+logger = log.get_logger()
+
+
+class UnsupportedPlatform(Exception):
+    pass
+
+
+class MITMNotFound(Exception):
+    pass
+
+
+def show_mitmdump_help():
+    proxy_port = application.config['proxy.port']
+    errmsg = f'Init mitmproxy failed.\nCan\'t start HTTP proxy server on {proxy_port}\nPlease install mitmproxy(https://mitmproxy.org/) and restart lyrebird\nOr using --no-mitm option for skip starting mitmproxy server\n'
+    logger.error(errmsg)
+    return errmsg
+
+
+def get_mitmdump_filename():
+    platform = sys.platform
+    if platform in ['linux', 'darwin']:
+        return 'mitmdump'
+    elif platform.startswith('win'):
+        return 'mitmdump.exe'
+    else:
+        raise UnsupportedPlatform(f'platform name: {platform}')
+
+
+def find_mitmdump_in_path():
+    mitmdump_path = shutil.which('mitmdump')
+    if not mitmdump_path:
+        return None
+
+    mitmdump = Path(mitmdump_path)
+    if mitmdump.exists() and mitmdump.is_file():
+        return mitmdump
+    else:
+        return None
+
+
+def find_mitmdump_in_lyrebird_home():
+    mitmdump = Path('~/.lyrebird/bin').expanduser().absolute()/get_mitmdump_filename()
+    if mitmdump.exists() and mitmdump.is_file():
+        return mitmdump
+    else:
+        return None
+
+
+def download_mitmproxy():
+    '''
+    Download mitmdump 8.1.1 from mitmproxy.org
+    New file will be write in to ~/.lyrebird/bin
+    Support Window Linux and OSX
+    '''
+    platform = sys.platform
+    if platform == 'linux':
+        download_url = 'https://snapshots.mitmproxy.org/8.1.1/mitmproxy-8.1.1-linux.tar.gz'
+    elif platform == 'darwin':
+        download_url = 'https://snapshots.mitmproxy.org/8.1.1/mitmproxy-8.1.1-osx.tar.gz'
+    elif platform.startswith('win'):
+        download_url = 'https://snapshots.mitmproxy.org/8.1.1/mitmproxy-8.1.1-windows.zip'
+    else:
+        raise UnsupportedPlatform(f'unsupport platform: {platform}')
+
+    resp = requests.get(download_url, stream=True)
+    content_length = int(resp.headers.get('content-length'))
+    click.secho(f'\nmitmdupm not found\nStart downloading mitmproxy: {download_url}')
+    with click.progressbar(length=content_length) as bar, tempfile.NamedTemporaryFile('w+b') as tempf:
+        for chunk in resp.iter_content(4*2048):
+            tempf.write(chunk)
+            bar.update(len(chunk))
+        tempf.flush()
+        tempf.seek(0)
+
+        tarf = tarfile.open(fileobj=tempf.file)
+        mitmdump_filename = get_mitmdump_filename()
+        tarf.extract(mitmdump_filename, str(Path('~/.lyrebird/bin/').expanduser().absolute()))
+        mitmdump_filepath = f'~/.lyrebird/bin/{mitmdump_filename}'
+        click.secho(f'\n🍺 Download completed: write to {mitmdump_filepath}')
+        return mitmdump_filepath
+
+
+def init_mitm():
+    # Find mitmproxy in sys path
+    mitmdump_path = find_mitmdump_in_path()
+    if not mitmdump_path:
+        # Find mitmproxy in ~/.lyrebird/bin
+        mitmdump_path = find_mitmdump_in_lyrebird_home()
+    if not mitmdump_path:
+        # Download mitmproxy and save in ~/.lyrebird/bin
+        mitmdump_path = download_mitmproxy()
+    if not mitmdump_path:
+        # Start HTTP proxy server failed
+        # mitmdump not found
+        show_mitmdump_help()
+        raise MITMNotFound
+    return mitmdump_path

lyrebird/mitm/proxy_server.py

@@ -3,14 +3,10 @@
 import subprocess
 import os
 import json
-import sys
-import shutil
-import tarfile
 import requests
-import click
-import tempfile
 import time
 from lyrebird.base_server import ProcessServer
+from lyrebird.mitm.mitm_installer import init_mitm
 
 """
 HTTP proxy server
@@ -20,71 +16,10 @@
 
 class LyrebirdProxyServer(ProcessServer):
 
-    def get_mitmdump_filename(self):
-        platform = sys.platform
-        if platform in ['linux', 'darwin']:
-            return 'mitmdump'
-        elif platform.startswith('win'):
-            return 'mitmdump.exe'
-        else:
-            raise UnsupportedPlatform(f'platform name: {platform}')
-
-    def find_mitmdump_in_path(self):
-        mitmdump_path = shutil.which('mitmdump')
-        if not mitmdump_path:
-            return None
-
-        mitmdump = Path(mitmdump_path)
-        if mitmdump.exists() and mitmdump.is_file():
-            return mitmdump
-        else:
-            return None
-
-    def find_mitmdump_in_lyrebird_home(self):
-        mitmdump = Path('~/.lyrebird/bin').expanduser().absolute()/self.get_mitmdump_filename()
-        if mitmdump.exists() and mitmdump.is_file():
-            return mitmdump
-        else:
-            return None
-
-    def download_mitmproxy(self):
-        '''
-        Download mitmdump 8.1.1 from mitmproxy.org
-        New file will be write in to ~/.lyrebird/bin
-        Support Window Linux and OSX
-        '''
-        platform = sys.platform
-        if platform == 'linux':
-            download_url = 'https://snapshots.mitmproxy.org/8.1.1/mitmproxy-8.1.1-linux.tar.gz'
-        elif platform == 'darwin':
-            download_url = 'https://snapshots.mitmproxy.org/8.1.1/mitmproxy-8.1.1-osx.tar.gz'
-        elif platform.startswith('win'):
-            download_url = 'https://snapshots.mitmproxy.org/8.1.1/mitmproxy-8.1.1-windows.zip'
-        else:
-            raise UnsupportedPlatform(f'unsupport platform: {platform}')
-
-        resp = requests.get(download_url, stream=True)
-        content_length = int(resp.headers.get('content-length'))
-        click.secho(f'\nmitmdupm not found\nStart downloading mitmproxy: {download_url}')
-        with click.progressbar(length=content_length) as bar, tempfile.NamedTemporaryFile('w+b') as tempf:
-            for chunk in resp.iter_content(4*2048):
-                tempf.write(chunk)
-                bar.update(len(chunk))
-            tempf.flush()
-            tempf.seek(0)
-
-            tarf = tarfile.open(fileobj=tempf.file)
-            mitmdump_filename = self.get_mitmdump_filename()
-            tarf.extract(mitmdump_filename, str(Path('~/.lyrebird/bin/').expanduser().absolute()))
-            mitmdump_filepath = f'~/.lyrebird/bin/{mitmdump_filename}'
-            click.secho(f'\n🍺 Download completed: write to {mitmdump_filepath}')
-            return mitmdump_filepath
-
-    def show_mitmdump_help(self, config, logger):
-        proxy_port = config['proxy.port']
-        errmsg = f'Download mitmproxy fail.\nCan\'t start HTTP proxy server on {proxy_port}\nPlease install mitmproxy(https://mitmproxy.org/) and restart lyrebird\n'
-        logger.error(errmsg)
-        return errmsg
+    def __init__(self):
+        super().__init__()
+        self.mitm_path = init_mitm()
+        self.kwargs['mitm_path'] = self.mitm_path
 
     def show_mitmdump_start_timeout_help(self, mitmdump_filepath, logger):
         logger.error(f'Start mitmdump failed.\nPlease check your mitmdump file {mitmdump_filepath}')
@@ -110,7 +45,7 @@ def wait_for_mitm_start(self, config, logger):
             except Exception:
                 continue
 
-    def start_mitmdump(self, queue, config, mitmdump_path, logger):
+    def start_mitmdump(self, config, logger, mitmdump_path):
         proxy_port = config.get('proxy.port', 4272)
         mock_port = config.get('mock.port', 9090)
         '''
@@ -155,20 +90,9 @@ def run(self, queue, config, *args, **kwargs):
         # Init logger
         log.init(config)
         logger = log.get_logger()
-        # Find mitmproxy in sys path
-        mitmdump_path = self.find_mitmdump_in_path()
-        if not mitmdump_path:
-            # Find mitmproxy in ~/.lyrebird/bin
-            mitmdump_path = self.find_mitmdump_in_lyrebird_home()
-        if not mitmdump_path:
-            # Download mitmproxy and save in ~/.lyrebird/bin
-            mitmdump_path = self.download_mitmproxy()
-        if not mitmdump_path:
-            # Start HTTP proxy server failed
-            # mitmdump not found
-            self.show_mitmdump_help(config, logger)
-            return
-        self.start_mitmdump(queue, config, mitmdump_path, logger)
+        # TODO
+        mitm_path = kwargs.get('mitm_path')
+        self.start_mitmdump(config, logger, mitm_path)
 
 
 class UnsupportedPlatform(Exception):

setup.py

@@ -35,6 +35,7 @@ def read_requirements(name):
         "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
         "License :: OSI Approved :: MIT License",
         "Operating System :: MacOS",
         "Operating System :: Microsoft :: Windows",