support py3.10; rm mitmproxy from install require

.github/workflows/tests.yml

@@ -6,7 +6,7 @@ jobs:
   unittest:
     strategy:
       matrix:
-        python-version: ["3.7", "3.8", "3.9"]
+        python-version: ["3.7", "3.8", "3.9", "3.10"]
     runs-on: macos-latest
     name: Python ${{ matrix.python-version }} tests
     steps:

.gitignore

@@ -114,6 +114,7 @@ celerybeat-schedule
 # virtualenv
 .venv
 venv/
+venv*/
 ENV/
 
 # Spyder project settings

README.md

@@ -1,7 +1,3 @@
-> 🔔 **We are hiring. 欢迎加入：**[工具链建设](https://zhaopin.meituan.com/job-detail?jobId=676386152) | [所有职位](https://zhaopin.meituan.com/job-list?jobFamily=203&jobFamilyGroup=29&keywords=%E5%88%B0%E5%BA%97%E5%B9%B3%E5%8F%B0%E6%8A%80%E6%9C%AF&pageNo=1)
-
----
-
 <p align="center"><a herf="https://meituan-dianping.github.io/lyrebird/"><img src="./image/lyrebird.png" width="220"></a></p>
 <h1 align="center">Lyrebird</h1>
 
@@ -105,6 +101,8 @@ lyrebird
 
 - 将移动设备的代理地址设为当前电脑地址，默认端口为 4272（IP 地址可查看 Lyrebird 启动时输出的日志）
 
+  _(注意 ⚠️：4272 端口上使用的 mitmproxy 服务为外置依赖，需要用户自行安装 mitmproxy。如果没有安装 lyrebird 会尝试自动安装该应用)_
+
 - 被测设备上用浏览器打开 http://mitm.it, 选择对应操作系统安装证书
 
 > Lyrebird 接入有两种方式:
@@ -142,6 +140,7 @@ b. 直连
 ---
 
 ## 插件
+
 > 文档在建中
 
 ---

frontend/src/views/Main.vue

@@ -80,6 +80,7 @@ export default {
     this.$io.removeListener('statusBarUpdate', this.loadAllStatusList)
     this.$io.removeListener('msgSuccess', this.successMessage)
     this.$io.removeListener('msgInfo', this.infoMessage)
+    this.$io.removeListener('msgError', this.errorMessage)
     this.$bus.$off('msg.success', this.successMessage)
     this.$bus.$off('msg.loading', this.loadingMessage)
     this.$bus.$off('msg.info', this.infoMessage)
@@ -95,6 +96,7 @@ export default {
     this.$io.on('statusBarUpdate', this.loadAllStatusList)
     this.$io.on('msgSuccess', this.successMessage)
     this.$io.on('msgInfo', this.infoMessage)
+    this.$io.on('msgError', this.errorMessage)
   },
   watch: {
     activeMenuItemIndex(newValue, oldValue) {

lyrebird/application.py

@@ -51,6 +51,7 @@ def make_fail_response(msg, **kwargs):
 
 encoders_decoders = None
 
+
 def start_server():
     for name in server:
         server[name].start()

lyrebird/base_server.py

@@ -3,6 +3,61 @@
 """
 
 from threading import Thread
+from multiprocessing import Process, Queue
+from lyrebird import application
+
+service_msg_queue = Queue()
+
+
+class ProcessServer:
+    def __init__(self):
+        self.server_process = None
+        self.running = False
+        self.name = None
+        self.event_thread = None
+
+    def run(self, queue, config, *args, **kwargs):
+        '''
+        queue
+        message queue for process server and main process
+
+        #1. Send event to main process,
+        {
+            "type": "event",
+            "channel": "",
+            "content": {}
+        }
+
+        #2. Send message to frontend
+        support channel: msgSuccess msgInfo msgError
+        {
+            "type": "ws",
+            "channel": "",
+            "content": ""
+        }
+
+        config
+        lyrebird config dict
+        '''
+        pass
+
+    def start(self, *args, **kwargs):
+        if self.running:
+            return
+
+        global service_msg_queue
+        config = application.config.raw()
+        self.server_process = Process(group=None, target=self.run,
+                                      args=[service_msg_queue, config, *args],
+                                      kwargs=kwargs,
+                                      daemon=True)
+        self.server_process.start()
+        self.running = True
+
+    def stop(self):
+        if self.server_process:
+            self.server_process.terminate()
+            self.server_process = None
 
 
 class ThreadServer:
@@ -37,3 +92,27 @@ def start(self, *args, **kwargs):
 
     def stop(self):
         pass
+
+
+class MultiProcessServerMessageDispatcher(ThreadServer):
+
+    def run(self):
+        global service_msg_queue
+        emit = application.server['mock'].socket_io.emit
+        publish = application.server['event'].publish
+
+        while True:
+            msg = service_msg_queue.get()
+            type = msg.get('type')
+            if type == 'event':
+                channel = msg.get('channel')
+                event = msg.get('content')
+                if channel and event:
+                    publish(channel, event)
+            elif type == 'ws':
+                ws_channel = msg.get('channel')
+                ws_msg = msg.get('content', '')
+                if ws_channel:
+                    emit(ws_channel, ws_msg)
+            else:
+                pass

lyrebird/log.py

@@ -1,7 +1,7 @@
 import logging
+from logging.handlers import TimedRotatingFileHandler
 from colorama import Fore, Style, Back
 from collections import namedtuple
-from lyrebird import application
 from pathlib import Path
 
 DEFAULT_LOG_PATH = '~/.lyrebird/lyrebird.log'
@@ -60,24 +60,27 @@ def make_file_handler(_log_path=None):
     )
     log_file = Path(_log_path).expanduser().absolute().resolve()
 
-    file_handler = logging.handlers.TimedRotatingFileHandler(log_file, backupCount=1, encoding='utf-8', when='midnight')
+    file_handler = TimedRotatingFileHandler(log_file, backupCount=1, encoding='utf-8', when='midnight')
     file_handler.setFormatter(file_formater)
     return file_handler
 
 
-def init():
+def init(config):
     global LOGGER_INITED
     if LOGGER_INITED:
         return
 
+    if not config:
+        config = {}
+
     logging.addLevelName(60, 'NOTICE')
 
     stream_handler = make_stream_handler()
 
-    log_path = application.config.get('log')
+    log_path = config.get('log')
     file_handler = make_file_handler(log_path)
 
-    verbose = application.config.get('verbose', 0)
+    verbose = config.get('verbose', 0)
     if verbose == 0:
         logger_level = logging.ERROR
     elif verbose == 1:

lyrebird/manager.py

@@ -23,8 +23,9 @@
 from lyrebird.mock.mock_server import LyrebirdMockServer
 from lyrebird.notice_center import NoticeCenter
 from lyrebird.plugins import PluginManager
-from lyrebird.proxy.proxy_server import LyrebirdProxyServer
+from lyrebird.mitm.proxy_server import LyrebirdProxyServer
 from lyrebird.task import BackgroundTaskServer
+from lyrebird.base_server import MultiProcessServerMessageDispatcher
 from lyrebird import utils
 
 logger = log.get_logger()
@@ -67,7 +68,7 @@ def main():
                         help='Set extra mock server port, default port is 9999')
     parser.add_argument('--proxy', dest='proxy', type=int, help='Set proxy server port, default port is 4272')
     parser.add_argument('--data', dest='data', help='Set data dir, default is "./data/"')
-    parser.add_argument('-b', '--no_browser', dest='no_browser',
+    parser.add_argument('-b', '--no-browser', dest='no_browser',
                         action='store_true', help='Start without open a browser')
     parser.add_argument('-c', '--config', action='append', dest='config',
                         help='Start with a config file. Default is "~/.lyrebird/conf.json"')
@@ -76,6 +77,7 @@ def main():
     parser.add_argument('--plugin', action='append', help='Set a plugin project path')
     parser.add_argument('--database', dest='database', help='Set a database path. Default is "~/.lyrebird/lyrebird.db"')
     parser.add_argument('--es', dest='extra_string', action='append', nargs=2, help='Set a custom config')
+    parser.add_argument('--no-mitm', dest='no_mitm', action='store_true', help='Start without mitmproxy on 4272')
 
     subparser = parser.add_subparsers(dest='sub_command')
 
@@ -96,7 +98,7 @@ def main():
     # init logger for main process
     application._cm.config['verbose'] = args.verbose
     application._cm.config['log'] = args.log
-    log.init()
+    log.init(application._cm.config)
 
     # Add exception hook
     def process_excepthook(exc_type, exc_value, tb):
@@ -168,9 +170,11 @@ def run(args: argparse.Namespace):
 
     # Main server
     application.server['event'] = EventServer()
-
+    # mutilprocess message dispatcher
+    application.server['dispather'] = MultiProcessServerMessageDispatcher()
     application.server['task'] = BackgroundTaskServer()
-    application.server['proxy'] = LyrebirdProxyServer()
+    if not args.no_mitm:
+        application.server['proxy'] = LyrebirdProxyServer()
     application.server['mock'] = LyrebirdMockServer()
     application.server['extra.mock'] = ExtraMockServer()
     application.server['db'] = LyrebirdDatabaseServer(path=args.database)

lyrebird/proxy/mitm_script.py → lyrebird/mitm/mitm_script.py

@@ -3,17 +3,12 @@
 
 Redirect request from proxy server to mock server
 """
-
-from urllib.parse import urlparse
-from mitmproxy import http
-from lyrebird import log
-import os
-import json
-import logging
 import re
+import json
+import os
+from mitmproxy import http
+from urllib.parse import urlparse
 
-_logger = log.get_logger()
-_logger.setLevel(logging.INFO)
 
 PROXY_PORT = int(os.environ.get('PROXY_PORT'))
 PROXY_FILTERS = json.loads(os.environ.get('PROXY_FILTERS'))
@@ -37,16 +32,14 @@ def to_mock_server(flow: http.HTTPFlow):
     # 获取的address是IPv6（内嵌IPv4地址表示法），需要获取IPv4地址，需要做以下处理
     if address.startswith('::ffff:'):
         address = address.split('::ffff:')[1]
-    
+
     flow.request.headers['Lyrebird-Client-Address'] = address
     flow.request.headers['Mitmproxy-Proxy'] = address
-    flow.request.headers['Proxy-Raw-Headers'] = json.dumps({name: flow.request.headers[name] for name in flow.request.headers}, ensure_ascii=False)
-
-    _logger.info('Redirect-> %s' % flow.request.url[:100])
+    flow.request.headers['Proxy-Raw-Headers'] = json.dumps({name: flow.request.headers[name]
+                                                           for name in flow.request.headers}, ensure_ascii=False)
 
 
 def request(flow: http.HTTPFlow):
-    _logger.info(flow.request.url[:100])
     if 'mitm.it' in flow.request.url:
         # Support mitm.it
         return