From 844de18acaf7623bac9a1c24da99338608e6c62e Mon Sep 17 00:00:00 2001 From: zhaoye Date: Tue, 13 Sep 2022 19:39:48 +0800 Subject: [PATCH] support py3.10; rm mitmproxy from install require --- .github/workflows/tests.yml | 2 +- .gitignore | 1 + README.md | 7 +- frontend/src/views/Main.vue | 2 + install_requires.txt | 65 -------- lyrebird/application.py | 1 + lyrebird/base_server.py | 79 ++++++++++ lyrebird/log.py | 13 +- lyrebird/manager.py | 14 +- lyrebird/{proxy => mitm}/__init__.py | 0 lyrebird/{proxy => mitm}/mitm_script.py | 21 +-- lyrebird/mitm/proxy_server.py | 177 ++++++++++++++++++++++ lyrebird/mock/extra_mock_server/server.py | 17 +-- lyrebird/mock/mock_server.py | 2 +- lyrebird/proxy/mitm_exec.py | 7 - lyrebird/proxy/proxy_server.py | 71 --------- requirements.dev.txt | 4 + requirements.txt | 15 +- requirements.txt.lock | 40 +++++ setup.py | 20 +-- dev.sh => setup.sh | 11 +- tests/test_mock_server.py | 3 +- 22 files changed, 370 insertions(+), 202 deletions(-) delete mode 100644 install_requires.txt rename lyrebird/{proxy => mitm}/__init__.py (100%) rename lyrebird/{proxy => mitm}/mitm_script.py (86%) create mode 100644 lyrebird/mitm/proxy_server.py delete mode 100644 lyrebird/proxy/mitm_exec.py delete mode 100644 lyrebird/proxy/proxy_server.py create mode 100644 requirements.dev.txt create mode 100644 requirements.txt.lock rename dev.sh => setup.sh (68%) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5dbed8ecb..cecdb238a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -6,7 +6,7 @@ jobs: unittest: strategy: matrix: - python-version: ["3.7", "3.8", "3.9"] + python-version: ["3.7", "3.8", "3.9", "3.10"] runs-on: macos-latest name: Python ${{ matrix.python-version }} tests steps: diff --git a/.gitignore b/.gitignore index abaec137b..333a52f54 100644 --- a/.gitignore +++ b/.gitignore @@ -114,6 +114,7 @@ celerybeat-schedule # virtualenv .venv venv/ +venv*/ ENV/ # Spyder project settings diff --git a/README.md b/README.md index ab720f0e8..7ad7e17a1 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,3 @@ -> 🔔 **We are hiring. 欢迎加入：**[工具链建设](https://zhaopin.meituan.com/job-detail?jobId=676386152) | [所有职位](https://zhaopin.meituan.com/job-list?jobFamily=203&jobFamilyGroup=29&keywords=%E5%88%B0%E5%BA%97%E5%B9%B3%E5%8F%B0%E6%8A%80%E6%9C%AF&pageNo=1) - ---- -

Lyrebird

@@ -105,6 +101,8 @@ lyrebird - 将移动设备的代理地址设为当前电脑地址，默认端口为 4272（IP 地址可查看 Lyrebird 启动时输出的日志） + _(注意 ⚠️：4272 端口上使用的 mitmproxy 服务为外置依赖，需要用户自行安装 mitmproxy。如果没有安装 lyrebird 会尝试自动安装该应用)_ + - 被测设备上用浏览器打开 http://mitm.it, 选择对应操作系统安装证书 > Lyrebird 接入有两种方式: @@ -142,6 +140,7 @@ b. 直连 --- ## 插件 + > 文档在建中 --- diff --git a/frontend/src/views/Main.vue b/frontend/src/views/Main.vue index 171eb9a8c..e4e12ebe4 100644 --- a/frontend/src/views/Main.vue +++ b/frontend/src/views/Main.vue @@ -80,6 +80,7 @@ export default { this.$io.removeListener('statusBarUpdate', this.loadAllStatusList) this.$io.removeListener('msgSuccess', this.successMessage) this.$io.removeListener('msgInfo', this.infoMessage) + this.$io.removeListener('msgError', this.errorMessage) this.$bus.$off('msg.success', this.successMessage) this.$bus.$off('msg.loading', this.loadingMessage) this.$bus.$off('msg.info', this.infoMessage) @@ -95,6 +96,7 @@ export default { this.$io.on('statusBarUpdate', this.loadAllStatusList) this.$io.on('msgSuccess', this.successMessage) this.$io.on('msgInfo', this.infoMessage) + this.$io.on('msgError', this.errorMessage) }, watch: { activeMenuItemIndex(newValue, oldValue) { diff --git a/install_requires.txt b/install_requires.txt deleted file mode 100644 index fd27b0324..000000000 --- a/install_requires.txt +++ /dev/null @@ -1,65 +0,0 @@ -aniso8601==9.0.1 -asgiref==3.3.4 -beautifulsoup4==4.7.1 -blinker==1.4 -Brotli==1.0.9 -certifi==2021.10.8 -cffi==1.15.0 -chardet==3.0.4 -click==7.1.2 -colorama==0.4.1 -cryptography==3.2.1 -Flask==1.1.1 -Flask-RESTful==0.3.7 -Flask-SocketIO==4.2.1 -h11==0.13.0 -h2==4.1.0 -hpack==4.0.0 -hyperframe==6.0.1 -idna==2.8 -itsdangerous==2.0.1 -Jinja2==3.0.3 -kaitaistruct==0.9 -ldap3==2.8.1 -MarkupSafe==2.0.1 -mitmproxy==5.3.0 -msgpack==1.0.3 -netifaces==0.11.0 -packaging==19.0 -passlib==1.7.4 -Pillow==9.1.1 -portpicker==1.3.1 -protobuf==3.13.0 -publicsuffix2==2.20191221 -pyasn1==0.4.8 -pycparser==2.21 -pyOpenSSL==19.1.0 -pyparsing==2.4.7 -pyperclip==1.8.2 -python-engineio==3.14.2 -python-socketio==4.6.1 -pytz==2022.1 -qrcode==7.3.1 -requests==2.21.0 -ruamel.yaml==0.16.13 -ruamel.yaml.clib==0.2.6 -six==1.16.0 -sortedcontainers==2.2.2 -soupsieve==2.3.1 -SQLAlchemy==1.3.22 -tornado==6.1 -typing-extensions==4.1.1 -urllib3==1.24.2 -urwid==2.1.2 -Werkzeug==2.0.3 -wsproto==0.15.0 -zstandard==0.14.1 -aiohttp==3.8.1 -async-timeout==4.0.2 -attrs==21.4.0 -multidict==6.0.2 -yarl==1.7.2 -frozenlist==1.3.0 -charset-normalizer==2.0.12 -aiosignal==1.2.0 -idna==2.8 diff --git a/lyrebird/application.py b/lyrebird/application.py index 5937c3d7d..52fb82d26 100644 --- a/lyrebird/application.py +++ b/lyrebird/application.py @@ -51,6 +51,7 @@ def make_fail_response(msg, **kwargs): encoders_decoders = None + def start_server(): for name in server: server[name].start() diff --git a/lyrebird/base_server.py b/lyrebird/base_server.py index 15455862d..a778315b8 100644 --- a/lyrebird/base_server.py +++ b/lyrebird/base_server.py @@ -3,6 +3,61 @@ """ from threading import Thread +from multiprocessing import Process, Queue +from lyrebird import application + +service_msg_queue = Queue() + + +class ProcessServer: + def __init__(self): + self.server_process = None + self.running = False + self.name = None + self.event_thread = None + + def run(self, queue, config, *args, **kwargs): + ''' + queue + message queue for process server and main process + + #1. Send event to main process, + { + "type": "event", + "channel": "", + "content": {} + } + + #2. Send message to frontend + support channel: msgSuccess msgInfo msgError + { + "type": "ws", + "channel": "", + "content": "" + } + + config + lyrebird config dict + ''' + pass + + def start(self, *args, **kwargs): + if self.running: + return + + global service_msg_queue + config = application.config.raw() + self.server_process = Process(group=None, target=self.run, + args=[service_msg_queue, config, *args], + kwargs=kwargs, + daemon=True) + self.server_process.start() + self.running = True + + def stop(self): + if self.server_process: + self.server_process.terminate() + self.server_process = None class ThreadServer: @@ -37,3 +92,27 @@ def start(self, *args, **kwargs): def stop(self): pass + + +class MultiProcessServerMessageDispatcher(ThreadServer): + + def run(self): + global service_msg_queue + emit = application.server['mock'].socket_io.emit + publish = application.server['event'].publish + + while True: + msg = service_msg_queue.get() + type = msg.get('type') + if type == 'event': + channel = msg.get('channel') + event = msg.get('content') + if channel and event: + publish(channel, event) + elif type == 'ws': + ws_channel = msg.get('channel') + ws_msg = msg.get('content', '') + if ws_channel: + emit(ws_channel, ws_msg) + else: + pass diff --git a/lyrebird/log.py b/lyrebird/log.py index 7639f3147..88679cfe2 100644 --- a/lyrebird/log.py +++ b/lyrebird/log.py @@ -1,7 +1,7 @@ import logging +from logging.handlers import TimedRotatingFileHandler from colorama import Fore, Style, Back from collections import namedtuple -from lyrebird import application from pathlib import Path DEFAULT_LOG_PATH = '~/.lyrebird/lyrebird.log' @@ -60,24 +60,27 @@ def make_file_handler(_log_path=None): ) log_file = Path(_log_path).expanduser().absolute().resolve() - file_handler = logging.handlers.TimedRotatingFileHandler(log_file, backupCount=1, encoding='utf-8', when='midnight') + file_handler = TimedRotatingFileHandler(log_file, backupCount=1, encoding='utf-8', when='midnight') file_handler.setFormatter(file_formater) return file_handler -def init(): +def init(config): global LOGGER_INITED if LOGGER_INITED: return + if not config: + config = {} + logging.addLevelName(60, 'NOTICE') stream_handler = make_stream_handler() - log_path = application.config.get('log') + log_path = config.get('log') file_handler = make_file_handler(log_path) - verbose = application.config.get('verbose', 0) + verbose = config.get('verbose', 0) if verbose == 0: logger_level = logging.ERROR elif verbose == 1: diff --git a/lyrebird/manager.py b/lyrebird/manager.py index 24860d151..1efb73e63 100644 --- a/lyrebird/manager.py +++ b/lyrebird/manager.py @@ -23,8 +23,9 @@ from lyrebird.mock.mock_server import LyrebirdMockServer from lyrebird.notice_center import NoticeCenter from lyrebird.plugins import PluginManager -from lyrebird.proxy.proxy_server import LyrebirdProxyServer +from lyrebird.mitm.proxy_server import LyrebirdProxyServer from lyrebird.task import BackgroundTaskServer +from lyrebird.base_server import MultiProcessServerMessageDispatcher from lyrebird import utils logger = log.get_logger() @@ -67,7 +68,7 @@ def main(): help='Set extra mock server port, default port is 9999') parser.add_argument('--proxy', dest='proxy', type=int, help='Set proxy server port, default port is 4272') parser.add_argument('--data', dest='data', help='Set data dir, default is "./data/"') - parser.add_argument('-b', '--no_browser', dest='no_browser', + parser.add_argument('-b', '--no-browser', dest='no_browser', action='store_true', help='Start without open a browser') parser.add_argument('-c', '--config', action='append', dest='config', help='Start with a config file. Default is "~/.lyrebird/conf.json"') @@ -76,6 +77,7 @@ def main(): parser.add_argument('--plugin', action='append', help='Set a plugin project path') parser.add_argument('--database', dest='database', help='Set a database path. Default is "~/.lyrebird/lyrebird.db"') parser.add_argument('--es', dest='extra_string', action='append', nargs=2, help='Set a custom config') + parser.add_argument('--no-mitm', dest='no_mitm', action='store_true', help='Start without mitmproxy on 4272') subparser = parser.add_subparsers(dest='sub_command') @@ -96,7 +98,7 @@ def main(): # init logger for main process application._cm.config['verbose'] = args.verbose application._cm.config['log'] = args.log - log.init() + log.init(application._cm.config) # Add exception hook def process_excepthook(exc_type, exc_value, tb): @@ -168,9 +170,11 @@ def run(args: argparse.Namespace): # Main server application.server['event'] = EventServer() - + # mutilprocess message dispatcher + application.server['dispather'] = MultiProcessServerMessageDispatcher() application.server['task'] = BackgroundTaskServer() - application.server['proxy'] = LyrebirdProxyServer() + if not args.no_mitm: + application.server['proxy'] = LyrebirdProxyServer() application.server['mock'] = LyrebirdMockServer() application.server['extra.mock'] = ExtraMockServer() application.server['db'] = LyrebirdDatabaseServer(path=args.database) diff --git a/lyrebird/proxy/__init__.py b/lyrebird/mitm/__init__.py similarity index 100% rename from lyrebird/proxy/__init__.py rename to lyrebird/mitm/__init__.py diff --git a/lyrebird/proxy/mitm_script.py b/lyrebird/mitm/mitm_script.py similarity index 86% rename from lyrebird/proxy/mitm_script.py rename to lyrebird/mitm/mitm_script.py index e82dd7725..657304bcb 100644 --- a/lyrebird/proxy/mitm_script.py +++ b/lyrebird/mitm/mitm_script.py @@ -3,17 +3,12 @@ Redirect request from proxy server to mock server """ - -from urllib.parse import urlparse -from mitmproxy import http -from lyrebird import log -import os -import json -import logging import re +import json +import os +from mitmproxy import http +from urllib.parse import urlparse -_logger = log.get_logger() -_logger.setLevel(logging.INFO) PROXY_PORT = int(os.environ.get('PROXY_PORT')) PROXY_FILTERS = json.loads(os.environ.get('PROXY_FILTERS')) @@ -37,16 +32,14 @@ def to_mock_server(flow: http.HTTPFlow): # 获取的address是IPv6（内嵌IPv4地址表示法），需要获取IPv4地址，需要做以下处理 if address.startswith('::ffff:'): address = address.split('::ffff:')[1] - + flow.request.headers['Lyrebird-Client-Address'] = address flow.request.headers['Mitmproxy-Proxy'] = address - flow.request.headers['Proxy-Raw-Headers'] = json.dumps({name: flow.request.headers[name] for name in flow.request.headers}, ensure_ascii=False) - - _logger.info('Redirect-> %s' % flow.request.url[:100]) + flow.request.headers['Proxy-Raw-Headers'] = json.dumps({name: flow.request.headers[name] + for name in flow.request.headers}, ensure_ascii=False) def request(flow: http.HTTPFlow): - _logger.info(flow.request.url[:100]) if 'mitm.it' in flow.request.url: # Support mitm.it return diff --git a/lyrebird/mitm/proxy_server.py b/lyrebird/mitm/proxy_server.py new file mode 100644 index 000000000..ed86f93ae --- /dev/null +++ b/lyrebird/mitm/proxy_server.py @@ -0,0 +1,177 @@ +from pathlib import Path +from colorama import Fore +from lyrebird import application +from lyrebird import log +import subprocess +import os +import json +import sys +import shutil +import tarfile +import requests +import click +import tempfile +import time +from lyrebird.base_server import ProcessServer + +""" +HTTP proxy server +Default port 4272 +""" + + +class LyrebirdProxyServer(ProcessServer): + + def get_mitmdump_filename(self): + platform = sys.platform + if platform in ['linux', 'darwin']: + return 'mitmdump' + elif platform.startswith('win'): + return 'mitmdump.exe' + else: + raise UnsupportedPlatform(f'platform name: {platform}') + + def find_mitmdump_in_path(self): + mitmdump_path = shutil.which('mitmdump') + if not mitmdump_path: + return None + + mitmdump = Path(mitmdump_path) + if mitmdump.exists() and mitmdump.is_file(): + return mitmdump + else: + return None + + def find_mitmdump_in_lyrebird_home(self): + mitmdump = Path('~/.lyrebird/bin').expanduser().absolute()/self.get_mitmdump_filename() + if mitmdump.exists() and mitmdump.is_file(): + return mitmdump + else: + return None + + def download_mitmproxy(self): + ''' + Download mitmdump 8.1.1 from mitmproxy.org + New file will be write in to ~/.lyrebird/bin + Support Window Linux and OSX + ''' + platform = sys.platform + if platform == 'linux': + download_url = 'https://snapshots.mitmproxy.org/8.1.1/mitmproxy-8.1.1-linux.tar.gz' + elif platform == 'darwin': + download_url = 'https://snapshots.mitmproxy.org/8.1.1/mitmproxy-8.1.1-osx.tar.gz' + elif platform.startswith('win'): + download_url = 'https://snapshots.mitmproxy.org/8.1.1/mitmproxy-8.1.1-windows.zip' + else: + raise UnsupportedPlatform(f'unsupport platform: {platform}') + + resp = requests.get(download_url, stream=True) + content_length = int(resp.headers.get('content-length')) + click.secho(f'\nmitmdupm not found\nStart downloading mitmproxy: {download_url}') + with click.progressbar(length=content_length) as bar, tempfile.NamedTemporaryFile('w+b') as tempf: + for chunk in resp.iter_content(4*2048): + tempf.write(chunk) + bar.update(len(chunk)) + tempf.flush() + tempf.seek(0) + + tarf = tarfile.open(fileobj=tempf.file) + mitmdump_filename = self.get_mitmdump_filename() + tarf.extract(mitmdump_filename, str(Path('~/.lyrebird/bin/').expanduser().absolute())) + mitmdump_filepath = f'~/.lyrebird/bin/{mitmdump_filename}' + click.secho(f'\n🍺 Download completed: write to {mitmdump_filepath}') + return mitmdump_filepath + + def show_mitmdump_help(self, config, logger): + proxy_port = config['proxy.port'] + errmsg = f'Download mitmproxy fail.\nCan\'t start HTTP proxy server on {proxy_port}\nPlease install mitmproxy(https://mitmproxy.org/) and restart lyrebird\n' + logger.error(errmsg) + return errmsg + + def show_mitmdump_start_timeout_help(self, mitmdump_filepath, logger): + logger.error(f'Start mitmdump failed.\nPlease check your mitmdump file {mitmdump_filepath}') + + def wait_for_mitm_start(self, config, logger): + timeout = 30 + wait_time_count = 0 + mock_port = config.get('mock.port') + proxy_port = config.get('proxy.port') + while True: + if wait_time_count >= timeout: + return False + + time.sleep(1) + wait_time_count += 1 + try: + resp = requests.get(f'http://127.0.0.1:{mock_port}/api/status', + proxies={'http': f'http://127.0.0.1:{proxy_port}'}) + if resp.status_code != 200: + continue + else: + return True + except Exception: + continue + + def start_mitmdump(self, queue, config, mitmdump_path, logger): + proxy_port = config.get('proxy.port', 4272) + mock_port = config.get('mock.port', 9090) + ''' + --ignore_hosts: + The ignore_hosts option allows you to specify a regex which is matched against a host:port + string (e.g. “example.com:443”) of a connection. Matching hosts are excluded from interception, + and passed on unmodified. + + # Ignore everything but sankuai.com, meituan.com and dianping.com: + --ignore-hosts '^(?!.*sankuai.*)(?!.*meituan.*)(?!.*dianping.*)' + + According to mitmproxy docs: https://docs.mitmproxy.org/stable/howto-ignoredomains/ + ''' + ignore_hosts = config.get('proxy.ignore_hosts', None) + + current_path = Path(__file__).parent + script_path = current_path/'mitm_script.py' + + mitm_arguments = [ + '-s', str(script_path), + '-p', str(proxy_port), + '--ssl-insecure', + '--no-http2', + '-q', + '--set', + 'block_global=false' + ] + if ignore_hosts: + mitm_arguments += ['--ignore-hosts', ignore_hosts] + mitmenv = os.environ + mitmenv['PROXY_PORT'] = str(mock_port) + mitmenv['PROXY_FILTERS'] = json.dumps(config.get('proxy.filters', [])) + logger.info('HTTP proxy server starting...') + subprocess.Popen(f'{str(mitmdump_path)} {" ".join(mitm_arguments)}', shell=True, env=mitmenv) + is_mitm_start = self.wait_for_mitm_start(config, logger) + if is_mitm_start: + logger.log(60, f'HTTP proxy server start on {proxy_port}') + else: + self.show_mitmdump_start_timeout_help(mitmdump_path, logger) + + def run(self, queue, config, *args, **kwargs): + # Init logger + log.init(config) + logger = log.get_logger() + # Find mitmproxy in sys path + mitmdump_path = self.find_mitmdump_in_path() + if not mitmdump_path: + # Find mitmproxy in ~/.lyrebird/bin + mitmdump_path = self.find_mitmdump_in_lyrebird_home() + if not mitmdump_path: + # Download mitmproxy and save in ~/.lyrebird/bin + mitmdump_path = self.download_mitmproxy() + if not mitmdump_path: + # Start HTTP proxy server failed + # mitmdump not found + errmsg = self.show_mitmdump_help(config, logger) + return + self.start_mitmdump(queue, config, mitmdump_path, logger) + + +class UnsupportedPlatform(Exception): + pass diff --git a/lyrebird/mock/extra_mock_server/server.py b/lyrebird/mock/extra_mock_server/server.py index 88e11bc80..7650f4e10 100644 --- a/lyrebird/mock/extra_mock_server/server.py +++ b/lyrebird/mock/extra_mock_server/server.py @@ -10,10 +10,9 @@ from typing import List, Set, Optional from lyrebird.mock.extra_mock_server.lyrebird_proxy_protocol import LyrebirdProxyContext -from lyrebird.log import get_logger - -logger = get_logger() +from lyrebird import log +logger = None lb_config = {} @@ -109,6 +108,10 @@ async def req_handler(request: web.Request): async def _run_app(config): + global logger + log.init(config) + logger = log.get_logger() + global lb_config lb_config = config @@ -125,12 +128,8 @@ async def _run_app(config): web_site = web.TCPSite(app_runner, '0.0.0.0', port) await web_site.start() - if print: - names = sorted(str(s.name) for s in app_runner.sites) - print( - "======== Running on {} ========\n" - "(Press CTRL+C to quit)".format(", ".join(names)) - ) + names = sorted(str(s.name) for s in app_runner.sites) + logger.log(60, f'Extra mock server start on {port}') # sleep forever by 1 hour intervals, # on Windows before Python 3.8 wake up every 1 second to handle diff --git a/lyrebird/mock/mock_server.py b/lyrebird/mock/mock_server.py index 34232894f..7e97f6e04 100644 --- a/lyrebird/mock/mock_server.py +++ b/lyrebird/mock/mock_server.py @@ -90,7 +90,7 @@ def on_app_error(error): def run(self): server_ip = application.config.get('ip') - _logger.warning(f'Core start on http://{server_ip}:{self.port}') + _logger.log(60, f'Core start on http://{server_ip}:{self.port}') self.socket_io.run(self.app, host='0.0.0.0', port=self.port, debug=self.debug, use_reloader=False) def stop(self): diff --git a/lyrebird/proxy/mitm_exec.py b/lyrebird/proxy/mitm_exec.py deleted file mode 100644 index 2844b43a5..000000000 --- a/lyrebird/proxy/mitm_exec.py +++ /dev/null @@ -1,7 +0,0 @@ -import re -import sys -from mitmproxy.tools.main import mitmdump - -if __name__ == '__main__': - sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0]) - sys.exit(mitmdump()) diff --git a/lyrebird/proxy/proxy_server.py b/lyrebird/proxy/proxy_server.py deleted file mode 100644 index 7bd862671..000000000 --- a/lyrebird/proxy/proxy_server.py +++ /dev/null @@ -1,71 +0,0 @@ -from pathlib import Path -from colorama import Fore -from lyrebird.base_server import ThreadServer -from lyrebird import application -from lyrebird.log import get_logger -import subprocess -import os -import json -import sys - - -""" -HTTP proxy server - -Default port 4272 -""" - -CURRENT_PATH = Path(__file__).parent -SCRIPT_FILE = CURRENT_PATH/'mitm_script.py' -MITMDUMP_FILE = CURRENT_PATH/'mitm_exec.py' - -logger = get_logger() - - -class LyrebirdProxyServer(): - - def __init__(self): - super().__init__() - - conf = application.config - self.proxy_port = str(conf.get('proxy.port', '4272')) if conf else '4272' - ''' - --ignore_hosts: - The ignore_hosts option allows you to specify a regex which is matched against a host:port - string (e.g. “example.com:443”) of a connection. Matching hosts are excluded from interception, - and passed on unmodified. - - # Ignore everything but sankuai.com, meituan.com and dianping.com: - --ignore-hosts '^(?!.*sankuai.*)(?!.*meituan.*)(?!.*dianping.*)' - - According to mitmproxy docs: https://docs.mitmproxy.org/stable/howto-ignoredomains/ - ''' - self.ignore_hosts = conf.get('proxy.ignore_hosts', None) - self._proxy_server_process = None - - def start(self): - server_ip = application.config.get('ip') - # info_msg(f'start on {server_ip}:{self.proxy_port}', f'{Fore.CYAN} ***请在被测设备上设置代理服务器地址***') - logger.warning(f'start on http://{server_ip}:{self.proxy_port} {Fore.CYAN} ***请在被测设备上设置代理服务器地址***') - mitm_arguments = [ - '-s', str(SCRIPT_FILE), - '-p', self.proxy_port, - '--ssl-insecure', - '--no-http2', - '-q', - '--set', - 'block_global=false' - ] - if self.ignore_hosts: - mitm_arguments += ['--ignore-hosts', self.ignore_hosts] - mitmenv = os.environ - mitmenv['PROXY_PORT'] = str(application.config.get('mock.port', 9090)) - mitmenv['PROXY_FILTERS'] = json.dumps(application.config.get('proxy.filters', [])) - self._proxy_server_process = subprocess.Popen(f'{sys.executable} {str(MITMDUMP_FILE)} {" ".join(mitm_arguments)}', shell=True, env=mitmenv) - - def stop(self): - if self._proxy_server_process: - self._proxy_server_process.terminate() - logger.warning('ProxyServer shutdown') - - diff --git a/requirements.dev.txt b/requirements.dev.txt new file mode 100644 index 000000000..9868b929e --- /dev/null +++ b/requirements.dev.txt @@ -0,0 +1,4 @@ +autopep8==1.7.0 +pip +pytest +pytest-cov diff --git a/requirements.txt b/requirements.txt index b2f91bdb9..608ed14a9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,14 @@ --e .[dev] \ No newline at end of file +beautifulsoup4==4.7.1 +colorama==0.4.1 +Flask==2.2.2 +Flask-RESTful==0.3.9 +Flask-SocketIO==5.3.0 +packaging==19.0 +Pillow==9.1.1 +portpicker==1.3.1 +qrcode==7.3.1 +requests==2.21.0 +SQLAlchemy==1.3.22 +aiohttp==3.8.1 +netifaces==0.11.0 + diff --git a/requirements.txt.lock b/requirements.txt.lock new file mode 100644 index 000000000..27079e6e8 --- /dev/null +++ b/requirements.txt.lock @@ -0,0 +1,40 @@ +aiohttp==3.8.1 +aiosignal==1.2.0 +aniso8601==9.0.1 +async-timeout==4.0.2 +attrs==22.1.0 +autopep8==1.7.0 +beautifulsoup4==4.7.1 +bidict==0.22.0 +certifi==2022.6.15.1 +chardet==3.0.4 +charset-normalizer==2.1.1 +click==8.1.3 +colorama==0.4.1 +Flask==2.2.2 +Flask-RESTful==0.3.9 +Flask-SocketIO==5.3.0 +frozenlist==1.3.1 +idna==2.8 +itsdangerous==2.1.2 +Jinja2==3.1.2 +MarkupSafe==2.1.1 +multidict==6.0.2 +netifaces==0.11.0 +packaging==19.0 +Pillow==9.1.1 +portpicker==1.3.1 +pycodestyle==2.9.1 +pyparsing==3.0.9 +python-engineio==4.3.4 +python-socketio==5.7.1 +pytz==2022.2.1 +qrcode==7.3.1 +requests==2.21.0 +six==1.16.0 +soupsieve==2.3.2.post1 +SQLAlchemy==1.3.22 +toml==0.10.2 +urllib3==1.24.3 +Werkzeug==2.2.2 +yarl==1.8.1 diff --git a/setup.py b/setup.py index 878516743..8c55842b3 100644 --- a/setup.py +++ b/setup.py @@ -8,12 +8,16 @@ os.path.join(here, "lyrebird", "version.py") )["VERSION"] + +def read_requirements(name): + with open(os.path.join(here, name), encoding='utf-8') as f: + require_str = f.read() + return require_str.split() + + with open(os.path.join(here, 'README.md'), encoding='utf-8') as f: long_description = f.read() -with open(os.path.join(here, 'install_requires.txt'), encoding='utf-8') as f: - install_requires_str = f.read() - install_requires = install_requires_str.split() setup( name='lyrebird', @@ -30,6 +34,7 @@ "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", "License :: OSI Approved :: MIT License", "Operating System :: MacOS", "Operating System :: Microsoft :: Windows", @@ -39,13 +44,8 @@ 'lyrebird = lyrebird.manager:main' ] }, - install_requires=install_requires, + install_requires=read_requirements('requirements.txt.lock'), extras_require={ - 'dev': [ - "autopep8", - "pylint", - "pytest", - "pytest-cov" - ] + 'dev': read_requirements('requirements.dev.txt') } ) diff --git a/dev.sh b/setup.sh similarity index 68% rename from dev.sh rename to setup.sh index 894f2be78..1d6f1507c 100644 --- a/dev.sh +++ b/setup.sh @@ -11,15 +11,10 @@ python3 -m venv --clear venv source ./venv/bin/activate # upgrage pip -pip install --upgrade pip +pip install pip -U -# install from requirements.txt -pip install -r requirements.txt - -# create data dir for debug -if [ ! -e "./data/" ]; then -mkdir ./data -fi +# install from freeze requirements +pip install -r requirements.txt.lock -U echo "***************************" echo " dev setup finish " diff --git a/tests/test_mock_server.py b/tests/test_mock_server.py index 5a3be4fe7..319a2fd14 100644 --- a/tests/test_mock_server.py +++ b/tests/test_mock_server.py @@ -24,6 +24,7 @@ MockConfigManager = NamedTuple('MockConfigManager', [('config', dict)]) + @pytest.fixture def client(): # application.config = conf @@ -39,7 +40,7 @@ def client(): def test_mock_api(client): resp = client.get('/mock/http://www.bing.com') - assert 200<= resp.status_code <= 400 + assert 200 <= resp.status_code <= 400 def test_status_api(client):