diff --git a/tools/run_server_tests.py b/tools/run_server_tests.py index af4b281bd..611ec66a5 100755 --- a/tools/run_server_tests.py +++ b/tools/run_server_tests.py @@ -135,9 +135,39 @@ def handle_test_scenario(data, root_tests_dir, test_lib_dir, server, benchmark, print(f"Mock server on port {mock_port} stopped.") -def main(root_tests_dir, test_lib_dir, specific_test=None, server="php-built-in", benchmark=False, valgrind=False, debug=False): - if specific_test: - test_dirs = [os.path.join(root_tests_dir, specific_test)] +def prepare_test_data_for_dir(test_dir, debug, server): + test_name = os.path.basename(os.path.normpath(test_dir)) + mock_port = generate_unique_port() + test_data = { + "test_name": test_name, + "test_dir": test_dir, + "mock_port": mock_port, + "server_port": generate_unique_port(), + "config_path": os.path.join(test_dir, "start_config.json"), + "env_path": os.path.join(test_dir, "env.json") + } + + env = { + "AIKIDO_LOG_LEVEL": "DEBUG" if debug else "ERROR", + "AIKIDO_DISK_LOGS": "1" if debug else "0", + "AIKIDO_TOKEN": "AIK_RUNTIME_MOCK", + "AIKIDO_ENDPOINT": f"http://localhost:{mock_port}/", + "AIKIDO_REALTIME_ENDPOINT": f"http://localhost:{mock_port}/", + } + env.update(load_env_from_json(test_data["env_path"])) + env = {k: v for k, v in env.items() if v != ""} + test_data["env"] = env + + server_process_test = servers[server][PROCESS_TEST] + if server_process_test is not None: + test_data = server_process_test(test_data) + return test_data + + +def main(root_tests_dir, test_lib_dir, specific_tests=None, server="php-built-in", benchmark=False, valgrind=False, debug=False): + if specific_tests: + specific_tests = specific_tests.split(",") # comma separated list of tests + test_dirs = [os.path.join(root_tests_dir, specific_test) for specific_test in specific_tests] else: test_dirs = [f.path for f in os.scandir(root_tests_dir) if f.is_dir()] @@ -147,30 +177,7 @@ def main(root_tests_dir, test_lib_dir, specific_test=None, server="php-built-in" tests_data = [] for test_dir in test_dirs: - mock_port = generate_unique_port() - test_data = { - "test_name": os.path.basename(os.path.normpath(test_dir)), - "test_dir": test_dir, - "mock_port": mock_port, - "server_port": generate_unique_port(), - "config_path": os.path.join(test_dir, "start_config.json"), - "env_path": os.path.join(test_dir, "env.json") - } - - env = { - "AIKIDO_LOG_LEVEL": "DEBUG" if debug else "ERROR", - "AIKIDO_DISK_LOGS": "1" if debug else "0", - "AIKIDO_TOKEN": "AIK_RUNTIME_MOCK", - "AIKIDO_ENDPOINT": f"http://localhost:{mock_port}/", - "AIKIDO_REALTIME_ENDPOINT": f"http://localhost:{mock_port}/", - } - env.update(load_env_from_json(test_data["env_path"])) - env = {k: v for k, v in env.items() if v != ""} - test_data["env"] = env - - server_process_test = servers[server][PROCESS_TEST] - if server_process_test is not None: - test_data = server_process_test(test_data) + test_data = prepare_test_data_for_dir(test_dir, debug, server) tests_data.append(test_data) if servers[server][2] is not None: @@ -191,9 +198,51 @@ def main(root_tests_dir, test_lib_dir, specific_test=None, server="php-built-in" if server_uninit is not None: server_uninit() - print_test_results("Passed tests:", passed_tests) - print_test_results("Failed tests:", failed_tests) - assert failed_tests == [], f"Found failed tests: {failed_tests}" + # Retry logic for failed tests: up to 3 attempts, with debug enabled + name_to_dir = {os.path.basename(os.path.normpath(d)): d for d in test_dirs} + to_retry = list(dict.fromkeys(failed_tests)) + + if len(to_retry): + print(f"Initial failed tests: {to_retry}") + + max_retries = 3 + attempt = 1 + while len(to_retry) and attempt <= max_retries: + print(f"Retry attempt {attempt}/{max_retries} for failed tests with debug enabled...") + + # Clear current failure list for this attempt; keep passed_tests accumulating + failed_tests.clear() + + retry_tests_data = [] + for test_name in to_retry: + if test_name not in name_to_dir: + continue + retry_tests_data.append(prepare_test_data_for_dir(name_to_dir[test_name], True, server)) + + threads = [] + for test_data in retry_tests_data: + args = (test_data, root_tests_dir, test_lib_dir, server, benchmark, valgrind, True) + thread = threading.Thread(target=handle_test_scenario, args=args) + threads.append(thread) + thread.start() + time.sleep(10) + + for thread in threads: + thread.join() + + # Tests that still failed after this attempt + to_retry = list(dict.fromkeys(failed_tests)) + if len(to_retry): + print(f"Still failing after attempt {attempt}: {to_retry}") + + attempt += 1 + + final_passed = sorted(set(passed_tests)) + final_failed = sorted(set(to_retry)) + + print_test_results("Passed tests:", final_passed) + print_test_results("Failed tests:", final_failed) + assert final_failed == [], f"Found failed tests: {final_failed}" print("All tests passed!") @@ -213,4 +262,4 @@ def main(root_tests_dir, test_lib_dir, specific_test=None, server="php-built-in" # Extract values from parsed arguments root_folder = os.path.abspath(args.root_folder_path) test_lib_dir = os.path.abspath(args.test_lib_dir) - main(root_folder, test_lib_dir, args.test, args.server, args.benchmark, args.valgrind, args.debug) + main(root_folder, test_lib_dir, args.test, args.server, args.benchmark, args.valgrind, args.debug) \ No newline at end of file diff --git a/tools/server_tests/nginx/main.py b/tools/server_tests/nginx/main.py index be4224287..7ab257c28 100644 --- a/tools/server_tests/nginx/main.py +++ b/tools/server_tests/nginx/main.py @@ -184,7 +184,27 @@ def nginx_php_fpm_pre_tests(): create_folder(php_fpm_run_dir) create_folder(f'{log_dir}/php-fpm') modify_nginx_conf(nginx_global_conf) - subprocess.run(['nginx'], check=True) + def start_nginx_with_retries(max_retries=5, delay_seconds=2): + for attempt in range(1, max_retries + 1): + try: + return subprocess.run(['nginx'], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + except subprocess.CalledProcessError as e: + stderr_text = ( + e.stderr.decode('utf-8', errors='ignore') + if isinstance(e.stderr, (bytes, bytearray)) else str(e.stderr) + ) + print(f"nginx start attempt {attempt} failed") + # Retry on bind/address-in-use races; nginx may still be releasing the port + if ('Address already in use' in stderr_text) or ('bind()' in stderr_text) or ('could not bind' in stderr_text): + if attempt < max_retries: + time.sleep(delay_seconds * attempt) + continue + print("Error running nginx:") + print("stdout:", e.stdout) + print("stderr:", e.stderr) + raise + + start_nginx_with_retries() print("nginx server restarted!") time.sleep(5)