Skip to content

[WIP]- Add retry logic for server tests in run_server_tests.py #286

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 7 commits into
base: main
Choose a base branch
from
Open

[WIP]- Add retry logic for server tests in run_server_tests.py #286

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
32e9084
Add retry logic for server tests in run_server_tests.py
Sep 25, 2025
3db0bcc
Implement retry logic for running server tests in run_server_tests.py
Sep 25, 2025
0fc804b
Implement retry logic for running server tests in run_server_tests.py
Sep 25, 2025
52616fd
Implement retry logic for running server tests in run_server_tests.py
Sep 25, 2025
c0f79c4
Implement retry logic for running server tests in run_server_tests.py
Sep 25, 2025
385107c
Implement retry logic for running server tests in run_server_tests.py
Sep 25, 2025
50832c2
Implement retry logic for running server tests in run_server_tests.py
Sep 26, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
143 changes: 89 additions & 54 deletions tools/run_server_tests.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@
used_ports = set()
passed_tests = []
failed_tests = []
retry_summary = {} # Track retry attempts per test

lock = threading.Lock()

Expand Down Expand Up @@ -76,66 +77,97 @@ def print_test_results(s, tests):
for t in tests:
print(f"\t- {t}")

def print_retry_summary():
if retry_summary:
print("\nRetry Summary:")
for test_name, attempts in retry_summary.items():
print(f"\t- {test_name}: passed after {attempts} retries")

def handle_test_scenario(data, root_tests_dir, test_lib_dir, server, benchmark, valgrind, debug):

def handle_test_scenario(data, root_tests_dir, test_lib_dir, server, benchmark, valgrind, debug, max_retries=0, retry_delay=5):
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Function call with 9 positional arguments makes parameter order unclear and error-prone. More info

test_name = data["test_name"]
mock_port = data["mock_port"]
server_port = data["server_port"]
try:
print(f"Running {test_name}...")
print(f"Starting mock server on port {mock_port} with start_config.json for {test_name}...")
mock_aikido_core = subprocess.Popen(["python3", "mock_aikido_core.py", str(mock_port), data["config_path"]])
time.sleep(5)

print(f"Starting {server} server on port {server_port} for {test_name}...")

for attempt in range(max_retries + 1):
server_process = None
mock_aikido_core = None

server_start = servers[server][START_SERVER]
server_process = server_start(data, test_lib_dir, valgrind)

time.sleep(20)

test_script_name = "test.py"
test_script_cwd = data["test_dir"]
if benchmark:
print(f"Running benchmark for {test_name}...")
test_script_name = "benchmark.py"
test_script_cwd = root_tests_dir
else:
print(f"Running test.py for {test_name}...")
try:
if attempt > 0:
print(f"Retry attempt {attempt}/{max_retries} for {test_name}...")
time.sleep(retry_delay)
else:
print(f"Running {test_name}...")

print(f"Starting mock server on port {mock_port} with start_config.json for {test_name}...")
mock_aikido_core = subprocess.Popen(["python3", "mock_aikido_core.py", str(mock_port), data["config_path"]])
time.sleep(5)

print(f"Starting {server} server on port {server_port} for {test_name}...")

subprocess.run(["python3", test_script_name, str(server_port), str(mock_port), test_name],
env=dict(os.environ, PYTHONPATH=f"{test_lib_dir}:$PYTHONPATH"),
cwd=test_script_cwd,
check=True, timeout=600, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

passed_tests.append(test_name)

except subprocess.CalledProcessError as e:
print(f"Error in testing scenario {test_name}:")
print(f"Exception output: {e.output}")
print(f"Test exit code: {e.returncode}")
print(f"Test stdout: {e.stdout.decode()}")
print(f"Test stderr: {e.stderr.decode()}")
failed_tests.append(test_name)

except subprocess.TimeoutExpired:
print(f"Error in testing scenario {test_name}:")
print(f"Execution timed out.")
failed_tests.append(test_name)

finally:
if server_process:
server_process.terminate()
server_process.wait()
print(f"PHP server on port {server_port} stopped.")
server_start = servers[server][START_SERVER]
server_process = server_start(data, test_lib_dir, valgrind)

time.sleep(20)

test_script_name = "test.py"
test_script_cwd = data["test_dir"]
if benchmark:
print(f"Running benchmark for {test_name}...")
test_script_name = "benchmark.py"
test_script_cwd = root_tests_dir
else:
print(f"Running test.py for {test_name}...")

subprocess.run(["python3", test_script_name, str(server_port), str(mock_port), test_name],
env=dict(os.environ, PYTHONPATH=f"{test_lib_dir}:$PYTHONPATH"),
cwd=test_script_cwd,
check=True, timeout=600, stdout=subprocess.PIPE, stderr=subprocess.PIPE)

# Test passed
if attempt > 0:
print(f"Test {test_name} passed on retry attempt {attempt}")
retry_summary[test_name] = attempt
passed_tests.append(test_name)
return # Exit the retry loop on success

except subprocess.CalledProcessError as e:
print(f"Error in testing scenario {test_name} (attempt {attempt + 1}/{max_retries + 1}):")
print(f"Exception output: {e.output}")
print(f"Test exit code: {e.returncode}")
print(f"Test stdout: {e.stdout.decode()}")
print(f"Test stderr: {e.stderr.decode()}")

if attempt == max_retries:
print(f"Test {test_name} failed after {max_retries + 1} attempts")
failed_tests.append(test_name)
else:
print(f"Test {test_name} failed, will retry in {retry_delay} seconds...")

except subprocess.TimeoutExpired:
print(f"Error in testing scenario {test_name} (attempt {attempt + 1}/{max_retries + 1}):")
print(f"Execution timed out.")

if attempt == max_retries:
print(f"Test {test_name} failed after {max_retries + 1} attempts")
failed_tests.append(test_name)
else:
print(f"Test {test_name} timed out, will retry in {retry_delay} seconds...")

finally:
if server_process:
server_process.terminate()
server_process.wait()
print(f"PHP server on port {server_port} stopped.")

if mock_aikido_core:
mock_aikido_core.terminate()
mock_aikido_core.wait()
print(f"Mock server on port {mock_port} stopped.")
if mock_aikido_core:
mock_aikido_core.terminate()
mock_aikido_core.wait()
print(f"Mock server on port {mock_port} stopped.")


def main(root_tests_dir, test_lib_dir, specific_test=None, server="php-built-in", benchmark=False, valgrind=False, debug=False):
def main(root_tests_dir, test_lib_dir, specific_test=None, server="php-built-in", benchmark=False, valgrind=False, debug=False, max_retries=0, retry_delay=5):
if specific_test:
test_dirs = [os.path.join(root_tests_dir, specific_test)]
else:
Expand Down Expand Up @@ -178,21 +210,22 @@ def main(root_tests_dir, test_lib_dir, specific_test=None, server="php-built-in"

threads = []
for test_data in tests_data:
args = (test_data, root_tests_dir, test_lib_dir, server, benchmark, valgrind, debug)
args = (test_data, root_tests_dir, test_lib_dir, server, benchmark, valgrind, debug, max_retries, retry_delay)
thread = threading.Thread(target=handle_test_scenario, args=args)
threads.append(thread)
thread.start()
time.sleep(10)

for thread in threads:
thread.join()

server_uninit = servers[server][UNINIT]
if server_uninit is not None:
server_uninit()

print_test_results("Passed tests:", passed_tests)
print_test_results("Failed tests:", failed_tests)
print_retry_summary()
assert failed_tests == [], f"Found failed tests: {failed_tests}"
print("All tests passed!")

Expand All @@ -206,11 +239,13 @@ def main(root_tests_dir, test_lib_dir, specific_test=None, server="php-built-in"
parser.add_argument("--valgrind", action="store_true", help="Enable valgrind.")
parser.add_argument("--debug", action="store_true", help="Enable debugging logs.")
parser.add_argument("--server", type=str, choices=["php-built-in", "apache-mod-php", "nginx-php-fpm"], default="php-built-in", help="Enable nginx & php-fpm testing.")
parser.add_argument("--max-retries", type=int, default=2, help="Maximum number of retries for failed tests (default: 2).")
parser.add_argument("--retry-delay", type=int, default=5, help="Delay in seconds between retry attempts (default: 5).")

# Parse arguments
args = parser.parse_args()

# Extract values from parsed arguments
root_folder = os.path.abspath(args.root_folder_path)
test_lib_dir = os.path.abspath(args.test_lib_dir)
main(root_folder, test_lib_dir, args.test, args.server, args.benchmark, args.valgrind, args.debug)
main(root_folder, test_lib_dir, args.test, args.server, args.benchmark, args.valgrind, args.debug, args.max_retries, args.retry_delay)
8 changes: 7 additions & 1 deletion tools/server_tests/nginx/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,13 @@ def nginx_php_fpm_pre_tests():
create_folder(php_fpm_run_dir)
create_folder(f'{log_dir}/php-fpm')
modify_nginx_conf(nginx_global_conf)
subprocess.run(['nginx'], check=True)
try:
subprocess.run(['nginx'], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
except subprocess.CalledProcessError as e:
print("Error running nginx:")
print("stdout:", e.stdout)
print("stderr:", e.stderr)
raise
print("nginx server restarted!")
time.sleep(5)

Expand Down
Loading