diff --git a/src/pystack/__main__.py b/src/pystack/__main__.py index ed78b0d3..f0ae7f86 100644 --- a/src/pystack/__main__.py +++ b/src/pystack/__main__.py @@ -160,6 +160,15 @@ def generate_cli_parser() -> argparse.ArgumentParser: help="Include native (C) frames from threads not registered with " "the interpreter (implies --native)", ) + remote_parser.add_argument( + "--native-last", + action="store_const", + dest="native_mode", + const=NativeReportingMode.LAST, + default=NativeReportingMode.OFF, + help="Include native (C) frames only after the last python frame " + "in the resulting stack trace", + ) remote_parser.add_argument( "--locals", action="store_true", @@ -206,6 +215,15 @@ def generate_cli_parser() -> argparse.ArgumentParser: help="Include native (C) frames from threads not registered with " "the interpreter (implies --native)", ) + core_parser.add_argument( + "--native-last", + action="store_const", + dest="native_mode", + const=NativeReportingMode.LAST, + default=NativeReportingMode.OFF, + help="Include native (C) frames only after the last python frame " + "in the resulting stack trace", + ) core_parser.add_argument( "--locals", action="store_true", @@ -269,7 +287,8 @@ def process_remote(parser: argparse.ArgumentParser, args: argparse.Namespace) -> method=StackMethod.ALL if args.exhaustive else StackMethod.AUTO, ): native = args.native_mode != NativeReportingMode.OFF - print_thread(thread, native) + native_last = args.native_mode == NativeReportingMode.LAST + print_thread(thread, native, native_last) def format_psinfo_information(psinfo: Dict[str, Any]) -> str: @@ -378,7 +397,8 @@ def process_core(parser: argparse.ArgumentParser, args: argparse.Namespace) -> N method=StackMethod.ALL if args.exhaustive else StackMethod.AUTO, ): native = args.native_mode != NativeReportingMode.OFF - print_thread(thread, native) + native_last = args.native_mode == NativeReportingMode.LAST + print_thread(thread, native, native_last) if __name__ == "__main__": # pragma: no cover diff --git a/src/pystack/_pystack.pyi b/src/pystack/_pystack.pyi index 36417c8a..67d6d031 100644 --- a/src/pystack/_pystack.pyi +++ b/src/pystack/_pystack.pyi @@ -26,6 +26,7 @@ class NativeReportingMode(enum.Enum): ALL: int OFF: int PYTHON: int + LAST: int class StackMethod(enum.Enum): ALL: int diff --git a/src/pystack/_pystack.pyx b/src/pystack/_pystack.pyx index a949a1ac..38e777cc 100644 --- a/src/pystack/_pystack.pyx +++ b/src/pystack/_pystack.pyx @@ -85,6 +85,7 @@ class NativeReportingMode(enum.Enum): OFF = 0 PYTHON = 1 ALL = 1000 + LAST = 2000 cdef api void log_with_python(const cppstring *message, int level) noexcept: diff --git a/src/pystack/traceback_formatter.py b/src/pystack/traceback_formatter.py index 134dce46..089e9908 100644 --- a/src/pystack/traceback_formatter.py +++ b/src/pystack/traceback_formatter.py @@ -11,8 +11,8 @@ from .types import frame_type -def print_thread(thread: PyThread, native: bool) -> None: - for line in format_thread(thread, native): +def print_thread(thread: PyThread, native: bool, native_last: bool = False) -> None: + for line in format_thread(thread, native, native_last): print(line, file=sys.stdout, flush=True) @@ -62,7 +62,9 @@ def _are_the_stacks_mergeable(thread: PyThread) -> bool: return n_eval_frames == n_entry_frames -def format_thread(thread: PyThread, native: bool) -> Iterable[str]: +def format_thread( + thread: PyThread, native: bool, native_last: bool = False +) -> Iterable[str]: current_frame: Optional[PyFrame] = thread.frame if current_frame is None and not native: yield f"The frame stack for thread {thread.tid} is empty" @@ -81,16 +83,20 @@ def format_thread(thread: PyThread, native: bool) -> Iterable[str]: yield from format_frame(current_frame) current_frame = current_frame.next else: - yield from _format_merged_stacks(thread, current_frame) + yield from _format_merged_stacks(thread, current_frame, native_last) yield "" def _format_merged_stacks( - thread: PyThread, current_frame: Optional[PyFrame] + thread: PyThread, + current_frame: Optional[PyFrame], + native_last: bool = False, ) -> Iterable[str]: + c_frames_list: list[str] = [] for frame in thread.native_frames: if frame_type(frame, thread.python_version) == NativeFrame.FrameType.EVAL: assert current_frame is not None + c_frames_list = [] yield from format_frame(current_frame) current_frame = current_frame.next while current_frame and not current_frame.is_entry: @@ -101,12 +107,18 @@ def _format_merged_stacks( continue elif frame_type(frame, thread.python_version) == NativeFrame.FrameType.OTHER: function = colored(frame.symbol, "yellow") - yield ( + formatted_c_frame = ( f' {colored("(C)", "blue")} File "{frame.path}",' f" line {frame.linenumber}," f" in {function} ({colored(frame.library, attrs=['faint'])})" ) + if native_last: + c_frames_list.append(formatted_c_frame) + else: + yield formatted_c_frame else: # pragma: no cover raise ValueError( f"Invalid frame type: {frame_type(frame, thread.python_version)}" ) + for c_frame in c_frames_list: + yield c_frame diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 86f55cc6..1a78d6b2 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -205,7 +205,9 @@ def test_process_remote_default(): locals=False, method=StackMethod.AUTO, ) - assert print_thread_mock.mock_calls == [call(thread, False) for thread in threads] + assert print_thread_mock.mock_calls == [ + call(thread, False, False) for thread in threads + ] def test_process_remote_no_block(): @@ -236,13 +238,55 @@ def test_process_remote_no_block(): locals=False, method=StackMethod.AUTO, ) - assert print_thread_mock.mock_calls == [call(thread, False) for thread in threads] + assert print_thread_mock.mock_calls == [ + call(thread, False, False) for thread in threads + ] + + +@pytest.mark.parametrize( + "argument, mode", + [ + ["--native", NativeReportingMode.PYTHON], + ["--native-all", NativeReportingMode.ALL], + ], +) +def test_process_remote_native(argument, mode): + # GIVEN + + argv = ["pystack", "remote", "31", argument] + + threads = [Mock(), Mock(), Mock()] + + # WHEN + + with patch( + "pystack.__main__.get_process_threads" + ) as get_process_threads_mock, patch( + "pystack.__main__.print_thread" + ) as print_thread_mock, patch( + "sys.argv", argv + ): + get_process_threads_mock.return_value = threads + main() + + # THEN + + get_process_threads_mock.assert_called_with( + 31, + stop_process=True, + native_mode=mode, + locals=False, + method=StackMethod.AUTO, + ) + assert print_thread_mock.mock_calls == [ + call(thread, True, False) for thread in threads + ] -def test_process_remote_native(): +def test_process_remote_native_last(): # GIVEN - argv = ["pystack", "remote", "31", "--native"] + argv = ["pystack", "remote", "31", "--native-last"] threads = [Mock(), Mock(), Mock()] @@ -263,11 +307,13 @@ def test_process_remote_native(): get_process_threads_mock.assert_called_with( 31, stop_process=True, - native_mode=NativeReportingMode.PYTHON, + native_mode=NativeReportingMode.LAST, locals=False, method=StackMethod.AUTO, ) - assert print_thread_mock.mock_calls == [call(thread, True) for thread in threads] + assert print_thread_mock.mock_calls == [ + call(thread, True, True) for thread in threads + ] def test_process_remote_locals(): @@ -298,7 +344,9 @@ def test_process_remote_locals(): locals=True, method=StackMethod.AUTO, ) - assert print_thread_mock.mock_calls == [call(thread, False) for thread in threads] + assert print_thread_mock.mock_calls == [ + call(thread, False, False) for thread in threads + ] def test_process_remote_native_no_block(capsys): @@ -355,7 +403,9 @@ def test_process_remote_exhaustive(): locals=False, method=StackMethod.ALL, ) - assert print_thread_mock.mock_calls == [call(thread, False) for thread in threads] + assert print_thread_mock.mock_calls == [ + call(thread, False, False) for thread in threads + ] @pytest.mark.parametrize( @@ -428,7 +478,9 @@ def test_process_core_default_without_executable(): locals=False, method=StackMethod.AUTO, ) - assert print_thread_mock.mock_calls == [call(thread, False) for thread in threads] + assert print_thread_mock.mock_calls == [ + call(thread, False, False) for thread in threads + ] def test_process_core_default_without_executable_and_executable_does_not_exist(capsys): @@ -517,7 +569,9 @@ def test_process_core_default_with_executable(): locals=False, method=StackMethod.AUTO, ) - assert print_thread_mock.mock_calls == [call(thread, False) for thread in threads] + assert print_thread_mock.mock_calls == [ + call(thread, False, False) for thread in threads + ] @pytest.mark.parametrize( @@ -562,7 +616,49 @@ def test_process_core_native(argument, mode): locals=False, method=StackMethod.AUTO, ) - assert print_thread_mock.mock_calls == [call(thread, True) for thread in threads] + assert print_thread_mock.mock_calls == [ + call(thread, True, False) for thread in threads + ] + + +def test_process_core_native_last(): + # GIVEN + + argv = ["pystack", "core", "corefile", "executable", "--native-last"] + + threads = [Mock(), Mock(), Mock()] + + # WHEN + + with patch( + "pystack.__main__.get_process_threads_for_core" + ) as get_process_threads_mock, patch( + "pystack.__main__.print_thread" + ) as print_thread_mock, patch( + "sys.argv", argv + ), patch( + "pathlib.Path.exists", return_value=True + ), patch( + "pystack.__main__.CoreFileAnalyzer" + ), patch( + "pystack.__main__.is_elf", return_value=True + ): + get_process_threads_mock.return_value = threads + main() + + # THEN + + get_process_threads_mock.assert_called_with( + Path("corefile"), + Path("executable"), + library_search_path="", + native_mode=NativeReportingMode.LAST, + locals=False, + method=StackMethod.AUTO, + ) + assert print_thread_mock.mock_calls == [ + call(thread, True, True) for thread in threads + ] def test_process_core_locals(): @@ -600,7 +696,9 @@ def test_process_core_locals(): locals=True, method=StackMethod.AUTO, ) - assert print_thread_mock.mock_calls == [call(thread, False) for thread in threads] + assert print_thread_mock.mock_calls == [ + call(thread, False, False) for thread in threads + ] def test_process_core_with_search_path(): @@ -645,7 +743,9 @@ def test_process_core_with_search_path(): locals=False, method=StackMethod.AUTO, ) - assert print_thread_mock.mock_calls == [call(thread, False) for thread in threads] + assert print_thread_mock.mock_calls == [ + call(thread, False, False) for thread in threads + ] def test_process_core_with_search_root(): @@ -691,7 +791,9 @@ def test_process_core_with_search_root(): locals=False, method=StackMethod.AUTO, ) - assert print_thread_mock.mock_calls == [call(thread, False) for thread in threads] + assert print_thread_mock.mock_calls == [ + call(thread, False, False) for thread in threads + ] def test_process_core_with_not_readable_search_root(): @@ -870,7 +972,9 @@ def test_process_core_exhaustive(): locals=False, method=StackMethod.ALL, ) - assert print_thread_mock.mock_calls == [call(thread, False) for thread in threads] + assert print_thread_mock.mock_calls == [ + call(thread, False, False) for thread in threads + ] def test_default_colored_output(): diff --git a/tests/unit/test_traceback_formatter.py b/tests/unit/test_traceback_formatter.py index 62d628ee..67e7e9c6 100644 --- a/tests/unit/test_traceback_formatter.py +++ b/tests/unit/test_traceback_formatter.py @@ -71,7 +71,7 @@ def test_traceback_formatter_no_native(): # WHEN - lines = list(format_thread(thread, native=False)) + lines = list(format_thread(thread, native=False, native_last=False)) # THEN @@ -98,7 +98,7 @@ def test_traceback_formatter_no_frames_no_native(): # WHEN - lines = list(format_thread(thread, native=False)) + lines = list(format_thread(thread, native=False, native_last=False)) # THEN @@ -126,7 +126,7 @@ def test_traceback_formatter_no_frames_native(): # WHEN - lines = list(format_thread(thread, native=True)) + lines = list(format_thread(thread, native=True, native_last=False)) # THEN assert lines == [ @@ -163,7 +163,7 @@ def test_traceback_formatter_no_frames_native_with_eval_frames(): # WHEN - lines = list(format_thread(thread, native=True)) + lines = list(format_thread(thread, native=True, native_last=False)) # THEN assert lines == [ @@ -225,7 +225,7 @@ def test_traceback_formatter_no_mergeable_native_frames(): # WHEN - lines = list(format_thread(thread, native=True)) + lines = list(format_thread(thread, native=True, native_last=False)) # THEN assert lines == [ @@ -294,7 +294,7 @@ def test_traceback_formatter_with_source(): with patch("builtins.open", mock_open(read_data=source_data)), patch( "os.path.exists", return_value=True ): - lines = list(format_thread(thread, native=False)) + lines = list(format_thread(thread, native=False, native_last=False)) # THEN @@ -369,7 +369,7 @@ def test_traceback_formatter_native_matching_simple_eval_frames(): # WHEN - lines = list(format_thread(thread, native=True)) + lines = list(format_thread(thread, native=True, native_last=False)) # THEN @@ -457,7 +457,7 @@ def test_traceback_formatter_native_matching_composite_eval_frames(): # WHEN - lines = list(format_thread(thread, native=True)) + lines = list(format_thread(thread, native=True, native_last=False)) # THEN @@ -571,7 +571,7 @@ def test_traceback_formatter_native_matching_eval_frames_ignore_frames(): # WHEN - lines = list(format_thread(thread, native=True)) + lines = list(format_thread(thread, native=True, native_last=False)) # THEN @@ -616,7 +616,7 @@ def test_traceback_formatter_gil_detection(): # WHEN - lines = list(format_thread(thread, native=False)) + lines = list(format_thread(thread, native=False, native_last=False)) # THEN @@ -655,7 +655,7 @@ def test_traceback_formatter_gc_detection_with_native(): # WHEN - lines = list(format_thread(thread, native=False)) + lines = list(format_thread(thread, native=False, native_last=False)) # THEN @@ -692,7 +692,7 @@ def test_traceback_formatter_gc_detection_without_native(): # WHEN - lines = list(format_thread(thread, native=False)) + lines = list(format_thread(thread, native=False, native_last=False)) # THEN @@ -738,7 +738,7 @@ def test_traceback_formatter_dropping_the_gil_detection(): # WHEN - lines = list(format_thread(thread, native=False)) + lines = list(format_thread(thread, native=False, native_last=False)) # THEN @@ -784,7 +784,7 @@ def test_traceback_formatter_taking_the_gil_detection(): # WHEN - lines = list(format_thread(thread, native=False)) + lines = list(format_thread(thread, native=False, native_last=False)) # THEN @@ -846,7 +846,7 @@ def test_traceback_formatter_native_not_matching_simple_eval_frames(): # WHEN - lines = list(format_thread(thread, native=False)) + lines = list(format_thread(thread, native=False, native_last=False)) # THEN @@ -923,7 +923,7 @@ def test_traceback_formatter_native_not_matching_composite_eval_frames(): # WHEN - lines = list(format_thread(thread, native=False)) + lines = list(format_thread(thread, native=False, native_last=False)) # THEN @@ -1004,7 +1004,7 @@ def test_traceback_formatter_mixed_inlined_frames(): # WHEN - lines = list(format_thread(thread, native=True)) + lines = list(format_thread(thread, native=True, native_last=False)) # THEN @@ -1086,7 +1086,7 @@ def test_traceback_formatter_all_inlined_frames(): # WHEN - lines = list(format_thread(thread, native=True)) + lines = list(format_thread(thread, native=True, native_last=False)) # THEN @@ -1104,6 +1104,89 @@ def test_traceback_formatter_all_inlined_frames(): assert lines == expected_lines +def test_traceback_formatter_native_last(): + # GIVEN + + codes = [ + PyCodeObject( + filename="file1.py", + scope="function1", + location=LocationInfo(1, 1, 0, 0), + ), + PyCodeObject( + filename="file2.py", + scope="function2", + location=LocationInfo(2, 2, 0, 0), + ), + PyCodeObject( + filename="file3.py", + scope="function3", + location=LocationInfo(3, 3, 0, 0), + ), + PyCodeObject( + filename="file4.py", + scope="function4", + location=LocationInfo(4, 4, 0, 0), + ), + PyCodeObject( + filename="file5.py", + scope="function5", + location=LocationInfo(5, 5, 0, 0), + ), + ] + + current_frame = None + for code in reversed(codes): + current_frame = PyFrame( + prev=None, + next=current_frame, + code=code, + arguments={}, + locals={}, + is_entry=False, + ) + current_frame.is_entry = True + + native_frames = [ + NativeFrame(0x0, "native_function1", "native_file1.c", 1, 0, "library.so"), + NativeFrame(0x1, "PyEval_EvalFrameEx", "Python/ceval.c", 123, 0, "library.so"), + NativeFrame(0x3, "native_function2", "native_file2.c", 2, 0, "library.so"), + NativeFrame( + 0x2, "_PyEval_EvalFrameDefault", "Python/ceval.c", 12, 0, "library.so" + ), + NativeFrame(0x4, "native_function3", "native_file3.c", 3, 0, "library.so"), + NativeFrame(0x6, "native_function4", "native_file4.c", 4, 0, "library.so"), + ] + + thread = PyThread( + tid=1, + frame=current_frame, + native_frames=native_frames, + holds_the_gil=False, + is_gc_collecting=False, + python_version=(3, 8), + ) + + # WHEN + + lines = list(format_thread(thread, native=True, native_last=True)) + + # THEN + + expected_lines = [ + "Traceback for thread 1 [] (most recent call last):", + ' (Python) File "file1.py", line 1, in function1', + ' (Python) File "file2.py", line 2, in function2', + ' (Python) File "file3.py", line 3, in function3', + ' (Python) File "file4.py", line 4, in function4', + ' (Python) File "file5.py", line 5, in function5', + ' (C) File "native_file3.c", line 3, in native_function3 (library.so)', + ' (C) File "native_file4.c", line 4, in native_function4 (library.so)', + "", + ] + assert lines == expected_lines + + def test_print_thread(capsys): # GIVEN thread = PyThread( @@ -1120,7 +1203,7 @@ def test_print_thread(capsys): "pystack.traceback_formatter.format_thread", return_value=("1", "2", "3"), ): - print_thread(thread, native=False) + print_thread(thread, native=False, native_last=False) # THEN @@ -1206,7 +1289,7 @@ def test_traceback_formatter_locals( with patch("builtins.open", mock_open(read_data=source_data)), patch( "os.path.exists", return_value=True ): - lines = list(format_thread(thread, native=False)) + lines = list(format_thread(thread, native=False, native_last=False)) # THEN print(lines) @@ -1244,7 +1327,7 @@ def test_traceback_formatter_thread_names(): # WHEN - lines = list(format_thread(thread, native=False)) + lines = list(format_thread(thread, native=False, native_last=False)) # THEN print(lines) @@ -1305,7 +1388,7 @@ def test_traceback_formatter_position_infomation(): "pystack.traceback_formatter.colored", side_effect=lambda x, *args, **kwargs: x, ) as colored_mock: - lines = list(format_thread(thread, native=False)) + lines = list(format_thread(thread, native=False, native_last=False)) # THEN