feat: implement streaming file I/O and refine async infrastructure

- Implement high-performance streaming file I/O in Rust (classic-file-io-core) with Python bindings.
- Add Python wrappers and sync adapters for streaming I/O in ClassicLib.FileIO.
- Refactor AsyncBridge and AsyncUtilities for improved stability and performance.
- Update PapyrusLog monitoring to utilize new async patterns.
- Add comprehensive tests for streaming I/O and updated async logic.
- Add codebase inefficiency report to documentation.

Author: evildarkarchon

ClassicLib/AsyncBridge.py

@@ -729,62 +729,25 @@ def smart_await[T](coro: Coroutine[Any, Any, T]) -> T:
     )
 
 
-def create_sync_wrapper[T](async_func: Callable[..., Coroutine[Any, Any, T]]) -> Callable[..., T]:
+def create_sync_wrapper[T](async_func: Callable[..., Coroutine[Any, Any, T]], strict: bool = False) -> Callable[..., T]:
     """
     Create a sync wrapper for an async function with context-aware execution.
 
     This wrapper automatically chooses the appropriate async execution method:
     - GUI mode: Uses AsyncBridge (Qt event loop integration)
     - CLI/TUI mode: Uses asyncio.run() (creates new event loop per call)
 
-    IMPORTANT - Appropriate Usage:
-    ✅ GUI workers (Qt threads, PySide6 slots)
-    ✅ Testing and benchmarking isolated async functions
-    ✅ One-off operations in sync contexts (initialization, cleanup)
-
-    ❌ DO NOT USE in production CLI main flow
-    ❌ DO NOT USE when already in async context
-    ❌ DO NOT USE for repeated operations in CLI (inefficient)
-
-    Best Practices:
-    - Production CLI code should be async-first (use asyncio.run() once at entry point)
-    - See CLASSIC_ScanLogs.py for reference async-first CLI pattern
-    - In CLI, call async methods directly with await instead of using wrappers
-    - Sync wrappers are primarily for GUI thread safety and testing purposes
-
-    Usage:
-        # Example 1: GUI worker (CORRECT)
-        class CrashLogsScanWorker(QThread):
-            def _perform_scan(self):
-                sync_scan = create_sync_wrapper(async_scan_function)
-                result = sync_scan()  # Uses AsyncBridge in GUI mode
-
-        # Example 2: Testing (CORRECT)
-        def test_async_function():
-            sync_wrapper = create_sync_wrapper(async_function)
-            result = sync_wrapper()  # Uses asyncio.run() in CLI mode
-
-        # Example 3: CLI production (INCORRECT - don't do this)
-        def main():
-            sync_wrapper = create_sync_wrapper(async_function)
-            result = sync_wrapper()  # Creates new event loop per call!
-
-        # Example 4: CLI production (CORRECT - do this instead)
-        async def main():
-            result = await async_function()  # Direct async, one event loop
-
-        if __name__ == "__main__":
-            asyncio.run(main())  # Single event loop at entry point
-
     Args:
         async_func: The async function to wrap
+        strict: If True, raises RuntimeError in CLI/TUI mode instead of falling back
+               to asyncio.run(). Use this for functions that must only be called
+               in GUI contexts to prevent performance "footguns".
 
     Returns:
-        A sync wrapper that works in both GUI and CLI modes
+        A sync wrapper that works in both GUI and CLI modes (unless strict=True)
 
-    Note:
-        The CLI/TUI mode asyncio.run() fallback is intentional for testing
-        and benchmarking. Production CLI code should not rely on this pattern.
+    Raises:
+        RuntimeError: If strict=True and called in CLI/TUI mode.
     """
     import asyncio
 
@@ -796,6 +759,15 @@ def wrapper(*args: Any, **kwargs: Any) -> T:
             # GUI mode: Use AsyncBridge for Qt event loop integration
             bridge = AsyncBridge.get_instance()
             return bridge.run_async(coro)
+        
+        # Strict mode check - prevent inefficient usage in CLI
+        if strict:
+            raise RuntimeError(
+                f"Strict mode: Cannot use sync wrapper for '{async_func.__name__}' in CLI/TUI mode.\n"
+                "This function creates a new event loop for every call, which is inefficient.\n"
+                "Use 'await' and call the async function directly instead."
+            )
+
         # CLI/TUI mode: Use standard asyncio.run()
         return asyncio.run(coro)
 

ClassicLib/FileIO/__init__.py

@@ -16,6 +16,7 @@
     read_crash_log_sync,
     read_file_sync,
     read_lines_sync,
+    stream_lines_sync,
     write_bytes_sync,
     write_crash_report_sync,
     write_file_sync,
@@ -31,6 +32,7 @@
     # Sync adapters
     "read_file_sync",
     "read_lines_sync",
+    "stream_lines_sync",
     "read_bytes_sync",
     "write_file_sync",
     "write_lines_sync",

ClassicLib/FileIO/core.py

@@ -27,6 +27,7 @@
     aiofiles = None  # type: ignore[assignment]
     AIOFILES_AVAILABLE = False
 
+from collections.abc import AsyncIterator, Iterator
 from itertools import starmap
 
 from ClassicLib.FileIO.path_utils import ensure_path
@@ -35,6 +36,7 @@
 # Import async utilities if available
 try:
     from ClassicLib.FileIO.Async import (
+        open_file_with_encoding_async,
         read_file_with_encoding_async,
     )
 
@@ -152,6 +154,59 @@ async def read_lines(self, path: Path | str) -> list[str]:
             content = await loop.run_in_executor(None, path.read_text, self.default_encoding, self.default_errors)
             return content.splitlines()
 
+    async def stream_lines(self, path: Path | str) -> AsyncIterator[str]:
+        """
+        Asynchronously streams the contents of a file line by line.
+
+        This method yields lines from the file one by one, which is memory-efficient
+        for large files. It utilizes automatic encoding detection if available.
+        Lines are stripped of trailing newline characters for consistency with read_lines().
+
+        Args:
+            path (Path | str): The path to the file to be read.
+
+        Yields:
+            str: A single line from the file.
+        """
+        path = FileIOCore._ensure_path(path)
+
+        # Use encoding detection if available
+        if ASYNC_ENCODING_AVAILABLE:
+            async with open_file_with_encoding_async(path) as f:
+                async for line in f:
+                    yield line.rstrip("\n")
+        elif AIOFILES_AVAILABLE:
+            assert aiofiles is not None
+            async with aiofiles.open(path, encoding=self.default_encoding, errors=self.default_errors) as f:
+                async for line in f:
+                    yield line.rstrip("\n")
+        else:
+            # Fallback: Read all lines and yield them (loses streaming benefit but maintains API)
+            lines = await self.read_lines(path)
+            for line in lines:
+                yield line
+
+    def stream_lines_sync(self, path: Path | str) -> Iterator[str]:
+        """
+        Synchronously streams the contents of a file line by line.
+
+        This method yields lines from the file one by one, using automatic encoding
+        detection. It is memory-efficient for large files.
+        Lines are stripped of trailing newline characters.
+
+        Args:
+            path (Path | str): The path to the file to be read.
+
+        Yields:
+            str: A single line from the file.
+        """
+        from ClassicLib.Utils.file_utils import open_file_with_encoding
+
+        path = FileIOCore._ensure_path(path)
+        with open_file_with_encoding(path) as f:
+            for line in f:
+                yield line.rstrip("\n")
+
     @staticmethod
     async def read_bytes(path: Path | str) -> bytes:
         """

ClassicLib/FileIO/sync_adapters.py

@@ -53,11 +53,13 @@ async def main():
     with FileIOCore for optimal performance.
 """
 
+from collections.abc import Iterator
 from pathlib import Path
 from typing import Any
 
 from ClassicLib.AsyncBridge import create_sync_wrapper
 from ClassicLib.integration.factory import get_file_io
+from ClassicLib.Utils.file_utils import open_file_with_encoding
 
 
 # Helper to get core lazily
@@ -112,3 +114,30 @@ async def _append_file(path: Path | str, content: str) -> None:
 read_crash_log_sync = create_sync_wrapper(_read_crash_log)
 write_crash_report_sync = create_sync_wrapper(_write_crash_report)
 append_file_sync = create_sync_wrapper(_append_file)
+
+
+def stream_lines_sync(path: Path | str) -> Iterator[str]:
+    """
+    Synchronously streams the contents of a file line by line.
+
+    This function yields lines from the file one by one, using automatic encoding
+    detection. It is memory-efficient for large files and does NOT use the
+    AsyncBridge or creating a new event loop, making it safe for simple sync loops.
+
+    It attempts to use the Rust-accelerated implementation if available via
+    get_file_io(), otherwise falls back to pure Python.
+
+    Args:
+        path (Path | str): The path to the file to be read.
+
+    Yields:
+        str: A single line from the file.
+    """
+    # Try to use FileIOCore (which might be Rust-accelerated)
+    io_core = _core()
+    if hasattr(io_core, "stream_lines_sync"):
+        yield from io_core.stream_lines_sync(path)
+    else:
+        # Fallback for standard Python FileIOCore or generic IO
+        with open_file_with_encoding(path) as f:
+            yield from f

ClassicLib/PapyrusLog.py

@@ -13,8 +13,7 @@
 
 from ClassicLib import GlobalRegistry
 from ClassicLib.Constants import YAML
-from ClassicLib.FileIO import read_lines_sync
-from ClassicLib.integration.status import is_rust_accelerated
+from ClassicLib.FileIO import stream_lines_sync
 from ClassicLib.Logger import logger
 from ClassicLib.YamlSettingsCache import yaml_settings
 
@@ -23,16 +22,16 @@ def papyrus_logging() -> tuple[str, int]:
     """
     Analyzes Papyrus log files, extracting various statistics and compiling a summary.
 
-    This function reads a Papyrus log file using Rust-accelerated file I/O if available,
+    This function reads a Papyrus log file using streaming file I/O to minimize memory usage,
     and computes key data such as the total number of dumps, stacks, warnings, and errors
     present in the log. It also calculates the ratio of dumps to stacks. If the log file
     is not found, the function provides user guidance on enabling and locating Papyrus
     logging.
 
-    Rust Acceleration:
-        - Uses read_lines_sync for 10x faster file I/O
-        - Automatic encoding detection (no chardet needed)
-        - Better error handling for malformed files
+    Optimization:
+        - Uses stream_lines_sync for memory-efficient line-by-line processing
+        - Automatic encoding detection
+        - Handles multi-gigabyte logs without OOM errors
 
     Returns:
         tuple[str, int]: A tuple containing a formatted string with log analysis
@@ -44,20 +43,15 @@ def papyrus_logging() -> tuple[str, int]:
     message_list: list[str] = []
     papyrus_path: Path | None = yaml_settings(Path, YAML.Game_Local, f"Game{GlobalRegistry.get_vr()}_Info.Docs_File_PapyrusLog")
 
-    # Log Rust acceleration status (only once at module level)
-    if not hasattr(papyrus_logging, "_logged_rust_status"):
-        if is_rust_accelerated("file_io"):
-            logger.debug("Papyrus log reading using Rust-accelerated file I/O (10x faster)")
-        else:
-            logger.debug("Papyrus log reading using Python file I/O implementation")
-        papyrus_logging._logged_rust_status = True  # type: ignore
+    # Log optimization status (only once at module level)
+    if not hasattr(papyrus_logging, "_logged_status"):
+        logger.debug("Papyrus log reading using Streaming I/O (Memory Efficient)")
+        papyrus_logging._logged_status = True  # type: ignore
 
     count_dumps = count_stacks = count_warnings = count_errors = 0
     if papyrus_path and papyrus_path.exists():
-        # Use Rust-accelerated read_lines_sync (automatic encoding detection, 10x faster)
-        papyrus_data: list[str] = read_lines_sync(papyrus_path)
-
-        for line in papyrus_data:
+        # Use streaming I/O (automatic encoding detection, memory efficient)
+        for line in stream_lines_sync(papyrus_path):
             if "Dumping Stacks" in line:
                 count_dumps += 1
             elif "Dumping Stack" in line:

ClassicLib/ScanLog/FormIDAnalyzer.py

@@ -128,7 +128,7 @@ def formid_match(self, formids_matches: list[str], crashlog_plugins: dict[str, s
             RuntimeError: If called in CLI/TUI mode (use FormIDAnalyzerCore instead)
         """
         # Use Phase 2 wrapper - errors in CLI/TUI, works in GUI
-        wrapper = create_sync_wrapper(self._core.formid_match)
+        wrapper = create_sync_wrapper(self._core.formid_match, strict=True)
         return wrapper(formids_matches, crashlog_plugins)
 
     def lookup_formid_value(self, formid: str, plugin: str) -> str | None:
@@ -153,5 +153,5 @@ def lookup_formid_value(self, formid: str, plugin: str) -> str | None:
             RuntimeError: If called in CLI/TUI mode (use FormIDAnalyzerCore instead)
         """
         # Use Phase 2 wrapper - errors in CLI/TUI, works in GUI
-        wrapper = create_sync_wrapper(self._core.lookup_formid_value)
+        wrapper = create_sync_wrapper(self._core.lookup_formid_value, strict=True)
         return wrapper(formid, plugin)

ClassicLib/ScanLog/ScanLogsExecutor.py

@@ -407,7 +407,7 @@ def scan_sync(self) -> ScanResult:
             RuntimeError: If called in CLI/TUI mode (use async methods)
         """
         # Create wrapper per call for proper instance method binding
-        wrapper = create_sync_wrapper(self.execute_scan)
+        wrapper = create_sync_wrapper(self.execute_scan, strict=True)
         return wrapper()