diff --git a/.gitignore b/.gitignore index 5a2dce5..5f73537 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .gitignore /venv -/.pytest_cache \ No newline at end of file +/.pytest_cache +__pycache__/ diff --git a/.jules/bolt.md b/.jules/bolt.md index 8780446..f8b3080 100644 --- a/.jules/bolt.md +++ b/.jules/bolt.md @@ -1,3 +1,7 @@ ## 2024-05-23 - [Regex Pre-compilation in Loops] **Learning:** Pre-compiling regular expressions (`re.compile`) at the module level provides a significant performance boost (measured ~1.8x speedup) when the regex is used inside a tight loop or a pandas `apply` function, compared to compiling it repeatedly or implicitly inside the loop. Vectorized string operations in Pandas are usually faster, but in complex logic cases (multiple prioritized regex groups + fallback logic), a simple pre-compiled regex with `apply` can sometimes be cleaner and sufficiently fast, or even faster if the vectorized approach requires multiple passes or expensive intermediate structures. **Action:** Always check for regex usage in loops or `apply` calls. If found, refactor to use module-level pre-compiled patterns. When considering vectorization, benchmark against the optimized loop version, as the overhead of complex vectorization might outweigh the benefits for moderate dataset sizes. + +## 2024-05-24 - [TextIOWrapper for Large File Uploads] +**Learning:** Using `io.TextIOWrapper` to wrap binary file streams (like `streamlit.UploadedFile` or `io.BytesIO`) is significantly more memory-efficient than `read().decode()` + `io.StringIO`. The latter creates two full copies of the data in memory (bytes and string) before parsing even begins. `TextIOWrapper` streams and buffers only what is needed. +**Action:** When handling large text-based file uploads (e.g., MGF, mzTab), always prefer `io.TextIOWrapper(file_buffer, encoding='utf-8')` over reading the full content. Ensure the downstream parser (like `pyteomics`) supports file-like objects. diff --git a/app.py b/app.py index aa7990e..c60f581 100644 --- a/app.py +++ b/app.py @@ -32,9 +32,9 @@ def run_streamlit_app(): # Process files only when both are uploaded if mgf_file and mztab_file: # Decode uploaded file contents (Streamlit files are bytes by default) - # Use StringIO to create file-like objects for pyteomics parsers - spectra = load_mgf(io.StringIO(mgf_file.read().decode('utf-8'))) - psm_df = load_mztab(io.StringIO(mztab_file.read().decode('utf-8'))) + # Use TextIOWrapper to wrap the file buffer to avoid reading into memory + spectra = load_mgf(io.TextIOWrapper(mgf_file, encoding='utf-8')) + psm_df = load_mztab(io.TextIOWrapper(mztab_file, encoding='utf-8')) # Create mappings between PSMs and spectra mapped = map_psms_to_spectra(spectra, psm_df)