From 3f6c11a9d19ba0b2aea44357796f37aa626d5eec Mon Sep 17 00:00:00 2001 From: Jorge Miguel Silva Date: Wed, 13 Aug 2025 15:57:18 +0100 Subject: [PATCH 1/3] docs: add metrics toggles and examples in README/usage; cli: add --metrics alias; report: respect dict-style enable flags for additional quality tables --- README.md | 18 +++++++++++++++++- docs/source/usage.rst | 18 +++++++++++++++++- src/phenoqc/cli.py | 2 +- src/phenoqc/reporting.py | 8 ++++++-- 4 files changed, 41 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 166a338..c728258 100644 --- a/README.md +++ b/README.md @@ -156,7 +156,7 @@ phenoqc \ - `--impute-params '{"n_neighbors": 5}'` (JSON) - `--impute-tuning on|off` - `--label-column class` and `--imbalance-threshold 0.10` -- `--quality-metrics imputation_bias redundancy` (or `all`) +- `--quality-metrics imputation_bias redundancy` (or `all`) (alias: `--metrics`) - Imputation-bias thresholds: `--bias-smd-threshold`, `--bias-var-low`, `--bias-var-high`, `--bias-ks-alpha` - Categorical bias thresholds: `--bias-psi-threshold`, `--bias-cramer-threshold` - Imputation stability diagnostics: `--impute-diagnostics on|off`, `--diag-repeats`, `--diag-mask-fraction`, `--diag-scoring` @@ -384,6 +384,22 @@ cache_expiry_days: 30 # optional: offline (forces cache/local ontologies only for the run) # offline: true +quality_metrics: + redundancy: { enable: true } + imputation_bias: { enable: true } + imputation_stability: { enable: true, repeats: 5, mask_fraction: 0.10, scoring: MAE } +class_distribution: + label_column: class + warn_threshold: 0.10 + +imputation_bias: + smd_threshold: 0.10 + var_ratio_low: 0.5 + var_ratio_high: 2.0 + ks_alpha: 0.05 + psi_threshold: 0.10 + cramer_threshold: 0.20 + imputation: strategy: knn params: diff --git a/docs/source/usage.rst b/docs/source/usage.rst index ae84d8b..98556fb 100644 --- a/docs/source/usage.rst +++ b/docs/source/usage.rst @@ -50,7 +50,7 @@ Parameters - ``--impute``: Strategy for missing data (mean, median, mode, knn, mice, svd, none) - ``--impute-params``: JSON object of parameters for the imputation strategy (e.g. ``{"n_neighbors": 5}``) - ``--impute-tuning {on,off}``: Enable quick tuning (mask-and-score) for imputation -- ``--quality-metrics``: Choose metrics (e.g., ``imputation_bias``); ``all`` enables all +- ``--quality-metrics`` (alias: ``--metrics``): Choose metrics (e.g., ``imputation_bias``); ``all`` enables all - ``--bias-smd-threshold``, ``--bias-var-low``, ``--bias-var-high``, ``--bias-ks-alpha``: thresholds for numeric bias diagnostics - ``--bias-psi-threshold``, ``--bias-cramer-threshold``: thresholds for categorical bias diagnostics (PSI, Cramér’s V) - ``--impute-diagnostics {on,off}``, ``--diag-repeats``, ``--diag-mask-fraction``, ``--diag-scoring``: stability diagnostics @@ -108,6 +108,22 @@ PhenoQC uses a YAML configuration file to define settings. Example ``config.yaml cache_expiry_days: 30 # offline: true # optional: force cached/local ontologies only for the run + quality_metrics: + redundancy: { enable: true } + imputation_bias: { enable: true } + imputation_stability: { enable: true, repeats: 5, mask_fraction: 0.10, scoring: MAE } + class_distribution: + label_column: class + warn_threshold: 0.10 + + imputation_bias: + smd_threshold: 0.10 + var_ratio_low: 0.5 + var_ratio_high: 2.0 + ks_alpha: 0.05 + psi_threshold: 0.10 + cramer_threshold: 0.20 + imputation: strategy: knn params: diff --git a/src/phenoqc/cli.py b/src/phenoqc/cli.py index ba5d19f..a9b9f62 100644 --- a/src/phenoqc/cli.py +++ b/src/phenoqc/cli.py @@ -65,7 +65,7 @@ def parse_arguments(): help='[Deprecated] Use --phenotype_columns instead' ) parser.add_argument( - '--quality-metrics', + '--quality-metrics', '--metrics', nargs='+', choices=QUALITY_METRIC_CHOICES + ['all'], help='Additional quality metrics to evaluate', diff --git a/src/phenoqc/reporting.py b/src/phenoqc/reporting.py index 053d6ac..41fe0c8 100644 --- a/src/phenoqc/reporting.py +++ b/src/phenoqc/reporting.py @@ -304,10 +304,14 @@ def build_dataframe_table(df: pd.DataFrame, title: str, max_rows: int = 50): if isinstance(quality_metrics_enabled, list): enabled_ids = {str(m).lower() for m in quality_metrics_enabled} elif isinstance(quality_metrics_enabled, dict): - # dictionary style not typically used for these metrics; treat any truthy flag as enabled + # dictionary style: check nested enable flags when present for mid in ["accuracy", "redundancy", "traceability", "timeliness"]: try: - if quality_metrics_enabled.get(mid): + block = quality_metrics_enabled.get(mid) + if isinstance(block, dict): + if block.get('enable', False): + enabled_ids.add(mid) + elif block: enabled_ids.add(mid) except Exception: pass From 7a6dc967e90865aba8444050c477501a538761e8 Mon Sep 17 00:00:00 2001 From: Jorge Miguel Silva Date: Thu, 14 Aug 2025 14:18:49 +0100 Subject: [PATCH 2/3] =?UTF-8?q?GUI:=20strategy-agnostic=20imputation=20pan?= =?UTF-8?q?el;=20fix=20quality=20metrics=20defaults;=20Reporting:=20add=20?= =?UTF-8?q?categorical=20PSI/Cram=C3=A9r=E2=80=99s=20V=20to=20bias=20rule?= =?UTF-8?q?=20header=20and=20per-row=20triggers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/phenoqc/gui/__init__.py | 6 +- src/phenoqc/gui/gui.py | 243 ++++++++++++++++++++++-------------- src/phenoqc/gui/views.py | 33 +++-- src/phenoqc/reporting.py | 24 +++- 4 files changed, 200 insertions(+), 106 deletions(-) diff --git a/src/phenoqc/gui/__init__.py b/src/phenoqc/gui/__init__.py index ff003d7..f2a99d9 100644 --- a/src/phenoqc/gui/__init__.py +++ b/src/phenoqc/gui/__init__.py @@ -1,3 +1,7 @@ -from .gui import main +def main(): + # Lazy import so test modules that import phenoqc.gui.views + # do not pull heavy GUI dependencies at import time. + from .gui import main as _main + return _main() __all__ = ["main"] diff --git a/src/phenoqc/gui/gui.py b/src/phenoqc/gui/gui.py index 9542345..0a9734b 100644 --- a/src/phenoqc/gui/gui.py +++ b/src/phenoqc/gui/gui.py @@ -19,6 +19,7 @@ from ..utils.ontology_utils import suggest_ontologies import glob import numpy as np +from typing import Optional def preserve_original_format_and_save(df_in_memory, original_filename, out_dir): base, ext = os.path.splitext(original_filename) @@ -703,98 +704,162 @@ def proceed_to_step(step_name): # Fixed supported strategies for UI; avoid case mismatch supported_strategies = ['none', 'mean', 'median', 'mode', 'knn', 'mice', 'svd'] - st.subheader("Configure Imputation Strategy") - # Determine default strategy from config['imputation'] if present - default_strategy_value = str(imputation_cfg.get('strategy') or 'none').lower() - try: - default_idx = supported_strategies.index(default_strategy_value) - except ValueError: - default_idx = 0 - global_strategy = st.selectbox( - "Default Imputation Strategy", - options=supported_strategies, - index=default_idx, - help="Used for columns without specific overrides" - ) - - st.subheader("Column-Specific Overrides") - column_strategies = {} - - per_col_cfg = imputation_cfg.get('per_column', {}) if imputation_cfg else {} - - for col in all_columns: - with st.expander(f"Column: {col}", expanded=False): - # We'll guess from config or fallback to global - suggested = ( - (per_col_cfg.get(col, {}) or {}).get('strategy') - or default_strategies.get(col) - or global_strategy - ) + # --- Strategy-agnostic, config-driven Imputation panel --- + st.subheader("Imputation") + + # Parameter specs per strategy + PARAM_SPECS = { + "none": {}, + "mean": {}, + "median": {}, + "mode": {}, + "knn": { + "n_neighbors": {"widget": "int", "default": 5, "min": 1, "max": 100}, + "weights": {"widget": "select", "options": ["uniform", "distance"], "default": "uniform"}, + "metric": {"widget": "select", "options": ["nan_euclidean"], "default": "nan_euclidean"}, + }, + "mice": { + "max_iter": {"widget": "int", "default": 10, "min": 1, "max": 200}, + "sample_posterior": {"widget": "bool", "default": False}, + "random_state": {"widget": "int", "default": 0, "min": 0, "max": 10000}, + }, + "svd": { + "rank": {"widget": "int", "default": 3, "min": 1, "max": 200}, + "max_iters": {"widget": "int", "default": 50, "min": 1, "max": 10000}, + "convergence_threshold": {"widget": "float", "default": 1e-4}, + }, + } - strategy_options = ['Use Default'] + supported_strategies - default_index = 0 - if suggested in strategy_options: - default_index = strategy_options.index(suggested) + def _render_params(spec: dict, initial: Optional[dict] = None) -> dict: + initial = initial or {} + values: dict = {} + for name, meta in spec.items(): + w = meta["widget"] + if w == "int": + values[name] = st.number_input( + name, + value=int(initial.get(name, meta.get("default", 0))), + min_value=int(meta.get("min", -10000)), + max_value=int(meta.get("max", 10000)), + step=1, + key=f"int_{name}", + ) + elif w == "float": + values[name] = st.number_input( + name, + value=float(initial.get(name, meta.get("default", 0.0))), + key=f"float_{name}", + ) + elif w == "bool": + values[name] = st.checkbox( + name, value=bool(initial.get(name, meta.get("default", False))), key=f"bool_{name}" + ) + elif w == "select": + options = meta["options"] + default = initial.get(name, meta.get("default", options[0])) + idx = options.index(default) if default in options else 0 + values[name] = st.selectbox(name, options, index=idx, key=f"sel_{name}") + return values + + # Existing config scaffold (if any) + impute_cfg = config.get("imputation", {}) or {} + existing_strategy = impute_cfg.get("strategy", "knn") + existing_params = impute_cfg.get("params", {}) + existing_overrides = impute_cfg.get("per_column", {}) + existing_tuning = impute_cfg.get("tuning", {}) + + # 4A) Global strategy + params + strategy = st.selectbox( + "Global strategy", + list(PARAM_SPECS.keys()), + index=max(0, list(PARAM_SPECS.keys()).index(existing_strategy) if existing_strategy in PARAM_SPECS else 0), + ) + params = _render_params(PARAM_SPECS[strategy], initial=existing_params) + + # 4B) Per-column overrides + st.markdown("**Per-column overrides (optional)**") + if existing_overrides and isinstance(existing_overrides, dict): + rows = [] + for col, ov in existing_overrides.items(): + rows.append({ + "column": col, + "strategy": ov.get("strategy", strategy), + "params": json.dumps(ov.get("params", {})), + }) + overrides_df = pd.DataFrame(rows) + else: + overrides_df = pd.DataFrame(columns=["column", "strategy", "params"]) + + overrides_df = st.data_editor( + overrides_df, + num_rows="dynamic", + use_container_width=True, + column_config={ + "column": st.column_config.TextColumn("column"), + "strategy": st.column_config.SelectboxColumn("strategy", options=list(PARAM_SPECS.keys())), + "params": st.column_config.TextColumn("params (JSON)", help='e.g. {"rank": 3}'), + }, + key="per_column_editor", + ) - strategy = st.selectbox( - f"Imputation strategy for {col}", - options=strategy_options, - index=default_index, - key=f"impute_{col}" - ) - if strategy != 'Use Default': - column_strategies[col] = strategy - - # Global parameter inputs (common params per strategy) - st.subheader("Imputation Parameters") - params: dict = {} - current_params = imputation_cfg.get('params', {}) if imputation_cfg else {} - if global_strategy == 'knn': - n_neighbors = st.number_input("KNN n_neighbors", min_value=1, max_value=100, value=int(current_params.get('n_neighbors', 5)), step=1) - weights = st.selectbox("KNN weights", options=['uniform', 'distance'], index=0 if current_params.get('weights', 'uniform') == 'uniform' else 1) - params.update({'n_neighbors': int(n_neighbors), 'weights': weights}) - elif global_strategy == 'mice': - max_iter = st.number_input("MICE max_iter", min_value=1, max_value=100, value=int(current_params.get('max_iter', 10)), step=1) - params.update({'max_iter': int(max_iter)}) - elif global_strategy == 'svd': - # Optional parameters for IterativeSVD - rank = st.number_input("SVD rank (optional, 0=auto)", min_value=0, max_value=500, value=int(current_params.get('rank', 0)), step=1) - if rank > 0: - params.update({'rank': int(rank)}) - - # Quick tuning controls (mask-and-score) - st.subheader("Quick Tuning (mask-and-score)") - tuning_defaults = imputation_cfg.get('tuning', {}) if imputation_cfg else {} - enable_tuning = st.checkbox("Enable tuning", value=bool(tuning_defaults.get('enable', False)), - help="Evaluate candidate parameters on masked observed cells and choose the best.") - tuning_cfg = {} - if enable_tuning: - mask_fraction = st.slider("Mask fraction", min_value=0.01, max_value=0.5, value=float(tuning_defaults.get('mask_fraction', 0.10)), step=0.01, key="tuning_mask_fraction") - scoring = st.selectbox("Scoring metric", options=['MAE', 'RMSE'], index=0 if str(tuning_defaults.get('scoring', 'MAE')).upper() == 'MAE' else 1, key="tuning_scoring") - max_cells = st.number_input("Max cells", min_value=1000, max_value=200000, value=int(tuning_defaults.get('max_cells', 50000)), step=1000) - random_state = st.number_input("Random state", min_value=0, max_value=10**9, value=int(tuning_defaults.get('random_state', 42)), step=1) - default_grid = tuning_defaults.get('grid', {}).get('n_neighbors', [3, 5, 7]) - grid_n = st.text_input("Grid for n_neighbors (comma-separated)", value=",".join(map(str, default_grid))) + per_column: dict = {} + for _, row in overrides_df.iterrows(): + col = str(row.get("column") or "").strip() + if not col: + continue + col_strategy = row.get("strategy") or strategy try: - grid_vals = [int(x.strip()) for x in grid_n.split(',') if x.strip()] + col_params = json.loads(row.get("params") or "{}") + if not isinstance(col_params, dict): + raise ValueError("params must be a JSON object") except Exception: - grid_vals = [3, 5, 7] - tuning_cfg = { - 'enable': True, - 'mask_fraction': float(mask_fraction), - 'scoring': scoring, - 'max_cells': int(max_cells), - 'random_state': int(random_state), - 'grid': {'n_neighbors': grid_vals} + col_params = {} + per_column[col] = {"strategy": col_strategy, "params": col_params} + + # 4C) Tuning (mask-and-score) + with st.expander("Tuning (mask-and-score)", expanded=False): + enable = st.checkbox("Enable tuning", value=bool(existing_tuning.get("enable", False))) + mask_fraction = st.slider("Mask fraction", 0.01, 0.50, float(existing_tuning.get("mask_fraction", 0.10))) + scoring = st.selectbox("Scoring", ["MAE", "RMSE"], index=0 if str(existing_tuning.get("scoring", "MAE")).upper()=="MAE" else 1) + max_cells = st.number_input("Max cells", min_value=1000, max_value=200000, value=int(existing_tuning.get("max_cells", 50000)), step=1000) + random_state = st.number_input("Random state", min_value=0, max_value=10**9, value=int(existing_tuning.get("random_state", 42)), step=1) + default_grid = {"knn": {"n_neighbors": [3, 5, 7]}, + "mice": {"max_iter": [5, 10, 15]}, + "svd": {"rank": [2, 3, 5]}}.get(strategy, {}) + grid_text = st.text_area( + "Parameter grid (JSON)", + value=json.dumps(existing_tuning.get("grid", default_grid), indent=2), + help="You can provide any param grid here; keys must match the selected strategy.", + ) + try: + grid = json.loads(grid_text) if grid_text.strip() else {} + if not isinstance(grid, dict): + raise ValueError("grid must be a JSON object") + except Exception: + st.warning("Invalid grid JSON; ignoring and using an empty grid.") + grid = {} + + tuning = { + "enable": enable, + "mask_fraction": mask_fraction, + "scoring": scoring, + "max_cells": int(max_cells), + "random_state": int(random_state), + "grid": grid, } - # Persist imputation block into config for ImputationEngine - st.session_state.setdefault('config', {}) - st.session_state['config']['imputation'] = { - 'strategy': None if global_strategy == 'Use Default' else str(global_strategy).lower(), - 'params': params or {}, - 'per_column': {c: {'strategy': s} for c, s in column_strategies.items()}, - 'tuning': tuning_cfg or {'enable': False}, + # Persist into the config dict the rest of the app uses + config["imputation"] = { + "strategy": strategy, + "params": params, + "per_column": per_column, + "tuning": tuning, + } + + # Back-compat: also surface a simplified mirror used later in the run step + st.session_state['imputation_config'] = { + 'global_strategy': strategy, + 'column_strategies': {c: v.get('strategy') for c, v in per_column.items()}, } # Imputation-bias diagnostics (optional) @@ -903,11 +968,7 @@ def proceed_to_step(step_name): else: st.session_state['config'].pop('mi_uncertainty', None) - # Retain older session fields (used by legacy flow) - st.session_state['imputation_config'] = { - 'global_strategy': global_strategy, - 'column_strategies': column_strategies - } + # Retain older session fields (used by legacy flow) — already set above st.markdown("---") st.success("Imputation configuration complete!") diff --git a/src/phenoqc/gui/views.py b/src/phenoqc/gui/views.py index e3151a8..43b3c15 100644 --- a/src/phenoqc/gui/views.py +++ b/src/phenoqc/gui/views.py @@ -7,17 +7,30 @@ def build_quality_metrics_widget(cfg: Dict) -> Dict: """Return widget configuration for quality metrics selection. - Parameters - ---------- - cfg : dict - Current configuration dictionary. - - Returns - ------- - dict - Dictionary with available options and currently selected metrics. + - Supports both list-style (e.g., ["accuracy", ...]) and dict-style + (e.g., {"imputation_bias": {...}, "imputation_stability": {...}}) + - Filters unknown keys not in QUALITY_OPTIONS + - Excludes diagnostics that are configured elsewhere in the UI (e.g., + imputation_stability), preventing Streamlit defaults from including + values that are not present in the options list. """ - selected = cfg.get("quality_metrics", []) if cfg else [] + selected_raw = (cfg or {}).get("quality_metrics", []) + # Normalize to a list of known metrics + if isinstance(selected_raw, dict): + # Take enabled metrics (or all present) that are part of QUALITY_METRIC_CHOICES + enabled_keys = [] + for k, v in selected_raw.items(): + try: + is_enabled = bool(v.get("enable", True)) if isinstance(v, dict) else bool(v) + except Exception: + is_enabled = True + if is_enabled and k in QUALITY_METRIC_CHOICES: + enabled_keys.append(k) + selected = enabled_keys + elif isinstance(selected_raw, list): + selected = [m for m in selected_raw if m in QUALITY_METRIC_CHOICES] + else: + selected = [] return {"options": QUALITY_OPTIONS, "selected": selected} def apply_quality_metrics_selection(cfg: Dict, selection: List[str]) -> Dict: diff --git a/src/phenoqc/reporting.py b/src/phenoqc/reporting.py index 41fe0c8..cd1555d 100644 --- a/src/phenoqc/reporting.py +++ b/src/phenoqc/reporting.py @@ -434,10 +434,13 @@ def build_dataframe_table(df: pd.DataFrame, title: str, max_rows: int = 50): pass # Show thresholds if provided if isinstance(bias_thresholds, dict) and bias_thresholds: - thr_text = \ - f"SMD≥{bias_thresholds.get('smd_threshold', 0.10)} | " \ - f"Var-ratio∉[{bias_thresholds.get('var_ratio_low', 0.5)},{bias_thresholds.get('var_ratio_high', 2.0)}] | " \ - f"KS p<{bias_thresholds.get('ks_alpha', 0.05)}" + thr_text = ( + f"SMD≥{bias_thresholds.get('smd_threshold', 0.10)} | " + f"Var-ratio∉[{bias_thresholds.get('var_ratio_low', 0.5)},{bias_thresholds.get('var_ratio_high', 2.0)}] | " + f"KS p<{bias_thresholds.get('ks_alpha', 0.05)} | " + f"PSI≥{bias_thresholds.get('psi_threshold', 0.10)} | " + f"CramérV≥{bias_thresholds.get('cramer_threshold', 0.20)}" + ) story.append(Paragraph(f"Warning rules: {thr_text}", styles['Normal'])) story.append(Spacer(1, 6)) # Only render bias tables if we have bias diagnostics @@ -468,6 +471,19 @@ def _trigger_text(row) -> str: reasons.append(f"KS p<{_ks_alpha}") except Exception: pass + # Categorical triggers + try: + psi_val = pd.to_numeric(row.get('psi'), errors='coerce') + if pd.notnull(psi_val) and float(psi_val) >= float(bias_thresholds.get('psi_threshold', 0.10)): + reasons.append(f"PSI≥{bias_thresholds.get('psi_threshold', 0.10)}") + except Exception: + pass + try: + cv_val = pd.to_numeric(row.get("cramers_v"), errors='coerce') + if pd.notnull(cv_val) and float(cv_val) >= float(bias_thresholds.get('cramer_threshold', 0.20)): + reasons.append(f"CramérV≥{bias_thresholds.get('cramer_threshold', 0.20)}") + except Exception: + pass return "; ".join(reasons) cols_desired = [ From 56de9b58e0aa6ce709de3fa52236cd9ddc136140 Mon Sep 17 00:00:00 2001 From: Jorge Miguel Silva Date: Thu, 14 Aug 2025 14:20:17 +0100 Subject: [PATCH 3/3] Scripts: add clinical_all_features_e2e end-to-end exam (dataset generation, full schema/config/custom mappings, online+offline runs, per-column imputation coverage, fallback output discovery) --- scripts/clinical_all_features_e2e.py | 439 ++++++++++++++++++ .../config/clinical_all_features_config.yaml | 118 +++++ .../clinical_all_features_custom_mapping.json | 11 + .../config/clinical_all_features_schema.json | 155 +++++++ 4 files changed, 723 insertions(+) create mode 100644 scripts/clinical_all_features_e2e.py create mode 100644 scripts/config/clinical_all_features_config.yaml create mode 100644 scripts/config/clinical_all_features_custom_mapping.json create mode 100644 scripts/config/clinical_all_features_schema.json diff --git a/scripts/clinical_all_features_e2e.py b/scripts/clinical_all_features_e2e.py new file mode 100644 index 0000000..9c07555 --- /dev/null +++ b/scripts/clinical_all_features_e2e.py @@ -0,0 +1,439 @@ +#!/usr/bin/env python3 +""" +clinical_all_features_e2e.py + +Comprehensive end-to-end script that: +- Generates a realistic clinical dataset with diverse data types +- Writes a full JSON Schema capturing types and constraints +- Writes a config.yaml covering all features: + - Ontologies and fuzzy mapping config (threshold, cache expiry) + - Imputation: global strategy, params, per-column overrides, tuning + - Protected columns, redundancy settings + - Quality metrics: accuracy, redundancy, traceability, timeliness, + imputation_bias, imputation_stability, imputation_uncertainty + - Class distribution +- Runs the PhenoQC CLI twice: + 1) Regular run + 2) Offline run to exercise cache/offline mapping path +- Verifies outputs exist and contain expected artifacts + +Outputs are placed under scripts/output/clinical_all_features/ + +Usage: + python scripts/clinical_all_features_e2e.py +""" + +import os +import sys +import json +import subprocess +from typing import Tuple +import glob + +import numpy as np +import pandas as pd + + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +PROJECT_ROOT = os.path.dirname(SCRIPT_DIR) +SRC_PATH = os.path.join(PROJECT_ROOT, "src") +if SRC_PATH not in sys.path: + sys.path.insert(0, SRC_PATH) + +OUT_BASE = os.path.join(SCRIPT_DIR, "output", "clinical_all_features") +RUN1_DIR = os.path.join(OUT_BASE, "run_online") +RUN2_DIR = os.path.join(OUT_BASE, "run_offline") +os.makedirs(RUN1_DIR, exist_ok=True) +os.makedirs(RUN2_DIR, exist_ok=True) + +DATA_PATH = os.path.join(OUT_BASE, "clinical_input.csv") +CONFIG_PATH = os.path.join(SCRIPT_DIR, "config", "clinical_all_features_config.yaml") +SCHEMA_PATH = os.path.join(SCRIPT_DIR, "config", "clinical_all_features_schema.json") +CUSTOM_MAPPING_PATH = os.path.join(SCRIPT_DIR, "config", "clinical_all_features_custom_mapping.json") +os.makedirs(os.path.join(SCRIPT_DIR, "config"), exist_ok=True) + + +def write_schema() -> None: + schema = { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Clinical All Features Schema", + "type": "object", + "properties": { + "PatientID": {"type": ["string", "null"]}, + "VisitID": {"type": ["string", "null"]}, + "Age": {"type": ["number", "null"], "minimum": 0, "maximum": 120}, + "Sex": {"type": ["string", "null"], "enum": ["M", "F", "Other", None]}, + "Height_cm": {"type": ["number", "null"], "minimum": 30, "maximum": 250}, + "Weight_kg": {"type": ["number", "null"], "minimum": 1, "maximum": 400}, + "BMI": {"type": ["number", "null"], "minimum": 5, "maximum": 100}, + "BP_systolic": {"type": ["number", "null"], "minimum": 50, "maximum": 250}, + "BP_diastolic": {"type": ["number", "null"], "minimum": 30, "maximum": 200}, + "Cholesterol_mgdl": {"type": ["number", "null"], "minimum": 50, "maximum": 600}, + "Glucose_mgdl": {"type": ["number", "null"], "minimum": 20, "maximum": 1000}, + "Creatinine_mgdl": {"type": ["number", "null"], "minimum": 0.1, "maximum": 20}, + "PrimaryPhenotype": {"type": ["string", "null"]}, + "SecondaryPhenotype": {"type": ["string", "null"]}, + "DiseaseCode": {"type": ["string", "null"]}, + "MedicationCode": {"type": ["string", "null"]}, + "Smoker": {"type": ["boolean", "null"]}, + "Pregnant": {"type": ["boolean", "null"]}, + "VisitDate": {"type": ["string", "null"]}, + "class": {"type": ["string", "null"]} + }, + "required": ["PatientID", "VisitID"], + } + with open(SCHEMA_PATH, "w", encoding="utf-8") as fh: + json.dump(schema, fh, indent=2) + + +def write_config() -> None: + cfg = f""" +ontologies: + HPO: + name: Human Phenotype Ontology + source: local + file: {os.path.join(PROJECT_ROOT, 'ontologies', 'hp.obo')} + format: obo + DO: + name: Disease Ontology + source: local + file: {os.path.join(PROJECT_ROOT, 'ontologies', 'doid.obo')} + format: obo + MPO: + name: Mammalian Phenotype Ontology + source: local + file: {os.path.join(PROJECT_ROOT, 'ontologies', 'mp.obo')} + format: obo + +default_ontologies: + - HPO + - DO + - MPO + +fuzzy_threshold: 82 +cache_expiry_days: 1 + +imputation: + strategy: knn + params: + n_neighbors: 5 + weights: uniform + metric: nan_euclidean + per_column: + Age: + strategy: mean + Height_cm: + strategy: median + Weight_kg: + strategy: knn + params: + n_neighbors: 7 + weights: distance + metric: nan_euclidean + BMI: + strategy: mean + BP_systolic: + strategy: mice + params: + max_iter: 10 + random_state: 11 + BP_diastolic: + strategy: mice + params: + max_iter: 10 + random_state: 11 + Glucose_mgdl: + strategy: knn + params: + n_neighbors: 5 + weights: uniform + metric: nan_euclidean + Creatinine_mgdl: + strategy: mice + params: + max_iter: 8 + random_state: 7 + Cholesterol_mgdl: + strategy: svd + params: + rank: 3 + max_iters: 50 + Sex: + strategy: mode + Smoker: + strategy: mode + Pregnant: + strategy: mode + tuning: + enable: true + mask_fraction: 0.1 + scoring: MAE + max_cells: 20000 + random_state: 42 + grid: + n_neighbors: [3, 5, 7, 9] + +protected_columns: + - PatientID + - VisitID + - VisitDate + - MedicationCode + +redundancy: + threshold: 0.98 + method: pearson + +quality_metrics: + imputation_bias: + enable: true + smd_threshold: 0.10 + var_ratio_low: 0.5 + var_ratio_high: 2.0 + ks_alpha: 0.05 + imputation_stability: + enable: true + repeats: 5 + mask_fraction: 0.1 + scoring: MAE + +class_distribution: + label_column: class + warn_threshold: 0.10 + +mi_uncertainty: + enable: true + repeats: 3 + params: + max_iter: 6 +""" + with open(CONFIG_PATH, "w", encoding="utf-8") as fh: + fh.write(cfg) + + +def write_custom_mapping() -> None: + # Pin a few terms explicitly to validate custom mapping path + mapping = { + "HP:0001250": {"HPO": "HP:0001250"}, # seizures as-is + "seizure": {"HPO": "HP:0001250"}, # normalized synonym + "DOID:9352": {"DO": "DOID:9352"} # coronary artery disease + } + with open(CUSTOM_MAPPING_PATH, "w", encoding="utf-8") as fh: + json.dump(mapping, fh, indent=2) + + +def create_dataset(n: int = 3000, seed: int = 11) -> pd.DataFrame: + rng = np.random.RandomState(seed) + pid = [f"P{str(i+1).zfill(6)}" for i in range(n)] + vid = [f"V{str(i+1).zfill(6)}" for i in range(n)] + + age = rng.randint(0, 100, size=n).astype(float) + sex = rng.choice(["M", "F", "Other", None], size=n, p=[0.48, 0.48, 0.02, 0.02]) + height = rng.normal(170, 10, n) + weight = rng.normal(75, 15, n) + 0.15 * (height - 170) + bmi = weight / ((height/100) ** 2) + bp_sys = rng.normal(120, 18, n) + bp_dia = rng.normal(78, 12, n) + chol = np.clip(rng.lognormal(mean=5.1, sigma=0.35, size=n), 80, 600) + gluc = np.clip(rng.normal(100, 25, n), 30, 1000) + creat = np.abs(rng.normal(1.0, 0.4, n)) + + # Missingness patterns + for arr, miss in [ + (height, 0.10), (weight, 0.15), (bmi, 0.10), (bp_sys, 0.07), (bp_dia, 0.07), (chol, 0.12), (gluc, 0.12), (creat, 0.2), (age, 0.03) + ]: + mask = rng.rand(n) < miss + arr[mask] = np.nan + + phenos1 = rng.choice(["HP:0001250", "HP:0001166", "HP:0000001", None], size=n, p=[0.25, 0.25, 0.25, 0.25]) + phenos2 = rng.choice(["HP:0100022", "HP:0002011", None], size=n, p=[0.3, 0.3, 0.4]) + diseases = rng.choice(["DOID:9352", "DOID:12365", None], size=n, p=[0.45, 0.4, 0.15]) + meds = rng.choice(["RXCUI:153666", "RXCUI:83367", None], size=n, p=[0.3, 0.3, 0.4]) + smoker = rng.choice([True, False, None], size=n, p=[0.25, 0.7, 0.05]) + pregnant = rng.choice([True, False, None], size=n, p=[0.02, 0.96, 0.02]) + + base_date = pd.to_datetime('2022-01-01') + dates = (base_date + pd.to_timedelta(rng.randint(0, 365, size=n), unit='D')).astype(str) + dates = dates.to_numpy(copy=True) + # inject some invalid dates + for ix in rng.choice(np.arange(n), size=max(10, n // 300), replace=False): + dates[ix] = "not_a_date" + + labels = rng.choice(["majority", "minority"], size=n, p=[0.88, 0.12]) + + df = pd.DataFrame({ + "PatientID": pid, + "VisitID": vid, + "Age": age, + "Sex": sex, + "Height_cm": height, + "Weight_kg": weight, + "BMI": bmi, + "BP_systolic": bp_sys, + "BP_diastolic": bp_dia, + "Cholesterol_mgdl": chol, + "Glucose_mgdl": gluc, + "Creatinine_mgdl": creat, + "PrimaryPhenotype": phenos1, + "SecondaryPhenotype": phenos2, + "DiseaseCode": diseases, + "MedicationCode": meds, + "Smoker": smoker, + "Pregnant": pregnant, + "VisitDate": dates, + "class": labels, + }) + + # Duplicates (traceability) + if n >= 3: + df.loc[1, ["PatientID", "VisitID"]] = df.loc[0, ["PatientID", "VisitID"]] + return df + + +def unique_output_name(base_path: str, output_dir: str, suffix: str) -> str: + from phenoqc.batch_processing import unique_output_name as _uon + return _uon(base_path, output_dir, suffix=suffix) + + +def run_cli(output_dir: str, offline: bool = False) -> Tuple[str, str, str, str, str]: + cmd = [ + sys.executable, "-m", "phenoqc", + "--input", DATA_PATH, + "--schema", SCHEMA_PATH, + "--config", CONFIG_PATH, + "--unique_identifiers", "PatientID", "VisitID", + "--phenotype_columns", '{"PrimaryPhenotype": ["HPO"], "SecondaryPhenotype": ["HPO"], "DiseaseCode": ["DO"]}', + "--custom_mappings", CUSTOM_MAPPING_PATH, + "--output", output_dir, + "--quality-metrics", "accuracy", "redundancy", "traceability", "timeliness", "imputation_bias", + "--redundancy-threshold", "0.98", + "--redundancy-method", "pearson", + "--label-column", "class", + "--imbalance-threshold", "0.10", + "--bias-smd-threshold", "0.10", + "--bias-var-low", "0.5", + "--bias-var-high", "2.0", + "--bias-ks-alpha", "0.05", + "--impute-diagnostics", "on", + "--mi-uncertainty", "on", + "--mi-repeats", "3", + "--mi-params", '{"max_iter": 6}', + ] + if offline: + cmd.append("--offline") + env = os.environ.copy() + env["PYTHONPATH"] = SRC_PATH + (os.pathsep + env.get("PYTHONPATH", "")) + print("[INFO] Running:", " ".join(cmd)) + proc = subprocess.run(cmd, capture_output=True, text=True, env=env) + print("[STDOUT]\n", proc.stdout) + print("[STDERR]\n", proc.stderr) + print("[INFO] Exit code:", proc.returncode) + assert proc.returncode == 0 + + processed_csv = unique_output_name(DATA_PATH, output_dir, suffix=".csv") + report_pdf = unique_output_name(DATA_PATH, output_dir, suffix="_report.pdf") + metrics_tsv = unique_output_name(DATA_PATH, output_dir, suffix="_quality_metrics.tsv") + metrics_json = unique_output_name(DATA_PATH, output_dir, suffix="_quality_metrics_summary.json") + qc_json = unique_output_name(DATA_PATH, output_dir, suffix="_qc_summary.json") + return processed_csv, report_pdf, metrics_tsv, metrics_json, qc_json + + +def _fallback_find(output_dir: str, base_name: str, suffix: str) -> str: + """Find generated file by pattern if exact unique_output_name is missing.""" + name_no_ext, ext = os.path.splitext(os.path.basename(base_name)) + # Processed CSV example pattern: clinical_input_*_csv.csv + if suffix == ".csv": + pattern = os.path.join(output_dir, f"{name_no_ext}_*_csv.csv") + elif suffix.endswith("_report.pdf"): + pattern = os.path.join(output_dir, f"{name_no_ext}_*{suffix}") + elif suffix.endswith("_quality_metrics.tsv"): + pattern = os.path.join(output_dir, f"{name_no_ext}_*{suffix}") + elif suffix.endswith("_quality_metrics_summary.json"): + pattern = os.path.join(output_dir, f"{name_no_ext}_*{suffix}") + elif suffix.endswith("_qc_summary.json"): + pattern = os.path.join(output_dir, f"{name_no_ext}_*{suffix}") + else: + pattern = os.path.join(output_dir, f"{name_no_ext}_*{suffix}") + matches = sorted(glob.glob(pattern)) + return matches[0] if matches else "" + + +def verify_outputs(processed_csv: str, report_pdf: str, metrics_tsv: str, metrics_json: str, qc_json: str) -> dict: + # Fallback discovery if direct paths are missing + out_dir = os.path.dirname(processed_csv) + base_path = processed_csv.replace("_csv.csv", ".csv") if processed_csv.endswith("_csv.csv") else processed_csv + if not os.path.exists(processed_csv): + candidate = _fallback_find(out_dir, base_path, ".csv") + assert candidate, processed_csv + processed_csv = candidate + if not os.path.exists(report_pdf): + candidate = _fallback_find(out_dir, base_path, "_report.pdf") + assert candidate, report_pdf + report_pdf = candidate + if not os.path.exists(metrics_tsv): + candidate = _fallback_find(out_dir, base_path, "_quality_metrics.tsv") + assert candidate, metrics_tsv + metrics_tsv = candidate + if not os.path.exists(metrics_json): + candidate = _fallback_find(out_dir, base_path, "_quality_metrics_summary.json") + assert candidate, metrics_json + metrics_json = candidate + if not os.path.exists(qc_json): + candidate = _fallback_find(out_dir, base_path, "_qc_summary.json") + assert candidate, qc_json + qc_json = candidate + + df = pd.read_csv(processed_csv) + # Expect mapping outputs and missing flag + assert "HPO_ID" in df.columns or "DO_ID" in df.columns + assert "MissingDataFlag" in df.columns + + with open(qc_json, "r", encoding="utf-8") as fh: + qc = json.load(fh) + assert "quality_scores" in qc and isinstance(qc["quality_scores"], dict) + assert "imputation" in qc and isinstance(qc["imputation"], dict) + + # Bias diagnostic rows + bias_rows = ( + qc.get("quality_metrics", {}) + .get("imputation_bias", {}) + .get("rows", []) + ) + assert isinstance(bias_rows, list) + assert len(bias_rows) >= 1 + + # Optional stability rows presence check + stab_rows = ( + qc.get("quality_metrics", {}) + .get("imputation_stability", {}) + .get("rows", []) + ) + assert isinstance(stab_rows, list) + return qc + + +def main() -> None: + df = create_dataset() + df.to_csv(DATA_PATH, index=False) + write_schema() + write_config() + write_custom_mapping() + + # Run 1: online (default) + r1 = run_cli(RUN1_DIR, offline=False) + qc1 = verify_outputs(*r1) + + # Run 2: offline to exercise cache/local mapping + r2 = run_cli(RUN2_DIR, offline=True) + qc2 = verify_outputs(*r2) + + # If class distribution present, ensure same structure + assert isinstance(qc1.get("class_distribution"), (dict, type(None))) + assert isinstance(qc2.get("class_distribution"), (dict, type(None))) + + print("[SUCCESS] Clinical all-features E2E completed. Outputs in:", OUT_BASE) + + +if __name__ == "__main__": + main() + + diff --git a/scripts/config/clinical_all_features_config.yaml b/scripts/config/clinical_all_features_config.yaml new file mode 100644 index 0000000..aa44f85 --- /dev/null +++ b/scripts/config/clinical_all_features_config.yaml @@ -0,0 +1,118 @@ + +ontologies: + HPO: + name: Human Phenotype Ontology + source: local + file: /Users/jorgemiguelsilva/Documents/PhenoQC-1/ontologies/hp.obo + format: obo + DO: + name: Disease Ontology + source: local + file: /Users/jorgemiguelsilva/Documents/PhenoQC-1/ontologies/doid.obo + format: obo + MPO: + name: Mammalian Phenotype Ontology + source: local + file: /Users/jorgemiguelsilva/Documents/PhenoQC-1/ontologies/mp.obo + format: obo + +default_ontologies: + - HPO + - DO + - MPO + +fuzzy_threshold: 82 +cache_expiry_days: 1 + +imputation: + strategy: knn + params: + n_neighbors: 5 + weights: uniform + metric: nan_euclidean + per_column: + Age: + strategy: mean + Height_cm: + strategy: median + Weight_kg: + strategy: knn + params: + n_neighbors: 7 + weights: distance + metric: nan_euclidean + BMI: + strategy: mean + BP_systolic: + strategy: mice + params: + max_iter: 10 + random_state: 11 + BP_diastolic: + strategy: mice + params: + max_iter: 10 + random_state: 11 + Glucose_mgdl: + strategy: knn + params: + n_neighbors: 5 + weights: uniform + metric: nan_euclidean + Creatinine_mgdl: + strategy: mice + params: + max_iter: 8 + random_state: 7 + Cholesterol_mgdl: + strategy: svd + params: + rank: 3 + max_iters: 50 + Sex: + strategy: mode + Smoker: + strategy: mode + Pregnant: + strategy: mode + tuning: + enable: true + mask_fraction: 0.1 + scoring: MAE + max_cells: 20000 + random_state: 42 + grid: + n_neighbors: [3, 5, 7, 9] + +protected_columns: + - PatientID + - VisitID + - VisitDate + - MedicationCode + +redundancy: + threshold: 0.98 + method: pearson + +quality_metrics: + imputation_bias: + enable: true + smd_threshold: 0.10 + var_ratio_low: 0.5 + var_ratio_high: 2.0 + ks_alpha: 0.05 + imputation_stability: + enable: true + repeats: 5 + mask_fraction: 0.1 + scoring: MAE + +class_distribution: + label_column: class + warn_threshold: 0.10 + +mi_uncertainty: + enable: true + repeats: 3 + params: + max_iter: 6 diff --git a/scripts/config/clinical_all_features_custom_mapping.json b/scripts/config/clinical_all_features_custom_mapping.json new file mode 100644 index 0000000..9b30293 --- /dev/null +++ b/scripts/config/clinical_all_features_custom_mapping.json @@ -0,0 +1,11 @@ +{ + "HP:0001250": { + "HPO": "HP:0001250" + }, + "seizure": { + "HPO": "HP:0001250" + }, + "DOID:9352": { + "DO": "DOID:9352" + } +} \ No newline at end of file diff --git a/scripts/config/clinical_all_features_schema.json b/scripts/config/clinical_all_features_schema.json new file mode 100644 index 0000000..f010a49 --- /dev/null +++ b/scripts/config/clinical_all_features_schema.json @@ -0,0 +1,155 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Clinical All Features Schema", + "type": "object", + "properties": { + "PatientID": { + "type": [ + "string", + "null" + ] + }, + "VisitID": { + "type": [ + "string", + "null" + ] + }, + "Age": { + "type": [ + "number", + "null" + ], + "minimum": 0, + "maximum": 120 + }, + "Sex": { + "type": [ + "string", + "null" + ], + "enum": [ + "M", + "F", + "Other", + null + ] + }, + "Height_cm": { + "type": [ + "number", + "null" + ], + "minimum": 30, + "maximum": 250 + }, + "Weight_kg": { + "type": [ + "number", + "null" + ], + "minimum": 1, + "maximum": 400 + }, + "BMI": { + "type": [ + "number", + "null" + ], + "minimum": 5, + "maximum": 100 + }, + "BP_systolic": { + "type": [ + "number", + "null" + ], + "minimum": 50, + "maximum": 250 + }, + "BP_diastolic": { + "type": [ + "number", + "null" + ], + "minimum": 30, + "maximum": 200 + }, + "Cholesterol_mgdl": { + "type": [ + "number", + "null" + ], + "minimum": 50, + "maximum": 600 + }, + "Glucose_mgdl": { + "type": [ + "number", + "null" + ], + "minimum": 20, + "maximum": 1000 + }, + "Creatinine_mgdl": { + "type": [ + "number", + "null" + ], + "minimum": 0.1, + "maximum": 20 + }, + "PrimaryPhenotype": { + "type": [ + "string", + "null" + ] + }, + "SecondaryPhenotype": { + "type": [ + "string", + "null" + ] + }, + "DiseaseCode": { + "type": [ + "string", + "null" + ] + }, + "MedicationCode": { + "type": [ + "string", + "null" + ] + }, + "Smoker": { + "type": [ + "boolean", + "null" + ] + }, + "Pregnant": { + "type": [ + "boolean", + "null" + ] + }, + "VisitDate": { + "type": [ + "string", + "null" + ] + }, + "class": { + "type": [ + "string", + "null" + ] + } + }, + "required": [ + "PatientID", + "VisitID" + ] +} \ No newline at end of file