Feat/imputation improvements

README.md

@@ -165,9 +165,10 @@ Highlights:
 
 ## Examples and Scripts
 
-- `scripts/quality_metrics_cli_test.py` – small demo with metrics
-- `scripts/imputation_params_cli_test.py` – runs with config-driven imputation and label column
-- `scripts/comprehensive_cli_test.py` – generates a comprehensive dataset and runs full pipeline
+- `scripts/e2e_small_quality_metrics_cli_test.py` – small demo focusing on quality metrics
+- `scripts/e2e_medium_cli_test.py` – mid-sized end-to-end pipeline run
+- `scripts/end_to_end_e2e_cli_test.py` – large end-to-end pipeline run
+- `scripts/imputation_params_cli_test.py` – imputation params and optional tuning
 
 ---
 

scripts/config/e2e_config.yaml

@@ -0,0 +1,56 @@
+
+ontologies:
+  HPO:
+    name: Human Phenotype Ontology
+    source: url
+    url: http://purl.obolibrary.org/obo/hp.obo
+    format: obo
+  DO:
+    name: Disease Ontology
+    source: url
+    url: http://purl.obolibrary.org/obo/doid.obo
+    format: obo
+  MPO:
+    name: Mammalian Phenotype Ontology
+    source: url
+    url: http://purl.obolibrary.org/obo/mp.obo
+    format: obo
+
+default_ontologies:
+  - HPO
+  - DO
+  - MPO
+
+fuzzy_threshold: 80
+cache_expiry_days: 30
+
+imputation:
+  strategy: knn
+  params:
+    n_neighbors: 5
+    weights: uniform
+  per_column:
+    Creatinine_mgdl:
+      strategy: mice
+      params:
+        max_iter: 5
+    Cholesterol_mgdl:
+      strategy: svd
+      params:
+        rank: 2
+  tuning:
+    enable: true
+    mask_fraction: 0.1
+    scoring: MAE
+    max_cells: 10000
+    random_state: 42
+    grid:
+      n_neighbors: [3, 5, 7]
+
+quality_metrics:
+  imputation_bias:
+    enable: true
+    smd_threshold: 0.10
+    var_ratio_low: 0.5
+    var_ratio_high: 2.0
+    ks_alpha: 0.05

scripts/config/e2e_schema.json

@@ -0,0 +1,96 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "End-to-End E2E Schema",
+  "type": "object",
+  "properties": {
+    "SampleID": {
+      "type": [
+        "string",
+        "null"
+      ]
+    },
+    "Height_cm": {
+      "type": [
+        "number",
+        "null"
+      ],
+      "minimum": 30,
+      "maximum": 250
+    },
+    "Weight_kg": {
+      "type": [
+        "number",
+        "null"
+      ],
+      "minimum": 1,
+      "maximum": 400
+    },
+    "Cholesterol_mgdl": {
+      "type": [
+        "number",
+        "null"
+      ],
+      "minimum": 50,
+      "maximum": 600
+    },
+    "BP_systolic": {
+      "type": [
+        "number",
+        "null"
+      ],
+      "minimum": 50,
+      "maximum": 250
+    },
+    "BP_diastolic": {
+      "type": [
+        "number",
+        "null"
+      ],
+      "minimum": 30,
+      "maximum": 200
+    },
+    "Glucose_mgdl": {
+      "type": [
+        "number",
+        "null"
+      ],
+      "minimum": 20,
+      "maximum": 800
+    },
+    "Creatinine_mgdl": {
+      "type": [
+        "number",
+        "null"
+      ],
+      "minimum": 0.1,
+      "maximum": 20
+    },
+    "PrimaryPhenotype": {
+      "type": [
+        "string",
+        "null"
+      ]
+    },
+    "DiseaseCode": {
+      "type": [
+        "string",
+        "null"
+      ]
+    },
+    "VisitDate": {
+      "type": [
+        "string",
+        "null"
+      ]
+    },
+    "class": {
+      "type": [
+        "string",
+        "null"
+      ]
+    }
+  },
+  "required": [
+    "SampleID"
+  ]
+}

scripts/config/super_config.yaml

@@ -0,0 +1,56 @@
+
+ontologies:
+  HPO:
+    name: Human Phenotype Ontology
+    source: url
+    url: http://purl.obolibrary.org/obo/hp.obo
+    format: obo
+  DO:
+    name: Disease Ontology
+    source: url
+    url: http://purl.obolibrary.org/obo/doid.obo
+    format: obo
+  MPO:
+    name: Mammalian Phenotype Ontology
+    source: url
+    url: http://purl.obolibrary.org/obo/mp.obo
+    format: obo
+
+default_ontologies:
+  - HPO
+  - DO
+  - MPO
+
+fuzzy_threshold: 80
+cache_expiry_days: 30
+
+imputation:
+  strategy: knn
+  params:
+    n_neighbors: 5
+    weights: uniform
+  per_column:
+    Creatinine_mgdl:
+      strategy: mice
+      params:
+        max_iter: 5
+    Cholesterol_mgdl:
+      strategy: svd
+      params:
+        rank: 2
+  tuning:
+    enable: true
+    mask_fraction: 0.1
+    scoring: MAE
+    max_cells: 10000
+    random_state: 42
+    grid:
+      n_neighbors: [3, 5, 7]
+
+quality_metrics:
+  imputation_bias:
+    enable: true
+    smd_threshold: 0.10
+    var_ratio_low: 0.5
+    var_ratio_high: 2.0
+    ks_alpha: 0.05

scripts/config/super_schema.json

@@ -0,0 +1,96 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "title": "End-to-End E2E Schema",
+  "type": "object",
+  "properties": {
+    "SampleID": {
+      "type": [
+        "string",
+        "null"
+      ]
+    },
+    "Height_cm": {
+      "type": [
+        "number",
+        "null"
+      ],
+      "minimum": 30,
+      "maximum": 250
+    },
+    "Weight_kg": {
+      "type": [
+        "number",
+        "null"
+      ],
+      "minimum": 1,
+      "maximum": 400
+    },
+    "Cholesterol_mgdl": {
+      "type": [
+        "number",
+        "null"
+      ],
+      "minimum": 50,
+      "maximum": 600
+    },
+    "BP_systolic": {
+      "type": [
+        "number",
+        "null"
+      ],
+      "minimum": 50,
+      "maximum": 250
+    },
+    "BP_diastolic": {
+      "type": [
+        "number",
+        "null"
+      ],
+      "minimum": 30,
+      "maximum": 200
+    },
+    "Glucose_mgdl": {
+      "type": [
+        "number",
+        "null"
+      ],
+      "minimum": 20,
+      "maximum": 800
+    },
+    "Creatinine_mgdl": {
+      "type": [
+        "number",
+        "null"
+      ],
+      "minimum": 0.1,
+      "maximum": 20
+    },
+    "PrimaryPhenotype": {
+      "type": [
+        "string",
+        "null"
+      ]
+    },
+    "DiseaseCode": {
+      "type": [
+        "string",
+        "null"
+      ]
+    },
+    "VisitDate": {
+      "type": [
+        "string",
+        "null"
+      ]
+    },
+    "class": {
+      "type": [
+        "string",
+        "null"
+      ]
+    }
+  },
+  "required": [
+    "SampleID"
+  ]
+}

scripts/comprehensive_cli_test.py → scripts/e2e_medium_cli_test.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 """
-Generate a comprehensive synthetic dataset and run PhenoQC with
+Generate a mid-sized synthetic dataset and run PhenoQC with
 config-driven imputation and class-imbalance reporting.
 
 This script verifies that:
@@ -29,12 +29,12 @@
 from phenoqc.batch_processing import unique_output_name
 
 
-OUT_DIR = os.path.join(SCRIPT_DIR, "output", "comprehensive")
+OUT_DIR = os.path.join(SCRIPT_DIR, "output", "e2e_medium")
 os.makedirs(OUT_DIR, exist_ok=True)
 
 SCHEMA_PATH = os.path.join(SCRIPT_DIR, "config", "schema.json")
 CONFIG_PATH = os.path.join(SCRIPT_DIR, "config", "config.yaml")
-DATA_PATH = os.path.join(OUT_DIR, "comprehensive_input.csv")
+DATA_PATH = os.path.join(OUT_DIR, "e2e_medium_input.csv")
 
 
 def create_comprehensive_data(n: int = 1000, seed: int = 42) -> pd.DataFrame:

scripts/quality_metrics_cli_test.py → scripts/e2e_small_quality_metrics_cli_test.py

@@ -1,14 +1,14 @@
 #!/usr/bin/env python3
 """
-quality_metrics_cli_test.py
+small_quality_metrics_cli_test.py
 
 A self-contained script demonstrating PhenoQC's ``--quality-metrics`` option.
 It synthesizes a small dataset containing issues for all supported metrics
 (accuracy, redundancy, traceability, timeliness) and runs the CLI with those
 checks enabled.
 
 Example:
-    python quality_metrics_cli_test.py
+  python small_quality_metrics_cli_test.py
 
 Outputs and temporary files are written under ``./output/quality_metrics``.
 """
@@ -138,7 +138,7 @@ def run_phenoqc(data_path: str, cfg_path: str, output_dir: str) -> None:
 
 
 def main() -> None:
-    out_dir = os.path.join(SCRIPT_DIR, "output", "quality_metrics")
+    out_dir = os.path.join(SCRIPT_DIR, "output", "e2e_small_quality_metrics")
     os.makedirs(out_dir, exist_ok=True)
 
     data_path = os.path.join(out_dir, "quality_metrics_input.csv")