Release experimental taxonomy, scoring, and download updates

README.md

@@ -23,14 +23,11 @@ For model submission requirements, see the
 To learn how to submit your model to **Bittensor Subnet 34** — including the entrance exam, full benchmark, and scoring pipeline — see the  
 👉 **[Discriminative Mining Guide](https://github.com/BitMind-AI/bitmind-subnet/blob/main/docs/Discriminative-Mining.md)**
 
-> **Note**: ONNX format is no longer accepted for competition. See [ONNX.md](./docs/ONNX.md) for legacy reference.
-
 ---
 
 ## Installation
 
-To run benchmarks, install the `gpu` extra (on macOS/Windows this
-automatically substitutes CPU onnxruntime via platform markers):
+To run benchmarks, install the `gpu` extra:
 
 ```bash
 cd gasbench
@@ -39,7 +36,7 @@ pip install -e '.[gpu]'
 
 The base install is intentionally minimal — just the dataset registry
 (`gasbench.dataset.config` and the bundled YAML configs) with no
-torch/onnxruntime. Use it when you only need dataset definitions
+PyTorch. Use it when you only need dataset definitions
 (e.g. bitmind-subnet's validator cache):
 
 ```bash
@@ -116,8 +113,6 @@ results = asyncio.run(evaluate_model())
 For full submission requirements, see:  
 👉 **[Safetensors Model Specification](./docs/Safetensors.md)** (required for competition)
 
-> **Note**: ONNX format is no longer accepted. See [ONNX.md](./docs/ONNX.md) for legacy reference.
-
 ---
 
 ## Metrics

docs/Safetensors.md

@@ -2,8 +2,6 @@
 
 This document defines the requirements for **safetensors** format models used with GASBench on Subnet 34.
 
-> **Note:** ONNX format is no longer accepted for competition submissions. All new models must use safetensors format.
-
 ---
 
 ## 1. Required Files
@@ -355,5 +353,3 @@ gascli d push --audio-model my_model.zip
 4. **Mismatched resize dimensions**: Ensure `preprocessing.resize` in config matches your model's expected input size.
 
 5. **Mismatched frame count**: Ensure `preprocessing.num_frames` matches the temporal dimension your model expects. If `frame_rate` is set, frames are sampled at that fps from frame 0; otherwise the first `num_frames` consecutive frames are used.
-
-5. **ONNX format**: ONNX is no longer accepted. Convert your model to safetensors format.

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "gasbench"
-version = "0.8.3"
+version = "0.9.0"
 description = "GASBench - ML model benchmark evaluation package"
 readme = "README.md"
 requires-python = ">=3.10"
@@ -22,9 +22,6 @@ gpu = [
     "numpy==2.4.2",
     "pillow==12.1.1",
     "opencv-python-headless==4.13.0.92",
-    "onnxruntime-gpu==1.24.2; platform_system == 'Linux'",
-    "onnxruntime>=1.22.0; platform_system == 'Darwin'",
-    "onnxruntime>=1.22.0; platform_system == 'Windows'",
     "decord==0.6.0; platform_system != 'Darwin'",
     "scikit-learn==1.8.0",
     "torch==2.8.0",
@@ -91,5 +88,6 @@ follow_imports = "silent"
 
 [tool.pytest.ini_options]
 testpaths = ["tests"]
+addopts = "-m 'not slow'"
 markers = ["slow: marks tests as slow (network I/O)"]
 filterwarnings = ["ignore::DeprecationWarning:pkg_resources.*"]

src/gasbench/__init__.py

@@ -9,7 +9,7 @@
     from gasbench import run_benchmark
 
     results = await run_benchmark(
-        model_path="path/to/model.onnx",
+        model_path="path/to/model_directory",
         modality="image",
         mode="debug"
     )
@@ -35,7 +35,7 @@
 
 # Benchmark exports resolve lazily (PEP 562) so that the base install —
 # which carries only the dataset registry and pyyaml — can import
-# gasbench.dataset.config without pulling in torch/onnxruntime. The full
+# gasbench.dataset.config without pulling in torch. The full
 # benchmark stack requires the `gpu` extra: pip install gasbench[gpu]
 _LAZY_EXPORTS = {
     "run_image_benchmark": ("gasbench.benchmarks.image_bench", "run_image_benchmark"),

src/gasbench/benchmark.py

@@ -36,14 +36,15 @@ async def run_benchmark(
     holdouts_only: bool = False,
     content_category: Optional[str] = None,
     score_composition: Optional[Dict[str, float]] = None,
+    multiclass_scoring: bool = False,
     n_aug_per_dataset: int = 0,
     aug_weight: float = 0.2,
     aug_cache_dir: Optional[str] = None,
     aug_cache_readonly: bool = False,
 ) -> Dict:
     """
     Args:
-        model_path: Path to ONNX model file
+        model_path: Path to a custom PyTorch model directory
         modality: Type of modality to test ("image" or "video")
         mode: Benchmark mode - "debug", "small", or "full" (default: "full")
         gasstation_only: If True, only use gasstation datasets
@@ -125,6 +126,7 @@ async def run_benchmark(
             holdouts_only,
             content_category,
             score_composition,
+            multiclass_scoring,
             n_aug_per_dataset=n_aug_per_dataset,
             aug_weight=aug_weight,
             aug_cache_dir=aug_cache_dir,
@@ -162,7 +164,7 @@ async def run_benchmark(
 async def load_model_for_benchmark(
     model_path: str, modality: str, benchmark_results: Dict
 ):
-    """Load and validate ONNX model for benchmarking."""
+    """Load and validate a model for benchmarking."""
 
     if not os.path.exists(model_path):
         benchmark_results["errors"].append(f"Model file not found: {model_path}")
@@ -199,7 +201,7 @@ async def load_model_for_benchmark(
 
     except Exception as e:
         logger.error(f"Failed to load model for inference: {e}")
-        benchmark_results["errors"].append(f"ONNX runtime error: {str(e)}")
+        benchmark_results["errors"].append(f"Model loading error: {str(e)}")
         benchmark_results["benchmark_completed"] = False
         return None, None
 
@@ -225,14 +227,15 @@ async def execute_benchmark(
     holdouts_only: bool = False,
     content_category: Optional[str] = None,
     score_composition: Optional[Dict[str, float]] = None,
+    multiclass_scoring: bool = False,
     n_aug_per_dataset: int = 0,
     aug_weight: float = 0.2,
     aug_cache_dir: Optional[str] = None,
     aug_cache_readonly: bool = False,
 ) -> float:
     """Execute the actual benchmark evaluation."""
 
-    logger.info(f"Running {modality} benchmark (mode={mode}, gasstation_only={gasstation_only}, download_latest_gasstation_data={download_latest_gasstation_data}, skip_missing={skip_missing}, holdout_weight={holdout_weight}, holdouts_only={holdouts_only}, content_category={content_category}, score_composition={score_composition})")
+    logger.info(f"Running {modality} benchmark (mode={mode}, gasstation_only={gasstation_only}, download_latest_gasstation_data={download_latest_gasstation_data}, skip_missing={skip_missing}, holdout_weight={holdout_weight}, holdouts_only={holdouts_only}, content_category={content_category}, score_composition={score_composition}, multiclass_scoring={multiclass_scoring})")
     if dataset_filters:
         logger.info(f"Dataset filters: {dataset_filters}")
     if modality == "image":
@@ -256,6 +259,7 @@ async def execute_benchmark(
             holdouts_only=holdouts_only,
             content_category=content_category,
             score_composition=score_composition,
+            multiclass_scoring=multiclass_scoring,
             n_aug_per_dataset=n_aug_per_dataset,
             aug_weight=aug_weight,
             aug_cache_dir=aug_cache_dir,
@@ -283,6 +287,7 @@ async def execute_benchmark(
             holdouts_only=holdouts_only,
             content_category=content_category,
             score_composition=score_composition,
+            multiclass_scoring=multiclass_scoring,
             n_aug_per_dataset=n_aug_per_dataset,
             aug_weight=aug_weight,
             aug_cache_dir=aug_cache_dir,
@@ -310,6 +315,7 @@ async def execute_benchmark(
             holdouts_only=holdouts_only,
             content_category=content_category,
             score_composition=score_composition,
+            multiclass_scoring=multiclass_scoring,
         )
         benchmark_score = benchmark_results.get("audio_results", {}).get("benchmark_score", 0.0)
     else:
@@ -506,6 +512,18 @@ def save_results_to_json(
                 "aug_binary_mcc": results.get("aug_binary_mcc"),
                 "aug_binary_ce": results.get("aug_binary_ce"),
                 "aug_binary_brier": results.get("aug_binary_brier"),
+                # Multiclass metrics are recorded on every run regardless of
+                # which scheme sn34_score was derived from, so a round can be
+                # compared under either. Keys must stay in sync with the dict
+                # built in recording.compute_metrics_from_df.
+                "num_classes": results.get("num_classes"),
+                "multiclass_scoring": results.get("multiclass_scoring"),
+                "gorodkin_mcc": results.get("gorodkin_mcc"),
+                "multiclass_brier": results.get("multiclass_brier"),
+                "binary_sn34_score": results.get("binary_sn34_score"),
+                "multiclass_sn34_score": results.get("multiclass_sn34_score"),
+                # int keys become strings through JSON; that is fine for a report.
+                "per_class_recall": results.get("per_class_recall"),
             }
 
             per_dataset = results.get("per_dataset_results", {})

src/gasbench/benchmarks/audio_bench.py

@@ -81,6 +81,7 @@ async def run_audio_benchmark(
     holdouts_only: bool = False,
     content_category: Optional[str] = None,
     score_composition: dict = None,
+    multiclass_scoring: bool = False,
 ) -> pd.DataFrame:
     """Test model on benchmark audio datasets for AI-generated content detection.
 
@@ -117,6 +118,7 @@ async def run_audio_benchmark(
             holdouts_only=holdouts_only,
             content_category=content_category,
             score_composition=score_composition,
+            multiclass_scoring=multiclass_scoring,
         )
 
         plan = build_plan(logger, run_config, input_specs)