Merge pull request #42 from nezihtopaloglu/add_wvmos_sigmos

Add wvmos sigmos

Author: ftshijt

.gitignore

@@ -169,3 +169,4 @@ fadtk/
 scoreq/
 fairseq/
 UTMOSv2/
+wvmos/

docs/supported_metrics.md

@@ -55,8 +55,9 @@ We include x mark if the metric is auto-installed in versa.
 | 48 | x | DNSMOS Pro: A Reduced-Size DNN for Probabilistic MOS of Speech  | pseudo_mos | dnsmos_pro_bvcc | [DNSMOSPro](https://github.com/fcumlin/DNSMOSPro/tree/main) | [paper](https://www.isca-archive.org/interspeech_2024/cumlin24_interspeech.html) |
 | 49 | x | DNSMOS Pro: A Reduced-Size DNN for Probabilistic MOS of Speech  | pseudo_mos | dnsmos_pro_nisqa | [DNSMOSPro](https://github.com/fcumlin/DNSMOSPro/tree/main) | [paper](https://www.isca-archive.org/interspeech_2024/cumlin24_interspeech.html) |
 | 50 | x | DNSMOS Pro: A Reduced-Size DNN for Probabilistic MOS of Speech  | pseudo_mos | dnsmos_pro_vcc2018 | [DNSMOSPro](https://github.com/fcumlin/DNSMOSPro/tree/main) | [paper](https://www.isca-archive.org/interspeech_2024/cumlin24_interspeech.html) |
-| 51 | x | VQScore (Self-Supervised Speech Quality Estimation and Enhancement Using Only Clean Speech)  | vqscore | vqscore | [VQScore](https://github.com/JasonSWFu/VQscore) | [paper](https://arxiv.org/abs/2402.16321) |
-
+| 51 |   | WV-MOS (MOS score prediction by fine-tuned wav2vec2.0 model) | wvmos | wvmos | [wvmos](https://github.com/AndreevP/wvmos) | [paper](https://arxiv.org/abs/2203.13086) |
+| 52 |   |SIG-MOS | sigmos | {SIGMOS_COL, SIGMOS_DISC, SIGMOS_LOUD, SIGMOS_REVERB, SIGMOS_SIG, SIGMOS_OVRL} | [sigmos](https://github.com/microsoft/SIG-Challenge/tree/main/ICASSP2024/sigmos) |[paper](https://arxiv.org/pdf/2309.07385) |
+| 53 | x | VQScore (Self-Supervised Speech Quality Estimation and Enhancement Using Only Clean Speech)  | vqscore | vqscore | [VQScore](https://github.com/JasonSWFu/VQscore) | [paper](https://arxiv.org/abs/2402.16321) |
 
 
 ### Dependent Metrics
@@ -68,7 +69,7 @@ We include x mark if the metric is auto-installed in versa.
 | 4 | x | Signal-to-interference  Ratio (SIR)  | signal_metric | sir | [espnet](https://github.com/espnet/espnet) | - |
 | 5 | x | Signal-to-artifact Ratio (SAR)  | signal_metric | sar | [espnet](https://github.com/espnet/espnet) | - |
 | 6 | x | Signal-to-distortion Ratio (SDR)  | signal_metric | sdr | [espnet](https://github.com/espnet/espnet) | - |
-| 7 | x | Convolutional scale-invariant signal-to-distortion ratio (CI-SDR)  | signal_metric | ci-sdr | [ci_sdr](https://github.com/fgnt/ci_sdr) | [paper](https://arxiv.org/abs/2011.15003) |
+| 7 | x | Convolutional scale-invariant signal-to-distortion ratio (CI-SDR)  | signal_metric | ci-sdr | [ci_sdr](https://github.com/fgnt/ci_sdr) | [paper](https://arxiv.(org/abs/2011.15003) |
 | 8 | x | Scale-invariant signal-to-noise ratio (SI-SNR)  | signal_metric | si-snr | [espnet](https://github.com/espnet/espnet) | [paper](https://arxiv.org/abs/1711.00541) |
 | 9 | x | Perceptual Evaluation of Speech Quality (PESQ)  | pesq | pesq | [pesq](https://pypi.org/project/pesq/) | [paper](https://ieeexplore.ieee.org/document/941023) |
 | 10 | x | Short-Time Objective Intelligibility (STOI)  | stoi | stoi | [pystoi](https://github.com/mpariente/pystoi) | [paper](https://ieeexplore.ieee.org/document/5495701) |

egs/separate_metrics/sigmos.yaml

@@ -0,0 +1,3 @@
+# sigmos (independent) metric
+
+- name: sigmos

egs/separate_metrics/wvmos.yaml

@@ -0,0 +1,3 @@
+# wvmos (independent) metric
+
+- name: wvmos

test/test_pipeline/test_sigmos.py

@@ -0,0 +1,67 @@
+import logging
+import math
+import os
+
+import yaml
+
+from versa.scorer_shared import (
+    find_files,
+    list_scoring,
+    load_score_modules,
+    load_summary,
+)
+
+TEST_INFO = {
+    "SIGMOS_COL": 1.3242647647857666,
+    "SIGMOS_DISC": 1.0382881164550781,
+    "SIGMOS_LOUD": 1.0047355890274048,
+    "SIGMOS_REVERB": 1.0245660543441772,
+    "SIGMOS_SIG": 1.0186278820037842,
+    "SIGMOS_OVRL": 1.0545676946640015,
+}
+
+
+def info_update():
+    # find files
+    if os.path.isdir("test/test_samples/test2"):
+        gen_files = find_files("test/test_samples/test2")
+
+    # find reference file
+    if os.path.isdir("test/test_samples/test1"):
+        gt_files = find_files("test/test_samples/test1")
+
+    logging.info("The number of utterances = %d" % len(gen_files))
+
+    with open("egs/separate_metrics/sigmos.yaml", "r", encoding="utf-8") as f:
+        score_config = yaml.full_load(f)
+
+    score_modules = load_score_modules(
+        score_config,
+        use_gt=(True if gt_files is not None else False),
+        use_gpu=False,
+    )
+
+    assert len(score_config) > 0, "no scoring function is provided"
+
+    score_info = list_scoring(
+        gen_files, score_modules, gt_files, output_file=None, io="soundfile"
+    )
+    summary = load_summary(score_info)
+    print("Summary: {}".format(load_summary(score_info)), flush=True)
+
+    for key in summary:
+        if math.isinf(TEST_INFO[key]) and math.isinf(summary[key]):
+            # for sir"
+            continue
+        # the plc mos is undeterministic
+        if abs(TEST_INFO[key] - summary[key]) > 1e-4 and key != "plcmos":
+            raise ValueError(
+                "Value issue in the test case, might be some issue in scorer {}".format(
+                    key
+                )
+            )
+    print("check successful", flush=True)
+
+
+if __name__ == "__main__":
+    info_update()

test/test_pipeline/test_wvmos.py

@@ -0,0 +1,60 @@
+import logging
+import math
+import os
+
+import yaml
+
+from versa.scorer_shared import (
+    find_files,
+    list_scoring,
+    load_score_modules,
+    load_summary,
+)
+
+TEST_INFO = {"wvmos": 0.621284008026123}
+
+
+def info_update():
+    # find files
+    if os.path.isdir("test/test_samples/test2"):
+        gen_files = find_files("test/test_samples/test2")
+
+    # find reference file
+    if os.path.isdir("test/test_samples/test1"):
+        gt_files = find_files("test/test_samples/test1")
+
+    logging.info("The number of utterances = %d" % len(gen_files))
+
+    with open("egs/separate_metrics/wvmos.yaml", "r", encoding="utf-8") as f:
+        score_config = yaml.full_load(f)
+
+    score_modules = load_score_modules(
+        score_config,
+        use_gt=(True if gt_files is not None else False),
+        use_gpu=False,
+    )
+
+    assert len(score_config) > 0, "no scoring function is provided"
+
+    score_info = list_scoring(
+        gen_files, score_modules, gt_files, output_file=None, io="soundfile"
+    )
+    summary = load_summary(score_info)
+    print("Summary: {}".format(load_summary(score_info)), flush=True)
+
+    for key in summary:
+        if math.isinf(TEST_INFO[key]) and math.isinf(summary[key]):
+            # for sir"
+            continue
+        # the plc mos is undeterministic
+        if abs(TEST_INFO[key] - summary[key]) > 1e-4 and key != "plcmos":
+            raise ValueError(
+                "Value issue in the test case, might be some issue in scorer {}".format(
+                    key
+                )
+            )
+    print("check successful", flush=True)
+
+
+if __name__ == "__main__":
+    info_update()

tools/install_wvmos.sh

@@ -0,0 +1,12 @@
+#/bin/bash
+
+if [ -d "wvmos" ]; then
+    rm -rf wvmos
+fi
+
+# # Clone and install wvmos
+git clone https://github.com/AndreevP/wvmos.git
+cd wvmos
+pip install -e .
+cd ..
+

versa/__init__.py

@@ -109,6 +109,8 @@
 from versa.utterance_metrics.squim import squim_metric, squim_metric_no_ref
 from versa.utterance_metrics.srmr import srmr_metric
 from versa.utterance_metrics.chroma_alignment import chroma_metric
+from versa.utterance_metrics.wvmos import wvmos_setup, wvmos_calculate
+from versa.utterance_metrics.sigmos import sigmos_setup, sigmos_calculate
 from versa.utterance_metrics.dpam_distance import dpam_metric, dpam_model_setup
 from versa.utterance_metrics.cdpam_distance import cdpam_metric, cdpam_model_setup
 from versa.utterance_metrics.vqscore import vqscore_metric, vqscore_setup

versa/scorer_shared.py

@@ -97,7 +97,6 @@ def load_score_modules(score_config, use_gt=True, use_gt_text=False, use_gpu=Fal
             logging.info("Initiate WARP-Q metric...")
 
         elif config["name"] == "nisqa":
-
             logging.info("Loading NISQA evaluation...")
             from versa.utterance_metrics.nisqa import nisqa_metric, nisqa_model_setup
 
@@ -257,7 +256,6 @@ def load_score_modules(score_config, use_gt=True, use_gt_text=False, use_gpu=Fal
             logging.info("Initiate singer evaluation successfully.")
 
         elif config["name"] == "sheet_ssqa":
-
             logging.info("Loading Sheet SSQA models for evaluation...")
             from versa import sheet_ssqa, sheet_ssqa_setup
 
@@ -287,7 +285,6 @@ def load_score_modules(score_config, use_gt=True, use_gt_text=False, use_gpu=Fal
             logging.info("Initiate torch squim (with reference) successfully")
 
         elif config["name"] == "squim_no_ref":
-
             logging.info("Loading squim metrics with reference")
             from versa import squim_metric_no_ref
 
@@ -466,7 +463,6 @@ def load_score_modules(score_config, use_gt=True, use_gt_text=False, use_gpu=Fal
             logging.info("Initiate se_snr successfully")
 
         elif config["name"] == "pam":
-
             logging.info("Loading pam metric without reference...")
             from versa.utterance_metrics.pam import pam_metric, pam_model_setup
 
@@ -494,7 +490,6 @@ def load_score_modules(score_config, use_gt=True, use_gt_text=False, use_gpu=Fal
             logging.info("Initiate vad metric successfully.")
 
         elif config["name"] == "asvspoof_score":
-
             logging.info("Loading asvspoof score metric without reference...")
             from versa.utterance_metrics.asvspoof_score import (
                 asvspoof_metric,
@@ -897,6 +892,29 @@ def load_score_modules(score_config, use_gt=True, use_gt_text=False, use_gpu=Fal
                     "scale_factor": config.get("scale_factor", 100),
                 },
             }
+        elif "wvmos" in config["name"]:
+            logging.info("Loading WVMOS metric")
+            from versa import wvmos_setup, wvmos_calculate
+
+            model = wvmos_setup(
+                use_gpu=use_gpu,
+            )
+            score_modules["wvmos"] = {
+                "module": wvmos_calculate,
+                "args": {"model": model},
+            }
+            logging.info("Initiate WVMOS metric successfully")
+        elif "sigmos" in config["name"]:
+            logging.info("Loading SIGMOS metric")
+            from versa import sigmos_setup, sigmos_calculate
+
+            model = sigmos_setup()
+
+            score_modules["sigmos"] = {
+                "module": sigmos_calculate,
+                "args": {"model": model},
+            }
+            logging.info("Initiate SIGMOS metric successfully")
         elif "vqscore" in config["name"]:
             logging.info("Loading VQScore model")
             from versa import vqscore_metric, vqscore_setup
@@ -1108,6 +1126,18 @@ def use_score_modules(score_modules, gen_wav, gt_wav, gen_sr, text=None):
                 gen_sr,
                 custom_prompt=score_modules[key]["prompt"],
             )
+        elif key == "wvmos":
+            score = score_modules[key]["module"](
+                score_modules[key]["args"]["model"],
+                gen_wav,
+                gen_sr,
+            )
+        elif key == "sigmos":
+            score = score_modules[key]["module"](
+                score_modules[key]["args"]["model"],
+                gen_wav,
+                gen_sr,
+            )
         elif key == "vqscore":
             score = score_modules[key]["module"](
                 score_modules[key]["args"]["model"], gen_wav, gen_sr