How to use whisper-diarization without diarization?

[Arche151]

I know the point of this repo is adding diarization to faster-whisper, but I am unironcally asking, if there is a way to use whisper-diarization without diarization.

I am using whisper-diarization to create subtitles and for cases, where I don't need speaker diarization I tried using faster-whisper directly as well as WhisperX. But transcription quality, segments length and punctuation all sucked in comparison to whisper-diarization.

So now I'm asking myself, how I can use this amazing repo for the creation of non-diarized srt-files.

Would appreciate any kind of help :)

[MahmoudAshraf97]

it's definitely possible, just remove the diarization part from the script and adjust the punctuation part for the new input

[Arche151]

@MahmoudAshraf97 Okay, I will try that! Thanks :)

[Arche151]

@MahmoudAshraf97

Is this modified diarize.py the right approach? (I have to confess, that Claude 4 Sonnet did the rewrite, since I haven't leaned Python coding yet).

import logging
import os
import re

import faster_whisper
import torch
import torchaudio

from ctc_forced_aligner import (
    generate_emissions,
    get_alignments,
    get_spans,
    load_alignment_model,
    postprocess_results,
    preprocess_text,
)
from deepmultilingualpunctuation import PunctuationModel

from helpers import (
    cleanup,
    find_numeral_symbol_tokens,
    langs_to_iso,
    process_language_arg,
    punct_model_langs,
    whisper_langs,
)

mtypes = {"cpu": "int8", "cuda": "float16"}

pid = os.getpid()
temp_outputs_dir = f"temp_outputs_{pid}"

# Initialize parser
parser = argparse.ArgumentParser()
parser.add_argument(
    "-a", "--audio", help="name of the target audio file", required=True
)
parser.add_argument(
    "--no-stem",
    action="store_false",
    dest="stemming",
    default=True,
    help="Disables source separation."
    "This helps with long files that don't contain a lot of music.",
)

parser.add_argument(
    "--suppress_numerals",
    action="store_true",
    dest="suppress_numerals",
    default=False,
    help="Suppresses Numerical Digits."
    "This helps the diarization accuracy but converts all digits into written text.",
)

parser.add_argument(
    "--whisper-model",
    dest="model_name",
    default="medium.en",
    help="name of the Whisper model to use",
)

parser.add_argument(
    "--batch-size",
    type=int,
    dest="batch_size",
    default=8,
    help="Batch size for batched inference, reduce if you run out of memory, "
    "set to 0 for original whisper longform inference",
)

parser.add_argument(
    "--language",
    type=str,
    default=None,
    choices=whisper_langs,
    help="Language spoken in the audio, specify None to perform language detection",
)

parser.add_argument(
    "--device",
    dest="device",
    default="cuda" if torch.cuda.is_available() else "cpu",
    help="if you have a GPU use 'cuda', otherwise 'cpu'",
)

args = parser.parse_args()
language = process_language_arg(args.language, args.model_name)

if args.stemming:
    # Isolate vocals from the rest of the audio

    return_code = os.system(
        f'python -m demucs.separate -n htdemucs --two-stems=vocals "{args.audio}" -o "{temp_outputs_dir}" --device "{args.device}"'
    )

    if return_code != 0:
        logging.warning(
            "Source splitting failed, using original audio file. "
            "Use --no-stem argument to disable it."
        )
        vocal_target = args.audio
    else:
        vocal_target = os.path.join(
            temp_outputs_dir,
            "htdemucs",
            os.path.splitext(os.path.basename(args.audio))[0],
            "vocals.wav",
        )
else:
    vocal_target = args.audio


# Transcribe the audio file

whisper_model = faster_whisper.WhisperModel(
    args.model_name, device=args.device, compute_type=mtypes[args.device]
)
whisper_pipeline = faster_whisper.BatchedInferencePipeline(whisper_model)
audio_waveform = faster_whisper.decode_audio(vocal_target)
suppress_tokens = (
    find_numeral_symbol_tokens(whisper_model.hf_tokenizer)
    if args.suppress_numerals
    else [-1]
)

if args.batch_size > 0:
    transcript_segments, info = whisper_pipeline.transcribe(
        audio_waveform,
        language,
        suppress_tokens=suppress_tokens,
        batch_size=args.batch_size,
    )
else:
    transcript_segments, info = whisper_model.transcribe(
        audio_waveform,
        language,
        suppress_tokens=suppress_tokens,
        vad_filter=True,
    )

full_transcript = "".join(segment.text for segment in transcript_segments)

# clear gpu vram
del whisper_model, whisper_pipeline
torch.cuda.empty_cache()

# Forced Alignment
alignment_model, alignment_tokenizer = load_alignment_model(
    args.device,
    dtype=torch.float16 if args.device == "cuda" else torch.float32,
)

emissions, stride = generate_emissions(
    alignment_model,
    torch.from_numpy(audio_waveform)
    .to(alignment_model.dtype)
    .to(alignment_model.device),
    batch_size=args.batch_size,
)

del alignment_model
torch.cuda.empty_cache()

tokens_starred, text_starred = preprocess_text(
    full_transcript,
    romanize=True,
    language=langs_to_iso[info.language],
)

segments, scores, blank_token = get_alignments(
    emissions,
    tokens_starred,
    alignment_tokenizer,
)

spans = get_spans(tokens_starred, segments, blank_token)

word_timestamps = postprocess_results(text_starred, spans, stride, scores)

# REMOVED: convert audio to mono for NeMo combatibility
# REMOVED: Initialize NeMo MSDD diarization model
# REMOVED: Reading timestamps <> Speaker Labels mapping

# Adjust punctuation part for new input (word_timestamps instead of wsm)
if info.language in punct_model_langs:
    # restoring punctuation in the transcript to help realign the sentences
    punct_model = PunctuationModel(model="kredor/punctuate-all")

    # Changed: get words directly from word_timestamps using 'text' key
    words_list = list(map(lambda x: x["text"], word_timestamps))

    labled_words = punct_model.predict(words_list, chunk_size=230)

    ending_puncts = ".?!"
    model_puncts = ".,;:!?"

    # We don't want to punctuate U.S.A. with a period. Right?
    is_acronym = lambda x: re.fullmatch(r"\b(?:[a-zA-Z]\.){2,}", x)

    # Changed: apply punctuation to word_timestamps using 'text' key
    for word_dict, labeled_tuple in zip(word_timestamps, labled_words):
        word = word_dict["text"]
        if (
            word
            and labeled_tuple[1] in ending_puncts
            and (word[-1] not in model_puncts or is_acronym(word))
        ):
            word += labeled_tuple[1]
            if word.endswith(".."):
                word = word.rstrip(".")
            word_dict["text"] = word

else:
    logging.warning(
        f"Punctuation restoration is not available for {info.language} language."
        " Using the original punctuation."
    )

# REMOVED: wsm = get_realigned_ws_mapping_with_punctuation(wsm)
# REMOVED: ssm = get_sentences_speaker_mapping(wsm, speaker_ts)

# Simple output without speaker information
output_base = os.path.splitext(args.audio)[0]
with open(f"{output_base}.txt", "w", encoding="utf-8-sig") as f:
    # Write simple transcript using 'text' key
    transcript = " ".join([word["text"] for word in word_timestamps])
    f.write(transcript)

# Simple SRT output based on word timestamps
with open(f"{os.path.splitext(args.audio)[0]}.srt", "w", encoding="utf-8-sig") as srt:
    # Group words into sentences/phrases for proper SRT segments
    segments = []
    current_segment = []
    current_start = None
    
    for word_info in word_timestamps:
        word = word_info["text"].strip()
        if not word:
            continue
            
        start_time = word_info["start"]
        end_time = word_info["end"]
        
        # Start new segment if needed
        if current_start is None:
            current_start = start_time
            
        current_segment.append(word)
        
        # End segment on punctuation or reasonable length
        should_end_segment = (
            word.endswith(('.', '!', '?')) or  # Sentence ending
            len(current_segment) >= 10 or      # Max 10 words per segment
            (len(current_segment) >= 5 and word.endswith((',', ';', ':')))  # Clause ending
        )
        
        if should_end_segment:
            segment_text = " ".join(current_segment)
            segments.append({
                "start": current_start,
                "end": end_time,
                "text": segment_text
            })
            current_segment = []
            current_start = None
    
    # Handle remaining words
    if current_segment:
        segment_text = " ".join(current_segment)
        segments.append({
            "start": current_start,
            "end": word_timestamps[-1]["end"],
            "text": segment_text
        })
    
    # Write SRT segments
    for i, segment in enumerate(segments, 1):
        start_ms = int(segment["start"] * 1000)
        end_ms = int(segment["end"] * 1000)
        
        start_h, start_remainder = divmod(start_ms, 3600000)
        start_m, start_remainder = divmod(start_remainder, 60000)
        start_s, start_ms = divmod(start_remainder, 1000)
        
        end_h, end_remainder = divmod(end_ms, 3600000)
        end_m, end_remainder = divmod(end_remainder, 60000)
        end_s, end_ms = divmod(end_remainder, 1000)
        
        srt.write(f"{i}\n")
        srt.write(f"{start_h:02d}:{start_m:02d}:{start_s:02d},{start_ms:03d} --> ")
        srt.write(f"{end_h:02d}:{end_m:02d}:{end_s:02d},{end_ms:03d}\n")
        srt.write(f"{segment['text']}\n\n")

cleanup(temp_outputs_dir)