bump to 0.2.14

README.md

@@ -45,7 +45,7 @@ pip install whisperlivekit
 #### Quick Start
 1. **Start the transcription server:**
    ```bash
-   whisperlivekit-server --model base --language en
+   wlk --model base --language en
    ```
 
 2. **Open your browser** and navigate to `http://localhost:8000`. Start speaking and watch your words appear in real-time!
@@ -53,6 +53,7 @@ pip install whisperlivekit
 
 > - See [tokenizer.py](https://github.com/QuentinFuxa/WhisperLiveKit/blob/main/whisperlivekit/simul_whisper/whisper/tokenizer.py) for the list of all available languages.
 > - For HTTPS requirements, see the **Parameters** section for SSL configuration options.
+> - The CLI entry point is exposed as both `wlk` and `whisperlivekit-server`; they are equivalent.
 
 #### Use it to capture audio from web pages.
 
@@ -68,13 +69,12 @@ Go to `chrome-extension` for instructions.
 
 | Optional | `pip install` |
 |-----------|-------------|
-| **Speaker diarization** | `git+https://github.com/NVIDIA/NeMo.git@main#egg=nemo_toolkit[asr]` |
+| **Windows/Linux optimizations** | `faster-whisper` |
 | **Apple Silicon optimizations** | `mlx-whisper` |
 | **Translation** | `nllw` |
+| **Speaker diarization** | `git+https://github.com/NVIDIA/NeMo.git@main#egg=nemo_toolkit[asr]` |
+| OpenAI API | `openai` |
 | *[Not recommanded]*  Speaker diarization with Diart | `diart` |
-| *[Not recommanded]*  Original Whisper backend | `whisper` |
-| *[Not recommanded]*  Improved timestamps backend | `whisper-timestamped` |
-| OpenAI API backend | `openai` |
 
 See  **Parameters & Configuration** below on how to use them.
 
@@ -86,10 +86,10 @@ See  **Parameters & Configuration** below on how to use them.
 
 ```bash
 # Large model and translate from french to danish
-whisperlivekit-server --model large-v3 --language fr --target-language da
+wlk --model large-v3 --language fr --target-language da
 
 # Diarization and server listening on */80 
-whisperlivekit-server --host 0.0.0.0 --port 80 --model medium --diarization --language fr
+wlk --host 0.0.0.0 --port 80 --model medium --diarization --language fr
 ```
 
 
@@ -140,12 +140,12 @@ async def websocket_endpoint(websocket: WebSocket):
 | Parameter | Description | Default |
 |-----------|-------------|---------|
 | `--model` | Whisper model size. List and recommandations [here](https://github.com/QuentinFuxa/WhisperLiveKit/blob/main/docs/available_models.md) | `small` |
-| `--model-path` | .pt file/directory containing whisper model. Overrides `--model`. Recommandations [here](https://github.com/QuentinFuxa/WhisperLiveKit/blob/main/docs/models_compatible_formats.md) | `None` |
+| `--model-path` | Local .pt file/directory **or** Hugging Face repo ID containing the Whisper model. Overrides `--model`. Recommandations [here](https://github.com/QuentinFuxa/WhisperLiveKit/blob/main/docs/models_compatible_formats.md) | `None` |
 | `--language` | List [here](https://github.com/QuentinFuxa/WhisperLiveKit/blob/main/whisperlivekit/simul_whisper/whisper/tokenizer.py). If you use `auto`, the model attempts to detect the language automatically, but it tends to bias towards English. | `auto` |
-| `--target-language` | If sets, translate to using NLLB. Ex: `fr`. [200 languages available](https://github.com/QuentinFuxa/WhisperLiveKit/blob/main/docs/supported_languages.md). If you want to translate to english, you should rather use `--task translate`, since Whisper can do it directly. | `None` |
+| `--target-language` | If sets, translates using [NLLW](https://github.com/QuentinFuxa/NoLanguageLeftWaiting). [200 languages available](https://github.com/QuentinFuxa/WhisperLiveKit/blob/main/docs/supported_languages.md). If you want to translate to english, you can also use `--direct-english-translation`. The STT model will try to directly output the translation. | `None` |
-| `--task` | Set to `translate` to translate *only* to english, using Whisper translation. | `transcribe` |
 | `--diarization` | Enable speaker identification | `False` |
-| `--backend` | Processing backend. You can switch to `faster-whisper` if  `simulstreaming` does not work correctly | `simulstreaming` |
+| `--backend-policy` | Streaming strategy: `1`/`simulstreaming` uses AlignAtt SimulStreaming, `2`/`localagreement` uses the LocalAgreement policy | `simulstreaming` |
+| `--backend` | Whisper implementation selector. `auto` picks MLX on macOS (if installed), otherwise Faster-Whisper, otherwise vanilla Whisper. You can also force `mlx-whisper`, `faster-whisper`, `whisper`, or `openai-api` (LocalAgreement only) | `auto` |
 | `--no-vac` | Disable Voice Activity Controller | `False` |
 | `--no-vad` | Disable Voice Activity Detection | `False` |
 | `--warmup-file` | Audio file path for model warmup | `jfk.wav` |

docs/models_compatible_formats.md

@@ -13,4 +13,7 @@ May optionally contain:
 - **`.bin` file** - faster-whisper model for encoder (requires faster-whisper)
 - **`weights.npz`** or **`weights.safetensors`** - for encoder (requires whisper-mlx)
 
-To improve speed/reduce allucinations, you may want to use `scripts/determine_alignment_heads.py` to determine the alignment heads to use for your model, and use the `--custom-alignment-heads` to pass them to WLK. If not, alignement heads are set to be all the heads of the last half layer of decoder.
+## Hugging Face Repo ID
+- Provide the repo ID (e.g. `openai/whisper-large-v3`) and WhisperLiveKit will download and cache the snapshot automatically. For gated repos, authenticate via `huggingface-cli login` first.
+
+To improve speed/reduce allucinations, you may want to use `scripts/determine_alignment_heads.py` to determine the alignment heads to use for your model, and use the `--custom-alignment-heads` to pass them to WLK. If not, alignement heads are set to be all the heads of the last half layer of decoder.

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "whisperlivekit"
-version = "0.2.13.post1"
+version = "0.2.14"
 description = "Real-time speech-to-text with speaker diarization using Whisper"
 readme = "README.md"
 authors = [
@@ -34,6 +34,7 @@ dependencies = [
     "websockets",
     "torchaudio>=2.0.0",
     "torch>=2.0.0",
+    "huggingface-hub>=0.25.0",
     "tqdm",
     "tiktoken",
     'triton>=2.0.0; platform_machine == "x86_64" and (sys_platform == "linux" or sys_platform == "linux2")'
@@ -48,21 +49,22 @@ Homepage = "https://github.com/QuentinFuxa/WhisperLiveKit"
 
 [project.scripts]
 whisperlivekit-server = "whisperlivekit.basic_server:main"
+wlk = "whisperlivekit.basic_server:main"
 
 [tool.setuptools]
 packages = [
     "whisperlivekit",
     "whisperlivekit.diarization",
     "whisperlivekit.simul_whisper",
-    "whisperlivekit.simul_whisper.whisper",
+    "whisperlivekit.whisper",
-    "whisperlivekit.simul_whisper.whisper.assets",
+    "whisperlivekit.whisper.assets",
-    "whisperlivekit.simul_whisper.whisper.normalizers",
+    "whisperlivekit.whisper.normalizers",
     "whisperlivekit.web",
-    "whisperlivekit.whisper_streaming_custom",
+    "whisperlivekit.local_agreement",
     "whisperlivekit.vad_models"
 ]
 
 [tool.setuptools.package-data]
 whisperlivekit = ["web/*.html", "web/*.css", "web/*.js", "web/src/*.svg"]
-"whisperlivekit.simul_whisper.whisper.assets" = ["*.tiktoken", "*.npz"]
+"whisperlivekit.whisper.assets" = ["*.tiktoken", "*.npz"]
 "whisperlivekit.vad_models" = ["*.jit", "*.onnx"]

scripts/convert_hf_whisper.py

@@ -0,0 +1,153 @@
+#!/usr/bin/env python3
+"""
+Convert a Hugging Face style Whisper checkpoint into a WhisperLiveKit .pt file.
+
+Optionally shrink the supported audio chunk length (in seconds) by trimming the
+encoder positional embeddings and updating the stored model dimensions.
+"""
+
+import argparse
+import json
+import os
+from pathlib import Path
+from typing import Dict, Tuple
+
+import torch
+
+from whisperlivekit.whisper.audio import HOP_LENGTH, SAMPLE_RATE
+from whisperlivekit.whisper.model import ModelDimensions
+from whisperlivekit.whisper.utils import exact_div
+from whisperlivekit.whisper import _convert_hf_state_dict
+
+
+def _load_state_dict(repo_path: Path) -> Dict[str, torch.Tensor]:
+    safetensor_path = repo_path / "model.safetensors"
+    bin_path = repo_path / "pytorch_model.bin"
+
+    if safetensor_path.is_file():
+        try:
+            from safetensors.torch import load_file  # type: ignore
+        except Exception as exc:  # pragma: no cover - import guard
+            raise RuntimeError(
+                "Install safetensors to load model.safetensors "
+                "(pip install safetensors)"
+            ) from exc
+        return load_file(str(safetensor_path))
+
+    if bin_path.is_file():
+        return torch.load(bin_path, map_location="cpu")
+
+    raise FileNotFoundError(
+        f"Could not find model.safetensors or pytorch_model.bin under {repo_path}"
+    )
+
+
+def _load_config(repo_path: Path) -> Dict:
+    config_path = repo_path / "config.json"
+    if not config_path.is_file():
+        raise FileNotFoundError(
+            f"Hugging Face checkpoint at {repo_path} is missing config.json"
+        )
+    with open(config_path, "r", encoding="utf-8") as fp:
+        return json.load(fp)
+
+
+def _derive_audio_ctx(chunk_length: float) -> Tuple[int, int]:
+    n_samples = int(round(chunk_length * SAMPLE_RATE))
+    expected_samples = chunk_length * SAMPLE_RATE
+    if abs(n_samples - expected_samples) > 1e-6:
+        raise ValueError(
+            "chunk_length must align with sample rate so that "
+            "chunk_length * SAMPLE_RATE is an integer"
+        )
+    n_frames = exact_div(n_samples, HOP_LENGTH)
+    n_audio_ctx = exact_div(n_frames, 2)
+    return n_frames, n_audio_ctx
+
+
+def _build_dims(config: Dict, chunk_length: float) -> Dict:
+    base_dims = ModelDimensions(
+        n_mels=config["num_mel_bins"],
+        n_audio_ctx=config["max_source_positions"],
+        n_audio_state=config["d_model"],
+        n_audio_head=config["encoder_attention_heads"],
+        n_audio_layer=config.get("encoder_layers") or config["num_hidden_layers"],
+        n_vocab=config["vocab_size"],
+        n_text_ctx=config["max_target_positions"],
+        n_text_state=config["d_model"],
+        n_text_head=config["decoder_attention_heads"],
+        n_text_layer=config["decoder_layers"],
+    ).__dict__.copy()
+
+    _, n_audio_ctx = _derive_audio_ctx(chunk_length)
+    base_dims["n_audio_ctx"] = n_audio_ctx
+    base_dims["chunk_length"] = chunk_length
+    return base_dims
+
+
+def _trim_positional_embedding(
+    state_dict: Dict[str, torch.Tensor], target_ctx: int
+) -> None:
+    key = "encoder.positional_embedding"
+    if key not in state_dict:
+        raise KeyError(f"{key} missing from converted state dict")
+
+    tensor = state_dict[key]
+    if tensor.shape[0] < target_ctx:
+        raise ValueError(
+            f"Cannot increase encoder ctx from {tensor.shape[0]} to {target_ctx}"
+        )
+    if tensor.shape[0] == target_ctx:
+        return
+    state_dict[key] = tensor[:target_ctx].contiguous()
+
+
+def convert_checkpoint(hf_path: Path, output_path: Path, chunk_length: float) -> None:
+    state_dict = _load_state_dict(hf_path)
+    converted = _convert_hf_state_dict(state_dict)
+
+    config = _load_config(hf_path)
+    dims = _build_dims(config, chunk_length)
+
+    _trim_positional_embedding(converted, dims["n_audio_ctx"])
+
+    package = {"dims": dims, "model_state_dict": converted}
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    torch.save(package, output_path)
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Convert Hugging Face Whisper checkpoint to WhisperLiveKit format."
+    )
+    parser.add_argument(
+        "hf_path",
+        type=str,
+        help="Path to the cloned Hugging Face repository (e.g. whisper-tiny.en)",
+    )
+    parser.add_argument(
+        "--output",
+        type=str,
+        default="converted-whisper.pt",
+        help="Destination path for the .pt file",
+    )
+    parser.add_argument(
+        "--chunk-length",
+        type=float,
+        default=30.0,
+        help="Audio chunk length in seconds to support (default: 30)",
+    )
+    return parser.parse_args()
+
+
+def main():
+    args = parse_args()
+    hf_path = Path(os.path.expanduser(args.hf_path)).resolve()
+    output_path = Path(os.path.expanduser(args.output)).resolve()
+
+    convert_checkpoint(hf_path, output_path, args.chunk_length)
+    print(f"Saved converted checkpoint to {output_path}")
+
+
+if __name__ == "__main__":
+    main()

whisperlivekit/audio_processor.py

@@ -387,10 +387,10 @@ class AudioProcessor:
                         tokens_to_process.append(additional_token)                
                 if tokens_to_process:
                     self.translation.insert_tokens(tokens_to_process)
-                    translation_validated_segments, translation_buffer = await asyncio.to_thread(self.translation.process)
+                    translation_validated_segments, buffer_translation = await asyncio.to_thread(self.translation.process)
                     async with self.lock:
                         self.state.translation_validated_segments = translation_validated_segments
-                        self.state.translation_buffer = translation_buffer
+                        self.state.buffer_translation = buffer_translation
                 self.translation_queue.task_done()
                 for _ in additional_tokens:
                     self.translation_queue.task_done()
@@ -438,6 +438,12 @@ class AudioProcessor:
 
                     async with self.lock:
                         self.state.end_attributed_speaker = state.end_attributed_speaker
+
+                buffer_translation_text = ''
+                if state.buffer_translation:
+                    raw_buffer_translation = getattr(state.buffer_translation, 'text', state.buffer_translation)
+                    if raw_buffer_translation:
+                        buffer_translation_text = raw_buffer_translation.strip()
                 
                 response_status = "active_transcription"
                 if not state.tokens and not buffer_transcription and not buffer_diarization:
@@ -455,6 +461,7 @@ class AudioProcessor:
                     lines=lines,
                     buffer_transcription=buffer_transcription.text.strip(),
                     buffer_diarization=buffer_diarization,
+                    buffer_translation=buffer_translation_text,
                     remaining_time_transcription=state.remaining_time_transcription,
                     remaining_time_diarization=state.remaining_time_diarization if self.args.diarization else 0
                 )

whisperlivekit/backend_support.py

@@ -0,0 +1,41 @@
+import importlib.util
+import logging
+import platform
+
+logger = logging.getLogger(__name__)
+
+
+def module_available(module_name):
+    """Return True if the given module can be imported."""
+    return importlib.util.find_spec(module_name) is not None
+
+
+def mlx_backend_available(warn_on_missing = False):
+    is_macos = platform.system() == "Darwin"
+    is_arm = platform.machine() == "arm64"
+    available = (
+        is_macos
+        and is_arm
+        and module_available("mlx_whisper")
+    )
+    if not available and warn_on_missing and is_macos and is_arm:
+        logger.warning(
+            "=" * 50
+            + "\nMLX Whisper not found but you are on Apple Silicon. "
+              "Consider installing mlx-whisper for better performance: "
+              "`pip install mlx-whisper`\n"
+            + "=" * 50
+        )
+    return available
+
+
+def faster_backend_available(warn_on_missing = False):
+    available = module_available("faster_whisper")
+    if not available and warn_on_missing and platform.system() != "Darwin":
+        logger.warning(
+            "=" * 50
+            + "\nFaster-Whisper not found. Consider installing faster-whisper "
+              "for better performance: `pip install faster-whisper`\n"
+            + "=" * 50
+        )
+    return available

whisperlivekit/core.py

@@ -1,11 +1,9 @@
-try:
+from whisperlivekit.local_agreement.whisper_online import backend_factory
-    from whisperlivekit.whisper_streaming_custom.whisper_online import backend_factory
+from whisperlivekit.simul_whisper import SimulStreamingASR
-    from whisperlivekit.whisper_streaming_custom.online_asr import OnlineASRProcessor
+from whisperlivekit.local_agreement.online_asr import OnlineASRProcessor
-except ImportError:
-    from .whisper_streaming_custom.whisper_online import backend_factory
-    from .whisper_streaming_custom.online_asr import OnlineASRProcessor
 from argparse import Namespace
 import sys
+import logging
 
 def update_with_kwargs(_dict, kwargs):
     _dict.update({
@@ -13,6 +11,9 @@ def update_with_kwargs(_dict, kwargs):
     })
     return _dict
 
+
+logger = logging.getLogger(__name__)
+
 class TranscriptionEngine:
     _instance = None
     _initialized = False
@@ -44,18 +45,20 @@ class TranscriptionEngine:
             "pcm_input": False,
             "disable_punctuation_split" : False,
             "diarization_backend": "sortformer",
+            "backend_policy": "simulstreaming",
+            "backend": "auto",
         }
         global_params = update_with_kwargs(global_params, kwargs)
 
         transcription_common_params = {
-            "backend": "simulstreaming",
             "warmup_file": None,
             "min_chunk_size": 0.5,
             "model_size": "tiny",
             "model_cache_dir": None,
             "model_dir": None,
+            "model_path": None,
             "lan": "auto",
-            "task": "transcribe",
+            "direct_english_translation": False,
         }
         transcription_common_params = update_with_kwargs(transcription_common_params, kwargs)                                            
 
@@ -81,10 +84,9 @@ class TranscriptionEngine:
             use_onnx = kwargs.get('vac_onnx', False)
             self.vac_model = load_silero_vad(onnx=use_onnx)
         
+        backend_policy = self.args.backend_policy
         if self.args.transcription:
-            if self.args.backend == "simulstreaming": 
+            if backend_policy == "simulstreaming":                 
-                from whisperlivekit.simul_whisper import SimulStreamingASR
-                
                 simulstreaming_params = {
                     "disable_fast_encoder": False,
                     "custom_alignment_heads": None,
@@ -98,14 +100,19 @@ class TranscriptionEngine:
                     "init_prompt": None,
                     "static_init_prompt": None,
                     "max_context_tokens": None,
-                    "model_path": './base.pt',
                     "preload_model_count": 1,
                 }
                 simulstreaming_params = update_with_kwargs(simulstreaming_params, kwargs)
                 
                 self.tokenizer = None        
                 self.asr = SimulStreamingASR(
-                    **transcription_common_params, **simulstreaming_params
+                    **transcription_common_params,
+                    **simulstreaming_params,
+                    backend=self.args.backend,
+                )
+                logger.info(
+                    "Using SimulStreaming policy with %s backend",
+                    getattr(self.asr, "encoder_backend", "whisper"),
                 )
             else:
                 
@@ -117,7 +124,13 @@ class TranscriptionEngine:
                 whisperstreaming_params = update_with_kwargs(whisperstreaming_params, kwargs)
                 
                 self.asr = backend_factory(
-                    **transcription_common_params, **whisperstreaming_params
+                    backend=self.args.backend,
+                    **transcription_common_params,
+                    **whisperstreaming_params,
+                )
+                logger.info(
+                    "Using LocalAgreement policy with %s backend",
+                    getattr(self.asr, "backend_choice", self.asr.__class__.__name__),
                 )
 
         if self.args.diarization:
@@ -138,7 +151,7 @@ class TranscriptionEngine:
         
         self.translation_model = None
         if self.args.target_language:
-            if self.args.lan == 'auto' and self.args.backend != "simulstreaming":
+            if self.args.lan == 'auto' and backend_policy != "simulstreaming":
                 raise Exception('Translation cannot be set with language auto when transcription backend is not simulstreaming')
             else:
                 try:
@@ -155,7 +168,7 @@ class TranscriptionEngine:
 
 
 def online_factory(args, asr):
-    if args.backend == "simulstreaming":    
+    if args.backend_policy == "simulstreaming":    
         from whisperlivekit.simul_whisper import SimulStreamingOnlineProcessor
         online = SimulStreamingOnlineProcessor(asr)
     else:

whisperlivekit/local_agreement/backends.py

@@ -6,6 +6,8 @@ import math
 from typing import List
 import numpy as np
 from whisperlivekit.timed_objects import ASRToken
+from whisperlivekit.model_paths import resolve_model_path, model_path_and_type
+from whisperlivekit.whisper.transcribe import transcribe as whisper_transcribe
 logger = logging.getLogger(__name__)
 class ASRBase:
     sep = " "  # join transcribe words with this character (" " for whisper_timestamped,
@@ -37,40 +39,60 @@ class ASRBase:
         raise NotImplementedError("must be implemented in the child class")
 
 
-class WhisperTimestampedASR(ASRBase):
+class WhisperASR(ASRBase):
-    """Uses whisper_timestamped as the backend."""
+    """Uses WhisperLiveKit's built-in Whisper implementation."""
     sep = " "
 
     def load_model(self, model_size=None, cache_dir=None, model_dir=None):
-        import whisper
+        from whisperlivekit.whisper import load_model as load_model
-        import whisper_timestamped
-        from whisper_timestamped import transcribe_timestamped
 
-        self.transcribe_timestamped = transcribe_timestamped
         if model_dir is not None:
-            logger.debug("ignoring model_dir, not implemented")
+            resolved_path = resolve_model_path(model_dir)
-        return whisper.load_model(model_size, download_root=cache_dir)
+            if resolved_path.is_dir():
+                pytorch_path, _, _ = model_path_and_type(resolved_path)
+                if pytorch_path is None:
+                    raise FileNotFoundError(
+                        f"No supported PyTorch checkpoint found under {resolved_path}"
+                    )
+                resolved_path = pytorch_path
+            logger.debug(f"Loading Whisper model from custom path {resolved_path}")
+            return load_model(str(resolved_path))
+
+        if model_size is None:
+            raise ValueError("Either model_size or model_dir must be set for WhisperASR")
+
+        return load_model(model_size, download_root=cache_dir)
 
     def transcribe(self, audio, init_prompt=""):
-        result = self.transcribe_timestamped(
+        options = dict(self.transcribe_kargs)
+        options.pop("vad", None)
+        options.pop("vad_filter", None)
+        language = self.original_language if self.original_language else None
+
+        result = whisper_transcribe(
             self.model,
             audio,
-            language=self.original_language,
+            language=language,
             initial_prompt=init_prompt,
-            verbose=None,
             condition_on_previous_text=True,
-            **self.transcribe_kargs,
+            word_timestamps=True,
+            **options,
         )
         return result
 
     def ts_words(self, r) -> List[ASRToken]:
         """
-        Converts the whisper_timestamped result to a list of ASRToken objects.
+        Converts the Whisper result to a list of ASRToken objects.
         """
         tokens = []
         for segment in r["segments"]:
             for word in segment["words"]:
-                token = ASRToken(word["start"], word["end"], word["text"])
+                token = ASRToken(
+                    word["start"],
+                    word["end"],
+                    word["word"],
+                    probability=word.get("probability"),
+                )
                 tokens.append(token)
         return tokens
 
@@ -78,11 +100,7 @@ class WhisperTimestampedASR(ASRBase):
         return [segment["end"] for segment in res["segments"]]
 
     def use_vad(self):
-        self.transcribe_kargs["vad"] = True
+        logger.warning("VAD is not currently supported for WhisperASR backend and will be ignored.")
-
-    def set_translate_task(self):
-        self.transcribe_kargs["task"] = "translate"
-
 
 class FasterWhisperASR(ASRBase):
     """Uses faster-whisper as the backend."""
@@ -92,9 +110,10 @@ class FasterWhisperASR(ASRBase):
         from faster_whisper import WhisperModel
 
         if model_dir is not None:
-            logger.debug(f"Loading whisper model from model_dir {model_dir}. "
+            resolved_path = resolve_model_path(model_dir)
+            logger.debug(f"Loading faster-whisper model from {resolved_path}. "
                          f"model_size and cache_dir parameters are not used.")
-            model_size_or_path = model_dir
+            model_size_or_path = str(resolved_path)
         elif model_size is not None:
             model_size_or_path = model_size
         else:
@@ -139,10 +158,6 @@ class FasterWhisperASR(ASRBase):
     def use_vad(self):
         self.transcribe_kargs["vad_filter"] = True
 
-    def set_translate_task(self):
-        self.transcribe_kargs["task"] = "translate"
-
-
 class MLXWhisper(ASRBase):
     """
     Uses MLX Whisper optimized for Apple Silicon.
@@ -154,8 +169,9 @@ class MLXWhisper(ASRBase):
         import mlx.core as mx
 
         if model_dir is not None:
-            logger.debug(f"Loading whisper model from model_dir {model_dir}. model_size parameter is not used.")
+            resolved_path = resolve_model_path(model_dir)
-            model_size_or_path = model_dir
+            logger.debug(f"Loading MLX Whisper model from {resolved_path}. model_size parameter is not used.")
+            model_size_or_path = str(resolved_path)
         elif model_size is not None:
             model_size_or_path = self.translate_model_name(model_size)
             logger.debug(f"Loading whisper model {model_size}. You use mlx whisper, so {model_size_or_path} will be used.")
@@ -218,10 +234,6 @@ class MLXWhisper(ASRBase):
     def use_vad(self):
         self.transcribe_kargs["vad_filter"] = True
 
-    def set_translate_task(self):
-        self.transcribe_kargs["task"] = "translate"
-
-
 class OpenaiApiASR(ASRBase):
     """Uses OpenAI's Whisper API for transcription."""
     def __init__(self, lan=None, temperature=0, logfile=sys.stderr):
@@ -232,7 +244,7 @@ class OpenaiApiASR(ASRBase):
         self.temperature = temperature
         self.load_model()
         self.use_vad_opt = False
-        self.task = "transcribe"
+        self.direct_english_translation = False
 
     def load_model(self, *args, **kwargs):
         from openai import OpenAI
@@ -274,7 +286,7 @@ class OpenaiApiASR(ASRBase):
             "temperature": self.temperature,
             "timestamp_granularities": ["word", "segment"],
         }
-        if self.task != "translate" and self.original_language:
+        if not self.direct_english_translation and self.original_language:
             params["language"] = self.original_language
         if prompt:
             params["prompt"] = prompt
@@ -285,6 +297,3 @@ class OpenaiApiASR(ASRBase):
 
     def use_vad(self):
         self.use_vad_opt = True
-
-    def set_translate_task(self):
-        self.task = "translate"

whisperlivekit/local_agreement/whisper_online.py

@@ -0,0 +1,199 @@
+#!/usr/bin/env python3
+import sys
+import numpy as np
+import librosa
+from functools import lru_cache
+import time
+import logging
+import platform
+from .backends import FasterWhisperASR, MLXWhisper, WhisperASR, OpenaiApiASR
+from whisperlivekit.warmup import warmup_asr
+from whisperlivekit.model_paths import resolve_model_path, model_path_and_type
+from whisperlivekit.backend_support import (
+    mlx_backend_available,
+    faster_backend_available,
+)
+
+logger = logging.getLogger(__name__)
+
+
+WHISPER_LANG_CODES = "af,am,ar,as,az,ba,be,bg,bn,bo,br,bs,ca,cs,cy,da,de,el,en,es,et,eu,fa,fi,fo,fr,gl,gu,ha,haw,he,hi,hr,ht,hu,hy,id,is,it,ja,jw,ka,kk,km,kn,ko,la,lb,ln,lo,lt,lv,mg,mi,mk,ml,mn,mr,ms,mt,my,ne,nl,nn,no,oc,pa,pl,ps,pt,ro,ru,sa,sd,si,sk,sl,sn,so,sq,sr,su,sv,sw,ta,te,tg,th,tk,tl,tr,tt,uk,ur,uz,vi,yi,yo,zh".split(
+    ","
+)
+
+
+def create_tokenizer(lan):
+    """returns an object that has split function that works like the one of MosesTokenizer"""
+
+    assert (
+        lan in WHISPER_LANG_CODES
+    ), "language must be Whisper's supported lang code: " + " ".join(WHISPER_LANG_CODES)
+
+    if lan == "uk":
+        import tokenize_uk
+
+        class UkrainianTokenizer:
+            def split(self, text):
+                return tokenize_uk.tokenize_sents(text)
+
+        return UkrainianTokenizer()
+
+    # supported by fast-mosestokenizer
+    if (
+        lan
+        in "as bn ca cs de el en es et fi fr ga gu hi hu is it kn lt lv ml mni mr nl or pa pl pt ro ru sk sl sv ta te yue zh".split()
+    ):
+        from mosestokenizer import MosesSentenceSplitter        
+
+        return MosesSentenceSplitter(lan)
+
+    # the following languages are in Whisper, but not in wtpsplit:
+    if (
+        lan
+        in "as ba bo br bs fo haw hr ht jw lb ln lo mi nn oc sa sd sn so su sw tk tl tt".split()
+    ):
+        logger.debug(
+            f"{lan} code is not supported by wtpsplit. Going to use None lang_code option."
+        )
+        lan = None
+
+    from wtpsplit import WtP
+
+    # downloads the model from huggingface on the first use
+    wtp = WtP("wtp-canine-s-12l-no-adapters")
+
+    class WtPtok:
+        def split(self, sent):
+            return wtp.split(sent, lang_code=lan)
+
+    return WtPtok()
+
+
+def backend_factory(
+            backend,
+            lan,
+            model_size,
+            model_cache_dir,
+            model_dir,
+            model_path,
+            direct_english_translation,
+            buffer_trimming,
+            buffer_trimming_sec,
+            confidence_validation,
+            warmup_file=None,
+            min_chunk_size=None,
+        ):
+    backend_choice = backend
+    custom_reference = model_path or model_dir
+    resolved_root = None
+    pytorch_checkpoint = None
+    has_mlx_weights = False
+    has_fw_weights = False
+
+    if custom_reference:
+        resolved_root = resolve_model_path(custom_reference)
+        if resolved_root.is_dir():
+            pytorch_checkpoint, has_mlx_weights, has_fw_weights = model_path_and_type(resolved_root)
+        else:
+            pytorch_checkpoint = resolved_root
+
+    if backend_choice == "openai-api":
+        logger.debug("Using OpenAI API.")
+        asr = OpenaiApiASR(lan=lan)
+    else:
+        backend_choice = _normalize_backend_choice(
+            backend_choice,
+            resolved_root,
+            has_mlx_weights,
+            has_fw_weights,
+        )
+
+        if backend_choice == "faster-whisper":
+            asr_cls = FasterWhisperASR
+            if resolved_root is not None and not resolved_root.is_dir():
+                raise ValueError("Faster-Whisper backend expects a directory with CTranslate2 weights.")
+            model_override = str(resolved_root) if resolved_root is not None else None
+        elif backend_choice == "mlx-whisper":
+            asr_cls = MLXWhisper
+            if resolved_root is not None and not resolved_root.is_dir():
+                raise ValueError("MLX Whisper backend expects a directory containing MLX weights.")
+            model_override = str(resolved_root) if resolved_root is not None else None
+        else:
+            asr_cls = WhisperASR
+            model_override = str(pytorch_checkpoint) if pytorch_checkpoint is not None else None
+            if custom_reference and model_override is None:
+                raise FileNotFoundError(
+                    f"No PyTorch checkpoint found under {resolved_root or custom_reference}"
+                )
+
+        t = time.time()
+        logger.info(f"Loading Whisper {model_size} model for language {lan} using backend {backend_choice}...")
+        asr = asr_cls(
+            model_size=model_size,
+            lan=lan,
+            cache_dir=model_cache_dir,
+            model_dir=model_override,
+        )
+        e = time.time()
+        logger.info(f"done. It took {round(e-t,2)} seconds.")
+
+    if direct_english_translation:
+        tgt_language = "en"  # Whisper translates into English
+    else:
+        tgt_language = lan  # Whisper transcribes in this language
+
+    # Create the tokenizer
+    if buffer_trimming == "sentence":
+        tokenizer = create_tokenizer(tgt_language)
+    else:
+        tokenizer = None
+    
+    warmup_asr(asr, warmup_file)
+    
+    asr.confidence_validation = confidence_validation
+    asr.tokenizer = tokenizer
+    asr.buffer_trimming = buffer_trimming
+    asr.buffer_trimming_sec = buffer_trimming_sec
+    asr.backend_choice = backend_choice
+    return asr
+
+
+def _normalize_backend_choice(
+    preferred_backend,
+    resolved_root,
+    has_mlx_weights,
+    has_fw_weights,
+):
+    backend_choice = preferred_backend
+
+    if backend_choice == "auto":
+        if mlx_backend_available(warn_on_missing=True) and (resolved_root is None or has_mlx_weights):
+            return "mlx-whisper"
+        if faster_backend_available(warn_on_missing=True) and (resolved_root is None or has_fw_weights):
+            return "faster-whisper"
+        return "whisper"
+
+    if backend_choice == "mlx-whisper":
+        if not mlx_backend_available():
+            raise RuntimeError("mlx-whisper backend requested but mlx-whisper is not installed.")
+        if resolved_root is not None and not has_mlx_weights:
+            raise FileNotFoundError(
+                f"mlx-whisper backend requested but no MLX weights were found under {resolved_root}"
+            )
+        if platform.system() != "Darwin":
+            logger.warning("mlx-whisper backend requested on a non-macOS system; this may fail.")
+        return backend_choice
+
+    if backend_choice == "faster-whisper":
+        if not faster_backend_available():
+            raise RuntimeError("faster-whisper backend requested but faster-whisper is not installed.")
+        if resolved_root is not None and not has_fw_weights:
+            raise FileNotFoundError(
+                f"faster-whisper backend requested but no Faster-Whisper weights were found under {resolved_root}"
+            )
+        return backend_choice
+
+    if backend_choice == "whisper":
+        return backend_choice
+
+    raise ValueError(f"Unknown backend '{preferred_backend}' for LocalAgreement.")

whisperlivekit/model_paths.py

@@ -0,0 +1,69 @@
+from pathlib import Path
+from typing import Optional, Tuple, Union
+
+
+def model_path_and_type(model_path: Union[str, Path]) -> Tuple[Optional[Path], bool, bool]:
+    """
+    Inspect the provided path and determine which model formats are available.
+
+    Returns:
+        pytorch_path: Path to a PyTorch checkpoint (if present).
+        compatible_whisper_mlx: True if MLX weights exist in this folder.
+        compatible_faster_whisper: True if Faster-Whisper (ctranslate2) weights exist.
+    """
+    path = Path(model_path)
+
+    compatible_whisper_mlx = False
+    compatible_faster_whisper = False
+    pytorch_path: Optional[Path] = None
+
+    if path.is_file() and path.suffix.lower() in [".pt", ".safetensors", ".bin"]:
+        pytorch_path = path
+        return pytorch_path, compatible_whisper_mlx, compatible_faster_whisper
+
+    if path.is_dir():
+        for file in path.iterdir():
+            if not file.is_file():
+                continue
+
+            filename = file.name.lower()
+            suffix = file.suffix.lower()
+
+            if filename in {"weights.npz", "weights.safetensors"}:
+                compatible_whisper_mlx = True
+            elif filename in {"model.bin", "encoder.bin", "decoder.bin"}:
+                compatible_faster_whisper = True
+            elif suffix in {".pt", ".safetensors"}:
+                pytorch_path = file
+            elif filename == "pytorch_model.bin":
+                pytorch_path = file
+
+        if pytorch_path is None:
+            fallback = path / "pytorch_model.bin"
+            if fallback.exists():
+                pytorch_path = fallback
+
+    return pytorch_path, compatible_whisper_mlx, compatible_faster_whisper
+
+
+def resolve_model_path(model_path: Union[str, Path]) -> Path:
+    """
+    Return a local path for the provided model reference.
+
+    If the path does not exist locally, it is treated as a Hugging Face repo id
+    and downloaded via snapshot_download.
+    """
+    path = Path(model_path).expanduser()
+    if path.exists():
+        return path
+
+    try:
+        from huggingface_hub import snapshot_download
+    except ImportError as exc:  # pragma: no cover - optional dependency guard
+        raise FileNotFoundError(
+            f"Model path '{model_path}' does not exist locally and huggingface_hub "
+            "is not installed to download it."
+        ) from exc
+
+    downloaded_path = Path(snapshot_download(repo_id=str(model_path)))
+    return downloaded_path

whisperlivekit/parse_args.py

@@ -114,11 +114,10 @@ def parse_args():
         help="Source language code, e.g. en,de,cs, or 'auto' for language detection.",
     )
     parser.add_argument(
-        "--task",
+        "--direct-english-translation",
-        type=str,
+        action="store_true",
-        default="transcribe",
+        default=False,
-        choices=["transcribe", "translate"],
+        help="Use Whisper to directly translate to english.",
-        help="Transcribe or translate.",
     )
     
     parser.add_argument(
@@ -130,11 +129,18 @@ def parse_args():
     )    
 
     parser.add_argument(
-        "--backend",
+        "--backend-policy",
         type=str,
         default="simulstreaming",
-        choices=["faster-whisper", "whisper_timestamped", "mlx-whisper", "openai-api", "simulstreaming"],
+        choices=["1", "2", "simulstreaming", "localagreement"],
-        help="Load only this backend for Whisper processing.",
+        help="Select the streaming policy: 1 or 'simulstreaming' for AlignAtt, 2 or 'localagreement' for LocalAgreement.",
+    )
+    parser.add_argument(
+        "--backend",
+        type=str,
+        default="auto",
+        choices=["auto", "mlx-whisper", "faster-whisper", "whisper", "openai-api"],
+        help="Select the Whisper backend implementation (auto: prefer MLX on macOS, otherwise Faster-Whisper, else Whisper). Use 'openai-api' with --backend-policy localagreement to call OpenAI's API.",
     )
     parser.add_argument(
         "--no-vac",
@@ -317,5 +323,10 @@ def parse_args():
     args.vad = not args.no_vad    
     delattr(args, 'no_transcription')
     delattr(args, 'no_vad')
+
+    if args.backend_policy == "1":
+        args.backend_policy = "simulstreaming"
+    elif args.backend_policy == "2":
+        args.backend_policy = "localagreement"
     
     return args

whisperlivekit/results_formater.py

@@ -57,14 +57,13 @@ def format_output(state, silence, args, sep):
     disable_punctuation_split = args.disable_punctuation_split
     tokens = state.tokens
     translation_validated_segments = state.translation_validated_segments # Here we will attribute the speakers only based on the timestamps of the segments
-    translation_buffer = state.translation_buffer
     last_validated_token = state.last_validated_token
     
     previous_speaker = 1
     undiarized_text = []
     tokens = handle_silences(tokens, state.beg_loop, silence)
-    last_punctuation = None
+    for i in range(last_validated_token, len(tokens)):
-    for i, token in enumerate(tokens[last_validated_token:]):
+        token = tokens[i]
         speaker = int(token.speaker)
         token.corrected_speaker = speaker
         if not diarization:
@@ -73,9 +72,9 @@ def format_output(state, silence, args, sep):
                 token.validated_speaker = True
         else:
                 if is_punctuation(token):
-                    last_punctuation = i  
+                    state.last_punctuation_index = i  
                 
-                if last_punctuation == i-1:
+                if state.last_punctuation_index == i-1:
                     if token.speaker != previous_speaker:
                         token.validated_speaker = True
                         # perfect, diarization perfectly aligned

whisperlivekit/simul_whisper/backend.py

@@ -2,64 +2,37 @@ import sys
 import numpy as np
 import logging
 from typing import List, Tuple, Optional
-import logging
 import platform
 from whisperlivekit.timed_objects import ASRToken, Transcript, ChangeSpeaker
 from whisperlivekit.warmup import load_file
-from .whisper import load_model, tokenizer
+from whisperlivekit.whisper import load_model, tokenizer
-from .whisper.audio import TOKENS_PER_SECOND
+from whisperlivekit.whisper.audio import TOKENS_PER_SECOND
 import os
 import gc
 from pathlib import Path
-logger = logging.getLogger(__name__)
+from whisperlivekit.model_paths import model_path_and_type, resolve_model_path
+from whisperlivekit.backend_support import (
+    mlx_backend_available,
+    faster_backend_available,
+)
 
 import torch
 from whisperlivekit.simul_whisper.config import AlignAttConfig
 from whisperlivekit.simul_whisper.simul_whisper import PaddedAlignAttWhisper
-from whisperlivekit.simul_whisper.whisper import tokenizer
 
-try:
+logger = logging.getLogger(__name__)
-    from .mlx_encoder import mlx_model_mapping, load_mlx_encoder
+
-    HAS_MLX_WHISPER = True
+
-except ImportError:
+HAS_MLX_WHISPER = mlx_backend_available(warn_on_missing=True)
-    if platform.system() == "Darwin" and platform.machine() == "arm64":
-        print(f"""{"="*50}\nMLX Whisper not found but you are on Apple Silicon. Consider installing mlx-whisper for better performance: `pip install mlx-whisper`\n{"="*50}""")
-    HAS_MLX_WHISPER = False
 if HAS_MLX_WHISPER:
-    HAS_FASTER_WHISPER = False
+    from .mlx_encoder import mlx_model_mapping, load_mlx_encoder
 else:
-    try:
+    mlx_model_mapping = {}
-        from faster_whisper import WhisperModel
+HAS_FASTER_WHISPER = faster_backend_available(warn_on_missing=not HAS_MLX_WHISPER)
-        HAS_FASTER_WHISPER = True
+if HAS_FASTER_WHISPER:
-    except ImportError:
+    from faster_whisper import WhisperModel
-        if platform.system() != "Darwin":
+else:
-            print(f"""{"="*50}\nFaster-Whisper not found but. Consider installing faster-whisper for better performance: `pip install faster-whisper`\n{"="*50}`""")
+    WhisperModel = None
-        HAS_FASTER_WHISPER = False
-
-def model_path_and_type(model_path):
-    path = Path(model_path)
-    
-    compatible_whisper_mlx = False
-    compatible_faster_whisper = False
-    pytorch_path = None
-    if path.is_file() and path.suffix.lower() in ['.pt', '.safetensors', '.bin']:
-        pytorch_path = path
-    elif path.is_dir():
-        for file in path.iterdir():
-            if file.is_file():
-                if file.name in ['weights.npz', "weights.safetensors"]:
-                    compatible_whisper_mlx = True
-                elif file.suffix.lower() == '.bin':
-                    compatible_faster_whisper = True
-                elif file.suffix.lower() == '.pt':
-                    pytorch_path = file
-                elif file.suffix.lower() == '.safetensors':
-                    pytorch_path = file
-        if pytorch_path is None:
-            if (model_path / Path("pytorch_model.bin")).exists():
-                pytorch_path = model_path / Path("pytorch_model.bin")
-    return pytorch_path, compatible_whisper_mlx, compatible_faster_whisper
-
 
 class SimulStreamingOnlineProcessor:
     SAMPLING_RATE = 16000
@@ -177,11 +150,22 @@ class SimulStreamingASR():
             self.decoder_type = 'greedy' if self.beams == 1 else 'beam'
 
         self.fast_encoder = False
+        self._resolved_model_path = None
+        self.encoder_backend = "whisper"
+        preferred_backend = getattr(self, "backend", "auto")
         self.pytorch_path, compatible_whisper_mlx, compatible_faster_whisper = None, True, True
         if self.model_path:
-            self.pytorch_path, compatible_whisper_mlx, compatible_faster_whisper = model_path_and_type(self.model_path)
+            resolved_model_path = resolve_model_path(self.model_path)
-            self.model_name = self.pytorch_path.stem
+            self._resolved_model_path = resolved_model_path
-            is_multilingual = not self.model_path.endswith(".en")
+            self.model_path = str(resolved_model_path)
+            self.pytorch_path, compatible_whisper_mlx, compatible_faster_whisper = model_path_and_type(resolved_model_path)
+            if self.pytorch_path:
+                self.model_name = self.pytorch_path.stem
+            else:
+                self.model_name = Path(self.model_path).stem
+                raise FileNotFoundError(
+                    f"No PyTorch checkpoint (.pt/.bin/.safetensors) found under {self.model_path}"
+                )
         elif self.model_size is not None:
             model_mapping = {
                 'tiny': './tiny.pt',
@@ -198,7 +182,19 @@ class SimulStreamingASR():
                 'large': './large-v3.pt'
             }
             self.model_name = self.model_size
-            is_multilingual = not self.model_name.endswith(".en")
+        else:
+            raise ValueError("Either model_size or model_path must be specified for SimulStreaming.")
+
+        is_multilingual = not self.model_name.endswith(".en")
+
+        self.encoder_backend = self._resolve_encoder_backend(
+            preferred_backend,
+            compatible_whisper_mlx,
+            compatible_faster_whisper,
+        )
+        self.fast_encoder = self.encoder_backend in ("mlx-whisper", "faster-whisper")
+        if self.encoder_backend == "whisper":
+            self.disable_fast_encoder = True
                     
         self.cfg = AlignAttConfig(
                 tokenizer_is_multilingual= is_multilingual,
@@ -210,7 +206,7 @@ class SimulStreamingASR():
                 cif_ckpt_path=self.cif_ckpt_path,
                 decoder_type="beam",
                 beam_size=self.beams,
-                task=self.task,
+                task=self.direct_english_translation,
                 never_fire=self.never_fire,
                 init_prompt=self.init_prompt,
                 max_context_tokens=self.max_context_tokens,
@@ -218,7 +214,7 @@ class SimulStreamingASR():
         )  
         
         # Set up tokenizer for translation if needed
-        if self.task == "translate":
+        if self.direct_english_translation:
             self.tokenizer = self.set_translate_task()
         else:
             self.tokenizer = None
@@ -227,32 +223,72 @@ class SimulStreamingASR():
             
     
         self.mlx_encoder, self.fw_encoder = None, None
-        if not self.disable_fast_encoder:
+        if self.encoder_backend == "mlx-whisper":
-            if HAS_MLX_WHISPER:
+            print('Simulstreaming will use MLX whisper to increase encoding speed.')
-                print('Simulstreaming will use MLX whisper to increase encoding speed.')
+            if self._resolved_model_path is not None:
-                if self.model_path and compatible_whisper_mlx:
+                mlx_model = str(self._resolved_model_path)
-                    mlx_model = self.model_path
+            else:
-                else:
+                mlx_model = mlx_model_mapping.get(self.model_name)
-                    mlx_model = mlx_model_mapping.get(self.model_name)
+            if not mlx_model:
-                if mlx_model:
+                raise FileNotFoundError(
-                    self.mlx_encoder = load_mlx_encoder(path_or_hf_repo=mlx_model)
+                    f"MLX Whisper backend requested but no compatible weights found for model '{self.model_name}'."
-                    self.fast_encoder = True
-            elif HAS_FASTER_WHISPER and compatible_faster_whisper:
-                print('Simulstreaming will use Faster Whisper for the encoder.')
-                if self.model_path and compatible_faster_whisper:
-                    fw_model = self.model_path
-                else:
-                    fw_model = self.model_name
-                self.fw_encoder = WhisperModel(
-                    fw_model,
-                    device='auto',
-                    compute_type='auto',
                 )
-                self.fast_encoder = True
+            self.mlx_encoder = load_mlx_encoder(path_or_hf_repo=mlx_model)
+        elif self.encoder_backend == "faster-whisper":
+            print('Simulstreaming will use Faster Whisper for the encoder.')
+            if self._resolved_model_path is not None:
+                fw_model = str(self._resolved_model_path)
+            else:
+                fw_model = self.model_name
+            self.fw_encoder = WhisperModel(
+                fw_model,
+                device='auto',
+                compute_type='auto',
+            )
 
         self.models = [self.load_model() for i in range(self.preload_model_count)]
 
 
+    def _resolve_encoder_backend(self, preferred_backend, compatible_whisper_mlx, compatible_faster_whisper):
+        choice = preferred_backend or "auto"
+        if self.disable_fast_encoder:
+            return "whisper"
+        if choice == "whisper":
+            return "whisper"
+        if choice == "mlx-whisper":
+            if not self._can_use_mlx(compatible_whisper_mlx):
+                raise RuntimeError("mlx-whisper backend requested but MLX Whisper is unavailable or incompatible with the provided model.")
+            return "mlx-whisper"
+        if choice == "faster-whisper":
+            if not self._can_use_faster(compatible_faster_whisper):
+                raise RuntimeError("faster-whisper backend requested but Faster-Whisper is unavailable or incompatible with the provided model.")
+            return "faster-whisper"
+        if choice == "openai-api":
+            raise ValueError("openai-api backend is only supported with the LocalAgreement policy.")
+        # auto mode
+        if platform.system() == "Darwin" and self._can_use_mlx(compatible_whisper_mlx):
+            return "mlx-whisper"
+        if self._can_use_faster(compatible_faster_whisper):
+            return "faster-whisper"
+        return "whisper"
+
+    def _has_custom_model_path(self):
+        return self._resolved_model_path is not None
+
+    def _can_use_mlx(self, compatible_whisper_mlx):
+        if not HAS_MLX_WHISPER:
+            return False
+        if self._has_custom_model_path():
+            return compatible_whisper_mlx
+        return self.model_name in mlx_model_mapping
+
+    def _can_use_faster(self, compatible_faster_whisper):
+        if not HAS_FASTER_WHISPER:
+            return False
+        if self._has_custom_model_path():
+            return compatible_faster_whisper
+        return True
+
     def load_model(self):
         whisper_model = load_model(
             name=self.pytorch_path if self.pytorch_path else self.model_name,

whisperlivekit/simul_whisper/beam.py

@@ -1,4 +1,4 @@
-from .whisper.decoding import PyTorchInference
+from whisperlivekit.whisper.decoding import PyTorchInference
 
 # extention of PyTorchInference for beam search
 class BeamPyTorchInference(PyTorchInference):

whisperlivekit/simul_whisper/config.py

@@ -1,5 +1,3 @@
-# This code was originally in simul_whisper/transcriber/simul_whisper.py . It is adapted a lot for SimulStreaming.
-
 from dataclasses import dataclass, field
 from typing import Literal
 

whisperlivekit/simul_whisper/simul_whisper.py

@@ -6,17 +6,21 @@ import logging
 import torch
 import torch.nn.functional as F
 
-from .whisper import load_model, DecodingOptions, tokenizer
+from whisperlivekit.whisper import load_model, DecodingOptions, tokenizer
 from .config import AlignAttConfig
 from whisperlivekit.timed_objects import ASRToken
-from .whisper.audio import log_mel_spectrogram, TOKENS_PER_SECOND, pad_or_trim, N_SAMPLES, N_FRAMES
+from whisperlivekit.whisper.audio import log_mel_spectrogram, TOKENS_PER_SECOND, pad_or_trim, N_SAMPLES, N_FRAMES
-from .whisper.timing import median_filter
+from whisperlivekit.whisper.timing import median_filter
-from .whisper.decoding import GreedyDecoder, BeamSearchDecoder, SuppressTokens, detect_language
+from whisperlivekit.whisper.decoding import GreedyDecoder, BeamSearchDecoder, SuppressTokens, detect_language
 from .beam import BeamPyTorchInference
 from .eow_detection import fire_at_boundary, load_cif
 import os
 from time import time
 from .token_buffer import TokenBuffer
+from whisperlivekit.backend_support import (
+    mlx_backend_available,
+    faster_backend_available,
+)
 
 import numpy as np
 from ..timed_objects import PUNCTUATION_MARKS
@@ -26,21 +30,18 @@ DEC_PAD = 50257
 logger = logging.getLogger(__name__)
 
 
-try:
+HAS_MLX_WHISPER = False
+HAS_FASTER_WHISPER = False
+
+if mlx_backend_available():
     from mlx_whisper.audio import log_mel_spectrogram as mlx_log_mel_spectrogram
     from mlx_whisper.transcribe import pad_or_trim as mlx_pad_or_trim
     HAS_MLX_WHISPER = True
-except ImportError:
+
-    HAS_MLX_WHISPER = False
+if faster_backend_available():
-if HAS_MLX_WHISPER:
+    from faster_whisper.audio import pad_or_trim as fw_pad_or_trim
-    HAS_FASTER_WHISPER = False
+    from faster_whisper.feature_extractor import FeatureExtractor
-else:
+    HAS_FASTER_WHISPER = True
-    try:
-        from faster_whisper.audio import pad_or_trim as fw_pad_or_trim
-        from faster_whisper.feature_extractor import FeatureExtractor
-        HAS_FASTER_WHISPER = True
-    except ImportError:
-        HAS_FASTER_WHISPER = False
 
 class PaddedAlignAttWhisper:
     def __init__(

whisperlivekit/timed_objects.py

@@ -151,6 +151,7 @@ class FrontData():
     lines: list[Line] = field(default_factory=list)
     buffer_transcription: str = ''
     buffer_diarization: str = ''
+    buffer_translation: str = ''
     remaining_time_transcription: float = 0.
     remaining_time_diarization: float = 0.
     
@@ -160,6 +161,7 @@ class FrontData():
             'lines': [line.to_dict() for line in self.lines if (line.text or line.speaker == -2)],
             'buffer_transcription': self.buffer_transcription,
             'buffer_diarization': self.buffer_diarization,
+            'buffer_translation': self.buffer_translation,
             'remaining_time_transcription': self.remaining_time_transcription,
             'remaining_time_diarization': self.remaining_time_diarization,
         }
@@ -176,11 +178,12 @@ class ChangeSpeaker:
 class State():
     tokens: list = field(default_factory=list)
     last_validated_token: int = 0
+    last_punctuation_index: Optional[int] = None
     translation_validated_segments: list = field(default_factory=list)
-    translation_buffer: list = field(default_factory=list)
+    buffer_translation: str = field(default_factory=Transcript)
     buffer_transcription: str = field(default_factory=Transcript)
     end_buffer: float = 0.0
     end_attributed_speaker: float = 0.0
     remaining_time_transcription: float = 0.0
     remaining_time_diarization: float = 0.0
-    beg_loop: Optional[int] = None
+    beg_loop: Optional[int] = None

whisperlivekit/web/live_transcription.css

@@ -490,6 +490,11 @@ label {
   margin-left: 4px;
 }
 
+.buffer_translation {
+  color: #a0a0a0;
+  margin-left: 6px;
+}
+
 .spinner {
   display: inline-block;
   width: 8px;

whisperlivekit/web/live_transcription.js

@@ -232,10 +232,11 @@ function setupWebSocket() {
         if (waitingForStop) {
           statusText.textContent = "Processing finalized or connection closed.";
           if (lastReceivedData) {
-            renderLinesWithBuffer(
+          renderLinesWithBuffer(
               lastReceivedData.lines || [],
               lastReceivedData.buffer_diarization || "",
               lastReceivedData.buffer_transcription || "",
+              lastReceivedData.buffer_translation || "",
               0,
               0,
               true
@@ -281,6 +282,7 @@ function setupWebSocket() {
             lastReceivedData.lines || [],
             lastReceivedData.buffer_diarization || "",
             lastReceivedData.buffer_transcription || "",
+            lastReceivedData.buffer_translation || "",
             0,
             0,
             true
@@ -301,6 +303,7 @@ function setupWebSocket() {
         lines = [],
         buffer_transcription = "",
         buffer_diarization = "",
+        buffer_translation = "",
         remaining_time_transcription = 0,
         remaining_time_diarization = 0,
         status = "active_transcription",
@@ -310,6 +313,7 @@ function setupWebSocket() {
         lines,
         buffer_diarization,
         buffer_transcription,
+        buffer_translation,
         remaining_time_diarization,
         remaining_time_transcription,
         false,
@@ -323,6 +327,7 @@ function renderLinesWithBuffer(
   lines,
   buffer_diarization,
   buffer_transcription,
+  buffer_translation,
   remaining_time_diarization,
   remaining_time_transcription,
   isFinalizing = false,
@@ -341,6 +346,7 @@ function renderLinesWithBuffer(
     lines: (lines || []).map((it) => ({ speaker: it.speaker, text: it.text, start: it.start, end: it.end, detected_language: it.detected_language })),
     buffer_transcription: buffer_transcription || "",
     buffer_diarization: buffer_diarization || "",
+    buffer_translation: buffer_translation,
     status: current_status,
     showLoading,
     showTransLag,
@@ -415,13 +421,22 @@ function renderLinesWithBuffer(
           }
         }
       }
-      
+      let translationContent = "";
       if (item.translation) {
+        translationContent += item.translation.trim();
+      }
+      if (idx === lines.length - 1 && buffer_translation) {
+        const bufferPiece = isFinalizing
+          ? buffer_translation
+          : `<span class="buffer_translation">${buffer_translation}</span>`;
+        translationContent += translationContent ? `${bufferPiece}` : bufferPiece;
+      }
+      if (translationContent.trim().length > 0) {
         currentLineText += `
             <div>
                 <div class="label_translation">
                     ${translationIcon}
-                    <span>${item.translation}</span>
+                    <span class="translation_text">${translationContent}</span>
                 </div>
             </div>`;
       }

whisperlivekit/whisper_streaming_custom/whisper_online.py

@@ -1,122 +0,0 @@
-#!/usr/bin/env python3
-import sys
-import numpy as np
-import librosa
-from functools import lru_cache
-import time
-import logging
-from .backends import FasterWhisperASR, MLXWhisper, WhisperTimestampedASR, OpenaiApiASR
-from whisperlivekit.warmup import warmup_asr
-
-logger = logging.getLogger(__name__)
-
-
-
-WHISPER_LANG_CODES = "af,am,ar,as,az,ba,be,bg,bn,bo,br,bs,ca,cs,cy,da,de,el,en,es,et,eu,fa,fi,fo,fr,gl,gu,ha,haw,he,hi,hr,ht,hu,hy,id,is,it,ja,jw,ka,kk,km,kn,ko,la,lb,ln,lo,lt,lv,mg,mi,mk,ml,mn,mr,ms,mt,my,ne,nl,nn,no,oc,pa,pl,ps,pt,ro,ru,sa,sd,si,sk,sl,sn,so,sq,sr,su,sv,sw,ta,te,tg,th,tk,tl,tr,tt,uk,ur,uz,vi,yi,yo,zh".split(
-    ","
-)
-
-
-def create_tokenizer(lan):
-    """returns an object that has split function that works like the one of MosesTokenizer"""
-
-    assert (
-        lan in WHISPER_LANG_CODES
-    ), "language must be Whisper's supported lang code: " + " ".join(WHISPER_LANG_CODES)
-
-    if lan == "uk":
-        import tokenize_uk
-
-        class UkrainianTokenizer:
-            def split(self, text):
-                return tokenize_uk.tokenize_sents(text)
-
-        return UkrainianTokenizer()
-
-    # supported by fast-mosestokenizer
-    if (
-        lan
-        in "as bn ca cs de el en es et fi fr ga gu hi hu is it kn lt lv ml mni mr nl or pa pl pt ro ru sk sl sv ta te yue zh".split()
-    ):
-        from mosestokenizer import MosesSentenceSplitter        
-
-        return MosesSentenceSplitter(lan)
-
-    # the following languages are in Whisper, but not in wtpsplit:
-    if (
-        lan
-        in "as ba bo br bs fo haw hr ht jw lb ln lo mi nn oc sa sd sn so su sw tk tl tt".split()
-    ):
-        logger.debug(
-            f"{lan} code is not supported by wtpsplit. Going to use None lang_code option."
-        )
-        lan = None
-
-    from wtpsplit import WtP
-
-    # downloads the model from huggingface on the first use
-    wtp = WtP("wtp-canine-s-12l-no-adapters")
-
-    class WtPtok:
-        def split(self, sent):
-            return wtp.split(sent, lang_code=lan)
-
-    return WtPtok()
-
-
-def backend_factory(
-            backend,
-            lan,
-            model_size,
-            model_cache_dir,
-            model_dir,
-            task,
-            buffer_trimming,
-            buffer_trimming_sec,
-            confidence_validation,
-            warmup_file=None,
-            min_chunk_size=None,
-        ):
-    backend = backend
-    if backend == "openai-api":
-        logger.debug("Using OpenAI API.")
-        asr = OpenaiApiASR(lan=lan)        
-    else:
-        if backend == "faster-whisper":
-            asr_cls = FasterWhisperASR
-        elif backend == "mlx-whisper":
-            asr_cls = MLXWhisper
-        else:
-            asr_cls = WhisperTimestampedASR
-
-        # Only for FasterWhisperASR and WhisperTimestampedASR
-
-        t = time.time()
-        logger.info(f"Loading Whisper {model_size} model for language {lan}...")
-        asr = asr_cls(
-            model_size=model_size,
-            lan=lan,
-            cache_dir=model_cache_dir,
-            model_dir=model_dir,
-        )
-        e = time.time()
-        logger.info(f"done. It took {round(e-t,2)} seconds.")
-
-    if task == "translate":
-        tgt_language = "en"  # Whisper translates into English
-    else:
-        tgt_language = lan  # Whisper transcribes in this language
-
-    # Create the tokenizer
-    if buffer_trimming == "sentence":
-        tokenizer = create_tokenizer(tgt_language)
-    else:
-        tokenizer = None
-    
-    warmup_asr(asr, warmup_file)
-    
-    asr.confidence_validation = confidence_validation
-    asr.tokenizer = tokenizer
-    asr.buffer_trimming = buffer_trimming
-    asr.buffer_trimming_sec = buffer_trimming_sec
-    return asr