From 5fdb08edae214fe75627145265780502c329384f Mon Sep 17 00:00:00 2001 From: silask Date: Mon, 30 Dec 2024 21:20:38 +0100 Subject: [PATCH 1/3] black formating --- silero_vad_iterator.py | 77 +++-- whisper_fastapi_online_server.py | 82 +++-- whisper_online.py | 577 +++++++++++++++++++++---------- 3 files changed, 492 insertions(+), 244 deletions(-) diff --git a/silero_vad_iterator.py b/silero_vad_iterator.py index 1eea7af..6f633da 100644 --- a/silero_vad_iterator.py +++ b/silero_vad_iterator.py @@ -6,15 +6,16 @@ import torch # Their licence is MIT, same as ours: https://github.com/snakers4/silero-vad/blob/f6b1294cb27590fb2452899df98fb234dfef1134/LICENSE -class VADIterator: - def __init__(self, - model, - threshold: float = 0.5, - sampling_rate: int = 16000, - min_silence_duration_ms: int = 500, # makes sense on one recording that I checked - speech_pad_ms: int = 100 # same - ): +class VADIterator: + def __init__( + self, + model, + threshold: float = 0.5, + sampling_rate: int = 16000, + min_silence_duration_ms: int = 500, # makes sense on one recording that I checked + speech_pad_ms: int = 100, # same + ): """ Class for stream imitation @@ -41,7 +42,9 @@ class VADIterator: self.sampling_rate = sampling_rate if sampling_rate not in [8000, 16000]: - raise ValueError('VADIterator does not support sampling rates other than [8000, 16000]') + raise ValueError( + "VADIterator does not support sampling rates other than [8000, 16000]" + ) self.min_silence_samples = sampling_rate * min_silence_duration_ms / 1000 self.speech_pad_samples = sampling_rate * speech_pad_ms / 1000 @@ -80,7 +83,13 @@ class VADIterator: if (speech_prob >= self.threshold) and not self.triggered: self.triggered = True speech_start = self.current_sample - self.speech_pad_samples - return {'start': int(speech_start) if not return_seconds else round(speech_start / self.sampling_rate, 1)} + return { + "start": ( + int(speech_start) + if not return_seconds + else round(speech_start / self.sampling_rate, 1) + ) + } if (speech_prob < self.threshold - 0.15) and self.triggered: if not self.temp_end: @@ -91,26 +100,35 @@ class VADIterator: speech_end = self.temp_end + self.speech_pad_samples self.temp_end = 0 self.triggered = False - return {'end': int(speech_end) if not return_seconds else round(speech_end / self.sampling_rate, 1)} + return { + "end": ( + int(speech_end) + if not return_seconds + else round(speech_end / self.sampling_rate, 1) + ) + } return None + ####################### -# because Silero now requires exactly 512-sized audio chunks +# because Silero now requires exactly 512-sized audio chunks import numpy as np + + class FixedVADIterator(VADIterator): - '''It fixes VADIterator by allowing to process any audio length, not only exactly 512 frames at once. - If audio to be processed at once is long and multiple voiced segments detected, - then __call__ returns the start of the first segment, and end (or middle, which means no end) of the last segment. - ''' + """It fixes VADIterator by allowing to process any audio length, not only exactly 512 frames at once. + If audio to be processed at once is long and multiple voiced segments detected, + then __call__ returns the start of the first segment, and end (or middle, which means no end) of the last segment. + """ def reset_states(self): super().reset_states() - self.buffer = np.array([],dtype=np.float32) + self.buffer = np.array([], dtype=np.float32) def __call__(self, x, return_seconds=False): - self.buffer = np.append(self.buffer, x) + self.buffer = np.append(self.buffer, x) ret = None while len(self.buffer) >= 512: r = super().__call__(self.buffer[:512], return_seconds=return_seconds) @@ -118,29 +136,28 @@ class FixedVADIterator(VADIterator): if ret is None: ret = r elif r is not None: - if 'end' in r: - ret['end'] = r['end'] # the latter end - if 'start' in r and 'end' in ret: # there is an earlier start. + if "end" in r: + ret["end"] = r["end"] # the latter end + if "start" in r and "end" in ret: # there is an earlier start. # Remove end, merging this segment with the previous one. - del ret['end'] + del ret["end"] return ret if ret != {} else None + if __name__ == "__main__": # test/demonstrate the need for FixedVADIterator: import torch - model, _ = torch.hub.load( - repo_or_dir='snakers4/silero-vad', - model='silero_vad' - ) + + model, _ = torch.hub.load(repo_or_dir="snakers4/silero-vad", model="silero_vad") vac = FixedVADIterator(model) -# vac = VADIterator(model) # the second case crashes with this + # vac = VADIterator(model) # the second case crashes with this # this works: for both - audio_buffer = np.array([0]*(512),dtype=np.float32) + audio_buffer = np.array([0] * (512), dtype=np.float32) vac(audio_buffer) - # this crashes on the non FixedVADIterator with + # this crashes on the non FixedVADIterator with # ops.prim.RaiseException("Input audio chunk is too short", "builtins.ValueError") - audio_buffer = np.array([0]*(512-1),dtype=np.float32) + audio_buffer = np.array([0] * (512 - 1), dtype=np.float32) vac(audio_buffer) diff --git a/whisper_fastapi_online_server.py b/whisper_fastapi_online_server.py index 91b73ff..93468d1 100644 --- a/whisper_fastapi_online_server.py +++ b/whisper_fastapi_online_server.py @@ -22,10 +22,21 @@ app.add_middleware( parser = argparse.ArgumentParser(description="Whisper FastAPI Online Server") -parser.add_argument("--host", type=str, default='localhost', help="The host address to bind the server to.") -parser.add_argument("--port", type=int, default=8000, help="The port number to bind the server to.") -parser.add_argument("--warmup-file", type=str, dest="warmup_file", - help="The path to a speech audio wav file to warm up Whisper so that the very first chunk processing is fast. It can be e.g. https://github.com/ggerganov/whisper.cpp/raw/master/samples/jfk.wav .") +parser.add_argument( + "--host", + type=str, + default="localhost", + help="The host address to bind the server to.", +) +parser.add_argument( + "--port", type=int, default=8000, help="The port number to bind the server to." +) +parser.add_argument( + "--warmup-file", + type=str, + dest="warmup_file", + help="The path to a speech audio wav file to warm up Whisper so that the very first chunk processing is fast. It can be e.g. https://github.com/ggerganov/whisper.cpp/raw/master/samples/jfk.wav .", +) add_shared_args(parser) args = parser.parse_args() @@ -35,29 +46,38 @@ asr, online = asr_factory(args) with open("src/live_transcription.html", "r") as f: html = f.read() + @app.get("/") async def get(): return HTMLResponse(html) + SAMPLE_RATE = 16000 CHANNELS = 1 SAMPLES_PER_SEC = SAMPLE_RATE * int(args.min_chunk_size) -BYTES_PER_SAMPLE = 2 # s16le = 2 bytes per sample +BYTES_PER_SAMPLE = 2 # s16le = 2 bytes per sample BYTES_PER_SEC = SAMPLES_PER_SEC * BYTES_PER_SAMPLE + async def start_ffmpeg_decoder(): """ Start an FFmpeg process in async streaming mode that reads WebM from stdin and outputs raw s16le PCM on stdout. Returns the process object. """ process = ( - ffmpeg - .input('pipe:0', format='webm') - .output('pipe:1', format='s16le', acodec='pcm_s16le', ac=CHANNELS, ar=str(SAMPLE_RATE)) + ffmpeg.input("pipe:0", format="webm") + .output( + "pipe:1", + format="s16le", + acodec="pcm_s16le", + ac=CHANNELS, + ar=str(SAMPLE_RATE), + ) .run_async(pipe_stdin=True, pipe_stdout=True, pipe_stderr=True) ) return process + @app.websocket("/asr") async def websocket_endpoint(websocket: WebSocket): await websocket.accept() @@ -65,6 +85,7 @@ async def websocket_endpoint(websocket: WebSocket): ffmpeg_process = await start_ffmpeg_decoder() pcm_buffer = bytearray() + # Continuously read decoded PCM from ffmpeg stdout in a background task async def ffmpeg_stdout_reader(): nonlocal pcm_buffer @@ -75,10 +96,16 @@ async def websocket_endpoint(websocket: WebSocket): try: elapsed_time = int(time() - beg) beg = time() - chunk = await loop.run_in_executor(None, ffmpeg_process.stdout.read, 32000*elapsed_time) - if not chunk: # The first chunk will be almost empty, FFmpeg is still starting up - chunk = await loop.run_in_executor(None, ffmpeg_process.stdout.read, 4096) - if not chunk: # FFmpeg might have closed + chunk = await loop.run_in_executor( + None, ffmpeg_process.stdout.read, 32000 * elapsed_time + ) + if ( + not chunk + ): # The first chunk will be almost empty, FFmpeg is still starting up + chunk = await loop.run_in_executor( + None, ffmpeg_process.stdout.read, 4096 + ) + if not chunk: # FFmpeg might have closed print("FFmpeg stdout closed.") break @@ -86,21 +113,29 @@ async def websocket_endpoint(websocket: WebSocket): if len(pcm_buffer) >= BYTES_PER_SEC: # Convert int16 -> float32 - pcm_array = np.frombuffer(pcm_buffer, dtype=np.int16).astype(np.float32) / 32768.0 + pcm_array = ( + np.frombuffer(pcm_buffer, dtype=np.int16).astype(np.float32) + / 32768.0 + ) pcm_buffer = bytearray() online.insert_audio_chunk(pcm_array) transcription = online.process_iter()[2] full_transcription += transcription if args.vac: - buffer = online.online.to_flush(online.online.transcript_buffer.buffer)[2] # We need to access the underlying online object to get the buffer + buffer = online.online.to_flush( + online.online.transcript_buffer.buffer + )[ + 2 + ] # We need to access the underlying online object to get the buffer else: buffer = online.to_flush(online.transcript_buffer.buffer)[2] - if buffer in full_transcription: # With VAC, the buffer is not updated until the next chunk is processed + if ( + buffer in full_transcription + ): # With VAC, the buffer is not updated until the next chunk is processed buffer = "" - await websocket.send_json({ - "transcription": transcription, - "buffer": buffer - }) + await websocket.send_json( + {"transcription": transcription, "buffer": buffer} + ) except Exception as e: print(f"Exception in ffmpeg_stdout_reader: {e}") break @@ -135,8 +170,11 @@ async def websocket_endpoint(websocket: WebSocket): pass ffmpeg_process.wait() - - + + if __name__ == "__main__": import uvicorn - uvicorn.run("whisper_fastapi_online_server:app", host=args.host, port=args.port, reload=True) \ No newline at end of file + + uvicorn.run( + "whisper_fastapi_online_server:app", host=args.host, port=args.port, reload=True + ) diff --git a/whisper_online.py b/whisper_online.py index ce61a52..afbee2d 100644 --- a/whisper_online.py +++ b/whisper_online.py @@ -12,26 +12,31 @@ import math logger = logging.getLogger(__name__) + @lru_cache(10**6) def load_audio(fname): a, _ = librosa.load(fname, sr=16000, dtype=np.float32) return a + def load_audio_chunk(fname, beg, end): audio = load_audio(fname) - beg_s = int(beg*16000) - end_s = int(end*16000) + beg_s = int(beg * 16000) + end_s = int(end * 16000) return audio[beg_s:end_s] # Whisper backend + class ASRBase: - sep = " " # join transcribe words with this character (" " for whisper_timestamped, - # "" for faster-whisper because it emits the spaces when neeeded) + sep = " " # join transcribe words with this character (" " for whisper_timestamped, + # "" for faster-whisper because it emits the spaces when neeeded) - def __init__(self, lan, modelsize=None, cache_dir=None, model_dir=None, logfile=sys.stderr): + def __init__( + self, lan, modelsize=None, cache_dir=None, model_dir=None, logfile=sys.stderr + ): self.logfile = logfile self.transcribe_kargs = {} @@ -42,7 +47,6 @@ class ASRBase: self.model = self.load_model(modelsize, cache_dir, model_dir) - def load_model(self, modelsize, cache_dir): raise NotImplemented("must be implemented in the child class") @@ -64,24 +68,30 @@ class WhisperTimestampedASR(ASRBase): import whisper import whisper_timestamped from whisper_timestamped import transcribe_timestamped + self.transcribe_timestamped = transcribe_timestamped if model_dir is not None: logger.debug("ignoring model_dir, not implemented") return whisper.load_model(modelsize, download_root=cache_dir) def transcribe(self, audio, init_prompt=""): - result = self.transcribe_timestamped(self.model, - audio, language=self.original_language, - initial_prompt=init_prompt, verbose=None, - condition_on_previous_text=True, **self.transcribe_kargs) + result = self.transcribe_timestamped( + self.model, + audio, + language=self.original_language, + initial_prompt=init_prompt, + verbose=None, + condition_on_previous_text=True, + **self.transcribe_kargs, + ) return result - - def ts_words(self,r): + + def ts_words(self, r): # return: transcribe result object to [(beg,end,"word1"), ...] o = [] for s in r["segments"]: for w in s["words"]: - t = (w["start"],w["end"],w["text"]) + t = (w["start"], w["end"], w["text"]) o.append(t) return o @@ -95,43 +105,55 @@ class WhisperTimestampedASR(ASRBase): self.transcribe_kargs["task"] = "translate" - - class FasterWhisperASR(ASRBase): - """Uses faster-whisper library as the backend. Works much faster, appx 4-times (in offline mode). For GPU, it requires installation with a specific CUDNN version. - """ + """Uses faster-whisper library as the backend. Works much faster, appx 4-times (in offline mode). For GPU, it requires installation with a specific CUDNN version.""" sep = "" def load_model(self, modelsize=None, cache_dir=None, model_dir=None): from faster_whisper import WhisperModel -# logging.getLogger("faster_whisper").setLevel(logger.level) + + # logging.getLogger("faster_whisper").setLevel(logger.level) if model_dir is not None: - logger.debug(f"Loading whisper model from model_dir {model_dir}. modelsize and cache_dir parameters are not used.") + logger.debug( + f"Loading whisper model from model_dir {model_dir}. modelsize and cache_dir parameters are not used." + ) model_size_or_path = model_dir elif modelsize is not None: model_size_or_path = modelsize else: raise ValueError("modelsize or model_dir parameter must be set") - # this worked fast and reliably on NVIDIA L40 - model = WhisperModel(model_size_or_path, device="cuda", compute_type="float16", download_root=cache_dir) + model = WhisperModel( + model_size_or_path, + device="cuda", + compute_type="float16", + download_root=cache_dir, + ) # or run on GPU with INT8 # tested: the transcripts were different, probably worse than with FP16, and it was slightly (appx 20%) slower - #model = WhisperModel(model_size, device="cuda", compute_type="int8_float16") + # model = WhisperModel(model_size, device="cuda", compute_type="int8_float16") # or run on CPU with INT8 # tested: works, but slow, appx 10-times than cuda FP16 -# model = WhisperModel(modelsize, device="cpu", compute_type="int8") #, download_root="faster-disk-cache-dir/") + # model = WhisperModel(modelsize, device="cpu", compute_type="int8") #, download_root="faster-disk-cache-dir/") return model def transcribe(self, audio, init_prompt=""): # tested: beam_size=5 is faster and better than 1 (on one 200 second document from En ESIC, min chunk 0.01) - segments, info = self.model.transcribe(audio, language=self.original_language, initial_prompt=init_prompt, beam_size=5, word_timestamps=True, condition_on_previous_text=True, **self.transcribe_kargs) - #print(info) # info contains language detection result + segments, info = self.model.transcribe( + audio, + language=self.original_language, + initial_prompt=init_prompt, + beam_size=5, + word_timestamps=True, + condition_on_previous_text=True, + **self.transcribe_kargs, + ) + # print(info) # info contains language detection result return list(segments) @@ -156,40 +178,45 @@ class FasterWhisperASR(ASRBase): def set_translate_task(self): self.transcribe_kargs["task"] = "translate" + class MLXWhisper(ASRBase): """ Uses MPX Whisper library as the backend, optimized for Apple Silicon. Models available: https://huggingface.co/collections/mlx-community/whisper-663256f9964fbb1177db93dc - Significantly faster than faster-whisper (without CUDA) on Apple M1. + Significantly faster than faster-whisper (without CUDA) on Apple M1. """ sep = " " def load_model(self, modelsize=None, cache_dir=None, model_dir=None): """ - Loads the MLX-compatible Whisper model. + Loads the MLX-compatible Whisper model. - Args: - modelsize (str, optional): The size or name of the Whisper model to load. - If provided, it will be translated to an MLX-compatible model path using the `translate_model_name` method. - Example: "large-v3-turbo" -> "mlx-community/whisper-large-v3-turbo". - cache_dir (str, optional): Path to the directory for caching models. - **Note**: This is not supported by MLX Whisper and will be ignored. - model_dir (str, optional): Direct path to a custom model directory. - If specified, it overrides the `modelsize` parameter. + Args: + modelsize (str, optional): The size or name of the Whisper model to load. + If provided, it will be translated to an MLX-compatible model path using the `translate_model_name` method. + Example: "large-v3-turbo" -> "mlx-community/whisper-large-v3-turbo". + cache_dir (str, optional): Path to the directory for caching models. + **Note**: This is not supported by MLX Whisper and will be ignored. + model_dir (str, optional): Direct path to a custom model directory. + If specified, it overrides the `modelsize` parameter. """ from mlx_whisper import transcribe if model_dir is not None: - logger.debug(f"Loading whisper model from model_dir {model_dir}. modelsize parameter is not used.") + logger.debug( + f"Loading whisper model from model_dir {model_dir}. modelsize parameter is not used." + ) model_size_or_path = model_dir elif modelsize is not None: model_size_or_path = self.translate_model_name(modelsize) - logger.debug(f"Loading whisper model {modelsize}. You use mlx whisper, so {model_size_or_path} will be used.") - + logger.debug( + f"Loading whisper model {modelsize}. You use mlx whisper, so {model_size_or_path} will be used." + ) + self.model_size_or_path = model_size_or_path return transcribe - + def translate_model_name(self, model_name): """ Translates a given model name to its corresponding MLX-compatible model path. @@ -214,7 +241,7 @@ class MLXWhisper(ASRBase): "large-v2": "mlx-community/whisper-large-v2-mlx", "large-v3": "mlx-community/whisper-large-v3-mlx", "large-v3-turbo": "mlx-community/whisper-large-v3-turbo", - "large": "mlx-community/whisper-large-mlx" + "large": "mlx-community/whisper-large-mlx", } # Retrieve the corresponding MLX model path @@ -223,8 +250,10 @@ class MLXWhisper(ASRBase): if mlx_model_path: return mlx_model_path else: - raise ValueError(f"Model name '{model_name}' is not recognized or not supported.") - + raise ValueError( + f"Model name '{model_name}' is not recognized or not supported." + ) + def transcribe(self, audio, init_prompt=""): segments = self.model( audio, @@ -233,11 +262,10 @@ class MLXWhisper(ASRBase): word_timestamps=True, condition_on_previous_text=True, path_or_hf_repo=self.model_size_or_path, - **self.transcribe_kargs + **self.transcribe_kargs, ) return segments.get("segments", []) - def ts_words(self, segments): """ Extract timestamped words from transcription segments and skips words with high no-speech probability. @@ -248,9 +276,9 @@ class MLXWhisper(ASRBase): for word in segment.get("words", []) if segment.get("no_speech_prob", 0) <= 0.9 ] - + def segments_end_ts(self, res): - return [s['end'] for s in res] + return [s["end"] for s in res] def use_vad(self): self.transcribe_kargs["vad_filter"] = True @@ -258,15 +286,18 @@ class MLXWhisper(ASRBase): def set_translate_task(self): self.transcribe_kargs["task"] = "translate" + class OpenaiApiASR(ASRBase): """Uses OpenAI's Whisper API for audio transcription.""" def __init__(self, lan=None, temperature=0, logfile=sys.stderr): self.logfile = logfile - self.modelname = "whisper-1" - self.original_language = None if lan == "auto" else lan # ISO-639-1 language code - self.response_format = "verbose_json" + self.modelname = "whisper-1" + self.original_language = ( + None if lan == "auto" else lan + ) # ISO-639-1 language code + self.response_format = "verbose_json" self.temperature = temperature self.load_model() @@ -278,10 +309,12 @@ class OpenaiApiASR(ASRBase): def load_model(self, *args, **kwargs): from openai import OpenAI + self.client = OpenAI() - self.transcribed_seconds = 0 # for logging how many seconds were processed by API, to know the cost - + self.transcribed_seconds = ( + 0 # for logging how many seconds were processed by API, to know the cost + ) def ts_words(self, segments): no_speech_segments = [] @@ -289,7 +322,9 @@ class OpenaiApiASR(ASRBase): for segment in segments.segments: # TODO: threshold can be set from outside if segment["no_speech_prob"] > 0.8: - no_speech_segments.append((segment.get("start"), segment.get("end"))) + no_speech_segments.append( + (segment.get("start"), segment.get("end")) + ) o = [] for word in segments.words: @@ -301,7 +336,6 @@ class OpenaiApiASR(ASRBase): o.append((start, end, word.word)) return o - def segments_end_ts(self, res): return [s.end for s in res.words] @@ -309,17 +343,19 @@ class OpenaiApiASR(ASRBase): # Write the audio data to a buffer buffer = io.BytesIO() buffer.name = "temp.wav" - sf.write(buffer, audio_data, samplerate=16000, format='WAV', subtype='PCM_16') + sf.write(buffer, audio_data, samplerate=16000, format="WAV", subtype="PCM_16") buffer.seek(0) # Reset buffer's position to the beginning - self.transcribed_seconds += math.ceil(len(audio_data)/16000) # it rounds up to the whole seconds + self.transcribed_seconds += math.ceil( + len(audio_data) / 16000 + ) # it rounds up to the whole seconds params = { "model": self.modelname, "file": buffer, "response_format": self.response_format, "temperature": self.temperature, - "timestamp_granularities": ["word", "segment"] + "timestamp_granularities": ["word", "segment"], } if self.task != "translate" and self.original_language: params["language"] = self.original_language @@ -333,7 +369,9 @@ class OpenaiApiASR(ASRBase): # Process transcription/translation transcript = proc.create(**params) - logger.debug(f"OpenAI API processed accumulated {self.transcribed_seconds} seconds") + logger.debug( + f"OpenAI API processed accumulated {self.transcribed_seconds} seconds" + ) return transcript @@ -344,8 +382,6 @@ class OpenaiApiASR(ASRBase): self.task = "translate" - - class HypothesisBuffer: def __init__(self, logfile=sys.stderr): @@ -361,20 +397,24 @@ class HypothesisBuffer: def insert(self, new, offset): # compare self.commited_in_buffer and new. It inserts only the words in new that extend the commited_in_buffer, it means they are roughly behind last_commited_time and new in content # the new tail is added to self.new - - new = [(a+offset,b+offset,t) for a,b,t in new] - self.new = [(a,b,t) for a,b,t in new if a > self.last_commited_time-0.1] + + new = [(a + offset, b + offset, t) for a, b, t in new] + self.new = [(a, b, t) for a, b, t in new if a > self.last_commited_time - 0.1] if len(self.new) >= 1: - a,b,t = self.new[0] + a, b, t = self.new[0] if abs(a - self.last_commited_time) < 1: if self.commited_in_buffer: # it's going to search for 1, 2, ..., 5 consecutive words (n-grams) that are identical in commited and new. If they are, they're dropped. cn = len(self.commited_in_buffer) nn = len(self.new) - for i in range(1,min(min(cn,nn),5)+1): # 5 is the maximum - c = " ".join([self.commited_in_buffer[-j][2] for j in range(1,i+1)][::-1]) - tail = " ".join(self.new[j-1][2] for j in range(1,i+1)) + for i in range(1, min(min(cn, nn), 5) + 1): # 5 is the maximum + c = " ".join( + [self.commited_in_buffer[-j][2] for j in range(1, i + 1)][ + ::-1 + ] + ) + tail = " ".join(self.new[j - 1][2] for j in range(1, i + 1)) if c == tail: words = [] for j in range(i): @@ -384,7 +424,7 @@ class HypothesisBuffer: break def flush(self): - # returns commited chunk = the longest common prefix of 2 last inserts. + # returns commited chunk = the longest common prefix of 2 last inserts. commit = [] while self.new: @@ -394,7 +434,7 @@ class HypothesisBuffer: break if nt == self.buffer[0][2]: - commit.append((na,nb,nt)) + commit.append((na, nb, nt)) self.last_commited_word = nt self.last_commited_time = nb self.buffer.pop(0) @@ -413,16 +453,19 @@ class HypothesisBuffer: def complete(self): return self.buffer + class OnlineASRProcessor: SAMPLING_RATE = 16000 - def __init__(self, asr, tokenizer=None, buffer_trimming=("segment", 15), logfile=sys.stderr): + def __init__( + self, asr, tokenizer=None, buffer_trimming=("segment", 15), logfile=sys.stderr + ): """asr: WhisperASR object tokenizer: sentence tokenizer object for the target language. Must have a method *split* that behaves like the one of MosesTokenizer. It can be None, if "segment" buffer trimming option is used, then tokenizer is not used at all. ("segment", 15) buffer_trimming: a pair of (option, seconds), where option is either "sentence" or "segment", and seconds is a number. Buffer is trimmed if it is longer than "seconds" threshold. Default is the most recommended option. - logfile: where to store the log. + logfile: where to store the log. """ self.asr = asr self.tokenizer = tokenizer @@ -434,7 +477,7 @@ class OnlineASRProcessor: def init(self, offset=None): """run this when starting or restarting processing""" - self.audio_buffer = np.array([],dtype=np.float32) + self.audio_buffer = np.array([], dtype=np.float32) self.transcript_buffer = HypothesisBuffer(logfile=self.logfile) self.buffer_time_offset = 0 if offset is not None: @@ -446,34 +489,38 @@ class OnlineASRProcessor: self.audio_buffer = np.append(self.audio_buffer, audio) def prompt(self): - """Returns a tuple: (prompt, context), where "prompt" is a 200-character suffix of commited text that is inside of the scrolled away part of audio buffer. + """Returns a tuple: (prompt, context), where "prompt" is a 200-character suffix of commited text that is inside of the scrolled away part of audio buffer. "context" is the commited text that is inside the audio buffer. It is transcribed again and skipped. It is returned only for debugging and logging reasons. """ - k = max(0,len(self.commited)-1) - while k > 0 and self.commited[k-1][1] > self.buffer_time_offset: + k = max(0, len(self.commited) - 1) + while k > 0 and self.commited[k - 1][1] > self.buffer_time_offset: k -= 1 p = self.commited[:k] - p = [t for _,_,t in p] + p = [t for _, _, t in p] prompt = [] l = 0 while p and l < 200: # 200 characters prompt size x = p.pop(-1) - l += len(x)+1 + l += len(x) + 1 prompt.append(x) non_prompt = self.commited[k:] - return self.asr.sep.join(prompt[::-1]), self.asr.sep.join(t for _,_,t in non_prompt) + return self.asr.sep.join(prompt[::-1]), self.asr.sep.join( + t for _, _, t in non_prompt + ) def process_iter(self): """Runs on the current audio buffer. - Returns: a tuple (beg_timestamp, end_timestamp, "text"), or (None, None, ""). + Returns: a tuple (beg_timestamp, end_timestamp, "text"), or (None, None, ""). The non-emty text is confirmed (committed) partial transcript. """ prompt, non_prompt = self.prompt() logger.debug(f"PROMPT: {prompt}") logger.debug(f"CONTEXT: {non_prompt}") - logger.debug(f"transcribing {len(self.audio_buffer)/self.SAMPLING_RATE:2.2f} seconds from {self.buffer_time_offset:2.2f}") + logger.debug( + f"transcribing {len(self.audio_buffer)/self.SAMPLING_RATE:2.2f} seconds from {self.buffer_time_offset:2.2f}" + ) res = self.asr.transcribe(self.audio_buffer, init_prompt=prompt) # transform to [(beg,end,"word1"), ...] @@ -490,33 +537,37 @@ class OnlineASRProcessor: # there is a newly confirmed text if o and self.buffer_trimming_way == "sentence": # trim the completed sentences - if len(self.audio_buffer)/self.SAMPLING_RATE > self.buffer_trimming_sec: # longer than this + if ( + len(self.audio_buffer) / self.SAMPLING_RATE > self.buffer_trimming_sec + ): # longer than this self.chunk_completed_sentence() - if self.buffer_trimming_way == "segment": s = self.buffer_trimming_sec # trim the completed segments longer than s, else: - s = 30 # if the audio buffer is longer than 30s, trim it - - if len(self.audio_buffer)/self.SAMPLING_RATE > s: + s = 30 # if the audio buffer is longer than 30s, trim it + + if len(self.audio_buffer) / self.SAMPLING_RATE > s: self.chunk_completed_segment(res) # alternative: on any word - #l = self.buffer_time_offset + len(self.audio_buffer)/self.SAMPLING_RATE - 10 + # l = self.buffer_time_offset + len(self.audio_buffer)/self.SAMPLING_RATE - 10 # let's find commited word that is less - #k = len(self.commited)-1 - #while k>0 and self.commited[k][1] > l: + # k = len(self.commited)-1 + # while k>0 and self.commited[k][1] > l: # k -= 1 - #t = self.commited[k][1] + # t = self.commited[k][1] logger.debug("chunking segment") - #self.chunk_at(t) + # self.chunk_at(t) - logger.debug(f"len of buffer now: {len(self.audio_buffer)/self.SAMPLING_RATE:2.2f}") + logger.debug( + f"len of buffer now: {len(self.audio_buffer)/self.SAMPLING_RATE:2.2f}" + ) return self.to_flush(o) def chunk_completed_sentence(self): - if self.commited == []: return + if self.commited == []: + return logger.debug(self.commited) sents = self.words_to_sentences(self.commited) for s in sents: @@ -532,7 +583,8 @@ class OnlineASRProcessor: self.chunk_at(chunk_at) def chunk_completed_segment(self, res): - if self.commited == []: return + if self.commited == []: + return ends = self.asr.segments_end_ts(res) @@ -540,10 +592,10 @@ class OnlineASRProcessor: if len(ends) > 1: - e = ends[-2]+self.buffer_time_offset + e = ends[-2] + self.buffer_time_offset while len(ends) > 2 and e > t: ends.pop(-1) - e = ends[-2]+self.buffer_time_offset + e = ends[-2] + self.buffer_time_offset if e <= t: logger.debug(f"--- segment chunked at {e:2.2f}") self.chunk_at(e) @@ -552,23 +604,18 @@ class OnlineASRProcessor: else: logger.debug(f"--- not enough segments to chunk") - - - - def chunk_at(self, time): - """trims the hypothesis and audio buffer at "time" - """ + """trims the hypothesis and audio buffer at "time" """ self.transcript_buffer.pop_commited(time) cut_seconds = time - self.buffer_time_offset - self.audio_buffer = self.audio_buffer[int(cut_seconds*self.SAMPLING_RATE):] + self.audio_buffer = self.audio_buffer[int(cut_seconds * self.SAMPLING_RATE) :] self.buffer_time_offset = time def words_to_sentences(self, words): """Uses self.tokenizer for sentence segmentation of words. Returns: [(beg,end,"sentence 1"),...] """ - + cwords = [w for w in words] t = " ".join(o[2] for o in cwords) s = self.tokenizer.split(t) @@ -579,15 +626,15 @@ class OnlineASRProcessor: sent = s.pop(0).strip() fsent = sent while cwords: - b,e,w = cwords.pop(0) + b, e, w = cwords.pop(0) w = w.strip() if beg is None and sent.startswith(w): beg = b elif end is None and sent == w: end = e - out.append((beg,end,fsent)) + out.append((beg, end, fsent)) break - sent = sent[len(w):].strip() + sent = sent[len(w) :].strip() return out def finish(self): @@ -597,11 +644,15 @@ class OnlineASRProcessor: o = self.transcript_buffer.complete() f = self.to_flush(o) logger.debug(f"last, noncommited: {f}") - self.buffer_time_offset += len(self.audio_buffer)/16000 + self.buffer_time_offset += len(self.audio_buffer) / 16000 return f - - def to_flush(self, sents, sep=None, offset=0, ): + def to_flush( + self, + sents, + sep=None, + offset=0, + ): # concatenates the timestamped words or sentences into one sequence that is flushed in one line # sents: [(beg1, end1, "sentence1"), ...] or [] if empty # return: (beg1,end-of-last-sentence,"concatenation of sentences") or (None, None, "") if empty @@ -614,15 +665,16 @@ class OnlineASRProcessor: else: b = offset + sents[0][0] e = offset + sents[-1][1] - return (b,e,t) + return (b, e, t) + class VACOnlineASRProcessor(OnlineASRProcessor): - '''Wraps OnlineASRProcessor with VAC (Voice Activity Controller). + """Wraps OnlineASRProcessor with VAC (Voice Activity Controller). - It works the same way as OnlineASRProcessor: it receives chunks of audio (e.g. 0.04 seconds), - it runs VAD and continuously detects whether there is speech or not. + It works the same way as OnlineASRProcessor: it receives chunks of audio (e.g. 0.04 seconds), + it runs VAD and continuously detects whether there is speech or not. When it detects end of speech (non-voice for 500ms), it makes OnlineASRProcessor to end the utterance immediately. - ''' + """ def __init__(self, online_chunk_size, *a, **kw): self.online_chunk_size = online_chunk_size @@ -631,12 +683,13 @@ class VACOnlineASRProcessor(OnlineASRProcessor): # VAC: import torch - model, _ = torch.hub.load( - repo_or_dir='snakers4/silero-vad', - model='silero_vad' - ) + + model, _ = torch.hub.load(repo_or_dir="snakers4/silero-vad", model="silero_vad") from silero_vad_iterator import FixedVADIterator - self.vac = FixedVADIterator(model) # we use the default options there: 500ms silence, 100ms padding, etc. + + self.vac = FixedVADIterator( + model + ) # we use the default options there: 500ms silence, 100ms padding, etc. self.logfile = self.online.logfile self.init() @@ -649,60 +702,65 @@ class VACOnlineASRProcessor(OnlineASRProcessor): self.is_currently_final = False self.status = None # or "voice" or "nonvoice" - self.audio_buffer = np.array([],dtype=np.float32) + self.audio_buffer = np.array([], dtype=np.float32) self.buffer_offset = 0 # in frames def clear_buffer(self): self.buffer_offset += len(self.audio_buffer) - self.audio_buffer = np.array([],dtype=np.float32) - + self.audio_buffer = np.array([], dtype=np.float32) def insert_audio_chunk(self, audio): res = self.vac(audio) self.audio_buffer = np.append(self.audio_buffer, audio) if res is not None: - frame = list(res.values())[0]-self.buffer_offset - if 'start' in res and 'end' not in res: - self.status = 'voice' + frame = list(res.values())[0] - self.buffer_offset + if "start" in res and "end" not in res: + self.status = "voice" send_audio = self.audio_buffer[frame:] - self.online.init(offset=(frame+self.buffer_offset)/self.SAMPLING_RATE) + self.online.init( + offset=(frame + self.buffer_offset) / self.SAMPLING_RATE + ) self.online.insert_audio_chunk(send_audio) self.current_online_chunk_buffer_size += len(send_audio) self.clear_buffer() - elif 'end' in res and 'start' not in res: - self.status = 'nonvoice' + elif "end" in res and "start" not in res: + self.status = "nonvoice" send_audio = self.audio_buffer[:frame] self.online.insert_audio_chunk(send_audio) self.current_online_chunk_buffer_size += len(send_audio) self.is_currently_final = True self.clear_buffer() else: - beg = res["start"]-self.buffer_offset - end = res["end"]-self.buffer_offset - self.status = 'nonvoice' + beg = res["start"] - self.buffer_offset + end = res["end"] - self.buffer_offset + self.status = "nonvoice" send_audio = self.audio_buffer[beg:end] - self.online.init(offset=(beg+self.buffer_offset)/self.SAMPLING_RATE) + self.online.init(offset=(beg + self.buffer_offset) / self.SAMPLING_RATE) self.online.insert_audio_chunk(send_audio) self.current_online_chunk_buffer_size += len(send_audio) self.is_currently_final = True self.clear_buffer() else: - if self.status == 'voice': + if self.status == "voice": self.online.insert_audio_chunk(self.audio_buffer) self.current_online_chunk_buffer_size += len(self.audio_buffer) self.clear_buffer() else: # We keep 1 second because VAD may later find start of voice in it. - # But we trim it to prevent OOM. - self.buffer_offset += max(0,len(self.audio_buffer)-self.SAMPLING_RATE) - self.audio_buffer = self.audio_buffer[-self.SAMPLING_RATE:] - + # But we trim it to prevent OOM. + self.buffer_offset += max( + 0, len(self.audio_buffer) - self.SAMPLING_RATE + ) + self.audio_buffer = self.audio_buffer[-self.SAMPLING_RATE :] def process_iter(self): if self.is_currently_final: return self.finish() - elif self.current_online_chunk_buffer_size > self.SAMPLING_RATE*self.online_chunk_size: + elif ( + self.current_online_chunk_buffer_size + > self.SAMPLING_RATE * self.online_chunk_size + ): self.current_online_chunk_buffer_size = 0 ret = self.online.process_iter() return ret @@ -717,37 +775,55 @@ class VACOnlineASRProcessor(OnlineASRProcessor): return ret +WHISPER_LANG_CODES = "af,am,ar,as,az,ba,be,bg,bn,bo,br,bs,ca,cs,cy,da,de,el,en,es,et,eu,fa,fi,fo,fr,gl,gu,ha,haw,he,hi,hr,ht,hu,hy,id,is,it,ja,jw,ka,kk,km,kn,ko,la,lb,ln,lo,lt,lv,mg,mi,mk,ml,mn,mr,ms,mt,my,ne,nl,nn,no,oc,pa,pl,ps,pt,ro,ru,sa,sd,si,sk,sl,sn,so,sq,sr,su,sv,sw,ta,te,tg,th,tk,tl,tr,tt,uk,ur,uz,vi,yi,yo,zh".split( + "," +) -WHISPER_LANG_CODES = "af,am,ar,as,az,ba,be,bg,bn,bo,br,bs,ca,cs,cy,da,de,el,en,es,et,eu,fa,fi,fo,fr,gl,gu,ha,haw,he,hi,hr,ht,hu,hy,id,is,it,ja,jw,ka,kk,km,kn,ko,la,lb,ln,lo,lt,lv,mg,mi,mk,ml,mn,mr,ms,mt,my,ne,nl,nn,no,oc,pa,pl,ps,pt,ro,ru,sa,sd,si,sk,sl,sn,so,sq,sr,su,sv,sw,ta,te,tg,th,tk,tl,tr,tt,uk,ur,uz,vi,yi,yo,zh".split(",") def create_tokenizer(lan): """returns an object that has split function that works like the one of MosesTokenizer""" - assert lan in WHISPER_LANG_CODES, "language must be Whisper's supported lang code: " + " ".join(WHISPER_LANG_CODES) + assert ( + lan in WHISPER_LANG_CODES + ), "language must be Whisper's supported lang code: " + " ".join(WHISPER_LANG_CODES) if lan == "uk": import tokenize_uk + class UkrainianTokenizer: def split(self, text): return tokenize_uk.tokenize_sents(text) + return UkrainianTokenizer() # supported by fast-mosestokenizer - if lan in "as bn ca cs de el en es et fi fr ga gu hi hu is it kn lt lv ml mni mr nl or pa pl pt ro ru sk sl sv ta te yue zh".split(): + if ( + lan + in "as bn ca cs de el en es et fi fr ga gu hi hu is it kn lt lv ml mni mr nl or pa pl pt ro ru sk sl sv ta te yue zh".split() + ): from mosestokenizer import MosesTokenizer + return MosesTokenizer(lan) # the following languages are in Whisper, but not in wtpsplit: - if lan in "as ba bo br bs fo haw hr ht jw lb ln lo mi nn oc sa sd sn so su sw tk tl tt".split(): - logger.debug(f"{lan} code is not supported by wtpsplit. Going to use None lang_code option.") + if ( + lan + in "as ba bo br bs fo haw hr ht jw lb ln lo mi nn oc sa sd sn so su sw tk tl tt".split() + ): + logger.debug( + f"{lan} code is not supported by wtpsplit. Going to use None lang_code option." + ) lan = None from wtpsplit import WtP + # downloads the model from huggingface on the first use wtp = WtP("wtp-canine-s-12l-no-adapters") + class WtPtok: def split(self, sent): return wtp.split(sent, lang_code=lan) + return WtPtok() @@ -755,19 +831,91 @@ def add_shared_args(parser): """shared args for simulation (this entry point) and server parser: argparse.ArgumentParser object """ - parser.add_argument('--min-chunk-size', type=float, default=1.0, help='Minimum audio chunk size in seconds. It waits up to this time to do processing. If the processing takes shorter time, it waits, otherwise it processes the whole segment that was received by this time.') - parser.add_argument('--model', type=str, default='large-v2', choices="tiny.en,tiny,base.en,base,small.en,small,medium.en,medium,large-v1,large-v2,large-v3,large,large-v3-turbo".split(","),help="Name size of the Whisper model to use (default: large-v2). The model is automatically downloaded from the model hub if not present in model cache dir.") - parser.add_argument('--model_cache_dir', type=str, default=None, help="Overriding the default model cache dir where models downloaded from the hub are saved") - parser.add_argument('--model_dir', type=str, default=None, help="Dir where Whisper model.bin and other files are saved. This option overrides --model and --model_cache_dir parameter.") - parser.add_argument('--lan', '--language', type=str, default='auto', help="Source language code, e.g. en,de,cs, or 'auto' for language detection.") - parser.add_argument('--task', type=str, default='transcribe', choices=["transcribe","translate"],help="Transcribe or translate.") - parser.add_argument('--backend', type=str, default="faster-whisper", choices=["faster-whisper", "whisper_timestamped", "mlx-whisper", "openai-api"],help='Load only this backend for Whisper processing.') - parser.add_argument('--vac', action="store_true", default=False, help='Use VAC = voice activity controller. Recommended. Requires torch.') - parser.add_argument('--vac-chunk-size', type=float, default=0.04, help='VAC sample size in seconds.') - parser.add_argument('--vad', action="store_true", default=False, help='Use VAD = voice activity detection, with the default parameters.') - parser.add_argument('--buffer_trimming', type=str, default="segment", choices=["sentence", "segment"],help='Buffer trimming strategy -- trim completed sentences marked with punctuation mark and detected by sentence segmenter, or the completed segments returned by Whisper. Sentence segmenter must be installed for "sentence" option.') - parser.add_argument('--buffer_trimming_sec', type=float, default=15, help='Buffer trimming length threshold in seconds. If buffer length is longer, trimming sentence/segment is triggered.') - parser.add_argument("-l", "--log-level", dest="log_level", choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], help="Set the log level", default='DEBUG') + parser.add_argument( + "--min-chunk-size", + type=float, + default=1.0, + help="Minimum audio chunk size in seconds. It waits up to this time to do processing. If the processing takes shorter time, it waits, otherwise it processes the whole segment that was received by this time.", + ) + parser.add_argument( + "--model", + type=str, + default="large-v2", + choices="tiny.en,tiny,base.en,base,small.en,small,medium.en,medium,large-v1,large-v2,large-v3,large,large-v3-turbo".split( + "," + ), + help="Name size of the Whisper model to use (default: large-v2). The model is automatically downloaded from the model hub if not present in model cache dir.", + ) + parser.add_argument( + "--model_cache_dir", + type=str, + default=None, + help="Overriding the default model cache dir where models downloaded from the hub are saved", + ) + parser.add_argument( + "--model_dir", + type=str, + default=None, + help="Dir where Whisper model.bin and other files are saved. This option overrides --model and --model_cache_dir parameter.", + ) + parser.add_argument( + "--lan", + "--language", + type=str, + default="auto", + help="Source language code, e.g. en,de,cs, or 'auto' for language detection.", + ) + parser.add_argument( + "--task", + type=str, + default="transcribe", + choices=["transcribe", "translate"], + help="Transcribe or translate.", + ) + parser.add_argument( + "--backend", + type=str, + default="faster-whisper", + choices=["faster-whisper", "whisper_timestamped", "mlx-whisper", "openai-api"], + help="Load only this backend for Whisper processing.", + ) + parser.add_argument( + "--vac", + action="store_true", + default=False, + help="Use VAC = voice activity controller. Recommended. Requires torch.", + ) + parser.add_argument( + "--vac-chunk-size", type=float, default=0.04, help="VAC sample size in seconds." + ) + parser.add_argument( + "--vad", + action="store_true", + default=False, + help="Use VAD = voice activity detection, with the default parameters.", + ) + parser.add_argument( + "--buffer_trimming", + type=str, + default="segment", + choices=["sentence", "segment"], + help='Buffer trimming strategy -- trim completed sentences marked with punctuation mark and detected by sentence segmenter, or the completed segments returned by Whisper. Sentence segmenter must be installed for "sentence" option.', + ) + parser.add_argument( + "--buffer_trimming_sec", + type=float, + default=15, + help="Buffer trimming length threshold in seconds. If buffer length is longer, trimming sentence/segment is triggered.", + ) + parser.add_argument( + "-l", + "--log-level", + dest="log_level", + choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], + help="Set the log level", + default="DEBUG", + ) + def asr_factory(args, logfile=sys.stderr): """ @@ -789,12 +937,17 @@ def asr_factory(args, logfile=sys.stderr): size = args.model t = time.time() logger.info(f"Loading Whisper {size} model for {args.lan}...") - asr = asr_cls(modelsize=size, lan=args.lan, cache_dir=args.model_cache_dir, model_dir=args.model_dir) + asr = asr_cls( + modelsize=size, + lan=args.lan, + cache_dir=args.model_cache_dir, + model_dir=args.model_dir, + ) e = time.time() logger.info(f"done. It took {round(e-t,2)} seconds.") # Apply common configurations - if getattr(args, 'vad', False): # Checks if VAD argument is present and True + if getattr(args, "vad", False): # Checks if VAD argument is present and True logger.info("Setting VAD filter") asr.use_vad() @@ -813,51 +966,82 @@ def asr_factory(args, logfile=sys.stderr): # Create the OnlineASRProcessor if args.vac: - - online = VACOnlineASRProcessor(args.min_chunk_size, asr,tokenizer,logfile=logfile,buffer_trimming=(args.buffer_trimming, args.buffer_trimming_sec)) + + online = VACOnlineASRProcessor( + args.min_chunk_size, + asr, + tokenizer, + logfile=logfile, + buffer_trimming=(args.buffer_trimming, args.buffer_trimming_sec), + ) else: - online = OnlineASRProcessor(asr,tokenizer,logfile=logfile,buffer_trimming=(args.buffer_trimming, args.buffer_trimming_sec)) + online = OnlineASRProcessor( + asr, + tokenizer, + logfile=logfile, + buffer_trimming=(args.buffer_trimming, args.buffer_trimming_sec), + ) return asr, online -def set_logging(args,logger,other="_server"): - logging.basicConfig(#format='%(name)s - format='%(levelname)s\t%(message)s') - logger.setLevel(args.log_level) - logging.getLogger("whisper_online"+other).setLevel(args.log_level) -# logging.getLogger("whisper_online_server").setLevel(args.log_level) +def set_logging(args, logger, other="_server"): + logging.basicConfig(format="%(levelname)s\t%(message)s") # format='%(name)s + logger.setLevel(args.log_level) + logging.getLogger("whisper_online" + other).setLevel(args.log_level) + + +# logging.getLogger("whisper_online_server").setLevel(args.log_level) if __name__ == "__main__": import argparse + parser = argparse.ArgumentParser() - parser.add_argument('audio_path', type=str, help="Filename of 16kHz mono channel wav, on which live streaming is simulated.") + parser.add_argument( + "audio_path", + type=str, + help="Filename of 16kHz mono channel wav, on which live streaming is simulated.", + ) add_shared_args(parser) - parser.add_argument('--start_at', type=float, default=0.0, help='Start processing audio at this time.') - parser.add_argument('--offline', action="store_true", default=False, help='Offline mode.') - parser.add_argument('--comp_unaware', action="store_true", default=False, help='Computationally unaware simulation.') - + parser.add_argument( + "--start_at", + type=float, + default=0.0, + help="Start processing audio at this time.", + ) + parser.add_argument( + "--offline", action="store_true", default=False, help="Offline mode." + ) + parser.add_argument( + "--comp_unaware", + action="store_true", + default=False, + help="Computationally unaware simulation.", + ) + args = parser.parse_args() # reset to store stderr to different file stream, e.g. open(os.devnull,"w") logfile = sys.stderr if args.offline and args.comp_unaware: - logger.error("No or one option from --offline and --comp_unaware are available, not both. Exiting.") + logger.error( + "No or one option from --offline and --comp_unaware are available, not both. Exiting." + ) sys.exit(1) -# if args.log_level: -# logging.basicConfig(format='whisper-%(levelname)s:%(name)s: %(message)s', -# level=getattr(logging, args.log_level)) + # if args.log_level: + # logging.basicConfig(format='whisper-%(levelname)s:%(name)s: %(message)s', + # level=getattr(logging, args.log_level)) - set_logging(args,logger) + set_logging(args, logger) audio_path = args.audio_path SAMPLING_RATE = 16000 - duration = len(load_audio(audio_path))/SAMPLING_RATE + duration = len(load_audio(audio_path)) / SAMPLING_RATE logger.info("Audio duration is: %2.2f seconds" % duration) asr, online = asr_factory(args, logfile=logfile) @@ -867,13 +1051,13 @@ if __name__ == "__main__": min_chunk = args.min_chunk_size # load the audio into the LRU cache before we start the timer - a = load_audio_chunk(audio_path,0,1) + a = load_audio_chunk(audio_path, 0, 1) # warm up the ASR because the very first transcribe takes much more time than the other asr.transcribe(a) beg = args.start_at - start = time.time()-beg + start = time.time() - beg def output_transcript(o, now=None): # output format in stdout is like: @@ -883,15 +1067,22 @@ if __name__ == "__main__": # - beg and end timestamp of the text segment, as estimated by Whisper model. The timestamps are not accurate, but they're useful anyway # - the next words: segment transcript if now is None: - now = time.time()-start + now = time.time() - start if o[0] is not None: - print("%1.4f %1.0f %1.0f %s" % (now*1000, o[0]*1000,o[1]*1000,o[2]),file=logfile,flush=True) - print("%1.4f %1.0f %1.0f %s" % (now*1000, o[0]*1000,o[1]*1000,o[2]),flush=True) + print( + "%1.4f %1.0f %1.0f %s" % (now * 1000, o[0] * 1000, o[1] * 1000, o[2]), + file=logfile, + flush=True, + ) + print( + "%1.4f %1.0f %1.0f %s" % (now * 1000, o[0] * 1000, o[1] * 1000, o[2]), + flush=True, + ) else: # No text, so no output pass - if args.offline: ## offline mode processing (for testing/debugging) + if args.offline: ## offline mode processing (for testing/debugging) a = load_audio(audio_path) online.insert_audio_chunk(a) try: @@ -901,10 +1092,10 @@ if __name__ == "__main__": else: output_transcript(o) now = None - elif args.comp_unaware: # computational unaware mode + elif args.comp_unaware: # computational unaware mode end = beg + min_chunk while True: - a = load_audio_chunk(audio_path,beg,end) + a = load_audio_chunk(audio_path, beg, end) online.insert_audio_chunk(a) try: o = online.process_iter() @@ -918,23 +1109,23 @@ if __name__ == "__main__": if end >= duration: break - + beg = end - + if end + min_chunk > duration: end = duration else: end += min_chunk now = duration - else: # online = simultaneous mode + else: # online = simultaneous mode end = 0 while True: now = time.time() - start - if now < end+min_chunk: - time.sleep(min_chunk+end-now) + if now < end + min_chunk: + time.sleep(min_chunk + end - now) end = time.time() - start - a = load_audio_chunk(audio_path,beg,end) + a = load_audio_chunk(audio_path, beg, end) beg = end online.insert_audio_chunk(a) @@ -946,7 +1137,9 @@ if __name__ == "__main__": else: output_transcript(o) now = time.time() - start - logger.debug(f"## last processed {end:.2f} s, now is {now:.2f}, the latency is {now-end:.2f}") + logger.debug( + f"## last processed {end:.2f} s, now is {now:.2f}, the latency is {now-end:.2f}" + ) if end >= duration: break From aa4d2599cc32b1db1aa68ca4ac8712931a4b78df Mon Sep 17 00:00:00 2001 From: silask Date: Mon, 30 Dec 2024 21:22:41 +0100 Subject: [PATCH 2/3] fix #7 --- whisper_online.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/whisper_online.py b/whisper_online.py index afbee2d..30a8f2f 100644 --- a/whisper_online.py +++ b/whisper_online.py @@ -459,16 +459,20 @@ class OnlineASRProcessor: SAMPLING_RATE = 16000 def __init__( - self, asr, tokenizer=None, buffer_trimming=("segment", 15), logfile=sys.stderr + self, + asr, + tokenize_method=None, + buffer_trimming=("segment", 15), + logfile=sys.stderr, ): """asr: WhisperASR object - tokenizer: sentence tokenizer object for the target language. Must have a method *split* that behaves like the one of MosesTokenizer. It can be None, if "segment" buffer trimming option is used, then tokenizer is not used at all. + tokenize_method: sentence tokenizer function for the target language. Must be a callable and behaves like the one of MosesTokenizer. It can be None, if "segment" buffer trimming option is used, then tokenizer is not used at all. ("segment", 15) buffer_trimming: a pair of (option, seconds), where option is either "sentence" or "segment", and seconds is a number. Buffer is trimmed if it is longer than "seconds" threshold. Default is the most recommended option. logfile: where to store the log. """ self.asr = asr - self.tokenizer = tokenizer + self.tokenize = tokenize_method self.logfile = logfile self.init() @@ -612,13 +616,13 @@ class OnlineASRProcessor: self.buffer_time_offset = time def words_to_sentences(self, words): - """Uses self.tokenizer for sentence segmentation of words. + """Uses self.tokenize for sentence segmentation of words. Returns: [(beg,end,"sentence 1"),...] """ cwords = [w for w in words] t = " ".join(o[2] for o in cwords) - s = self.tokenizer.split(t) + s = self.tokenize(t) out = [] while s: beg = None From 6a04ddbed20bdf4ec8a108bbddcb979b85f1f730 Mon Sep 17 00:00:00 2001 From: silask Date: Mon, 30 Dec 2024 21:23:20 +0100 Subject: [PATCH 3/3] only print translated text not timestamps --- whisper_online.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/whisper_online.py b/whisper_online.py index 30a8f2f..b65c36a 100644 --- a/whisper_online.py +++ b/whisper_online.py @@ -534,9 +534,9 @@ class OnlineASRProcessor: o = self.transcript_buffer.flush() self.commited.extend(o) completed = self.to_flush(o) - logger.debug(f">>>>COMPLETE NOW: {completed}") + logger.debug(f">>>>COMPLETE NOW: {completed[2]}") the_rest = self.to_flush(self.transcript_buffer.complete()) - logger.debug(f"INCOMPLETE: {the_rest}") + logger.debug(f"INCOMPLETE: {the_rest[2]}") # there is a newly confirmed text @@ -572,7 +572,7 @@ class OnlineASRProcessor: def chunk_completed_sentence(self): if self.commited == []: return - logger.debug(self.commited) + logger.debug("COMPLETED SENTENCE: ", [s[2] for s in self.commited]) sents = self.words_to_sentences(self.commited) for s in sents: logger.debug(f"\t\tSENT: {s}")