From c30969fe271794eb9fedc90c7b01f9a9f4461b0f Mon Sep 17 00:00:00 2001 From: Tijs Zwinkels Date: Wed, 24 Jan 2024 15:31:18 +0100 Subject: [PATCH 01/25] OpenAI Whisper API backend --- whisper_online.py | 76 +++++++++++++++++++++++++++++++++++++++- whisper_online_server.py | 2 ++ 2 files changed, 77 insertions(+), 1 deletion(-) diff --git a/whisper_online.py b/whisper_online.py index 36bdbd6..860c82d 100644 --- a/whisper_online.py +++ b/whisper_online.py @@ -4,6 +4,8 @@ import numpy as np import librosa from functools import lru_cache import time +import io +import soundfile as sf @@ -142,6 +144,76 @@ class FasterWhisperASR(ASRBase): self.transcribe_kargs["task"] = "translate" +class OpenaiApiASR(ASRBase): + """Uses OpenAI's Whisper API for audio transcription.""" + + def __init__(self, modelsize=None, lan=None, cache_dir=None, model_dir=None, response_format="verbose_json", temperature=0): + self.modelname = "whisper-1" # modelsize is not used but kept for interface consistency + self.language = lan # ISO-639-1 language code + self.response_format = response_format + self.temperature = temperature + self.model = self.load_model(modelsize, cache_dir, model_dir) + + def load_model(self, *args, **kwargs): + from openai import OpenAI + self.client = OpenAI() + # Since we're using the OpenAI API, there's no model to load locally. + print("Model configuration is set to use the OpenAI Whisper API.") + + def ts_words(self, segments): + o = [] + for segment in segments: + # Skip segments containing no speech + if segment["no_speech_prob"] > 0.8: + continue + + # Splitting the text into words and filtering out empty strings + words = [word.strip() for word in segment["text"].split() if word.strip()] + + if not words: + continue + + # Assign start and end times for each word + # We only have timestamps per segment, so interpolating start and end-times + # assuming equal duration per word + segment_duration = segment["end"] - segment["start"] + duration_per_word = segment_duration / len(words) + start_time = segment["start"] + for word in words: + end_time = start_time + duration_per_word + o.append((start_time, end_time, word)) + start_time = end_time + + return o + + + def segments_end_ts(self, res): + return [s["end"] for s in res] + + def transcribe(self, audio_data, prompt=None, *args, **kwargs): + # Write the audio data to a buffer + buffer = io.BytesIO() + buffer.name = "temp.wav" + sf.write(buffer, audio_data, samplerate=16000, format='WAV', subtype='PCM_16') + buffer.seek(0) # Reset buffer's position to the beginning + + # Prepare transcription parameters + transcription_params = { + "model": self.modelname, + "file": buffer, + "response_format": self.response_format, + "temperature": self.temperature + } + if self.language: + transcription_params["language"] = self.language + if prompt: + transcription_params["prompt"] = prompt + + # Perform the transcription + transcript = self.client.audio.transcriptions.create(**transcription_params) + + return transcript.segments + class HypothesisBuffer: @@ -453,7 +525,7 @@ def add_shared_args(parser): parser.add_argument('--model_dir', type=str, default=None, help="Dir where Whisper model.bin and other files are saved. This option overrides --model and --model_cache_dir parameter.") parser.add_argument('--lan', '--language', type=str, default='en', help="Language code for transcription, e.g. en,de,cs.") parser.add_argument('--task', type=str, default='transcribe', choices=["transcribe","translate"],help="Transcribe or translate.") - parser.add_argument('--backend', type=str, default="faster-whisper", choices=["faster-whisper", "whisper_timestamped"],help='Load only this backend for Whisper processing.') + parser.add_argument('--backend', type=str, default="faster-whisper", choices=["faster-whisper", "whisper_timestamped", "openai-api"],help='Load only this backend for Whisper processing.') parser.add_argument('--vad', action="store_true", default=False, help='Use VAD = voice activity detection, with the default parameters.') parser.add_argument('--buffer_trimming', type=str, default="segment", choices=["sentence", "segment"],help='Buffer trimming strategy -- trim completed sentences marked with punctuation mark and detected by sentence segmenter, or the completed segments returned by Whisper. Sentence segmenter must be installed for "sentence" option.') parser.add_argument('--buffer_trimming_sec', type=float, default=15, help='Buffer trimming length threshold in seconds. If buffer length is longer, trimming sentence/segment is triggered.') @@ -493,6 +565,8 @@ if __name__ == "__main__": if args.backend == "faster-whisper": asr_cls = FasterWhisperASR + elif args.backend == "openai-api": + asr_cls = OpenaiApiASR else: asr_cls = WhisperTimestampedASR diff --git a/whisper_online_server.py b/whisper_online_server.py index b2f5120..13a85de 100644 --- a/whisper_online_server.py +++ b/whisper_online_server.py @@ -29,6 +29,8 @@ print(f"Loading Whisper {size} model for {language}...",file=sys.stderr,end=" ", if args.backend == "faster-whisper": from faster_whisper import WhisperModel asr_cls = FasterWhisperASR +elif args.backend == "openai-api": + asr_cls = OpenaiApiASR else: import whisper import whisper_timestamped From ab27bfb361da9ce38569a82f7002f3652571a5d9 Mon Sep 17 00:00:00 2001 From: Tijs Zwinkels Date: Thu, 25 Jan 2024 10:08:21 +0100 Subject: [PATCH 02/25] Update documentation to include openai-api backend --- README.md | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 584f3d0..8b2ffb8 100644 --- a/README.md +++ b/README.md @@ -31,14 +31,19 @@ Please, cite us. [Bibtex citation](http://www.afnlp.org/conferences/ijcnlp2023/p ## Installation -1) ``pip install librosa`` -- audio processing library +1) ``pip install librosa soundfile`` -- audio processing library 2) Whisper backend. -Two alternative backends are integrated. The most recommended one is [faster-whisper](https://github.com/guillaumekln/faster-whisper) with GPU support. Follow their instructions for NVIDIA libraries -- we succeeded with CUDNN 8.5.0 and CUDA 11.7. Install with `pip install faster-whisper`. + Several alternative backends are integrated. The most recommended one is [faster-whisper](https://github.com/guillaumekln/faster-whisper) with GPU support. Follow their instructions for NVIDIA libraries -- we succeeded with CUDNN 8.5.0 and CUDA 11.7. Install with `pip install faster-whisper`. Alternative, less restrictive, but slower backend is [whisper-timestamped](https://github.com/linto-ai/whisper-timestamped): `pip install git+https://github.com/linto-ai/whisper-timestamped` +Thirdly, it's also possible to run this software from the [OpenAI Whisper API](https://platform.openai.com/docs/api-reference/audio/createTranscription). This solution is fast and requires no GPU, just a small VM will suffice, but you will need to pay OpenAI for api access. Also note that, since each audio fragment is processed multiple times, the [price](https://openai.com/pricing) will be higher than obvious from the pricing page, so keep an eye on costs while using. Setting a higher chunk-size will reduce costs significantly. +Install with: `pip install openai` + +For running with the openai-api backend, make sure that your [OpenAI api key](https://platform.openai.com/api-keys) is set in the `OPENAI_API_KEY` environment variable. For example, before running, do: `export OPENAI_API_KEY=sk-xxx` with *sk-xxx* replaced with your api key. + The backend is loaded only when chosen. The unused one does not have to be installed. 3) Optional, not recommended: sentence segmenter (aka sentence tokenizer) @@ -69,7 +74,7 @@ In case of installation issues of opus-fast-mosestokenizer, especially on Window ``` usage: whisper_online.py [-h] [--min-chunk-size MIN_CHUNK_SIZE] [--model {tiny.en,tiny,base.en,base,small.en,small,medium.en,medium,large-v1,large-v2,large-v3,large}] [--model_cache_dir MODEL_CACHE_DIR] [--model_dir MODEL_DIR] [--lan LAN] [--task {transcribe,translate}] - [--backend {faster-whisper,whisper_timestamped}] [--vad] [--buffer_trimming {sentence,segment}] [--buffer_trimming_sec BUFFER_TRIMMING_SEC] [--start_at START_AT] [--offline] [--comp_unaware] + [--backend {faster-whisper,whisper_timestamped,openai-api}] [--vad] [--buffer_trimming {sentence,segment}] [--buffer_trimming_sec BUFFER_TRIMMING_SEC] [--start_at START_AT] [--offline] [--comp_unaware] audio_path positional arguments: @@ -89,7 +94,7 @@ options: Language code for transcription, e.g. en,de,cs. --task {transcribe,translate} Transcribe or translate. - --backend {faster-whisper,whisper_timestamped} + --backend {faster-whisper,whisper_timestamped,openai-api} Load only this backend for Whisper processing. --vad Use VAD = voice activity detection, with the default parameters. --buffer_trimming {sentence,segment} From 50f1b94856cd916d7e5bca1650fc8fc8ff3104f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominik=20Mach=C3=A1=C4=8Dek?= Date: Thu, 25 Jan 2024 16:49:25 +0100 Subject: [PATCH 03/25] missing features in openai-api, PR #52 --- whisper_online.py | 88 ++++++++++++++++++++++++++++++----------------- 1 file changed, 56 insertions(+), 32 deletions(-) diff --git a/whisper_online.py b/whisper_online.py index 860c82d..e4bfff8 100644 --- a/whisper_online.py +++ b/whisper_online.py @@ -6,8 +6,7 @@ from functools import lru_cache import time import io import soundfile as sf - - +import math @lru_cache def load_audio(fname): @@ -147,24 +146,34 @@ class FasterWhisperASR(ASRBase): class OpenaiApiASR(ASRBase): """Uses OpenAI's Whisper API for audio transcription.""" - def __init__(self, modelsize=None, lan=None, cache_dir=None, model_dir=None, response_format="verbose_json", temperature=0): - self.modelname = "whisper-1" # modelsize is not used but kept for interface consistency + def __init__(self, lan=None, response_format="verbose_json", temperature=0, logfile=sys.stderr): + self.logfile = logfile + + self.modelname = "whisper-1" self.language = lan # ISO-639-1 language code self.response_format = response_format self.temperature = temperature - self.model = self.load_model(modelsize, cache_dir, model_dir) + + self.load_model() + + self.use_vad = False + + # reset the task in set_translate_task + self.task = "transcribe" def load_model(self, *args, **kwargs): from openai import OpenAI self.client = OpenAI() - # Since we're using the OpenAI API, there's no model to load locally. - print("Model configuration is set to use the OpenAI Whisper API.") + + self.transcribed_seconds = 0 # for logging how many seconds were processed by API, to know the cost + def ts_words(self, segments): o = [] for segment in segments: - # Skip segments containing no speech - if segment["no_speech_prob"] > 0.8: + # If VAD on, skip segments containing no speech. + # TODO: threshold can be set from outside + if self.use_vad and segment["no_speech_prob"] > 0.8: continue # Splitting the text into words and filtering out empty strings @@ -197,23 +206,39 @@ class OpenaiApiASR(ASRBase): sf.write(buffer, audio_data, samplerate=16000, format='WAV', subtype='PCM_16') buffer.seek(0) # Reset buffer's position to the beginning - # Prepare transcription parameters - transcription_params = { + self.transcribed_seconds += math.ceil(len(audio_data)/16000) # it rounds up to the whole seconds + + params = { "model": self.modelname, "file": buffer, "response_format": self.response_format, "temperature": self.temperature } - if self.language: + if self.task != "translate" and self.language: transcription_params["language"] = self.language if prompt: transcription_params["prompt"] = prompt - # Perform the transcription - transcript = self.client.audio.transcriptions.create(**transcription_params) + if self.task == "translate": + proc = self.client.audio.translations + else: + proc = self.client.audio.transcriptions + + # Process transcription/translation + + transcript = proc.create(**params) + print(f"OpenAI API processed accumulated {self.transcribed_seconds} seconds ",file=self.logfile) return transcript.segments + def use_vad(self): + self.use_vad = True + + def set_translate_task(self): + self.task = "translate" + + + class HypothesisBuffer: @@ -557,20 +582,27 @@ if __name__ == "__main__": duration = len(load_audio(audio_path))/SAMPLING_RATE print("Audio duration is: %2.2f seconds" % duration, file=logfile) - size = args.model language = args.lan - t = time.time() - print(f"Loading Whisper {size} model for {language}...",file=logfile,end=" ",flush=True) - - if args.backend == "faster-whisper": - asr_cls = FasterWhisperASR - elif args.backend == "openai-api": - asr_cls = OpenaiApiASR + if args.backend == "openai-api": + print("Using OpenAI API.",file=logfile) + asr = OpenaiApiASR(lan=language) else: - asr_cls = WhisperTimestampedASR + if args.backend == "faster-whisper": + asr_cls = FasterWhisperASR + else: + asr_cls = WhisperTimestampedASR - asr = asr_cls(modelsize=size, lan=language, cache_dir=args.model_cache_dir, model_dir=args.model_dir) + size = args.model + t = time.time() + print(f"Loading Whisper {size} model for {language}...",file=logfile,end=" ",flush=True) + asr = asr_cls(modelsize=size, lan=language, cache_dir=args.model_cache_dir, model_dir=args.model_dir) + e = time.time() + print(f"done. It took {round(e-t,2)} seconds.",file=logfile) + + if args.vad: + print("setting VAD filter",file=logfile) + asr.use_vad() if args.task == "translate": asr.set_translate_task() @@ -578,14 +610,6 @@ if __name__ == "__main__": else: tgt_language = language # Whisper transcribes in this language - - e = time.time() - print(f"done. It took {round(e-t,2)} seconds.",file=logfile) - - if args.vad: - print("setting VAD filter",file=logfile) - asr.use_vad() - min_chunk = args.min_chunk_size if args.buffer_trimming == "sentence": From d65fd8a649cb8f577e32b3cff193b85818ae7bad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominik=20Mach=C3=A1=C4=8Dek?= Date: Thu, 25 Jan 2024 17:53:07 +0100 Subject: [PATCH 04/25] fixes --- whisper_online.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/whisper_online.py b/whisper_online.py index e4bfff8..2941920 100644 --- a/whisper_online.py +++ b/whisper_online.py @@ -215,9 +215,9 @@ class OpenaiApiASR(ASRBase): "temperature": self.temperature } if self.task != "translate" and self.language: - transcription_params["language"] = self.language + params["language"] = self.language if prompt: - transcription_params["prompt"] = prompt + params["prompt"] = prompt if self.task == "translate": proc = self.client.audio.translations @@ -227,7 +227,7 @@ class OpenaiApiASR(ASRBase): # Process transcription/translation transcript = proc.create(**params) - print(f"OpenAI API processed accumulated {self.transcribed_seconds} seconds ",file=self.logfile) + print(f"OpenAI API processed accumulated {self.transcribed_seconds} seconds",file=self.logfile) return transcript.segments From cd221a31988b3426a9f2fb86af33629e2f56c8ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominik=20Mach=C3=A1=C4=8Dek?= Date: Tue, 6 Feb 2024 14:29:30 +0100 Subject: [PATCH 05/25] auto language detection #56 --- whisper_online.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/whisper_online.py b/whisper_online.py index 36bdbd6..59d41e7 100644 --- a/whisper_online.py +++ b/whisper_online.py @@ -30,7 +30,10 @@ class ASRBase: self.logfile = logfile self.transcribe_kargs = {} - self.original_language = lan + if lan == "auto": + self.original_language = None + else: + self.original_language = lan self.model = self.load_model(modelsize, cache_dir, model_dir) @@ -118,8 +121,11 @@ class FasterWhisperASR(ASRBase): return model def transcribe(self, audio, init_prompt=""): + # tested: beam_size=5 is faster and better than 1 (on one 200 second document from En ESIC, min chunk 0.01) segments, info = self.model.transcribe(audio, language=self.original_language, initial_prompt=init_prompt, beam_size=5, word_timestamps=True, condition_on_previous_text=True, **self.transcribe_kargs) + #print(info) # info contains language detection result + return list(segments) def ts_words(self, segments): @@ -451,7 +457,7 @@ def add_shared_args(parser): parser.add_argument('--model', type=str, default='large-v2', choices="tiny.en,tiny,base.en,base,small.en,small,medium.en,medium,large-v1,large-v2,large-v3,large".split(","),help="Name size of the Whisper model to use (default: large-v2). The model is automatically downloaded from the model hub if not present in model cache dir.") parser.add_argument('--model_cache_dir', type=str, default=None, help="Overriding the default model cache dir where models downloaded from the hub are saved") parser.add_argument('--model_dir', type=str, default=None, help="Dir where Whisper model.bin and other files are saved. This option overrides --model and --model_cache_dir parameter.") - parser.add_argument('--lan', '--language', type=str, default='en', help="Language code for transcription, e.g. en,de,cs.") + parser.add_argument('--lan', '--language', type=str, default='en', help="Source language code, e.g. en,de,cs, or 'auto' for language detection.") parser.add_argument('--task', type=str, default='transcribe', choices=["transcribe","translate"],help="Transcribe or translate.") parser.add_argument('--backend', type=str, default="faster-whisper", choices=["faster-whisper", "whisper_timestamped"],help='Load only this backend for Whisper processing.') parser.add_argument('--vad', action="store_true", default=False, help='Use VAD = voice activity detection, with the default parameters.') From b66c61cf7a217716b6235e62d9cffce2ce3abd35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominik=20Mach=C3=A1=C4=8Dek?= Date: Tue, 6 Feb 2024 14:31:24 +0100 Subject: [PATCH 06/25] README update auto language detection --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 584f3d0..26b6525 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,7 @@ options: --model_dir MODEL_DIR Dir where Whisper model.bin and other files are saved. This option overrides --model and --model_cache_dir parameter. --lan LAN, --language LAN - Language code for transcription, e.g. en,de,cs. + Source language code, e.g. en,de,cs, or 'auto' for language detection. --task {transcribe,translate} Transcribe or translate. --backend {faster-whisper,whisper_timestamped} From f412812082164a33e97ac1226d915c549e345619 Mon Sep 17 00:00:00 2001 From: Tijs Zwinkels Date: Wed, 24 Jan 2024 15:31:18 +0100 Subject: [PATCH 07/25] OpenAI Whisper API backend --- whisper_online.py | 76 +++++++++++++++++++++++++++++++++++++++- whisper_online_server.py | 2 ++ 2 files changed, 77 insertions(+), 1 deletion(-) diff --git a/whisper_online.py b/whisper_online.py index 59d41e7..edab195 100644 --- a/whisper_online.py +++ b/whisper_online.py @@ -4,6 +4,8 @@ import numpy as np import librosa from functools import lru_cache import time +import io +import soundfile as sf @@ -148,6 +150,76 @@ class FasterWhisperASR(ASRBase): self.transcribe_kargs["task"] = "translate" +class OpenaiApiASR(ASRBase): + """Uses OpenAI's Whisper API for audio transcription.""" + + def __init__(self, modelsize=None, lan=None, cache_dir=None, model_dir=None, response_format="verbose_json", temperature=0): + self.modelname = "whisper-1" # modelsize is not used but kept for interface consistency + self.language = lan # ISO-639-1 language code + self.response_format = response_format + self.temperature = temperature + self.model = self.load_model(modelsize, cache_dir, model_dir) + + def load_model(self, *args, **kwargs): + from openai import OpenAI + self.client = OpenAI() + # Since we're using the OpenAI API, there's no model to load locally. + print("Model configuration is set to use the OpenAI Whisper API.") + + def ts_words(self, segments): + o = [] + for segment in segments: + # Skip segments containing no speech + if segment["no_speech_prob"] > 0.8: + continue + + # Splitting the text into words and filtering out empty strings + words = [word.strip() for word in segment["text"].split() if word.strip()] + + if not words: + continue + + # Assign start and end times for each word + # We only have timestamps per segment, so interpolating start and end-times + # assuming equal duration per word + segment_duration = segment["end"] - segment["start"] + duration_per_word = segment_duration / len(words) + start_time = segment["start"] + for word in words: + end_time = start_time + duration_per_word + o.append((start_time, end_time, word)) + start_time = end_time + + return o + + + def segments_end_ts(self, res): + return [s["end"] for s in res] + + def transcribe(self, audio_data, prompt=None, *args, **kwargs): + # Write the audio data to a buffer + buffer = io.BytesIO() + buffer.name = "temp.wav" + sf.write(buffer, audio_data, samplerate=16000, format='WAV', subtype='PCM_16') + buffer.seek(0) # Reset buffer's position to the beginning + + # Prepare transcription parameters + transcription_params = { + "model": self.modelname, + "file": buffer, + "response_format": self.response_format, + "temperature": self.temperature + } + if self.language: + transcription_params["language"] = self.language + if prompt: + transcription_params["prompt"] = prompt + + # Perform the transcription + transcript = self.client.audio.transcriptions.create(**transcription_params) + + return transcript.segments + class HypothesisBuffer: @@ -459,7 +531,7 @@ def add_shared_args(parser): parser.add_argument('--model_dir', type=str, default=None, help="Dir where Whisper model.bin and other files are saved. This option overrides --model and --model_cache_dir parameter.") parser.add_argument('--lan', '--language', type=str, default='en', help="Source language code, e.g. en,de,cs, or 'auto' for language detection.") parser.add_argument('--task', type=str, default='transcribe', choices=["transcribe","translate"],help="Transcribe or translate.") - parser.add_argument('--backend', type=str, default="faster-whisper", choices=["faster-whisper", "whisper_timestamped"],help='Load only this backend for Whisper processing.') + parser.add_argument('--backend', type=str, default="faster-whisper", choices=["faster-whisper", "whisper_timestamped", "openai-api"],help='Load only this backend for Whisper processing.') parser.add_argument('--vad', action="store_true", default=False, help='Use VAD = voice activity detection, with the default parameters.') parser.add_argument('--buffer_trimming', type=str, default="segment", choices=["sentence", "segment"],help='Buffer trimming strategy -- trim completed sentences marked with punctuation mark and detected by sentence segmenter, or the completed segments returned by Whisper. Sentence segmenter must be installed for "sentence" option.') parser.add_argument('--buffer_trimming_sec', type=float, default=15, help='Buffer trimming length threshold in seconds. If buffer length is longer, trimming sentence/segment is triggered.') @@ -499,6 +571,8 @@ if __name__ == "__main__": if args.backend == "faster-whisper": asr_cls = FasterWhisperASR + elif args.backend == "openai-api": + asr_cls = OpenaiApiASR else: asr_cls = WhisperTimestampedASR diff --git a/whisper_online_server.py b/whisper_online_server.py index b2f5120..13a85de 100644 --- a/whisper_online_server.py +++ b/whisper_online_server.py @@ -29,6 +29,8 @@ print(f"Loading Whisper {size} model for {language}...",file=sys.stderr,end=" ", if args.backend == "faster-whisper": from faster_whisper import WhisperModel asr_cls = FasterWhisperASR +elif args.backend == "openai-api": + asr_cls = OpenaiApiASR else: import whisper import whisper_timestamped From 6ec1f65fe2ebb9151792fb285826fb16efcf53ed Mon Sep 17 00:00:00 2001 From: Tijs Zwinkels Date: Thu, 25 Jan 2024 10:08:21 +0100 Subject: [PATCH 08/25] Update documentation to include openai-api backend --- README.md | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 26b6525..35f3fac 100644 --- a/README.md +++ b/README.md @@ -31,14 +31,19 @@ Please, cite us. [Bibtex citation](http://www.afnlp.org/conferences/ijcnlp2023/p ## Installation -1) ``pip install librosa`` -- audio processing library +1) ``pip install librosa soundfile`` -- audio processing library 2) Whisper backend. -Two alternative backends are integrated. The most recommended one is [faster-whisper](https://github.com/guillaumekln/faster-whisper) with GPU support. Follow their instructions for NVIDIA libraries -- we succeeded with CUDNN 8.5.0 and CUDA 11.7. Install with `pip install faster-whisper`. + Several alternative backends are integrated. The most recommended one is [faster-whisper](https://github.com/guillaumekln/faster-whisper) with GPU support. Follow their instructions for NVIDIA libraries -- we succeeded with CUDNN 8.5.0 and CUDA 11.7. Install with `pip install faster-whisper`. Alternative, less restrictive, but slower backend is [whisper-timestamped](https://github.com/linto-ai/whisper-timestamped): `pip install git+https://github.com/linto-ai/whisper-timestamped` +Thirdly, it's also possible to run this software from the [OpenAI Whisper API](https://platform.openai.com/docs/api-reference/audio/createTranscription). This solution is fast and requires no GPU, just a small VM will suffice, but you will need to pay OpenAI for api access. Also note that, since each audio fragment is processed multiple times, the [price](https://openai.com/pricing) will be higher than obvious from the pricing page, so keep an eye on costs while using. Setting a higher chunk-size will reduce costs significantly. +Install with: `pip install openai` + +For running with the openai-api backend, make sure that your [OpenAI api key](https://platform.openai.com/api-keys) is set in the `OPENAI_API_KEY` environment variable. For example, before running, do: `export OPENAI_API_KEY=sk-xxx` with *sk-xxx* replaced with your api key. + The backend is loaded only when chosen. The unused one does not have to be installed. 3) Optional, not recommended: sentence segmenter (aka sentence tokenizer) @@ -69,7 +74,7 @@ In case of installation issues of opus-fast-mosestokenizer, especially on Window ``` usage: whisper_online.py [-h] [--min-chunk-size MIN_CHUNK_SIZE] [--model {tiny.en,tiny,base.en,base,small.en,small,medium.en,medium,large-v1,large-v2,large-v3,large}] [--model_cache_dir MODEL_CACHE_DIR] [--model_dir MODEL_DIR] [--lan LAN] [--task {transcribe,translate}] - [--backend {faster-whisper,whisper_timestamped}] [--vad] [--buffer_trimming {sentence,segment}] [--buffer_trimming_sec BUFFER_TRIMMING_SEC] [--start_at START_AT] [--offline] [--comp_unaware] + [--backend {faster-whisper,whisper_timestamped,openai-api}] [--vad] [--buffer_trimming {sentence,segment}] [--buffer_trimming_sec BUFFER_TRIMMING_SEC] [--start_at START_AT] [--offline] [--comp_unaware] audio_path positional arguments: @@ -89,7 +94,7 @@ options: Source language code, e.g. en,de,cs, or 'auto' for language detection. --task {transcribe,translate} Transcribe or translate. - --backend {faster-whisper,whisper_timestamped} + --backend {faster-whisper,whisper_timestamped,openai-api} Load only this backend for Whisper processing. --vad Use VAD = voice activity detection, with the default parameters. --buffer_trimming {sentence,segment} From f8b2ae07b870077e8e3b88fed4e23ec0e118d66c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominik=20Mach=C3=A1=C4=8Dek?= Date: Thu, 25 Jan 2024 16:49:25 +0100 Subject: [PATCH 09/25] missing features in openai-api, PR #52 --- whisper_online.py | 88 ++++++++++++++++++++++++++++++----------------- 1 file changed, 56 insertions(+), 32 deletions(-) diff --git a/whisper_online.py b/whisper_online.py index edab195..e0515de 100644 --- a/whisper_online.py +++ b/whisper_online.py @@ -6,8 +6,7 @@ from functools import lru_cache import time import io import soundfile as sf - - +import math @lru_cache def load_audio(fname): @@ -153,24 +152,34 @@ class FasterWhisperASR(ASRBase): class OpenaiApiASR(ASRBase): """Uses OpenAI's Whisper API for audio transcription.""" - def __init__(self, modelsize=None, lan=None, cache_dir=None, model_dir=None, response_format="verbose_json", temperature=0): - self.modelname = "whisper-1" # modelsize is not used but kept for interface consistency + def __init__(self, lan=None, response_format="verbose_json", temperature=0, logfile=sys.stderr): + self.logfile = logfile + + self.modelname = "whisper-1" self.language = lan # ISO-639-1 language code self.response_format = response_format self.temperature = temperature - self.model = self.load_model(modelsize, cache_dir, model_dir) + + self.load_model() + + self.use_vad = False + + # reset the task in set_translate_task + self.task = "transcribe" def load_model(self, *args, **kwargs): from openai import OpenAI self.client = OpenAI() - # Since we're using the OpenAI API, there's no model to load locally. - print("Model configuration is set to use the OpenAI Whisper API.") + + self.transcribed_seconds = 0 # for logging how many seconds were processed by API, to know the cost + def ts_words(self, segments): o = [] for segment in segments: - # Skip segments containing no speech - if segment["no_speech_prob"] > 0.8: + # If VAD on, skip segments containing no speech. + # TODO: threshold can be set from outside + if self.use_vad and segment["no_speech_prob"] > 0.8: continue # Splitting the text into words and filtering out empty strings @@ -203,23 +212,39 @@ class OpenaiApiASR(ASRBase): sf.write(buffer, audio_data, samplerate=16000, format='WAV', subtype='PCM_16') buffer.seek(0) # Reset buffer's position to the beginning - # Prepare transcription parameters - transcription_params = { + self.transcribed_seconds += math.ceil(len(audio_data)/16000) # it rounds up to the whole seconds + + params = { "model": self.modelname, "file": buffer, "response_format": self.response_format, "temperature": self.temperature } - if self.language: + if self.task != "translate" and self.language: transcription_params["language"] = self.language if prompt: transcription_params["prompt"] = prompt - # Perform the transcription - transcript = self.client.audio.transcriptions.create(**transcription_params) + if self.task == "translate": + proc = self.client.audio.translations + else: + proc = self.client.audio.transcriptions + + # Process transcription/translation + + transcript = proc.create(**params) + print(f"OpenAI API processed accumulated {self.transcribed_seconds} seconds ",file=self.logfile) return transcript.segments + def use_vad(self): + self.use_vad = True + + def set_translate_task(self): + self.task = "translate" + + + class HypothesisBuffer: @@ -563,20 +588,27 @@ if __name__ == "__main__": duration = len(load_audio(audio_path))/SAMPLING_RATE print("Audio duration is: %2.2f seconds" % duration, file=logfile) - size = args.model language = args.lan - t = time.time() - print(f"Loading Whisper {size} model for {language}...",file=logfile,end=" ",flush=True) - - if args.backend == "faster-whisper": - asr_cls = FasterWhisperASR - elif args.backend == "openai-api": - asr_cls = OpenaiApiASR + if args.backend == "openai-api": + print("Using OpenAI API.",file=logfile) + asr = OpenaiApiASR(lan=language) else: - asr_cls = WhisperTimestampedASR + if args.backend == "faster-whisper": + asr_cls = FasterWhisperASR + else: + asr_cls = WhisperTimestampedASR - asr = asr_cls(modelsize=size, lan=language, cache_dir=args.model_cache_dir, model_dir=args.model_dir) + size = args.model + t = time.time() + print(f"Loading Whisper {size} model for {language}...",file=logfile,end=" ",flush=True) + asr = asr_cls(modelsize=size, lan=language, cache_dir=args.model_cache_dir, model_dir=args.model_dir) + e = time.time() + print(f"done. It took {round(e-t,2)} seconds.",file=logfile) + + if args.vad: + print("setting VAD filter",file=logfile) + asr.use_vad() if args.task == "translate": asr.set_translate_task() @@ -584,14 +616,6 @@ if __name__ == "__main__": else: tgt_language = language # Whisper transcribes in this language - - e = time.time() - print(f"done. It took {round(e-t,2)} seconds.",file=logfile) - - if args.vad: - print("setting VAD filter",file=logfile) - asr.use_vad() - min_chunk = args.min_chunk_size if args.buffer_trimming == "sentence": From 2270014219c89d3f4243438f3775aea763421d41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominik=20Mach=C3=A1=C4=8Dek?= Date: Thu, 25 Jan 2024 17:53:07 +0100 Subject: [PATCH 10/25] fixes --- whisper_online.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/whisper_online.py b/whisper_online.py index e0515de..d60cc84 100644 --- a/whisper_online.py +++ b/whisper_online.py @@ -221,9 +221,9 @@ class OpenaiApiASR(ASRBase): "temperature": self.temperature } if self.task != "translate" and self.language: - transcription_params["language"] = self.language + params["language"] = self.language if prompt: - transcription_params["prompt"] = prompt + params["prompt"] = prompt if self.task == "translate": proc = self.client.audio.translations @@ -233,7 +233,7 @@ class OpenaiApiASR(ASRBase): # Process transcription/translation transcript = proc.create(**params) - print(f"OpenAI API processed accumulated {self.transcribed_seconds} seconds ",file=self.logfile) + print(f"OpenAI API processed accumulated {self.transcribed_seconds} seconds",file=self.logfile) return transcript.segments From 531418ad07431fec99ec88b3efb73a7d476bfe9e Mon Sep 17 00:00:00 2001 From: Tijs Zwinkels Date: Thu, 8 Feb 2024 23:33:52 +0100 Subject: [PATCH 11/25] Interpolate word timestamps based on word character length --- whisper_online.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/whisper_online.py b/whisper_online.py index d60cc84..477297a 100644 --- a/whisper_online.py +++ b/whisper_online.py @@ -190,12 +190,14 @@ class OpenaiApiASR(ASRBase): # Assign start and end times for each word # We only have timestamps per segment, so interpolating start and end-times - # assuming equal duration per word + + segment_duration = segment["end"] - segment["start"] - duration_per_word = segment_duration / len(words) + total_characters = sum(len(word) for word in words) + duration_per_character = segment_duration / total_characters start_time = segment["start"] for word in words: - end_time = start_time + duration_per_word + end_time = start_time + duration_per_character * len(word) o.append((start_time, end_time, word)) start_time = end_time From 3696fef2b1d753f0ce2e27a892fb7e1954687d0f Mon Sep 17 00:00:00 2001 From: Tijs Zwinkels Date: Sat, 10 Feb 2024 14:07:15 +0100 Subject: [PATCH 12/25] Use OpenAI api word-level timestamps --- whisper_online.py | 36 ++++++++++-------------------------- 1 file changed, 10 insertions(+), 26 deletions(-) diff --git a/whisper_online.py b/whisper_online.py index 477297a..80cd68b 100644 --- a/whisper_online.py +++ b/whisper_online.py @@ -176,30 +176,14 @@ class OpenaiApiASR(ASRBase): def ts_words(self, segments): o = [] - for segment in segments: - # If VAD on, skip segments containing no speech. - # TODO: threshold can be set from outside - if self.use_vad and segment["no_speech_prob"] > 0.8: - continue + # If VAD on, skip segments containing no speech. + # TODO: threshold can be set from outside + # TODO: Make VAD work again with word-level timestamps + #if self.use_vad and segment["no_speech_prob"] > 0.8: + # continue - # Splitting the text into words and filtering out empty strings - words = [word.strip() for word in segment["text"].split() if word.strip()] - - if not words: - continue - - # Assign start and end times for each word - # We only have timestamps per segment, so interpolating start and end-times - - - segment_duration = segment["end"] - segment["start"] - total_characters = sum(len(word) for word in words) - duration_per_character = segment_duration / total_characters - start_time = segment["start"] - for word in words: - end_time = start_time + duration_per_character * len(word) - o.append((start_time, end_time, word)) - start_time = end_time + for word in segments: + o.append((word.get("start"), word.get("end"), word.get("word"))) return o @@ -220,7 +204,8 @@ class OpenaiApiASR(ASRBase): "model": self.modelname, "file": buffer, "response_format": self.response_format, - "temperature": self.temperature + "temperature": self.temperature, + "timestamp_granularities": ["word"] } if self.task != "translate" and self.language: params["language"] = self.language @@ -233,11 +218,10 @@ class OpenaiApiASR(ASRBase): proc = self.client.audio.transcriptions # Process transcription/translation - transcript = proc.create(**params) print(f"OpenAI API processed accumulated {self.transcribed_seconds} seconds",file=self.logfile) - return transcript.segments + return transcript.words def use_vad(self): self.use_vad = True From f0a24cd5e102a2f8215613ac295bddc78463cef8 Mon Sep 17 00:00:00 2001 From: Tijs Zwinkels Date: Sat, 10 Feb 2024 15:29:18 +0100 Subject: [PATCH 13/25] Make --vad work with --backend openai-api --- whisper_online.py | 38 ++++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/whisper_online.py b/whisper_online.py index 80cd68b..1bb2a28 100644 --- a/whisper_online.py +++ b/whisper_online.py @@ -162,7 +162,7 @@ class OpenaiApiASR(ASRBase): self.load_model() - self.use_vad = False + self.use_vad_opt = False # reset the task in set_translate_task self.task = "transcribe" @@ -175,21 +175,27 @@ class OpenaiApiASR(ASRBase): def ts_words(self, segments): - o = [] - # If VAD on, skip segments containing no speech. - # TODO: threshold can be set from outside - # TODO: Make VAD work again with word-level timestamps - #if self.use_vad and segment["no_speech_prob"] > 0.8: - # continue + no_speech_segments = [] + if self.use_vad_opt: + for segment in segments.segments: + # TODO: threshold can be set from outside + if segment["no_speech_prob"] > 0.8: + no_speech_segments.append((segment.get("start"), segment.get("end"))) - for word in segments: - o.append((word.get("start"), word.get("end"), word.get("word"))) + o = [] + for word in segments.words: + start = word.get("start") + end = word.get("end") + if any(s[0] <= start <= s[1] for s in no_speech_segments): + # print("Skipping word", word.get("word"), "because it's in a no-speech segment") + continue + o.append((start, end, word.get("word"))) return o def segments_end_ts(self, res): - return [s["end"] for s in res] + return [s["end"] for s in res.words] def transcribe(self, audio_data, prompt=None, *args, **kwargs): # Write the audio data to a buffer @@ -205,7 +211,7 @@ class OpenaiApiASR(ASRBase): "file": buffer, "response_format": self.response_format, "temperature": self.temperature, - "timestamp_granularities": ["word"] + "timestamp_granularities": ["word", "segment"] } if self.task != "translate" and self.language: params["language"] = self.language @@ -221,10 +227,10 @@ class OpenaiApiASR(ASRBase): transcript = proc.create(**params) print(f"OpenAI API processed accumulated {self.transcribed_seconds} seconds",file=self.logfile) - return transcript.words + return transcript def use_vad(self): - self.use_vad = True + self.use_vad_opt = True def set_translate_task(self): self.task = "translate" @@ -592,9 +598,9 @@ if __name__ == "__main__": e = time.time() print(f"done. It took {round(e-t,2)} seconds.",file=logfile) - if args.vad: - print("setting VAD filter",file=logfile) - asr.use_vad() + if args.vad: + print("setting VAD filter",file=logfile) + asr.use_vad() if args.task == "translate": asr.set_translate_task() From 922ad18ebcc117765de13692c49f024a09b7bfb9 Mon Sep 17 00:00:00 2001 From: Tijs Zwinkels Date: Wed, 14 Feb 2024 17:29:45 +0100 Subject: [PATCH 14/25] Make OpenAI backend work with language autodetect --- whisper_online.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/whisper_online.py b/whisper_online.py index 1bb2a28..8c1916e 100644 --- a/whisper_online.py +++ b/whisper_online.py @@ -152,12 +152,12 @@ class FasterWhisperASR(ASRBase): class OpenaiApiASR(ASRBase): """Uses OpenAI's Whisper API for audio transcription.""" - def __init__(self, lan=None, response_format="verbose_json", temperature=0, logfile=sys.stderr): + def __init__(self, lan=None, temperature=0, logfile=sys.stderr): self.logfile = logfile self.modelname = "whisper-1" - self.language = lan # ISO-639-1 language code - self.response_format = response_format + self.original_language = None if lan == "auto" else lan # ISO-639-1 language code + self.response_format = "verbose_json" self.temperature = temperature self.load_model() @@ -213,8 +213,8 @@ class OpenaiApiASR(ASRBase): "temperature": self.temperature, "timestamp_granularities": ["word", "segment"] } - if self.task != "translate" and self.language: - params["language"] = self.language + if self.task != "translate" and self.original_language: + params["language"] = self.original_language if prompt: params["prompt"] = prompt From 9fcd403439d95330b178354fd37d9e38af9a2d66 Mon Sep 17 00:00:00 2001 From: Tijs Zwinkels Date: Thu, 15 Feb 2024 22:24:43 +0100 Subject: [PATCH 15/25] Use automatic language detection by default (instead of English) --- whisper_online.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/whisper_online.py b/whisper_online.py index 8c1916e..7a6887d 100644 --- a/whisper_online.py +++ b/whisper_online.py @@ -546,7 +546,7 @@ def add_shared_args(parser): parser.add_argument('--model', type=str, default='large-v2', choices="tiny.en,tiny,base.en,base,small.en,small,medium.en,medium,large-v1,large-v2,large-v3,large".split(","),help="Name size of the Whisper model to use (default: large-v2). The model is automatically downloaded from the model hub if not present in model cache dir.") parser.add_argument('--model_cache_dir', type=str, default=None, help="Overriding the default model cache dir where models downloaded from the hub are saved") parser.add_argument('--model_dir', type=str, default=None, help="Dir where Whisper model.bin and other files are saved. This option overrides --model and --model_cache_dir parameter.") - parser.add_argument('--lan', '--language', type=str, default='en', help="Source language code, e.g. en,de,cs, or 'auto' for language detection.") + parser.add_argument('--lan', '--language', type=str, default='auto', help="Source language code, e.g. en,de,cs, or 'auto' for language detection.") parser.add_argument('--task', type=str, default='transcribe', choices=["transcribe","translate"],help="Transcribe or translate.") parser.add_argument('--backend', type=str, default="faster-whisper", choices=["faster-whisper", "whisper_timestamped", "openai-api"],help='Load only this backend for Whisper processing.') parser.add_argument('--vad', action="store_true", default=False, help='Use VAD = voice activity detection, with the default parameters.') From 80eb0baf5d242e952b83f79550c2388f7e0515f4 Mon Sep 17 00:00:00 2001 From: Aleksei Scripnic Date: Wed, 3 Jan 2024 10:06:05 +0000 Subject: [PATCH 16/25] Removed duplicate variable self.last_chunked_at I tried to find the difference between self.last_chunked_at and self.buffer_time_offset, and it took me a while to understand that they are exactly the same. I think it's better to get rid of one of the duplicates to make the code more readable. --- whisper_online.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/whisper_online.py b/whisper_online.py index 7672cc8..fd66319 100644 --- a/whisper_online.py +++ b/whisper_online.py @@ -328,7 +328,6 @@ class OnlineASRProcessor: self.transcript_buffer = HypothesisBuffer(logfile=self.logfile) self.commited = [] - self.last_chunked_at = 0 self.silence_iters = 0 @@ -340,7 +339,7 @@ class OnlineASRProcessor: "context" is the commited text that is inside the audio buffer. It is transcribed again and skipped. It is returned only for debugging and logging reasons. """ k = max(0,len(self.commited)-1) - while k > 0 and self.commited[k-1][1] > self.last_chunked_at: + while k > 0 and self.commited[k-1][1] > self.buffer_time_offset: k -= 1 p = self.commited[:k] @@ -451,7 +450,6 @@ class OnlineASRProcessor: cut_seconds = time - self.buffer_time_offset self.audio_buffer = self.audio_buffer[int(cut_seconds*self.SAMPLING_RATE):] self.buffer_time_offset = time - self.last_chunked_at = time def words_to_sentences(self, words): """Uses self.tokenizer for sentence segmentation of words. From db8b7d28837d69070feded662d63d02b5da6ddab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominik=20Mach=C3=A1=C4=8Dek?= Date: Tue, 20 Feb 2024 14:36:14 +0100 Subject: [PATCH 17/25] removed unused variable --- whisper_online.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/whisper_online.py b/whisper_online.py index fd66319..2794506 100644 --- a/whisper_online.py +++ b/whisper_online.py @@ -329,8 +329,6 @@ class OnlineASRProcessor: self.transcript_buffer = HypothesisBuffer(logfile=self.logfile) self.commited = [] - self.silence_iters = 0 - def insert_audio_chunk(self, audio): self.audio_buffer = np.append(self.audio_buffer, audio) From 24926c98e0891d2972a8b438bd0898b472d5f128 Mon Sep 17 00:00:00 2001 From: koiking213 Date: Tue, 20 Feb 2024 22:46:04 +0900 Subject: [PATCH 18/25] specify audio dtype --- whisper_online.py | 2 +- whisper_online_server.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/whisper_online.py b/whisper_online.py index 2794506..bcaeec4 100644 --- a/whisper_online.py +++ b/whisper_online.py @@ -11,7 +11,7 @@ import math @lru_cache def load_audio(fname): a, _ = librosa.load(fname, sr=16000) - return a + return a.astype('float32') def load_audio_chunk(fname, beg, end): audio = load_audio(fname) diff --git a/whisper_online_server.py b/whisper_online_server.py index 13a85de..a609ef0 100644 --- a/whisper_online_server.py +++ b/whisper_online_server.py @@ -138,7 +138,7 @@ class ServerProcessor: break sf = soundfile.SoundFile(io.BytesIO(raw_bytes), channels=1,endian="LITTLE",samplerate=SAMPLING_RATE, subtype="PCM_16",format="RAW") audio, _ = librosa.load(sf,sr=SAMPLING_RATE) - out.append(audio) + out.append(audio.astype('float32')) if not out: return None return np.concatenate(out) From 4405c451ce419539246b056112d44da70850c641 Mon Sep 17 00:00:00 2001 From: koiking213 Date: Tue, 20 Feb 2024 23:29:25 +0900 Subject: [PATCH 19/25] specify dtype for librosa.load, instead of cast --- whisper_online.py | 4 ++-- whisper_online_server.py | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/whisper_online.py b/whisper_online.py index bcaeec4..d79396a 100644 --- a/whisper_online.py +++ b/whisper_online.py @@ -10,8 +10,8 @@ import math @lru_cache def load_audio(fname): - a, _ = librosa.load(fname, sr=16000) - return a.astype('float32') + a, _ = librosa.load(fname, sr=16000, dtype=np.float32) + return a def load_audio_chunk(fname, beg, end): audio = load_audio(fname) diff --git a/whisper_online_server.py b/whisper_online_server.py index a609ef0..0cdc97d 100644 --- a/whisper_online_server.py +++ b/whisper_online_server.py @@ -4,6 +4,7 @@ from whisper_online import * import sys import argparse import os +import numpy as np parser = argparse.ArgumentParser() # server options @@ -137,8 +138,8 @@ class ServerProcessor: if not raw_bytes: break sf = soundfile.SoundFile(io.BytesIO(raw_bytes), channels=1,endian="LITTLE",samplerate=SAMPLING_RATE, subtype="PCM_16",format="RAW") - audio, _ = librosa.load(sf,sr=SAMPLING_RATE) - out.append(audio.astype('float32')) + audio, _ = librosa.load(sf,sr=SAMPLING_RATE,dtype=np.float32) + out.append(audio) if not out: return None return np.concatenate(out) From 5929a828969f9462f69efd39441be762bb09f58f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominik=20Mach=C3=A1=C4=8Dek?= Date: Mon, 11 Mar 2024 12:38:44 +0100 Subject: [PATCH 20/25] Update README.md bibtex update --- README.md | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 35f3fac..16d3a23 100644 --- a/README.md +++ b/README.md @@ -14,18 +14,23 @@ Demo video: https://player.vimeo.com/video/840442741 [Slides](http://ufallab.ms.mff.cuni.cz/~machacek/pre-prints/AACL23-2.11.2023-Turning-Whisper-oral.pdf) -- 15 minutes oral presentation at IJCNLP-AACL 2023 -Please, cite us. [Bibtex citation](http://www.afnlp.org/conferences/ijcnlp2023/proceedings/main-demo/cdrom/bib/2023.ijcnlp-demo.3.bib): +Please, cite us. [ACL Anthology](https://aclanthology.org/2023.ijcnlp-demo.3/), [Bibtex citation](https://aclanthology.org/2023.ijcnlp-demo.3.bib): ``` -@InProceedings{machacek-dabre-bojar:2023:ijcnlp, - author = {Macháček, Dominik and Dabre, Raj and Bojar, Ondřej}, - title = {Turning Whisper into Real-Time Transcription System}, - booktitle = {System Demonstrations}, - month = {November}, - year = {2023}, - address = {Bali, Indonesia}, - publisher = {Asian Federation of Natural Language Processing}, - pages = {17--24}, +@inproceedings{machacek-etal-2023-turning, + title = "Turning Whisper into Real-Time Transcription System", + author = "Mach{\'a}{\v{c}}ek, Dominik and + Dabre, Raj and + Bojar, Ond{\v{r}}ej", + editor = "Saha, Sriparna and + Sujaini, Herry", + booktitle = "Proceedings of the 13th International Joint Conference on Natural Language Processing and the 3rd Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics: System Demonstrations", + month = nov, + year = "2023", + address = "Bali, Indonesia", + publisher = "Association for Computational Linguistics", + url = "https://aclanthology.org/2023.ijcnlp-demo.3", + pages = "17--24", } ``` From 8896389ea3e7b142b21acc7a2ade9d42bd39cedd Mon Sep 17 00:00:00 2001 From: Tijs Zwinkels Date: Wed, 20 Mar 2024 15:29:10 +0100 Subject: [PATCH 21/25] Fix crash when using openai-api with whisper_online_server + refactored creation of the ASR into a factory method --- whisper_online.py | 53 ++++++++++++++++++++++++---------------- whisper_online_server.py | 25 +------------------ 2 files changed, 33 insertions(+), 45 deletions(-) diff --git a/whisper_online.py b/whisper_online.py index d79396a..c90babb 100644 --- a/whisper_online.py +++ b/whisper_online.py @@ -548,6 +548,37 @@ def add_shared_args(parser): parser.add_argument('--buffer_trimming', type=str, default="segment", choices=["sentence", "segment"],help='Buffer trimming strategy -- trim completed sentences marked with punctuation mark and detected by sentence segmenter, or the completed segments returned by Whisper. Sentence segmenter must be installed for "sentence" option.') parser.add_argument('--buffer_trimming_sec', type=float, default=15, help='Buffer trimming length threshold in seconds. If buffer length is longer, trimming sentence/segment is triggered.') +def asr_factory(args, logfile=sys.stderr): + """ + Creates and configures an ASR instance based on the specified backend and arguments. + """ + backend = args.backend + if backend == "openai-api": + print("Using OpenAI API.", file=logfile) + asr = OpenaiApiASR(lan=args.lan) + else: + if backend == "faster-whisper": + from faster_whisper import FasterWhisperASR + asr_cls = FasterWhisperASR + else: + from whisper_timestamped import WhisperTimestampedASR + asr_cls = WhisperTimestampedASR + + # Only for FasterWhisperASR and WhisperTimestampedASR + size = args.model + t = time.time() + print(f"Loading Whisper {size} model for {args.lan}...", file=logfile, end=" ", flush=True) + asr = asr_cls(modelsize=size, lan=args.lan, cache_dir=args.model_cache_dir, model_dir=args.model_dir) + e = time.time() + print(f"done. It took {round(e-t,2)} seconds.", file=logfile) + + # Apply common configurations + if getattr(args, 'vad', False): # Checks if VAD argument is present and True + print("Setting VAD filter", file=logfile) + asr.use_vad() + + return asr + ## main: if __name__ == "__main__": @@ -575,28 +606,8 @@ if __name__ == "__main__": duration = len(load_audio(audio_path))/SAMPLING_RATE print("Audio duration is: %2.2f seconds" % duration, file=logfile) + asr = asr_factory(args, logfile=logfile) language = args.lan - - if args.backend == "openai-api": - print("Using OpenAI API.",file=logfile) - asr = OpenaiApiASR(lan=language) - else: - if args.backend == "faster-whisper": - asr_cls = FasterWhisperASR - else: - asr_cls = WhisperTimestampedASR - - size = args.model - t = time.time() - print(f"Loading Whisper {size} model for {language}...",file=logfile,end=" ",flush=True) - asr = asr_cls(modelsize=size, lan=language, cache_dir=args.model_cache_dir, model_dir=args.model_dir) - e = time.time() - print(f"done. It took {round(e-t,2)} seconds.",file=logfile) - - if args.vad: - print("setting VAD filter",file=logfile) - asr.use_vad() - if args.task == "translate": asr.set_translate_task() tgt_language = "en" # Whisper translates into English diff --git a/whisper_online_server.py b/whisper_online_server.py index 0cdc97d..7f81caa 100644 --- a/whisper_online_server.py +++ b/whisper_online_server.py @@ -24,36 +24,13 @@ SAMPLING_RATE = 16000 size = args.model language = args.lan -t = time.time() -print(f"Loading Whisper {size} model for {language}...",file=sys.stderr,end=" ",flush=True) - -if args.backend == "faster-whisper": - from faster_whisper import WhisperModel - asr_cls = FasterWhisperASR -elif args.backend == "openai-api": - asr_cls = OpenaiApiASR -else: - import whisper - import whisper_timestamped -# from whisper_timestamped_model import WhisperTimestampedASR - asr_cls = WhisperTimestampedASR - -asr = asr_cls(modelsize=size, lan=language, cache_dir=args.model_cache_dir, model_dir=args.model_dir) - +asr = asr_factory(args) if args.task == "translate": asr.set_translate_task() tgt_language = "en" else: tgt_language = language -e = time.time() -print(f"done. It took {round(e-t,2)} seconds.",file=sys.stderr) - -if args.vad: - print("setting VAD filter",file=sys.stderr) - asr.use_vad() - - min_chunk = args.min_chunk_size if args.buffer_trimming == "sentence": From 006de3e7b044533bd89c0f228a7ce388bed41a51 Mon Sep 17 00:00:00 2001 From: Tijs Zwinkels Date: Wed, 20 Mar 2024 16:02:24 +0100 Subject: [PATCH 22/25] Fix imports Now, the ASR implementations do their own imports. No need to import in the factory --- whisper_online.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/whisper_online.py b/whisper_online.py index c90babb..a00547e 100644 --- a/whisper_online.py +++ b/whisper_online.py @@ -58,6 +58,7 @@ class WhisperTimestampedASR(ASRBase): def load_model(self, modelsize=None, cache_dir=None, model_dir=None): import whisper + import whisper_timestamped from whisper_timestamped import transcribe_timestamped self.transcribe_timestamped = transcribe_timestamped if model_dir is not None: @@ -558,10 +559,8 @@ def asr_factory(args, logfile=sys.stderr): asr = OpenaiApiASR(lan=args.lan) else: if backend == "faster-whisper": - from faster_whisper import FasterWhisperASR asr_cls = FasterWhisperASR else: - from whisper_timestamped import WhisperTimestampedASR asr_cls = WhisperTimestampedASR # Only for FasterWhisperASR and WhisperTimestampedASR From b3647da0875f9f4f5d2e57b0a57efdc1cf5cfa10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominik=20Mach=C3=A1=C4=8Dek?= Date: Fri, 29 Mar 2024 09:19:59 +0100 Subject: [PATCH 23/25] Update README.md PDF link --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 16d3a23..11cc555 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,9 @@ Demonstration paper, by Dominik Macháček, Raj Dabre, Ondřej Bojar, 2023 Abstract: Whisper is one of the recent state-of-the-art multilingual speech recognition and translation models, however, it is not designed for real time transcription. In this paper, we build on top of Whisper and create Whisper-Streaming, an implementation of real-time speech transcription and translation of Whisper-like models. Whisper-Streaming uses local agreement policy with self-adaptive latency to enable streaming transcription. We show that Whisper-Streaming achieves high quality and 3.3 seconds latency on unsegmented long-form speech transcription test set, and we demonstrate its robustness and practical usability as a component in live transcription service at a multilingual conference. -Paper in proceedings: http://www.afnlp.org/conferences/ijcnlp2023/proceedings/main-demo/cdrom/pdf/2023.ijcnlp-demo.3.pdf +Paper PDF: +https://aclanthology.org/2023.ijcnlp-demo.3.pdf + Demo video: https://player.vimeo.com/video/840442741 From 8223afee7888607879870ae11acd246e4f7a6ae2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20G=C3=A5rdhus?= Date: Fri, 29 Mar 2024 19:35:30 +0100 Subject: [PATCH 24/25] Update README.md Add Python syntax highlighting to code chunk --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 11cc555..ae63e19 100644 --- a/README.md +++ b/README.md @@ -159,7 +159,7 @@ The code whisper_online.py is nicely commented, read it as the full documentatio This pseudocode describes the interface that we suggest for your implementation. You can implement any features that you need for your application. -``` +```python from whisper_online import * src_lan = "en" # source language From d497503b5c86b6e4cf21eb9f1f4eaa4e60e1b1f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominik=20Mach=C3=A1=C4=8Dek?= Date: Wed, 10 Apr 2024 18:13:07 +0200 Subject: [PATCH 25/25] COntributions at README.md + nicer formatting + #77 --- README.md | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index ae63e19..707e58d 100644 --- a/README.md +++ b/README.md @@ -3,16 +3,12 @@ Whisper realtime streaming for long speech-to-text transcription and translation **Turning Whisper into Real-Time Transcription System** -Demonstration paper, by Dominik Macháček, Raj Dabre, Ondřej Bojar, 2023 +Demonstration paper, by [Dominik Macháček](https://ufal.mff.cuni.cz/dominik-machacek), [Raj Dabre](https://prajdabre.github.io/), [Ondřej Bojar](https://ufal.mff.cuni.cz/ondrej-bojar), 2023 -Abstract: Whisper is one of the recent state-of-the-art multilingual speech recognition and translation models, however, it is not designed for real time transcription. In this paper, we build on top of Whisper and create Whisper-Streaming, an implementation of real-time speech transcription and translation of Whisper-like models. Whisper-Streaming uses local agreement policy with self-adaptive latency to enable streaming transcription. We show that Whisper-Streaming achieves high quality and 3.3 seconds latency on unsegmented long-form speech transcription test set, and we demonstrate its robustness and practical usability as a component in live transcription service at a multilingual conference. +Abstract: Whisper is one of the recent state-of-the-art multilingual speech recognition and translation models, however, it is not designed for real-time transcription. In this paper, we build on top of Whisper and create Whisper-Streaming, an implementation of real-time speech transcription and translation of Whisper-like models. Whisper-Streaming uses local agreement policy with self-adaptive latency to enable streaming transcription. We show that Whisper-Streaming achieves high quality and 3.3 seconds latency on unsegmented long-form speech transcription test set, and we demonstrate its robustness and practical usability as a component in live transcription service at a multilingual conference. -Paper PDF: -https://aclanthology.org/2023.ijcnlp-demo.3.pdf - - -Demo video: https://player.vimeo.com/video/840442741 +[Paper PDF](https://aclanthology.org/2023.ijcnlp-demo.3.pdf), [Demo video](https://player.vimeo.com/video/840442741) [Slides](http://ufallab.ms.mff.cuni.cz/~machacek/pre-prints/AACL23-2.11.2023-Turning-Whisper-oral.pdf) -- 15 minutes oral presentation at IJCNLP-AACL 2023 @@ -228,12 +224,20 @@ In more detail: we use the init prompt, we handle the inaccurate timestamps, we re-process confirmed sentence prefixes and skip them, making sure they don't overlap, and we limit the processing buffer window. -Contributions are welcome. - ### Performance evaluation [See the paper.](http://www.afnlp.org/conferences/ijcnlp2023/proceedings/main-demo/cdrom/pdf/2023.ijcnlp-demo.3.pdf) +### Contributions + +Contributions are welcome. We acknowledge especially: + +- [The GitHub contributors](https://github.com/ufal/whisper_streaming/graphs/contributors) for their pull requests with new features and bugfixes. +- [The translation of this repo into Chinese.](https://github.com/Gloridust/whisper_streaming_CN) +- [Ondřej Plátek](https://opla.cz/) for the paper pre-review. +- [Peter Polák](https://ufal.mff.cuni.cz/peter-polak) for the original idea. +- The UEDIN team of the [ELITR project](https://elitr.eu) for the original line_packet.py. + ## Contact