From cd221a31988b3426a9f2fb86af33629e2f56c8ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominik=20Mach=C3=A1=C4=8Dek?= Date: Tue, 6 Feb 2024 14:29:30 +0100 Subject: [PATCH 01/11] auto language detection #56 --- whisper_online.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/whisper_online.py b/whisper_online.py index 36bdbd6..59d41e7 100644 --- a/whisper_online.py +++ b/whisper_online.py @@ -30,7 +30,10 @@ class ASRBase: self.logfile = logfile self.transcribe_kargs = {} - self.original_language = lan + if lan == "auto": + self.original_language = None + else: + self.original_language = lan self.model = self.load_model(modelsize, cache_dir, model_dir) @@ -118,8 +121,11 @@ class FasterWhisperASR(ASRBase): return model def transcribe(self, audio, init_prompt=""): + # tested: beam_size=5 is faster and better than 1 (on one 200 second document from En ESIC, min chunk 0.01) segments, info = self.model.transcribe(audio, language=self.original_language, initial_prompt=init_prompt, beam_size=5, word_timestamps=True, condition_on_previous_text=True, **self.transcribe_kargs) + #print(info) # info contains language detection result + return list(segments) def ts_words(self, segments): @@ -451,7 +457,7 @@ def add_shared_args(parser): parser.add_argument('--model', type=str, default='large-v2', choices="tiny.en,tiny,base.en,base,small.en,small,medium.en,medium,large-v1,large-v2,large-v3,large".split(","),help="Name size of the Whisper model to use (default: large-v2). The model is automatically downloaded from the model hub if not present in model cache dir.") parser.add_argument('--model_cache_dir', type=str, default=None, help="Overriding the default model cache dir where models downloaded from the hub are saved") parser.add_argument('--model_dir', type=str, default=None, help="Dir where Whisper model.bin and other files are saved. This option overrides --model and --model_cache_dir parameter.") - parser.add_argument('--lan', '--language', type=str, default='en', help="Language code for transcription, e.g. en,de,cs.") + parser.add_argument('--lan', '--language', type=str, default='en', help="Source language code, e.g. en,de,cs, or 'auto' for language detection.") parser.add_argument('--task', type=str, default='transcribe', choices=["transcribe","translate"],help="Transcribe or translate.") parser.add_argument('--backend', type=str, default="faster-whisper", choices=["faster-whisper", "whisper_timestamped"],help='Load only this backend for Whisper processing.') parser.add_argument('--vad', action="store_true", default=False, help='Use VAD = voice activity detection, with the default parameters.') From b66c61cf7a217716b6235e62d9cffce2ce3abd35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominik=20Mach=C3=A1=C4=8Dek?= Date: Tue, 6 Feb 2024 14:31:24 +0100 Subject: [PATCH 02/11] README update auto language detection --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 584f3d0..26b6525 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,7 @@ options: --model_dir MODEL_DIR Dir where Whisper model.bin and other files are saved. This option overrides --model and --model_cache_dir parameter. --lan LAN, --language LAN - Language code for transcription, e.g. en,de,cs. + Source language code, e.g. en,de,cs, or 'auto' for language detection. --task {transcribe,translate} Transcribe or translate. --backend {faster-whisper,whisper_timestamped} From f412812082164a33e97ac1226d915c549e345619 Mon Sep 17 00:00:00 2001 From: Tijs Zwinkels Date: Wed, 24 Jan 2024 15:31:18 +0100 Subject: [PATCH 03/11] OpenAI Whisper API backend --- whisper_online.py | 76 +++++++++++++++++++++++++++++++++++++++- whisper_online_server.py | 2 ++ 2 files changed, 77 insertions(+), 1 deletion(-) diff --git a/whisper_online.py b/whisper_online.py index 59d41e7..edab195 100644 --- a/whisper_online.py +++ b/whisper_online.py @@ -4,6 +4,8 @@ import numpy as np import librosa from functools import lru_cache import time +import io +import soundfile as sf @@ -148,6 +150,76 @@ class FasterWhisperASR(ASRBase): self.transcribe_kargs["task"] = "translate" +class OpenaiApiASR(ASRBase): + """Uses OpenAI's Whisper API for audio transcription.""" + + def __init__(self, modelsize=None, lan=None, cache_dir=None, model_dir=None, response_format="verbose_json", temperature=0): + self.modelname = "whisper-1" # modelsize is not used but kept for interface consistency + self.language = lan # ISO-639-1 language code + self.response_format = response_format + self.temperature = temperature + self.model = self.load_model(modelsize, cache_dir, model_dir) + + def load_model(self, *args, **kwargs): + from openai import OpenAI + self.client = OpenAI() + # Since we're using the OpenAI API, there's no model to load locally. + print("Model configuration is set to use the OpenAI Whisper API.") + + def ts_words(self, segments): + o = [] + for segment in segments: + # Skip segments containing no speech + if segment["no_speech_prob"] > 0.8: + continue + + # Splitting the text into words and filtering out empty strings + words = [word.strip() for word in segment["text"].split() if word.strip()] + + if not words: + continue + + # Assign start and end times for each word + # We only have timestamps per segment, so interpolating start and end-times + # assuming equal duration per word + segment_duration = segment["end"] - segment["start"] + duration_per_word = segment_duration / len(words) + start_time = segment["start"] + for word in words: + end_time = start_time + duration_per_word + o.append((start_time, end_time, word)) + start_time = end_time + + return o + + + def segments_end_ts(self, res): + return [s["end"] for s in res] + + def transcribe(self, audio_data, prompt=None, *args, **kwargs): + # Write the audio data to a buffer + buffer = io.BytesIO() + buffer.name = "temp.wav" + sf.write(buffer, audio_data, samplerate=16000, format='WAV', subtype='PCM_16') + buffer.seek(0) # Reset buffer's position to the beginning + + # Prepare transcription parameters + transcription_params = { + "model": self.modelname, + "file": buffer, + "response_format": self.response_format, + "temperature": self.temperature + } + if self.language: + transcription_params["language"] = self.language + if prompt: + transcription_params["prompt"] = prompt + + # Perform the transcription + transcript = self.client.audio.transcriptions.create(**transcription_params) + + return transcript.segments + class HypothesisBuffer: @@ -459,7 +531,7 @@ def add_shared_args(parser): parser.add_argument('--model_dir', type=str, default=None, help="Dir where Whisper model.bin and other files are saved. This option overrides --model and --model_cache_dir parameter.") parser.add_argument('--lan', '--language', type=str, default='en', help="Source language code, e.g. en,de,cs, or 'auto' for language detection.") parser.add_argument('--task', type=str, default='transcribe', choices=["transcribe","translate"],help="Transcribe or translate.") - parser.add_argument('--backend', type=str, default="faster-whisper", choices=["faster-whisper", "whisper_timestamped"],help='Load only this backend for Whisper processing.') + parser.add_argument('--backend', type=str, default="faster-whisper", choices=["faster-whisper", "whisper_timestamped", "openai-api"],help='Load only this backend for Whisper processing.') parser.add_argument('--vad', action="store_true", default=False, help='Use VAD = voice activity detection, with the default parameters.') parser.add_argument('--buffer_trimming', type=str, default="segment", choices=["sentence", "segment"],help='Buffer trimming strategy -- trim completed sentences marked with punctuation mark and detected by sentence segmenter, or the completed segments returned by Whisper. Sentence segmenter must be installed for "sentence" option.') parser.add_argument('--buffer_trimming_sec', type=float, default=15, help='Buffer trimming length threshold in seconds. If buffer length is longer, trimming sentence/segment is triggered.') @@ -499,6 +571,8 @@ if __name__ == "__main__": if args.backend == "faster-whisper": asr_cls = FasterWhisperASR + elif args.backend == "openai-api": + asr_cls = OpenaiApiASR else: asr_cls = WhisperTimestampedASR diff --git a/whisper_online_server.py b/whisper_online_server.py index b2f5120..13a85de 100644 --- a/whisper_online_server.py +++ b/whisper_online_server.py @@ -29,6 +29,8 @@ print(f"Loading Whisper {size} model for {language}...",file=sys.stderr,end=" ", if args.backend == "faster-whisper": from faster_whisper import WhisperModel asr_cls = FasterWhisperASR +elif args.backend == "openai-api": + asr_cls = OpenaiApiASR else: import whisper import whisper_timestamped From 6ec1f65fe2ebb9151792fb285826fb16efcf53ed Mon Sep 17 00:00:00 2001 From: Tijs Zwinkels Date: Thu, 25 Jan 2024 10:08:21 +0100 Subject: [PATCH 04/11] Update documentation to include openai-api backend --- README.md | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 26b6525..35f3fac 100644 --- a/README.md +++ b/README.md @@ -31,14 +31,19 @@ Please, cite us. [Bibtex citation](http://www.afnlp.org/conferences/ijcnlp2023/p ## Installation -1) ``pip install librosa`` -- audio processing library +1) ``pip install librosa soundfile`` -- audio processing library 2) Whisper backend. -Two alternative backends are integrated. The most recommended one is [faster-whisper](https://github.com/guillaumekln/faster-whisper) with GPU support. Follow their instructions for NVIDIA libraries -- we succeeded with CUDNN 8.5.0 and CUDA 11.7. Install with `pip install faster-whisper`. + Several alternative backends are integrated. The most recommended one is [faster-whisper](https://github.com/guillaumekln/faster-whisper) with GPU support. Follow their instructions for NVIDIA libraries -- we succeeded with CUDNN 8.5.0 and CUDA 11.7. Install with `pip install faster-whisper`. Alternative, less restrictive, but slower backend is [whisper-timestamped](https://github.com/linto-ai/whisper-timestamped): `pip install git+https://github.com/linto-ai/whisper-timestamped` +Thirdly, it's also possible to run this software from the [OpenAI Whisper API](https://platform.openai.com/docs/api-reference/audio/createTranscription). This solution is fast and requires no GPU, just a small VM will suffice, but you will need to pay OpenAI for api access. Also note that, since each audio fragment is processed multiple times, the [price](https://openai.com/pricing) will be higher than obvious from the pricing page, so keep an eye on costs while using. Setting a higher chunk-size will reduce costs significantly. +Install with: `pip install openai` + +For running with the openai-api backend, make sure that your [OpenAI api key](https://platform.openai.com/api-keys) is set in the `OPENAI_API_KEY` environment variable. For example, before running, do: `export OPENAI_API_KEY=sk-xxx` with *sk-xxx* replaced with your api key. + The backend is loaded only when chosen. The unused one does not have to be installed. 3) Optional, not recommended: sentence segmenter (aka sentence tokenizer) @@ -69,7 +74,7 @@ In case of installation issues of opus-fast-mosestokenizer, especially on Window ``` usage: whisper_online.py [-h] [--min-chunk-size MIN_CHUNK_SIZE] [--model {tiny.en,tiny,base.en,base,small.en,small,medium.en,medium,large-v1,large-v2,large-v3,large}] [--model_cache_dir MODEL_CACHE_DIR] [--model_dir MODEL_DIR] [--lan LAN] [--task {transcribe,translate}] - [--backend {faster-whisper,whisper_timestamped}] [--vad] [--buffer_trimming {sentence,segment}] [--buffer_trimming_sec BUFFER_TRIMMING_SEC] [--start_at START_AT] [--offline] [--comp_unaware] + [--backend {faster-whisper,whisper_timestamped,openai-api}] [--vad] [--buffer_trimming {sentence,segment}] [--buffer_trimming_sec BUFFER_TRIMMING_SEC] [--start_at START_AT] [--offline] [--comp_unaware] audio_path positional arguments: @@ -89,7 +94,7 @@ options: Source language code, e.g. en,de,cs, or 'auto' for language detection. --task {transcribe,translate} Transcribe or translate. - --backend {faster-whisper,whisper_timestamped} + --backend {faster-whisper,whisper_timestamped,openai-api} Load only this backend for Whisper processing. --vad Use VAD = voice activity detection, with the default parameters. --buffer_trimming {sentence,segment} From f8b2ae07b870077e8e3b88fed4e23ec0e118d66c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominik=20Mach=C3=A1=C4=8Dek?= Date: Thu, 25 Jan 2024 16:49:25 +0100 Subject: [PATCH 05/11] missing features in openai-api, PR #52 --- whisper_online.py | 88 ++++++++++++++++++++++++++++++----------------- 1 file changed, 56 insertions(+), 32 deletions(-) diff --git a/whisper_online.py b/whisper_online.py index edab195..e0515de 100644 --- a/whisper_online.py +++ b/whisper_online.py @@ -6,8 +6,7 @@ from functools import lru_cache import time import io import soundfile as sf - - +import math @lru_cache def load_audio(fname): @@ -153,24 +152,34 @@ class FasterWhisperASR(ASRBase): class OpenaiApiASR(ASRBase): """Uses OpenAI's Whisper API for audio transcription.""" - def __init__(self, modelsize=None, lan=None, cache_dir=None, model_dir=None, response_format="verbose_json", temperature=0): - self.modelname = "whisper-1" # modelsize is not used but kept for interface consistency + def __init__(self, lan=None, response_format="verbose_json", temperature=0, logfile=sys.stderr): + self.logfile = logfile + + self.modelname = "whisper-1" self.language = lan # ISO-639-1 language code self.response_format = response_format self.temperature = temperature - self.model = self.load_model(modelsize, cache_dir, model_dir) + + self.load_model() + + self.use_vad = False + + # reset the task in set_translate_task + self.task = "transcribe" def load_model(self, *args, **kwargs): from openai import OpenAI self.client = OpenAI() - # Since we're using the OpenAI API, there's no model to load locally. - print("Model configuration is set to use the OpenAI Whisper API.") + + self.transcribed_seconds = 0 # for logging how many seconds were processed by API, to know the cost + def ts_words(self, segments): o = [] for segment in segments: - # Skip segments containing no speech - if segment["no_speech_prob"] > 0.8: + # If VAD on, skip segments containing no speech. + # TODO: threshold can be set from outside + if self.use_vad and segment["no_speech_prob"] > 0.8: continue # Splitting the text into words and filtering out empty strings @@ -203,23 +212,39 @@ class OpenaiApiASR(ASRBase): sf.write(buffer, audio_data, samplerate=16000, format='WAV', subtype='PCM_16') buffer.seek(0) # Reset buffer's position to the beginning - # Prepare transcription parameters - transcription_params = { + self.transcribed_seconds += math.ceil(len(audio_data)/16000) # it rounds up to the whole seconds + + params = { "model": self.modelname, "file": buffer, "response_format": self.response_format, "temperature": self.temperature } - if self.language: + if self.task != "translate" and self.language: transcription_params["language"] = self.language if prompt: transcription_params["prompt"] = prompt - # Perform the transcription - transcript = self.client.audio.transcriptions.create(**transcription_params) + if self.task == "translate": + proc = self.client.audio.translations + else: + proc = self.client.audio.transcriptions + + # Process transcription/translation + + transcript = proc.create(**params) + print(f"OpenAI API processed accumulated {self.transcribed_seconds} seconds ",file=self.logfile) return transcript.segments + def use_vad(self): + self.use_vad = True + + def set_translate_task(self): + self.task = "translate" + + + class HypothesisBuffer: @@ -563,20 +588,27 @@ if __name__ == "__main__": duration = len(load_audio(audio_path))/SAMPLING_RATE print("Audio duration is: %2.2f seconds" % duration, file=logfile) - size = args.model language = args.lan - t = time.time() - print(f"Loading Whisper {size} model for {language}...",file=logfile,end=" ",flush=True) - - if args.backend == "faster-whisper": - asr_cls = FasterWhisperASR - elif args.backend == "openai-api": - asr_cls = OpenaiApiASR + if args.backend == "openai-api": + print("Using OpenAI API.",file=logfile) + asr = OpenaiApiASR(lan=language) else: - asr_cls = WhisperTimestampedASR + if args.backend == "faster-whisper": + asr_cls = FasterWhisperASR + else: + asr_cls = WhisperTimestampedASR - asr = asr_cls(modelsize=size, lan=language, cache_dir=args.model_cache_dir, model_dir=args.model_dir) + size = args.model + t = time.time() + print(f"Loading Whisper {size} model for {language}...",file=logfile,end=" ",flush=True) + asr = asr_cls(modelsize=size, lan=language, cache_dir=args.model_cache_dir, model_dir=args.model_dir) + e = time.time() + print(f"done. It took {round(e-t,2)} seconds.",file=logfile) + + if args.vad: + print("setting VAD filter",file=logfile) + asr.use_vad() if args.task == "translate": asr.set_translate_task() @@ -584,14 +616,6 @@ if __name__ == "__main__": else: tgt_language = language # Whisper transcribes in this language - - e = time.time() - print(f"done. It took {round(e-t,2)} seconds.",file=logfile) - - if args.vad: - print("setting VAD filter",file=logfile) - asr.use_vad() - min_chunk = args.min_chunk_size if args.buffer_trimming == "sentence": From 2270014219c89d3f4243438f3775aea763421d41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominik=20Mach=C3=A1=C4=8Dek?= Date: Thu, 25 Jan 2024 17:53:07 +0100 Subject: [PATCH 06/11] fixes --- whisper_online.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/whisper_online.py b/whisper_online.py index e0515de..d60cc84 100644 --- a/whisper_online.py +++ b/whisper_online.py @@ -221,9 +221,9 @@ class OpenaiApiASR(ASRBase): "temperature": self.temperature } if self.task != "translate" and self.language: - transcription_params["language"] = self.language + params["language"] = self.language if prompt: - transcription_params["prompt"] = prompt + params["prompt"] = prompt if self.task == "translate": proc = self.client.audio.translations @@ -233,7 +233,7 @@ class OpenaiApiASR(ASRBase): # Process transcription/translation transcript = proc.create(**params) - print(f"OpenAI API processed accumulated {self.transcribed_seconds} seconds ",file=self.logfile) + print(f"OpenAI API processed accumulated {self.transcribed_seconds} seconds",file=self.logfile) return transcript.segments From 531418ad07431fec99ec88b3efb73a7d476bfe9e Mon Sep 17 00:00:00 2001 From: Tijs Zwinkels Date: Thu, 8 Feb 2024 23:33:52 +0100 Subject: [PATCH 07/11] Interpolate word timestamps based on word character length --- whisper_online.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/whisper_online.py b/whisper_online.py index d60cc84..477297a 100644 --- a/whisper_online.py +++ b/whisper_online.py @@ -190,12 +190,14 @@ class OpenaiApiASR(ASRBase): # Assign start and end times for each word # We only have timestamps per segment, so interpolating start and end-times - # assuming equal duration per word + + segment_duration = segment["end"] - segment["start"] - duration_per_word = segment_duration / len(words) + total_characters = sum(len(word) for word in words) + duration_per_character = segment_duration / total_characters start_time = segment["start"] for word in words: - end_time = start_time + duration_per_word + end_time = start_time + duration_per_character * len(word) o.append((start_time, end_time, word)) start_time = end_time From 3696fef2b1d753f0ce2e27a892fb7e1954687d0f Mon Sep 17 00:00:00 2001 From: Tijs Zwinkels Date: Sat, 10 Feb 2024 14:07:15 +0100 Subject: [PATCH 08/11] Use OpenAI api word-level timestamps --- whisper_online.py | 36 ++++++++++-------------------------- 1 file changed, 10 insertions(+), 26 deletions(-) diff --git a/whisper_online.py b/whisper_online.py index 477297a..80cd68b 100644 --- a/whisper_online.py +++ b/whisper_online.py @@ -176,30 +176,14 @@ class OpenaiApiASR(ASRBase): def ts_words(self, segments): o = [] - for segment in segments: - # If VAD on, skip segments containing no speech. - # TODO: threshold can be set from outside - if self.use_vad and segment["no_speech_prob"] > 0.8: - continue + # If VAD on, skip segments containing no speech. + # TODO: threshold can be set from outside + # TODO: Make VAD work again with word-level timestamps + #if self.use_vad and segment["no_speech_prob"] > 0.8: + # continue - # Splitting the text into words and filtering out empty strings - words = [word.strip() for word in segment["text"].split() if word.strip()] - - if not words: - continue - - # Assign start and end times for each word - # We only have timestamps per segment, so interpolating start and end-times - - - segment_duration = segment["end"] - segment["start"] - total_characters = sum(len(word) for word in words) - duration_per_character = segment_duration / total_characters - start_time = segment["start"] - for word in words: - end_time = start_time + duration_per_character * len(word) - o.append((start_time, end_time, word)) - start_time = end_time + for word in segments: + o.append((word.get("start"), word.get("end"), word.get("word"))) return o @@ -220,7 +204,8 @@ class OpenaiApiASR(ASRBase): "model": self.modelname, "file": buffer, "response_format": self.response_format, - "temperature": self.temperature + "temperature": self.temperature, + "timestamp_granularities": ["word"] } if self.task != "translate" and self.language: params["language"] = self.language @@ -233,11 +218,10 @@ class OpenaiApiASR(ASRBase): proc = self.client.audio.transcriptions # Process transcription/translation - transcript = proc.create(**params) print(f"OpenAI API processed accumulated {self.transcribed_seconds} seconds",file=self.logfile) - return transcript.segments + return transcript.words def use_vad(self): self.use_vad = True From f0a24cd5e102a2f8215613ac295bddc78463cef8 Mon Sep 17 00:00:00 2001 From: Tijs Zwinkels Date: Sat, 10 Feb 2024 15:29:18 +0100 Subject: [PATCH 09/11] Make --vad work with --backend openai-api --- whisper_online.py | 38 ++++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/whisper_online.py b/whisper_online.py index 80cd68b..1bb2a28 100644 --- a/whisper_online.py +++ b/whisper_online.py @@ -162,7 +162,7 @@ class OpenaiApiASR(ASRBase): self.load_model() - self.use_vad = False + self.use_vad_opt = False # reset the task in set_translate_task self.task = "transcribe" @@ -175,21 +175,27 @@ class OpenaiApiASR(ASRBase): def ts_words(self, segments): - o = [] - # If VAD on, skip segments containing no speech. - # TODO: threshold can be set from outside - # TODO: Make VAD work again with word-level timestamps - #if self.use_vad and segment["no_speech_prob"] > 0.8: - # continue + no_speech_segments = [] + if self.use_vad_opt: + for segment in segments.segments: + # TODO: threshold can be set from outside + if segment["no_speech_prob"] > 0.8: + no_speech_segments.append((segment.get("start"), segment.get("end"))) - for word in segments: - o.append((word.get("start"), word.get("end"), word.get("word"))) + o = [] + for word in segments.words: + start = word.get("start") + end = word.get("end") + if any(s[0] <= start <= s[1] for s in no_speech_segments): + # print("Skipping word", word.get("word"), "because it's in a no-speech segment") + continue + o.append((start, end, word.get("word"))) return o def segments_end_ts(self, res): - return [s["end"] for s in res] + return [s["end"] for s in res.words] def transcribe(self, audio_data, prompt=None, *args, **kwargs): # Write the audio data to a buffer @@ -205,7 +211,7 @@ class OpenaiApiASR(ASRBase): "file": buffer, "response_format": self.response_format, "temperature": self.temperature, - "timestamp_granularities": ["word"] + "timestamp_granularities": ["word", "segment"] } if self.task != "translate" and self.language: params["language"] = self.language @@ -221,10 +227,10 @@ class OpenaiApiASR(ASRBase): transcript = proc.create(**params) print(f"OpenAI API processed accumulated {self.transcribed_seconds} seconds",file=self.logfile) - return transcript.words + return transcript def use_vad(self): - self.use_vad = True + self.use_vad_opt = True def set_translate_task(self): self.task = "translate" @@ -592,9 +598,9 @@ if __name__ == "__main__": e = time.time() print(f"done. It took {round(e-t,2)} seconds.",file=logfile) - if args.vad: - print("setting VAD filter",file=logfile) - asr.use_vad() + if args.vad: + print("setting VAD filter",file=logfile) + asr.use_vad() if args.task == "translate": asr.set_translate_task() From 922ad18ebcc117765de13692c49f024a09b7bfb9 Mon Sep 17 00:00:00 2001 From: Tijs Zwinkels Date: Wed, 14 Feb 2024 17:29:45 +0100 Subject: [PATCH 10/11] Make OpenAI backend work with language autodetect --- whisper_online.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/whisper_online.py b/whisper_online.py index 1bb2a28..8c1916e 100644 --- a/whisper_online.py +++ b/whisper_online.py @@ -152,12 +152,12 @@ class FasterWhisperASR(ASRBase): class OpenaiApiASR(ASRBase): """Uses OpenAI's Whisper API for audio transcription.""" - def __init__(self, lan=None, response_format="verbose_json", temperature=0, logfile=sys.stderr): + def __init__(self, lan=None, temperature=0, logfile=sys.stderr): self.logfile = logfile self.modelname = "whisper-1" - self.language = lan # ISO-639-1 language code - self.response_format = response_format + self.original_language = None if lan == "auto" else lan # ISO-639-1 language code + self.response_format = "verbose_json" self.temperature = temperature self.load_model() @@ -213,8 +213,8 @@ class OpenaiApiASR(ASRBase): "temperature": self.temperature, "timestamp_granularities": ["word", "segment"] } - if self.task != "translate" and self.language: - params["language"] = self.language + if self.task != "translate" and self.original_language: + params["language"] = self.original_language if prompt: params["prompt"] = prompt From 9fcd403439d95330b178354fd37d9e38af9a2d66 Mon Sep 17 00:00:00 2001 From: Tijs Zwinkels Date: Thu, 15 Feb 2024 22:24:43 +0100 Subject: [PATCH 11/11] Use automatic language detection by default (instead of English) --- whisper_online.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/whisper_online.py b/whisper_online.py index 8c1916e..7a6887d 100644 --- a/whisper_online.py +++ b/whisper_online.py @@ -546,7 +546,7 @@ def add_shared_args(parser): parser.add_argument('--model', type=str, default='large-v2', choices="tiny.en,tiny,base.en,base,small.en,small,medium.en,medium,large-v1,large-v2,large-v3,large".split(","),help="Name size of the Whisper model to use (default: large-v2). The model is automatically downloaded from the model hub if not present in model cache dir.") parser.add_argument('--model_cache_dir', type=str, default=None, help="Overriding the default model cache dir where models downloaded from the hub are saved") parser.add_argument('--model_dir', type=str, default=None, help="Dir where Whisper model.bin and other files are saved. This option overrides --model and --model_cache_dir parameter.") - parser.add_argument('--lan', '--language', type=str, default='en', help="Source language code, e.g. en,de,cs, or 'auto' for language detection.") + parser.add_argument('--lan', '--language', type=str, default='auto', help="Source language code, e.g. en,de,cs, or 'auto' for language detection.") parser.add_argument('--task', type=str, default='transcribe', choices=["transcribe","translate"],help="Transcribe or translate.") parser.add_argument('--backend', type=str, default="faster-whisper", choices=["faster-whisper", "whisper_timestamped", "openai-api"],help='Load only this backend for Whisper processing.') parser.add_argument('--vad', action="store_true", default=False, help='Use VAD = voice activity detection, with the default parameters.')