From 6fa008080accb4f0d6dc5a84211445bddba5910f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dominik=20Mach=C3=A1=C4=8Dek?= Date: Wed, 3 Jan 2024 17:55:33 +0100 Subject: [PATCH] VAC - performance tests pending - TODO: timestamps after refresh are decreasing --- voice_activity_controller.py | 49 ++++++++++++------------------------ whisper_online_server.py | 10 +++++--- whisper_online_vac.py | 48 ++++++++++++++++------------------- 3 files changed, 44 insertions(+), 63 deletions(-) diff --git a/voice_activity_controller.py b/voice_activity_controller.py index d478234..ccfbea7 100644 --- a/voice_activity_controller.py +++ b/voice_activity_controller.py @@ -1,18 +1,5 @@ import torch import numpy as np -# import sounddevice as sd -import torch -import numpy as np -import datetime - - -def int2float(sound): - abs_max = np.abs(sound).max() - sound = sound.astype('float32') - if abs_max > 0: - sound *= 1/32768 - sound = sound.squeeze() # depends on the use case - return sound class VoiceActivityController: def __init__( @@ -22,10 +9,10 @@ class VoiceActivityController: min_speech_to_final_ms = 100, min_silence_duration_ms = 100, use_vad_result = True, - activity_detected_callback=None, +# activity_detected_callback=None, threshold =0.3 ): - self.activity_detected_callback=activity_detected_callback +# self.activity_detected_callback=activity_detected_callback self.model, self.utils = torch.hub.load( repo_or_dir='snakers4/silero-vad', model='silero_vad' @@ -42,7 +29,6 @@ class VoiceActivityController: self.min_silence_samples = sampling_rate * min_silence_duration_ms / 1000 self.use_vad_result = use_vad_result - self.last_marked_chunk = None self.threshold = threshold self.reset_states() @@ -55,7 +41,13 @@ class VoiceActivityController: self.speech_len = 0 def apply_vad(self, audio): -# x = int2float(audio) + """ + returns: triple + (voice_audio, + speech_in_wav, + silence_in_wav) + + """ x = audio if not torch.is_tensor(x): try: @@ -64,16 +56,16 @@ class VoiceActivityController: raise TypeError("Audio cannot be casted to tensor. Cast it manually") speech_prob = self.model(x, self.sampling_rate).item() + print("speech_prob",speech_prob) window_size_samples = len(x[0]) if x.dim() == 2 else len(x) self.current_sample += window_size_samples - - if (speech_prob >= self.threshold): + if speech_prob >= self.threshold: # speech is detected self.temp_end = 0 return audio, window_size_samples, 0 - else : + else: # silence detected, counting w if not self.temp_end: self.temp_end = self.current_sample @@ -84,14 +76,12 @@ class VoiceActivityController: def detect_speech_iter(self, data, audio_in_int16 = False): -# audio_block = np.frombuffer(data, dtype=np.int16) if not audio_in_int16 else data audio_block = data wav = audio_block - print(wav, len(wav), type(wav), wav.dtype) - is_final = False voice_audio, speech_in_wav, last_silent_in_wav = self.apply_vad(wav) + print("speech, last silence",speech_in_wav, last_silent_in_wav) if speech_in_wav > 0 : @@ -101,27 +91,20 @@ class VoiceActivityController: # self.activity_detected_callback() self.last_silence_len += last_silent_in_wav + print("self.last_silence_len",self.last_silence_len, self.final_silence_limit,self.last_silence_len>= self.final_silence_limit) + print("self.speech_len, final_speech_limit",self.speech_len , self.final_speech_limit,self.speech_len >= self.final_speech_limit) if self.last_silence_len>= self.final_silence_limit and self.speech_len >= self.final_speech_limit: + for i in range(10): print("TADY!!!") is_final = True self.last_silence_len= 0 self.speech_len = 0 -# return voice_audio.tobytes(), is_final return voice_audio, is_final - - def detect_user_speech(self, audio_stream, audio_in_int16 = False): self.last_silence_len= 0 self.speech_len = 0 for data in audio_stream: # replace with your condition of choice yield self.detect_speech_iter(data, audio_in_int16) - - - - - - - diff --git a/whisper_online_server.py b/whisper_online_server.py index b2f5120..e699c65 100644 --- a/whisper_online_server.py +++ b/whisper_online_server.py @@ -9,7 +9,8 @@ parser = argparse.ArgumentParser() # server options parser.add_argument("--host", type=str, default='localhost') parser.add_argument("--port", type=int, default=43007) - +parser.add_argument('--vac', action="store_true", default=False, help='Use VAC = voice activity controller.') +parser.add_argument('--vac-chunk-size', type=float, default=0.04, help='VAC sample size in seconds.') # options from whisper_online add_shared_args(parser) @@ -57,8 +58,11 @@ if args.buffer_trimming == "sentence": tokenizer = create_tokenizer(tgt_language) else: tokenizer = None -online = OnlineASRProcessor(asr,tokenizer,buffer_trimming=(args.buffer_trimming, args.buffer_trimming_sec)) - +if not args.vac: + online = OnlineASRProcessor(asr,tokenizer,buffer_trimming=(args.buffer_trimming, args.buffer_trimming_sec)) +else: + from whisper_online_vac import * + online = VACOnlineASRProcessor(min_chunk, asr,tokenizer,buffer_trimming=(args.buffer_trimming, args.buffer_trimming_sec)) demo_audio_path = "cs-maji-2.16k.wav" diff --git a/whisper_online_vac.py b/whisper_online_vac.py index 7001d58..daf66a9 100644 --- a/whisper_online_vac.py +++ b/whisper_online_vac.py @@ -7,52 +7,46 @@ SAMPLING_RATE = 16000 class VACOnlineASRProcessor(OnlineASRProcessor): - def __init__(self, *a, **kw): - self.online = OnlineASRProcessor(*a, **kw) - self.vac = VoiceActivityController(use_vad_result = True) + def __init__(self, online_chunk_size, *a, **kw): + self.online_chunk_size = online_chunk_size + + self.online = OnlineASRProcessor(*a, **kw) + self.vac = VoiceActivityController(use_vad_result = False) - self.is_currently_final = False self.logfile = self.online.logfile - #self.vac_buffer = io.BytesIO() - #self.vac_stream = self.vac.detect_user_speech(self.vac_buffer, audio_in_int16=False) - - self.audio_log = open("audio_log.wav","wb") + self.init() def init(self): self.online.init() self.vac.reset_states() + self.current_online_chunk_buffer_size = 0 + self.is_currently_final = False + def insert_audio_chunk(self, audio): - print(audio, len(audio), type(audio), audio.dtype) r = self.vac.detect_speech_iter(audio,audio_in_int16=False) - raw_bytes, is_final = r - print("is_final",is_final) - print("raw_bytes", raw_bytes[:10], len(raw_bytes), type(raw_bytes)) -# self.audio_log.write(raw_bytes) - #sf = soundfile.SoundFile(io.BytesIO(raw_bytes), channels=1,endian="LITTLE",samplerate=SAMPLING_RATE, subtype="PCM_16",format="RAW") - #audio, _ = librosa.load(sf,sr=SAMPLING_RATE) - audio = raw_bytes - print("po překonvertování", audio, len(audio), type(audio), audio.dtype) + audio, is_final = r + print(is_final) self.is_currently_final = is_final self.online.insert_audio_chunk(audio) -# self.audio_log.write(audio) - self.audio_log.flush() - - print("inserted",file=self.logfile) + self.current_online_chunk_buffer_size += len(audio) def process_iter(self): if self.is_currently_final: return self.finish() - else: - print(self.online.audio_buffer) + elif self.current_online_chunk_buffer_size > SAMPLING_RATE*self.online_chunk_size: + self.current_online_chunk_buffer_size = 0 ret = self.online.process_iter() - print("tady",file=self.logfile) return ret + else: + print("no online update, only VAD", file=self.logfile) + return (None, None, "") def finish(self): ret = self.online.finish() self.online.init() + self.current_online_chunk_buffer_size = 0 return ret @@ -67,7 +61,7 @@ if __name__ == "__main__": parser.add_argument('--start_at', type=float, default=0.0, help='Start processing audio at this time.') parser.add_argument('--offline', action="store_true", default=False, help='Offline mode.') parser.add_argument('--comp_unaware', action="store_true", default=False, help='Computationally unaware simulation.') - + parser.add_argument('--vac-chunk-size', type=float, default=0.04, help='VAC sample size in seconds.') args = parser.parse_args() # reset to store stderr to different file stream, e.g. open(os.devnull,"w") @@ -111,12 +105,12 @@ if __name__ == "__main__": asr.use_vad() - min_chunk = args.min_chunk_size + min_chunk = args.vac_chunk_size if args.buffer_trimming == "sentence": tokenizer = create_tokenizer(tgt_language) else: tokenizer = None - online = VACOnlineASRProcessor(asr,tokenizer,logfile=logfile,buffer_trimming=(args.buffer_trimming, args.buffer_trimming_sec)) + online = VACOnlineASRProcessor(args.min_chunk_size, asr,tokenizer,logfile=logfile,buffer_trimming=(args.buffer_trimming, args.buffer_trimming_sec)) # load the audio into the LRU cache before we start the timer