diff --git a/voice_activity_controller.py b/voice_activity_controller.py index 3ccc29a..d478234 100644 --- a/voice_activity_controller.py +++ b/voice_activity_controller.py @@ -51,8 +51,12 @@ class VoiceActivityController: self.temp_end = 0 self.current_sample = 0 + self.last_silence_len= 0 + self.speech_len = 0 + def apply_vad(self, audio): - x = int2float(audio) +# x = int2float(audio) + x = audio if not torch.is_tensor(x): try: x = torch.Tensor(x) @@ -79,38 +83,42 @@ class VoiceActivityController: return np.array([], dtype=np.float16) if self.use_vad_result else audio, 0, window_size_samples + def detect_speech_iter(self, data, audio_in_int16 = False): +# audio_block = np.frombuffer(data, dtype=np.int16) if not audio_in_int16 else data + audio_block = data + wav = audio_block + + print(wav, len(wav), type(wav), wav.dtype) + + is_final = False + voice_audio, speech_in_wav, last_silent_in_wav = self.apply_vad(wav) + + + if speech_in_wav > 0 : + self.last_silence_len= 0 + self.speech_len += speech_in_wav +# if self.activity_detected_callback is not None: +# self.activity_detected_callback() + + self.last_silence_len += last_silent_in_wav + if self.last_silence_len>= self.final_silence_limit and self.speech_len >= self.final_speech_limit: + + is_final = True + self.last_silence_len= 0 + self.speech_len = 0 + +# return voice_audio.tobytes(), is_final + return voice_audio, is_final def detect_user_speech(self, audio_stream, audio_in_int16 = False): - last_silence_len= 0 - speech_len = 0 + self.last_silence_len= 0 + self.speech_len = 0 for data in audio_stream: # replace with your condition of choice - - - audio_block = np.frombuffer(data, dtype=np.int16) if not audio_in_int16 else data - wav = audio_block - - is_final = False - voice_audio, speech_in_wav, last_silent_in_wav = self.apply_vad(wav) - - - if speech_in_wav > 0 : - last_silence_len= 0 - speech_len += speech_in_wav - if self.activity_detected_callback is not None: - self.activity_detected_callback() - - last_silence_len += last_silent_in_wav - if last_silence_len>= self.final_silence_limit and speech_len >= self.final_speech_limit: - - is_final = True - last_silence_len= 0 - speech_len = 0 - - yield voice_audio.tobytes(), is_final - + yield self.detect_speech_iter(data, audio_in_int16) + diff --git a/whisper_online_vac.py b/whisper_online_vac.py new file mode 100644 index 0000000..7001d58 --- /dev/null +++ b/whisper_online_vac.py @@ -0,0 +1,209 @@ +from whisper_online import * +from voice_activity_controller import * +import soundfile +import io + +SAMPLING_RATE = 16000 + +class VACOnlineASRProcessor(OnlineASRProcessor): + + def __init__(self, *a, **kw): + self.online = OnlineASRProcessor(*a, **kw) + self.vac = VoiceActivityController(use_vad_result = True) + + self.is_currently_final = False + self.logfile = self.online.logfile + + #self.vac_buffer = io.BytesIO() + #self.vac_stream = self.vac.detect_user_speech(self.vac_buffer, audio_in_int16=False) + + self.audio_log = open("audio_log.wav","wb") + + def init(self): + self.online.init() + self.vac.reset_states() + + def insert_audio_chunk(self, audio): + print(audio, len(audio), type(audio), audio.dtype) + r = self.vac.detect_speech_iter(audio,audio_in_int16=False) + raw_bytes, is_final = r + print("is_final",is_final) + print("raw_bytes", raw_bytes[:10], len(raw_bytes), type(raw_bytes)) +# self.audio_log.write(raw_bytes) + #sf = soundfile.SoundFile(io.BytesIO(raw_bytes), channels=1,endian="LITTLE",samplerate=SAMPLING_RATE, subtype="PCM_16",format="RAW") + #audio, _ = librosa.load(sf,sr=SAMPLING_RATE) + audio = raw_bytes + print("po překonvertování", audio, len(audio), type(audio), audio.dtype) + self.is_currently_final = is_final + self.online.insert_audio_chunk(audio) +# self.audio_log.write(audio) + self.audio_log.flush() + + print("inserted",file=self.logfile) + + def process_iter(self): + if self.is_currently_final: + return self.finish() + else: + print(self.online.audio_buffer) + ret = self.online.process_iter() + print("tady",file=self.logfile) + return ret + + def finish(self): + ret = self.online.finish() + self.online.init() + return ret + + + + +if __name__ == "__main__": + + import argparse + parser = argparse.ArgumentParser() + parser.add_argument('audio_path', type=str, help="Filename of 16kHz mono channel wav, on which live streaming is simulated.") + add_shared_args(parser) + parser.add_argument('--start_at', type=float, default=0.0, help='Start processing audio at this time.') + parser.add_argument('--offline', action="store_true", default=False, help='Offline mode.') + parser.add_argument('--comp_unaware', action="store_true", default=False, help='Computationally unaware simulation.') + + args = parser.parse_args() + + # reset to store stderr to different file stream, e.g. open(os.devnull,"w") + logfile = sys.stderr + + if args.offline and args.comp_unaware: + print("No or one option from --offline and --comp_unaware are available, not both. Exiting.",file=logfile) + sys.exit(1) + + audio_path = args.audio_path + + SAMPLING_RATE = 16000 + duration = len(load_audio(audio_path))/SAMPLING_RATE + print("Audio duration is: %2.2f seconds" % duration, file=logfile) + + size = args.model + language = args.lan + + t = time.time() + print(f"Loading Whisper {size} model for {language}...",file=logfile,end=" ",flush=True) + + if args.backend == "faster-whisper": + asr_cls = FasterWhisperASR + else: + asr_cls = WhisperTimestampedASR + + asr = asr_cls(modelsize=size, lan=language, cache_dir=args.model_cache_dir, model_dir=args.model_dir) + + if args.task == "translate": + asr.set_translate_task() + tgt_language = "en" # Whisper translates into English + else: + tgt_language = language # Whisper transcribes in this language + + + e = time.time() + print(f"done. It took {round(e-t,2)} seconds.",file=logfile) + + if args.vad: + print("setting VAD filter",file=logfile) + asr.use_vad() + + + min_chunk = args.min_chunk_size + if args.buffer_trimming == "sentence": + tokenizer = create_tokenizer(tgt_language) + else: + tokenizer = None + online = VACOnlineASRProcessor(asr,tokenizer,logfile=logfile,buffer_trimming=(args.buffer_trimming, args.buffer_trimming_sec)) + + + # load the audio into the LRU cache before we start the timer + a = load_audio_chunk(audio_path,0,1) + + # warm up the ASR, because the very first transcribe takes much more time than the other + asr.transcribe(a) + + beg = args.start_at + start = time.time()-beg + + def output_transcript(o, now=None): + # output format in stdout is like: + # 4186.3606 0 1720 Takhle to je + # - the first three words are: + # - emission time from beginning of processing, in milliseconds + # - beg and end timestamp of the text segment, as estimated by Whisper model. The timestamps are not accurate, but they're useful anyway + # - the next words: segment transcript + if now is None: + now = time.time()-start + if o[0] is not None: + print("%1.4f %1.0f %1.0f %s" % (now*1000, o[0]*1000,o[1]*1000,o[2]),file=logfile,flush=True) + print("%1.4f %1.0f %1.0f %s" % (now*1000, o[0]*1000,o[1]*1000,o[2]),flush=True) + else: + print(o,file=logfile,flush=True) + + if args.offline: ## offline mode processing (for testing/debugging) + a = load_audio(audio_path) + online.insert_audio_chunk(a) + try: + o = online.process_iter() + except AssertionError: + print("assertion error",file=logfile) + pass + else: + output_transcript(o) + now = None + elif args.comp_unaware: # computational unaware mode + end = beg + min_chunk + while True: + a = load_audio_chunk(audio_path,beg,end) + online.insert_audio_chunk(a) + try: + o = online.process_iter() + except AssertionError: + print("assertion error",file=logfile) + pass + else: + output_transcript(o, now=end) + + print(f"## last processed {end:.2f}s",file=logfile,flush=True) + + if end >= duration: + break + + beg = end + + if end + min_chunk > duration: + end = duration + else: + end += min_chunk + now = duration + + else: # online = simultaneous mode + end = 0 + while True: + now = time.time() - start + if now < end+min_chunk: + time.sleep(min_chunk+end-now) + end = time.time() - start + a = load_audio_chunk(audio_path,beg,end) + beg = end + online.insert_audio_chunk(a) + + try: + o = online.process_iter() + except AssertionError: + print("assertion error",file=logfile) + pass + else: + output_transcript(o) + now = time.time() - start + print(f"## last processed {end:.2f} s, now is {now:.2f}, the latency is {now-end:.2f}",file=logfile,flush=True) + + if end >= duration: + break + now = None + + o = online.finish() + output_transcript(o, now=now)