LLM/WhisperLiveKit
1
0
Fork 0
mirror of https://github.com/QuentinFuxa/WhisperLiveKit.git synced 2026-03-07 22:33:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

VAC

Browse Source 
- performance tests pending
- TODO: timestamps after refresh are decreasing
This commit is contained in:
Dominik Macháček
2024-01-03 17:55:33 +01:00
parent d543411bbd
commit 6fa008080a
3 changed files with 44 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
voice_activity_controller.py
View File

@@ -1,18 +1,5 @@
import torch
import numpy as np
# import sounddevice as sd
import torch
import numpy as np
import datetime
def int2float(sound):
abs_max = np.abs(sound).max()
sound = sound.astype('float32')
if abs_max > 0:
sound *= 1/32768
sound = sound.squeeze() # depends on the use case
return sound
class VoiceActivityController:
def __init__(
@@ -22,10 +9,10 @@ class VoiceActivityController:
min_speech_to_final_ms = 100,
min_silence_duration_ms = 100,
use_vad_result = True,
activity_detected_callback=None,
# activity_detected_callback=None,
threshold =0.3
):
self.activity_detected_callback=activity_detected_callback
# self.activity_detected_callback=activity_detected_callback
self.model, self.utils = torch.hub.load(
repo_or_dir='snakers4/silero-vad',
model='silero_vad'
@@ -42,7 +29,6 @@ class VoiceActivityController:
self.min_silence_samples = sampling_rate * min_silence_duration_ms / 1000
self.use_vad_result = use_vad_result
self.last_marked_chunk = None
self.threshold = threshold
self.reset_states()
@@ -55,7 +41,13 @@ class VoiceActivityController:
self.speech_len = 0
def apply_vad(self, audio):
# x = int2float(audio)
"""
returns: triple
(voice_audio,
speech_in_wav,
silence_in_wav)
"""
x = audio
if not torch.is_tensor(x):
try:
@@ -64,16 +56,16 @@ class VoiceActivityController:
raise TypeError("Audio cannot be casted to tensor. Cast it manually")
speech_prob = self.model(x, self.sampling_rate).item()
print("speech_prob",speech_prob)
window_size_samples = len(x[0]) if x.dim() == 2 else len(x)
self.current_sample += window_size_samples
if (speech_prob >= self.threshold):
if speech_prob >= self.threshold: # speech is detected
self.temp_end = 0
return audio, window_size_samples, 0
else :
else: # silence detected, counting w
if not self.temp_end:
self.temp_end = self.current_sample
@@ -84,14 +76,12 @@ class VoiceActivityController:
def detect_speech_iter(self, data, audio_in_int16 = False):
# audio_block = np.frombuffer(data, dtype=np.int16) if not audio_in_int16 else data
audio_block = data
wav = audio_block
print(wav, len(wav), type(wav), wav.dtype)
is_final = False
voice_audio, speech_in_wav, last_silent_in_wav = self.apply_vad(wav)
print("speech, last silence",speech_in_wav, last_silent_in_wav)
if speech_in_wav > 0 :
@@ -101,27 +91,20 @@ class VoiceActivityController:
# self.activity_detected_callback()
self.last_silence_len += last_silent_in_wav
print("self.last_silence_len",self.last_silence_len, self.final_silence_limit,self.last_silence_len>= self.final_silence_limit)
print("self.speech_len, final_speech_limit",self.speech_len , self.final_speech_limit,self.speech_len >= self.final_speech_limit)
if self.last_silence_len>= self.final_silence_limit and self.speech_len >= self.final_speech_limit:
for i in range(10): print("TADY!!!")
is_final = True
self.last_silence_len= 0
self.speech_len = 0
# return voice_audio.tobytes(), is_final
return voice_audio, is_final
def detect_user_speech(self, audio_stream, audio_in_int16 = False):
self.last_silence_len= 0
self.speech_len = 0
for data in audio_stream: # replace with your condition of choice
yield self.detect_speech_iter(data, audio_in_int16)

10
whisper_online_server.py
View File

@@ -9,7 +9,8 @@ parser = argparse.ArgumentParser()
# server options
parser.add_argument("--host", type=str, default='localhost')
parser.add_argument("--port", type=int, default=43007)
parser.add_argument('--vac', action="store_true", default=False, help='Use VAC = voice activity controller.')
parser.add_argument('--vac-chunk-size', type=float, default=0.04, help='VAC sample size in seconds.')
# options from whisper_online
add_shared_args(parser)
@@ -57,8 +58,11 @@ if args.buffer_trimming == "sentence":
tokenizer = create_tokenizer(tgt_language)
else:
tokenizer = None
online = OnlineASRProcessor(asr,tokenizer,buffer_trimming=(args.buffer_trimming, args.buffer_trimming_sec))
if not args.vac:
online = OnlineASRProcessor(asr,tokenizer,buffer_trimming=(args.buffer_trimming, args.buffer_trimming_sec))
else:
from whisper_online_vac import *
online = VACOnlineASRProcessor(min_chunk, asr,tokenizer,buffer_trimming=(args.buffer_trimming, args.buffer_trimming_sec))
demo_audio_path = "cs-maji-2.16k.wav"

48
whisper_online_vac.py
View File

@@ -7,52 +7,46 @@ SAMPLING_RATE = 16000
class VACOnlineASRProcessor(OnlineASRProcessor):
def __init__(self, *a, **kw):
self.online = OnlineASRProcessor(*a, **kw)
self.vac = VoiceActivityController(use_vad_result = True)
def __init__(self, online_chunk_size, *a, **kw):
self.online_chunk_size = online_chunk_size
self.online = OnlineASRProcessor(*a, **kw)
self.vac = VoiceActivityController(use_vad_result = False)
self.is_currently_final = False
self.logfile = self.online.logfile
#self.vac_buffer = io.BytesIO()
#self.vac_stream = self.vac.detect_user_speech(self.vac_buffer, audio_in_int16=False)
self.audio_log = open("audio_log.wav","wb")
self.init()
def init(self):
self.online.init()
self.vac.reset_states()
self.current_online_chunk_buffer_size = 0
self.is_currently_final = False
def insert_audio_chunk(self, audio):
print(audio, len(audio), type(audio), audio.dtype)
r = self.vac.detect_speech_iter(audio,audio_in_int16=False)
raw_bytes, is_final = r
print("is_final",is_final)
print("raw_bytes", raw_bytes[:10], len(raw_bytes), type(raw_bytes))
# self.audio_log.write(raw_bytes)
#sf = soundfile.SoundFile(io.BytesIO(raw_bytes), channels=1,endian="LITTLE",samplerate=SAMPLING_RATE, subtype="PCM_16",format="RAW")
#audio, _ = librosa.load(sf,sr=SAMPLING_RATE)
audio = raw_bytes
print("po překonvertování", audio, len(audio), type(audio), audio.dtype)
audio, is_final = r
print(is_final)
self.is_currently_final = is_final
self.online.insert_audio_chunk(audio)
# self.audio_log.write(audio)
self.audio_log.flush()
print("inserted",file=self.logfile)
self.current_online_chunk_buffer_size += len(audio)
def process_iter(self):
if self.is_currently_final:
return self.finish()
else:
print(self.online.audio_buffer)
elif self.current_online_chunk_buffer_size > SAMPLING_RATE*self.online_chunk_size:
self.current_online_chunk_buffer_size = 0
ret = self.online.process_iter()
print("tady",file=self.logfile)
return ret
else:
print("no online update, only VAD", file=self.logfile)
return (None, None, "")
def finish(self):
ret = self.online.finish()
self.online.init()
self.current_online_chunk_buffer_size = 0
return ret
@@ -67,7 +61,7 @@ if __name__ == "__main__":
parser.add_argument('--start_at', type=float, default=0.0, help='Start processing audio at this time.')
parser.add_argument('--offline', action="store_true", default=False, help='Offline mode.')
parser.add_argument('--comp_unaware', action="store_true", default=False, help='Computationally unaware simulation.')
parser.add_argument('--vac-chunk-size', type=float, default=0.04, help='VAC sample size in seconds.')
args = parser.parse_args()
# reset to store stderr to different file stream, e.g. open(os.devnull,"w")
@@ -111,12 +105,12 @@ if __name__ == "__main__":
asr.use_vad()
min_chunk = args.min_chunk_size
min_chunk = args.vac_chunk_size
if args.buffer_trimming == "sentence":
tokenizer = create_tokenizer(tgt_language)
else:
tokenizer = None
online = VACOnlineASRProcessor(asr,tokenizer,logfile=logfile,buffer_trimming=(args.buffer_trimming, args.buffer_trimming_sec))
online = VACOnlineASRProcessor(args.min_chunk_size, asr,tokenizer,logfile=logfile,buffer_trimming=(args.buffer_trimming, args.buffer_trimming_sec))
# load the audio into the LRU cache before we start the timer