From c7b3bb5e582169e9b616b397245e45f9b6b33eee Mon Sep 17 00:00:00 2001 From: Alexander Lindberg Date: Tue, 9 Sep 2025 11:18:55 +0300 Subject: [PATCH 1/3] Fix regression with faster-whisper encoder_feature --- whisperlivekit/simul_whisper/simul_whisper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/whisperlivekit/simul_whisper/simul_whisper.py b/whisperlivekit/simul_whisper/simul_whisper.py index 3b0e7c5..5707e41 100644 --- a/whisperlivekit/simul_whisper/simul_whisper.py +++ b/whisperlivekit/simul_whisper/simul_whisper.py @@ -408,7 +408,7 @@ class PaddedAlignAttWhisper: content_mel_len = int(audio_length_seconds * 100)//2 mel_padded_2 = self.fw_feature_extractor(waveform=input_segments.numpy(), padding=N_SAMPLES)[None, :] mel = fw_pad_or_trim(mel_padded_2, N_FRAMES, axis=-1) - encoder_feature_ctranslate = np.array(self.fw_encoder.encode(mel)) + encoder_feature_ctranslate = self.fw_encoder.encode(mel) encoder_feature = torch.as_tensor(encoder_feature_ctranslate, device=self.device) else: # mel + padding to 30s From 1f7798c7c10dc9e1d3a9caf9bc6973cecdd0ec3e Mon Sep 17 00:00:00 2001 From: Quentin Fuxa Date: Tue, 9 Sep 2025 12:16:52 +0200 Subject: [PATCH 2/3] condition on encoder_feature_ctranslate type --- whisperlivekit/simul_whisper/simul_whisper.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/whisperlivekit/simul_whisper/simul_whisper.py b/whisperlivekit/simul_whisper/simul_whisper.py index 5707e41..437247d 100644 --- a/whisperlivekit/simul_whisper/simul_whisper.py +++ b/whisperlivekit/simul_whisper/simul_whisper.py @@ -409,6 +409,8 @@ class PaddedAlignAttWhisper: mel_padded_2 = self.fw_feature_extractor(waveform=input_segments.numpy(), padding=N_SAMPLES)[None, :] mel = fw_pad_or_trim(mel_padded_2, N_FRAMES, axis=-1) encoder_feature_ctranslate = self.fw_encoder.encode(mel) + if type(encoder_feature_ctranslate).__module__ == 'ctranslate2._ext': + encoder_feature_ctranslate = np.array(encoder_feature_ctranslate) encoder_feature = torch.as_tensor(encoder_feature_ctranslate, device=self.device) else: # mel + padding to 30s From 3358877054c5e6ab75d9b50175d340d3ecf716d5 Mon Sep 17 00:00:00 2001 From: Quentin Fuxa Date: Tue, 9 Sep 2025 15:44:16 +0200 Subject: [PATCH 3/3] Fix StorageView conversion for CPU/GPU compatibility --- whisperlivekit/simul_whisper/simul_whisper.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/whisperlivekit/simul_whisper/simul_whisper.py b/whisperlivekit/simul_whisper/simul_whisper.py index 437247d..b537510 100644 --- a/whisperlivekit/simul_whisper/simul_whisper.py +++ b/whisperlivekit/simul_whisper/simul_whisper.py @@ -409,9 +409,12 @@ class PaddedAlignAttWhisper: mel_padded_2 = self.fw_feature_extractor(waveform=input_segments.numpy(), padding=N_SAMPLES)[None, :] mel = fw_pad_or_trim(mel_padded_2, N_FRAMES, axis=-1) encoder_feature_ctranslate = self.fw_encoder.encode(mel) - if type(encoder_feature_ctranslate).__module__ == 'ctranslate2._ext': + if self.device == 'cpu': #it seems that on gpu, passing StorageView to torch.as_tensor fails and wrapping in the array works encoder_feature_ctranslate = np.array(encoder_feature_ctranslate) - encoder_feature = torch.as_tensor(encoder_feature_ctranslate, device=self.device) + try: + encoder_feature = torch.as_tensor(encoder_feature_ctranslate, device=self.device) + except TypeError: # Normally the cpu condition should prevent having exceptions, but just in case: + encoder_feature = torch.as_tensor(np.array(encoder_feature_ctranslate), device=self.device) else: # mel + padding to 30s mel_padded = log_mel_spectrogram(input_segments, n_mels=self.model.dims.n_mels, padding=N_SAMPLES,