LLM/WhisperLiveKit
1
0
Fork 0
mirror of https://github.com/QuentinFuxa/WhisperLiveKit.git synced 2026-03-07 14:23:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #342 from mnicnc404/fix/whisper-tokenizer-index-error

Browse Source 
fix(whisper/tokenizer): prevent IndexError from crashing multilingual…
This commit is contained in:
Quentin Fuxa
2026-03-02 20:36:58 +01:00
committed by GitHub
parent 4917406e06 b63f54e838
commit c112ceffb6
1 changed files with 9 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
whisperlivekit/whisper/tokenizer.py
View File

@@ -296,10 +296,15 @@ class Tokenizer:
current_tokens.append(token)
decoded = self.decode_with_timestamps(current_tokens)
if (
replacement_char not in decoded
or decoded_full[unicode_offset + decoded.index(replacement_char)]
== replacement_char
try:
replacement_char_index = decoded.index(replacement_char)
replacement_char_index += unicode_offset
except ValueError:
replacement_char_index = None
if replacement_char_index is None or (
replacement_char_index < len(decoded_full)
and decoded_full[replacement_char_index] == replacement_char
):
words.append(decoded)
word_tokens.append(current_tokens)