fixes #227 , auto language dectection v0.1 - simulstreaming only - when diarization and auto

whisperlivekit/audio_processor.py

@@ -66,6 +66,8 @@ class AudioProcessor:
         self.beg_loop = None #to deal with a potential little lag at the websocket initialization, this is now set in process_audio
         self.sep = " "  # Default separator
         self.last_response_content = FrontData()
+        self.last_detected_speaker = None
+        self.speaker_languages = {}
         
         # Models and processing
         self.asr = models.asr
@@ -333,7 +335,7 @@ class AudioProcessor:
                 await diarization_obj.diarize(pcm_array)
                 
                 async with self.lock:
-                    self.tokens = diarization_obj.assign_speakers_to_tokens(
+                    self.tokens, last_segment = diarization_obj.assign_speakers_to_tokens(
                         self.tokens,
                         use_punctuation_split=self.args.punctuation_split
                     )
@@ -341,7 +343,12 @@ class AudioProcessor:
                         self.end_attributed_speaker = max(self.tokens[-1].end, self.end_attributed_speaker)
                     if buffer_diarization:
                         self.buffer_diarization = buffer_diarization
-                
+
+                    # if last_segment is not None and last_segment.speaker != self.last_detected_speaker:
+                    #     if not self.speaker_languages.get(last_segment.speaker, None):
+                    #         self.last_detected_speaker = last_segment.speaker
+                    #         self.online.on_new_speaker(last_segment)
+       
                 self.diarization_queue.task_done()
                 
             except Exception as e:
@@ -552,20 +559,20 @@ class AudioProcessor:
             if task and not task.done():
                 task.cancel()
             
-            created_tasks = [t for t in self.all_tasks_for_cleanup if t]
-            if created_tasks:
-                await asyncio.gather(*created_tasks, return_exceptions=True)
-            logger.info("All processing tasks cancelled or finished.")
+        created_tasks = [t for t in self.all_tasks_for_cleanup if t]
+        if created_tasks:
+            await asyncio.gather(*created_tasks, return_exceptions=True)
+        logger.info("All processing tasks cancelled or finished.")
 
-            if not self.is_pcm_input and self.ffmpeg_manager:
-                try:
-                    await self.ffmpeg_manager.stop()
-                    logger.info("FFmpeg manager stopped.")
-                except Exception as e:
-                    logger.warning(f"Error stopping FFmpeg manager: {e}")
-            if self.args.diarization and hasattr(self, 'dianization') and hasattr(self.diarization, 'close'):
-                self.diarization.close()
-            logger.info("AudioProcessor cleanup complete.")
+        if not self.is_pcm_input and self.ffmpeg_manager:
+            try:
+                await self.ffmpeg_manager.stop()
+                logger.info("FFmpeg manager stopped.")
+            except Exception as e:
+                logger.warning(f"Error stopping FFmpeg manager: {e}")
+        if self.args.diarization and hasattr(self, 'dianization') and hasattr(self.diarization, 'close'):
+            self.diarization.close()
+        logger.info("AudioProcessor cleanup complete.")
 
 
     async def process_audio(self, message):

whisperlivekit/diarization/diart_backend.py

@@ -242,7 +242,7 @@ class DiartDiarization:
                         token.speaker = extract_number(segment.speaker) + 1
         else:
             tokens = add_speaker_to_tokens(segments, tokens)
-        return tokens
+        return tokens, segments[-1]
         
 def concatenate_speakers(segments):
     segments_concatenated = [{"speaker": 1, "begin": 0.0, "end": 0.0}]

whisperlivekit/diarization/sortformer_backend.py

@@ -289,13 +289,14 @@ class SortformerDiarizationOnline:
             
         Returns:
             List of tokens with speaker assignments
+            Last speaker_segment
         """
         with self.segment_lock:
             segments = self.speaker_segments.copy()
         
         if not segments or not tokens:
             logger.debug("No segments or tokens available for speaker assignment")
-            return tokens
+            return tokens, None
         
         logger.debug(f"Assigning speakers to {len(tokens)} tokens using {len(segments)} segments")
         use_punctuation_split = False
@@ -312,7 +313,7 @@ class SortformerDiarizationOnline:
             # Use punctuation-aware assignment (similar to diart_backend)
             tokens = self._add_speaker_to_tokens_with_punctuation(segments, tokens)
         
-        return tokens
+        return tokens, segments[-1]
 
     def _add_speaker_to_tokens_with_punctuation(self, segments: List[SpeakerSegment], tokens: list) -> list:
         """

whisperlivekit/timed_objects.py

@@ -92,16 +92,22 @@ class Silence():
 @dataclass
 class Line(TimedText):
     translation: str = ''
+    detected_language: str = None
     
     def to_dict(self):
-        return {
+        _dict = {
             'speaker': int(self.speaker),
             'text': self.text,
-            'translation': self.translation,
             'start': format_time(self.start),
             'end': format_time(self.end),
         }
-        
+        if self.translation:
+            _dict['translation'] = self.translation
+        if self.detected_language:
+            _dict['detected_language'] = self.detected_language
+        return _dict
+    
+
 @dataclass  
 class FrontData():
     status: str = ''

whisperlivekit/web/live_transcription.css

@@ -346,7 +346,7 @@ label {
 
 .label_diarization {
   background-color: var(--chip-bg);
-  border-radius: 8px 8px 8px 8px;
+  border-radius: 100px;
   padding: 2px 10px;
   margin-left: 10px;
   display: inline-block;
@@ -358,7 +358,7 @@ label {
 
 .label_transcription {
   background-color: var(--chip-bg);
-  border-radius: 8px 8px 8px 8px;
+  border-radius: 100px;
   padding: 2px 10px;
   display: inline-block;
   white-space: nowrap;
@@ -370,16 +370,20 @@ label {
 
 .label_translation {
   background-color: var(--chip-bg);
+  display: inline-flex;
   border-radius: 10px;
   padding: 4px 8px;
   margin-top: 4px;
   font-size: 14px;
   color: var(--text);
-  display: flex;
   align-items: flex-start;
   gap: 4px;
 }
 
+.lag-diarization-value {
+    margin-left: 10px;
+}
+
 .label_translation img {
   margin-top: 2px;
 }
@@ -391,7 +395,7 @@ label {
 
 #timeInfo {
   color: var(--muted);
-  margin-left: 10px;
+  margin-left: 0px;
 }
 
 .textcontent {
@@ -514,3 +518,49 @@ label {
     padding: 10px;
   }
 }
+
+.label_language {
+  background-color: var(--chip-bg);
+  margin-bottom: 0px;
+  margin-top: 5px;
+  height: 18.5px;
+  border-radius: 100px;
+  padding: 2px 8px;
+  margin-left: 10px;
+  display: inline-flex;
+  align-items: center;
+  gap: 4px;
+  font-size: 14px;
+  color: var(--muted);
+}
+
+.label_language img {
+  width: 12px;
+  height: 12px;
+}
+
+.silence-icon {
+  width: 14px;
+  height: 14px;
+  vertical-align: text-bottom;
+}
+
+.speaker-icon {
+  width: 16px;
+  height: 16px;
+  vertical-align: text-bottom;
+}
+
+.speaker-badge {
+  display: inline-flex;
+  align-items: center;
+  justify-content: center;
+  width: 16px;
+  height: 16px;
+  margin-left: -5px;
+  border-radius: 50%;
+  font-size: 11px;
+  line-height: 1;
+  font-weight: 800;
+  color: var(--muted);
+}

whisperlivekit/web/live_transcription.js

@@ -306,7 +306,7 @@ function renderLinesWithBuffer(
   const showTransLag = !isFinalizing && remaining_time_transcription > 0;
   const showDiaLag = !isFinalizing && !!buffer_diarization && remaining_time_diarization > 0;
   const signature = JSON.stringify({
-    lines: (lines || []).map((it) => ({ speaker: it.speaker, text: it.text, start: it.start, end: it.end })),
+    lines: (lines || []).map((it) => ({ speaker: it.speaker, text: it.text, start: it.start, end: it.end, detected_language: it.detected_language })),
     buffer_transcription: buffer_transcription || "",
     buffer_diarization: buffer_diarization || "",
     status: current_status,
@@ -335,13 +335,20 @@ function renderLinesWithBuffer(
 
       let speakerLabel = "";
       if (item.speaker === -2) {
-        speakerLabel = `<span class="silence">Silence<span id='timeInfo'>${timeInfo}</span></span>`;
+        const silenceIcon = `<img class="silence-icon" src="/web/src/silence.svg" alt="Silence" />`;
+        speakerLabel = `<span class="silence">${silenceIcon}<span id='timeInfo'>${timeInfo}</span></span>`;
       } else if (item.speaker == 0 && !isFinalizing) {
         speakerLabel = `<span class='loading'><span class="spinner"></span><span id='timeInfo'><span class="loading-diarization-value">${fmt1(
           remaining_time_diarization
         )}</span> second(s) of audio are undergoing diarization</span></span>`;
       } else if (item.speaker !== 0) {
-        speakerLabel = `<span id="speaker">Speaker ${item.speaker}<span id='timeInfo'>${timeInfo}</span></span>`;
+        const speakerIcon = `<img class="speaker-icon" src="/web/src/speaker.svg" alt="Speaker ${item.speaker}" />`;
+        const speakerNum = `<span class="speaker-badge">${item.speaker}</span>`;
+        speakerLabel = `<span id="speaker">${speakerIcon}${speakerNum}<span id='timeInfo'>${timeInfo}</span></span>`;
+
+        if (item.detected_language) {
+          speakerLabel += `<span class="label_language"><img src="/web/src/language.svg" alt="Detected language" width="12" height="12" /><span>${item.detected_language}</span></span>`;
+        }
       }
 
       let currentLineText = item.text || "";

whisperlivekit/web/src/language.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" height="24px" viewBox="0 -960 960 960" width="24px" fill="#5f6368"><path d="M480-80q-82 0-155-31.5t-127.5-86Q143-252 111.5-325T80-480q0-83 31.5-155.5t86-127Q252-817 325-848.5T480-880q83 0 155.5 31.5t127 86q54.5 54.5 86 127T880-480q0 82-31.5 155t-86 127.5q-54.5 54.5-127 86T480-80Zm0-82q26-36 45-75t31-83H404q12 44 31 83t45 75Zm-104-16q-18-33-31.5-68.5T322-320H204q29 50 72.5 87t99.5 55Zm208 0q56-18 99.5-55t72.5-87H638q-9 38-22.5 73.5T584-178ZM170-400h136q-3-20-4.5-39.5T300-480q0-21 1.5-40.5T306-560H170q-5 20-7.5 39.5T160-480q0 21 2.5 40.5T170-400Zm216 0h188q3-20 4.5-39.5T580-480q0-21-1.5-40.5T574-560H386q-3 20-4.5 39.5T380-480q0 21 1.5 40.5T386-400Zm268 0h136q5-20 7.5-39.5T800-480q0-21-2.5-40.5T790-560H654q3 20 4.5 39.5T660-480q0 21-1.5 40.5T654-400Zm-16-240h118q-29-50-72.5-87T584-782q18 33 31.5 68.5T638-640Zm-234 0h152q-12-44-31-83t-45-75q-26 36-45 75t-31 83Zm-200 0h118q9-38 22.5-73.5T376-782q-56 18-99.5 55T204-640Z"/></svg>

whisperlivekit/web/src/silence.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" height="24px" viewBox="0 -960 960 960" width="24px" fill="#5f6368"><path d="M514-556 320-752q9-3 19-5.5t21-2.5q66 0 113 47t47 113q0 11-1.5 22t-4.5 22ZM40-200v-32q0-33 17-62t47-44q51-26 115-44t141-18q26 0 49.5 2.5T456-392l-56-54q-9 3-19 4.5t-21 1.5q-66 0-113-47t-47-113q0-11 1.5-21t4.5-19L84-764q-11-11-11-28t11-28q12-12 28.5-12t27.5 12l675 685q11 11 11.5 27.5T816-80q-11 13-28 12.5T759-80L641-200h39q0 33-23.5 56.5T600-120H120q-33 0-56.5-23.5T40-200Zm80 0h480v-32q0-14-4.5-19.5T580-266q-36-18-92.5-36T360-320q-71 0-127.5 18T140-266q-9 5-14.5 14t-5.5 20v32Zm240 0Zm560-400q0 69-24.5 131.5T829-355q-12 14-30 15t-32-13q-13-13-12-31t12-33q30-38 46.5-85t16.5-98q0-51-16.5-97T767-781q-12-15-12.5-33t12.5-32q13-14 31.5-13.5T829-845q42 51 66.5 113.5T920-600Zm-182 0q0 32-10 61.5T700-484q-11 15-29.5 15.5T638-482q-13-13-13.5-31.5T633-549q6-11 9.5-24t3.5-27q0-14-3.5-27t-9.5-25q-9-17-8.5-35t13.5-31q14-14 32.5-13.5T700-716q18 25 28 54.5t10 61.5Z"/></svg>

whisperlivekit/web/src/speaker.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" height="24px" viewBox="0 -960 960 960" width="24px" fill="#5f6368"><path d="M480-480q-66 0-113-47t-47-113q0-66 47-113t113-47q66 0 113 47t47 113q0 66-47 113t-113 47ZM160-240v-32q0-34 17.5-62.5T224-378q62-31 126-46.5T480-440q66 0 130 15.5T736-378q29 15 46.5 43.5T800-272v32q0 33-23.5 56.5T720-160H240q-33 0-56.5-23.5T160-240Zm80 0h480v-32q0-11-5.5-20T700-306q-54-27-109-40.5T480-360q-56 0-111 13.5T260-306q-9 5-14.5 14t-5.5 20v32Zm240-320q33 0 56.5-23.5T560-640q0-33-23.5-56.5T480-720q-33 0-56.5 23.5T400-640q0 33 23.5 56.5T480-560Zm0-80Zm0 400Z"/></svg>