diff --git a/whisperlivekit/simul_whisper/simul_whisper.py b/whisperlivekit/simul_whisper/simul_whisper.py
index 250798f..ce8123a 100644
--- a/whisperlivekit/simul_whisper/simul_whisper.py
+++ b/whisperlivekit/simul_whisper/simul_whisper.py
@@ -262,7 +262,6 @@ class PaddedAlignAttWhisper:
             self.last_attend_frame -= int(TOKENS_PER_SECOND*removed_len)
             self.segments = self.segments[1:]
             if len(self.tokens) > 1: # When warming up, we can have a too long segments_len while not having any tokens yet
-                self.context.append_token_ids(self.tokens[1][0,:])
                 self.context.append_token_ids(self.tokens[1][0,:])
                 self.tokens = [self.initial_tokens] + self.tokens[2:]
         return removed_len