From 068db0311e89f59ba4d1762ba472d33e8e49a5ef Mon Sep 17 00:00:00 2001 From: Vik Paruchuri Date: Wed, 12 Nov 2025 16:06:02 -0500 Subject: [PATCH] Add a small sleep --- chandra/model/vllm.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/chandra/model/vllm.py b/chandra/model/vllm.py index a74b852..a2e5816 100644 --- a/chandra/model/vllm.py +++ b/chandra/model/vllm.py @@ -1,5 +1,6 @@ import base64 import io +import time from concurrent.futures import ThreadPoolExecutor from itertools import repeat from typing import List @@ -112,6 +113,7 @@ def generate_vllm( print( f"Detected vllm error, retrying generation (attempt {retries + 1})..." ) + time.sleep(2 * (retries + 1)) # Sleeping can help under load return True if ( @@ -122,6 +124,7 @@ def generate_vllm( print( f"Detected vllm error, retrying generation (attempt {retries + 1})..." ) + time.sleep(2 * (retries + 1)) # Sleeping can help under load return True return False