LLM/chandra
1
0
Fork 0
mirror of https://github.com/datalab-to/chandra.git synced 2025-11-29 08:33:13 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add a small sleep

Browse Source
This commit is contained in:
Vik Paruchuri
2025-11-12 16:06:02 -05:00
parent aafbb70ce8
commit 068db0311e
1 changed files with 3 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
chandra/model/vllm.py
View File

@@ -1,5 +1,6 @@
import base64 import base64
import io import io
import time
from concurrent.futures import ThreadPoolExecutor from concurrent.futures import ThreadPoolExecutor
from itertools import repeat from itertools import repeat
from typing import List from typing import List
@@ -112,6 +113,7 @@ def generate_vllm(
print( print(
f"Detected vllm error, retrying generation (attempt {retries + 1})..." f"Detected vllm error, retrying generation (attempt {retries + 1})..."
) )
time.sleep(2 * (retries + 1)) # Sleeping can help under load
return True return True
if ( if (
@@ -122,6 +124,7 @@ def generate_vllm(
print( print(
f"Detected vllm error, retrying generation (attempt {retries + 1})..." f"Detected vllm error, retrying generation (attempt {retries + 1})..."
) )
time.sleep(2 * (retries + 1)) # Sleeping can help under load
return True return True
return False return False