1 Commits

Author SHA1 Message Date
Tarun Menta
eaa31d169d Fix missing setting 2025-10-21 11:13:26 -04:00
2 changed files with 2 additions and 2 deletions

View File

@@ -73,7 +73,7 @@ def load_model():
"dtype": settings.TORCH_DTYPE,
"device_map": device_map,
}
if settings.TORCH_ATTN:
if settings.TORCH_ATTN_IMPLEMENTATION:
kwargs["attn_implementation"] = settings.TORCH_ATTN_IMPLEMENTATION
model = Qwen3VLForConditionalGeneration.from_pretrained(

View File

@@ -13,7 +13,7 @@ class Settings(BaseSettings):
MODEL_CHECKPOINT: str = "datalab-to/chandra"
TORCH_DEVICE: str | None = None
MAX_OUTPUT_TOKENS: int = 8192
TORCH_ATTN: str | None = None
TORCH_ATTN_IMPLEMENTATION: str | None = None
# vLLM server settings
VLLM_API_KEY: str = "EMPTY"