diff --git a/application/api/answer/routes.py b/application/api/answer/routes.py index 8488a56d..7f88ba0f 100644 --- a/application/api/answer/routes.py +++ b/application/api/answer/routes.py @@ -42,6 +42,8 @@ elif settings.LLM_NAME == "anthropic": gpt_model = "claude-2" elif settings.LLM_NAME == "groq": gpt_model = "llama3-8b-8192" +elif settings.LLM_NAME == "novita": + gpt_model = "deepseek/deepseek-r1" if settings.MODEL_NAME: # in case there is particular model name configured gpt_model = settings.MODEL_NAME diff --git a/application/llm/llm_creator.py b/application/llm/llm_creator.py index f32089de..9f1305ba 100644 --- a/application/llm/llm_creator.py +++ b/application/llm/llm_creator.py @@ -7,7 +7,7 @@ from application.llm.anthropic import AnthropicLLM from application.llm.docsgpt_provider import DocsGPTAPILLM from application.llm.premai import PremAILLM from application.llm.google_ai import GoogleLLM - +from application.llm.novita import NovitaLLM class LLMCreator: llms = { @@ -20,7 +20,8 @@ class LLMCreator: "docsgpt": DocsGPTAPILLM, "premai": PremAILLM, "groq": GroqLLM, - "google": GoogleLLM + "google": GoogleLLM, + "novita": NovitaLLM } @classmethod diff --git a/application/llm/novita.py b/application/llm/novita.py new file mode 100644 index 00000000..8d6ac042 --- /dev/null +++ b/application/llm/novita.py @@ -0,0 +1,32 @@ +from application.llm.base import BaseLLM +from openai import OpenAI + + +class NovitaLLM(BaseLLM): + def __init__(self, api_key=None, user_api_key=None, *args, **kwargs): + super().__init__(*args, **kwargs) + self.client = OpenAI(api_key=api_key, base_url="https://api.novita.ai/v3/openai") + self.api_key = api_key + self.user_api_key = user_api_key + + def _raw_gen(self, baseself, model, messages, stream=False, tools=None, **kwargs): + if tools: + response = self.client.chat.completions.create( + model=model, messages=messages, stream=stream, tools=tools, **kwargs + ) + return response.choices[0] + else: + response = self.client.chat.completions.create( + model=model, messages=messages, stream=stream, **kwargs + ) + return response.choices[0].message.content + + def _raw_gen_stream( + self, baseself, model, messages, stream=True, tools=None, **kwargs + ): + response = self.client.chat.completions.create( + model=model, messages=messages, stream=stream, **kwargs + ) + for line in response: + if line.choices[0].delta.content is not None: + yield line.choices[0].delta.content diff --git a/setup.sh b/setup.sh index 7775e24e..31ed3e42 100755 --- a/setup.sh +++ b/setup.sh @@ -148,6 +148,7 @@ prompt_cloud_api_provider_options() { echo -e "${YELLOW}4) Groq${NC}" echo -e "${YELLOW}5) HuggingFace Inference API${NC}" echo -e "${YELLOW}6) Azure OpenAI${NC}" + echo -e "${YELLOW}7) Novita${NC}" echo -e "${YELLOW}b) Back to Main Menu${NC}" echo read -p "$(echo -e "${DEFAULT_FG}Choose option (1-6, or b): ${NC}")" provider_choice @@ -428,6 +429,12 @@ connect_cloud_api_provider() { model_name="gpt-4o" get_api_key break ;; + 7) # Novita + provider_name="Novita" + llm_name="novita" + model_name="deepseek/deepseek-r1" + get_api_key + break ;; b|B) clear; return ;; # Clear screen and Back to Main Menu *) echo -e "\n${RED}Invalid choice. Please choose 1-6, or b.${NC}" ; sleep 1 ;; esac