From 0001963a04e5e47c5ce9f58445840a60d2ebf8c0 Mon Sep 17 00:00:00 2001
From: kom-senapati <komnoob123@gmail.com>
Date: Sat, 5 Oct 2024 18:36:12 +0530
Subject: [PATCH 1/2] feat: groq llms added

---
 application/api/answer/routes.py |  2 ++
 application/llm/groq.py          | 46 ++++++++++++++++++++++++++++++++
 application/llm/llm_creator.py   |  2 ++
 3 files changed, 50 insertions(+)
 create mode 100644 application/llm/groq.py

diff --git a/application/api/answer/routes.py b/application/api/answer/routes.py
index 35b95174..9a22db84 100644
--- a/application/api/answer/routes.py
+++ b/application/api/answer/routes.py
@@ -40,6 +40,8 @@ if settings.LLM_NAME == "openai":
     gpt_model = "gpt-3.5-turbo"
 elif settings.LLM_NAME == "anthropic":
     gpt_model = "claude-2"
+elif settings.LLM_NAME == "groq":
+    gpt_model = "llama3-8b-8192"
 
 if settings.MODEL_NAME:  # in case there is particular model name configured
     gpt_model = settings.MODEL_NAME
diff --git a/application/llm/groq.py b/application/llm/groq.py
new file mode 100644
index 00000000..5a29e8d8
--- /dev/null
+++ b/application/llm/groq.py
@@ -0,0 +1,46 @@
+from application.llm.base import BaseLLM
+from application.core.settings import settings
+
+
+
+class GroqLLM(BaseLLM):
+
+    def __init__(self, api_key=None, user_api_key=None, *args, **kwargs):
+        from openai import OpenAI
+
+        super().__init__(*args, **kwargs)
+        self.client = OpenAI(api_key=api_key, base_url="https://api.groq.com/openai/v1")
+        self.api_key = api_key
+        self.user_api_key = user_api_key
+
+    def _raw_gen(
+        self,
+        baseself,
+        model,
+        messages,
+        stream=False,
+        **kwargs
+    ):  
+        response = self.client.chat.completions.create(
+            model=model, messages=messages, stream=stream, **kwargs
+        )
+
+        return response.choices[0].message.content
+
+    def _raw_gen_stream(
+        self,
+        baseself,
+        model,
+        messages,
+        stream=True,
+        **kwargs
+    ):  
+        response = self.client.chat.completions.create(
+            model=model, messages=messages, stream=stream, **kwargs
+        )
+
+        for line in response:
+            # import sys
+            # print(line.choices[0].delta.content, file=sys.stderr)
+            if line.choices[0].delta.content is not None:
+                yield line.choices[0].delta.content
diff --git a/application/llm/llm_creator.py b/application/llm/llm_creator.py
index 7960778b..6a19de10 100644
--- a/application/llm/llm_creator.py
+++ b/application/llm/llm_creator.py
@@ -1,3 +1,4 @@
+from application.llm.groq import GroqLLM
 from application.llm.openai import OpenAILLM, AzureOpenAILLM
 from application.llm.sagemaker import SagemakerAPILLM
 from application.llm.huggingface import HuggingFaceLLM
@@ -17,6 +18,7 @@ class LLMCreator:
         "anthropic": AnthropicLLM,
         "docsgpt": DocsGPTAPILLM,
         "premai": PremAILLM,
+        "groq": GroqLLM
     }
 
     @classmethod

From dc0f26d3d8038d1343cfd14ff957344b9ad93105 Mon Sep 17 00:00:00 2001
From: kom-senapati <komnoob123@gmail.com>
Date: Sat, 5 Oct 2024 19:05:20 +0530
Subject: [PATCH 2/2] fix: unused import of setting

---
 application/llm/groq.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/application/llm/groq.py b/application/llm/groq.py
index 5a29e8d8..b5731a90 100644
--- a/application/llm/groq.py
+++ b/application/llm/groq.py
@@ -1,5 +1,4 @@
 from application.llm.base import BaseLLM
-from application.core.settings import settings