Merge branch 'main' of https://github.com/manishmadan2882/docsgpt

application/agents/tools/read_webpage.py

@@ -0,0 +1,83 @@
+import requests
+from markdownify import markdownify
+from application.agents.tools.base import Tool
+from urllib.parse import urlparse
+
+class ReadWebpageTool(Tool):
+    """
+    Read Webpage (browser)
+    A tool to fetch the HTML content of a URL and convert it to Markdown.
+    """
+
+    def __init__(self, config=None):
+        """
+        Initializes the tool.
+        :param config: Optional configuration dictionary. Not used by this tool.
+        """
+        self.config = config
+
+    def execute_action(self, action_name: str, **kwargs) -> str:
+        """
+        Executes the specified action. For this tool, the only action is 'read_webpage'.
+
+        :param action_name: The name of the action to execute. Should be 'read_webpage'.
+        :param kwargs: Keyword arguments, must include 'url'.
+        :return: The Markdown content of the webpage or an error message.
+        """
+        if action_name != "read_webpage":
+            return f"Error: Unknown action '{action_name}'. This tool only supports 'read_webpage'."
+
+        url = kwargs.get("url")
+        if not url:
+            return "Error: URL parameter is missing."
+
+        # Ensure the URL has a scheme (if not, default to http)
+        parsed_url = urlparse(url)
+        if not parsed_url.scheme:
+            url = "http://" + url
+        
+        try:
+            response = requests.get(url, timeout=10, headers={'User-Agent': 'DocsGPT-Agent/1.0'})
+            response.raise_for_status()  # Raise an exception for HTTP errors (4xx or 5xx)
+            
+            html_content = response.text
+            #soup = BeautifulSoup(html_content, 'html.parser')
+            
+            
+            markdown_content = markdownify(html_content, heading_style="ATX", newline_style="BACKSLASH")
+            
+            return markdown_content
+
+        except requests.exceptions.RequestException as e:
+            return f"Error fetching URL {url}: {e}"
+        except Exception as e:
+            return f"Error processing URL {url}: {e}"
+
+    def get_actions_metadata(self):
+        """
+        Returns metadata for the actions supported by this tool.
+        """
+        return [
+            {
+                "name": "read_webpage",
+                "description": "Fetches the HTML content of a given URL and returns it as clean Markdown text. Input must be a valid URL.",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "url": {
+                            "type": "string",
+                            "description": "The fully qualified URL of the webpage to read (e.g., 'https://www.example.com').",
+                        }
+                    },
+                    "required": ["url"],
+                    "additionalProperties": False,
+                },
+            }
+        ]
+
+    def get_config_requirements(self):
+        """
+        Returns a dictionary describing the configuration requirements for the tool.
+        This tool does not require any specific configuration.
+        """
+        return {}

application/api/answer/routes.py

@@ -472,6 +472,7 @@ class Stream(Resource):
                     decoded_token = request.decoded_token
                 else:
                     decoded_token = {"sub": data_key.get("user")}
+                    is_shared_usage = False
 
             elif "active_docs" in data:
                 source = {"active_docs": data["active_docs"]}
@@ -524,7 +525,8 @@ class Stream(Resource):
                 user_api_key=user_api_key,
                 decoded_token=decoded_token,
             )
-
+            is_shared_usage_val = data.get("is_shared_usage", False)
+            is_shared_token_val = data.get("shared_token", None)
             return Response(
                 complete_stream(
                     question=question,
@@ -537,8 +539,8 @@ class Stream(Resource):
                     index=index,
                     should_save_conversation=save_conv,
                     agent_id=agent_id,
-                    is_shared_usage=is_shared_usage,
-                    shared_token=shared_token,
+                    is_shared_usage=is_shared_usage_val,
+                    shared_token=is_shared_token_val,
                 ),
                 mimetype="text/event-stream",
             )

application/llm/openai.py

@@ -13,10 +13,11 @@ class OpenAILLM(BaseLLM):
         from openai import OpenAI
 
         super().__init__(*args, **kwargs)
-        if settings.OPENAI_BASE_URL:
+        if isinstance(settings.OPENAI_BASE_URL, str) and settings.OPENAI_BASE_URL.strip():
             self.client = OpenAI(api_key=api_key, base_url=settings.OPENAI_BASE_URL)
         else:
-            self.client = OpenAI(api_key=api_key)
+            DEFAULT_OPENAI_API_BASE = "https://api.openai.com/v1"
+            self.client = OpenAI(api_key=api_key, base_url=DEFAULT_OPENAI_API_BASE)
         self.api_key = api_key
         self.user_api_key = user_api_key
         self.storage = StorageCreator.get_storage()

application/prompts/chat_combine_creative.txt

@@ -1,9 +1,15 @@
-You are a DocsGPT, friendly and helpful AI assistant by Arc53 that provides help with documents. You give thorough answers with code examples if possible.
-Use the following pieces of context to help answer the users question. If its not relevant to the question, provide friendly responses.
-You have access to chat history, and can use it to help answer the question.
-When using code examples, use the following format:
+You are a helpful AI assistant, DocsGPT. You are proactive and helpful. Try to use tools, if they are available to you, 
+be proactive and fill in missing information.
+Users can Upload documents for your context as attachments or sources via UI using the Conversation input box.
+If appropriate, your answers can include code examples, formatted as follows:
 ```(language)
 (code)
 ```
+Users are also able to see charts and diagrams if you use them with valid mermaid syntax in your responses.
+Try to respond with mermaid charts if visualization helps with users queries.
+You effectively utilize chat history, ensuring relevant and tailored responses. 
+Try to use additional provided context if it's available, otherwise use your knowledge and tool capabilities.
+Allow yourself to be very creative and use your imagination.
 ----------------
+Possible additional context from uploaded sources:
 {summaries}

application/prompts/chat_combine_default.txt

@@ -1,9 +1,14 @@
-You are a helpful AI assistant, DocsGPT, specializing in document assistance, designed to offer detailed and informative responses. 
+You are a helpful AI assistant, DocsGPT. You are proactive and helpful. Try to use tools, if they are available to you, 
+be proactive and fill in missing information.
+Users can Upload documents for your context as attachments or sources via UI using the Conversation input box.
 If appropriate, your answers can include code examples, formatted as follows:
 ```(language)
 (code)
 ```
+Users are also able to see charts and diagrams if you use them with valid mermaid syntax in your responses.
+Try to respond with mermaid charts if visualization helps with users queries.
 You effectively utilize chat history, ensuring relevant and tailored responses. 
-If a question doesn't align with your context, you provide friendly and helpful replies.
+Try to use additional provided context if it's available, otherwise use your knowledge and tool capabilities.
 ----------------
+Possible additional context from uploaded sources:
 {summaries}

application/prompts/chat_combine_strict.txt

@@ -1,13 +1,17 @@
-You are an AI Assistant, DocsGPT, adept at offering document assistance. 
-Your expertise lies in providing answer on top of provided context. 
-You can leverage the chat history if needed.
-Answer the question based on the context below. 
-Keep the answer concise. Respond "Irrelevant context" if not sure about the answer.
-If question is not related to the context, respond "Irrelevant context".
-When using code examples, use the following format:
+You are a helpful AI assistant, DocsGPT. You are proactive and helpful. Try to use tools, if they are available to you, 
+be proactive and fill in missing information.
+Users can Upload documents for your context as attachments or sources via UI using the Conversation input box.
+If appropriate, your answers can include code examples, formatted as follows:
 ```(language)
 (code)
 ```
- ----------------
- Context:
- {summaries}
+Users are also able to see charts and diagrams if you use them with valid mermaid syntax in your responses.
+Try to respond with mermaid charts if visualization helps with users queries.
+You effectively utilize chat history, ensuring relevant and tailored responses. 
+Use context provided below or use available tools tool capabilities to answer user queries.
+If you dont have enough information from the context or tools, answer "I don't know" or "I don't have enough information".
+Never make up information or provide false information!
+Allow yourself to be very creative and use your imagination.
+----------------
+Context from uploaded sources:
+{summaries}

deployment/docker-compose.yaml

@@ -12,6 +12,7 @@ services:
       - backend
 
   backend:
+    user: root
     build: ../application
     environment:
       - API_KEY=$API_KEY
@@ -27,13 +28,14 @@ services:
       - "7091:7091"
     volumes:
       - ../application/indexes:/app/application/indexes
-      - ../application/inputs:/app/application/inputs
+      - ../application/inputs:/app/inputs
       - ../application/vectors:/app/application/vectors
     depends_on:
       - redis
       - mongo
 
   worker:
+    user: root
     build: ../application
     command: celery -A application.app.celery worker -l INFO -B
     environment:
@@ -45,6 +47,10 @@ services:
       - MONGO_URI=mongodb://mongo:27017/docsgpt
       - API_URL=http://backend:7091
       - CACHE_REDIS_URL=redis://redis:6379/2
+    volumes:
+      - ../application/indexes:/app/application/indexes
+      - ../application/inputs:/app/inputs
+      - ../application/vectors:/app/application/vectors
     depends_on:
       - redis
       - mongo

docs/pages/_app.mdx

@@ -4,7 +4,7 @@ export default function MyApp({ Component, pageProps }) {
   return (
     <>
       <Component {...pageProps} />
-        <DocsGPTWidget showSources={true} apiKey="6dd66edf-d374-4904-93af-ab0c6d41ee56" theme="dark" size="medium" />
+        <DocsGPTWidget showSources={true} apiKey="5d8270cb-735f-484e-9dc9-5b407f24e652" theme="dark" size="medium" />
     </>
   )
 }

frontend/public/toolIcons/tool_read_webpage.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" height="24px" viewBox="0 -960 960 960" width="24px" fill="#e3e3e3"><path d="M480-80q-82 0-155-31.5t-127.5-86Q143-252 111.5-325T80-480q0-83 31.5-155.5t86-127Q252-817 325-848.5T480-880q83 0 155.5 31.5t127 86q54.5 54.5 86 127T880-480q0 82-31.5 155t-86 127.5q-54.5 54.5-127 86T480-80Zm0-82q26-36 45-75t31-83H404q12 44 31 83t45 75Zm-104-16q-18-33-31.5-68.5T322-320H204q29 50 72.5 87t99.5 55Zm208 0q56-18 99.5-55t72.5-87H638q-9 38-22.5 73.5T584-178ZM170-400h136q-3-20-4.5-39.5T300-480q0-21 1.5-40.5T306-560H170q-5 20-7.5 39.5T160-480q0 21 2.5 40.5T170-400Zm216 0h188q3-20 4.5-39.5T580-480q0-21-1.5-40.5T574-560H386q-3 20-4.5 39.5T380-480q0 21 1.5 40.5T386-400Zm268 0h136q5-20 7.5-39.5T800-480q0-21-2.5-40.5T790-560H654q3 20 4.5 39.5T660-480q0 21-1.5 40.5T654-400Zm-16-240h118q-29-50-72.5-87T584-782q18 33 31.5 68.5T638-640Zm-234 0h152q-12-44-31-83t-45-75q-26 36-45 75t-31 83Zm-200 0h118q9-38 22.5-73.5T376-782q-56 18-99.5 55T204-640Z"/></svg>

setup.sh

@@ -240,7 +240,7 @@ serve_local_ollama() {
     echo "LLM_NAME=openai" >> .env
     echo "MODEL_NAME=$model_name" >> .env
     echo "VITE_API_STREAMING=true" >> .env
-    echo "OPENAI_BASE_URL=http://host.docker.internal:11434/v1" >> .env
+    echo "OPENAI_BASE_URL=http://ollama:11434/v1" >> .env
     echo "EMBEDDINGS_NAME=huggingface_sentence-transformers/all-mpnet-base-v2" >> .env
     echo -e "${GREEN}.env file configured for Ollama ($(echo "$docker_compose_file_suffix" | tr '[:lower:]' '[:upper:]')${NC}${GREEN}).${NC}"
     echo -e "${YELLOW}Note: MODEL_NAME is set to '${BOLD}$model_name${NC}${YELLOW}'. You can change it later in the .env file.${NC}"