diff --git a/application/agents/tools/read_webpage.py b/application/agents/tools/read_webpage.py new file mode 100644 index 00000000..e87c79e3 --- /dev/null +++ b/application/agents/tools/read_webpage.py @@ -0,0 +1,83 @@ +import requests +from markdownify import markdownify +from application.agents.tools.base import Tool +from urllib.parse import urlparse + +class ReadWebpageTool(Tool): + """ + Read Webpage (browser) + A tool to fetch the HTML content of a URL and convert it to Markdown. + """ + + def __init__(self, config=None): + """ + Initializes the tool. + :param config: Optional configuration dictionary. Not used by this tool. + """ + self.config = config + + def execute_action(self, action_name: str, **kwargs) -> str: + """ + Executes the specified action. For this tool, the only action is 'read_webpage'. + + :param action_name: The name of the action to execute. Should be 'read_webpage'. + :param kwargs: Keyword arguments, must include 'url'. + :return: The Markdown content of the webpage or an error message. + """ + if action_name != "read_webpage": + return f"Error: Unknown action '{action_name}'. This tool only supports 'read_webpage'." + + url = kwargs.get("url") + if not url: + return "Error: URL parameter is missing." + + # Ensure the URL has a scheme (if not, default to http) + parsed_url = urlparse(url) + if not parsed_url.scheme: + url = "http://" + url + + try: + response = requests.get(url, timeout=10, headers={'User-Agent': 'DocsGPT-Agent/1.0'}) + response.raise_for_status() # Raise an exception for HTTP errors (4xx or 5xx) + + html_content = response.text + #soup = BeautifulSoup(html_content, 'html.parser') + + + markdown_content = markdownify(html_content, heading_style="ATX", newline_style="BACKSLASH") + + return markdown_content + + except requests.exceptions.RequestException as e: + return f"Error fetching URL {url}: {e}" + except Exception as e: + return f"Error processing URL {url}: {e}" + + def get_actions_metadata(self): + """ + Returns metadata for the actions supported by this tool. + """ + return [ + { + "name": "read_webpage", + "description": "Fetches the HTML content of a given URL and returns it as clean Markdown text. Input must be a valid URL.", + "parameters": { + "type": "object", + "properties": { + "url": { + "type": "string", + "description": "The fully qualified URL of the webpage to read (e.g., 'https://www.example.com').", + } + }, + "required": ["url"], + "additionalProperties": False, + }, + } + ] + + def get_config_requirements(self): + """ + Returns a dictionary describing the configuration requirements for the tool. + This tool does not require any specific configuration. + """ + return {} diff --git a/application/api/answer/routes.py b/application/api/answer/routes.py index 2d889b34..e5a33874 100644 --- a/application/api/answer/routes.py +++ b/application/api/answer/routes.py @@ -472,6 +472,7 @@ class Stream(Resource): decoded_token = request.decoded_token else: decoded_token = {"sub": data_key.get("user")} + is_shared_usage = False elif "active_docs" in data: source = {"active_docs": data["active_docs"]} @@ -524,7 +525,8 @@ class Stream(Resource): user_api_key=user_api_key, decoded_token=decoded_token, ) - + is_shared_usage_val = data.get("is_shared_usage", False) + is_shared_token_val = data.get("shared_token", None) return Response( complete_stream( question=question, @@ -537,8 +539,8 @@ class Stream(Resource): index=index, should_save_conversation=save_conv, agent_id=agent_id, - is_shared_usage=is_shared_usage, - shared_token=shared_token, + is_shared_usage=is_shared_usage_val, + shared_token=is_shared_token_val, ), mimetype="text/event-stream", ) diff --git a/application/llm/openai.py b/application/llm/openai.py index c918768d..e363130b 100644 --- a/application/llm/openai.py +++ b/application/llm/openai.py @@ -13,10 +13,11 @@ class OpenAILLM(BaseLLM): from openai import OpenAI super().__init__(*args, **kwargs) - if settings.OPENAI_BASE_URL: + if isinstance(settings.OPENAI_BASE_URL, str) and settings.OPENAI_BASE_URL.strip(): self.client = OpenAI(api_key=api_key, base_url=settings.OPENAI_BASE_URL) else: - self.client = OpenAI(api_key=api_key) + DEFAULT_OPENAI_API_BASE = "https://api.openai.com/v1" + self.client = OpenAI(api_key=api_key, base_url=DEFAULT_OPENAI_API_BASE) self.api_key = api_key self.user_api_key = user_api_key self.storage = StorageCreator.get_storage() diff --git a/application/prompts/chat_combine_creative.txt b/application/prompts/chat_combine_creative.txt index 2f9a61c9..9ce1f12e 100644 --- a/application/prompts/chat_combine_creative.txt +++ b/application/prompts/chat_combine_creative.txt @@ -1,9 +1,15 @@ -You are a DocsGPT, friendly and helpful AI assistant by Arc53 that provides help with documents. You give thorough answers with code examples if possible. -Use the following pieces of context to help answer the users question. If its not relevant to the question, provide friendly responses. -You have access to chat history, and can use it to help answer the question. -When using code examples, use the following format: +You are a helpful AI assistant, DocsGPT. You are proactive and helpful. Try to use tools, if they are available to you, +be proactive and fill in missing information. +Users can Upload documents for your context as attachments or sources via UI using the Conversation input box. +If appropriate, your answers can include code examples, formatted as follows: ```(language) (code) ``` +Users are also able to see charts and diagrams if you use them with valid mermaid syntax in your responses. +Try to respond with mermaid charts if visualization helps with users queries. +You effectively utilize chat history, ensuring relevant and tailored responses. +Try to use additional provided context if it's available, otherwise use your knowledge and tool capabilities. +Allow yourself to be very creative and use your imagination. ---------------- +Possible additional context from uploaded sources: {summaries} \ No newline at end of file diff --git a/application/prompts/chat_combine_default.txt b/application/prompts/chat_combine_default.txt index fdf0b2c2..e505370f 100644 --- a/application/prompts/chat_combine_default.txt +++ b/application/prompts/chat_combine_default.txt @@ -1,9 +1,14 @@ -You are a helpful AI assistant, DocsGPT, specializing in document assistance, designed to offer detailed and informative responses. +You are a helpful AI assistant, DocsGPT. You are proactive and helpful. Try to use tools, if they are available to you, +be proactive and fill in missing information. +Users can Upload documents for your context as attachments or sources via UI using the Conversation input box. If appropriate, your answers can include code examples, formatted as follows: ```(language) (code) ``` +Users are also able to see charts and diagrams if you use them with valid mermaid syntax in your responses. +Try to respond with mermaid charts if visualization helps with users queries. You effectively utilize chat history, ensuring relevant and tailored responses. -If a question doesn't align with your context, you provide friendly and helpful replies. +Try to use additional provided context if it's available, otherwise use your knowledge and tool capabilities. ---------------- +Possible additional context from uploaded sources: {summaries} \ No newline at end of file diff --git a/application/prompts/chat_combine_strict.txt b/application/prompts/chat_combine_strict.txt index b6de4630..6aa2ef5e 100644 --- a/application/prompts/chat_combine_strict.txt +++ b/application/prompts/chat_combine_strict.txt @@ -1,13 +1,17 @@ -You are an AI Assistant, DocsGPT, adept at offering document assistance. -Your expertise lies in providing answer on top of provided context. -You can leverage the chat history if needed. -Answer the question based on the context below. -Keep the answer concise. Respond "Irrelevant context" if not sure about the answer. -If question is not related to the context, respond "Irrelevant context". -When using code examples, use the following format: +You are a helpful AI assistant, DocsGPT. You are proactive and helpful. Try to use tools, if they are available to you, +be proactive and fill in missing information. +Users can Upload documents for your context as attachments or sources via UI using the Conversation input box. +If appropriate, your answers can include code examples, formatted as follows: ```(language) (code) ``` - ---------------- - Context: - {summaries} \ No newline at end of file +Users are also able to see charts and diagrams if you use them with valid mermaid syntax in your responses. +Try to respond with mermaid charts if visualization helps with users queries. +You effectively utilize chat history, ensuring relevant and tailored responses. +Use context provided below or use available tools tool capabilities to answer user queries. +If you dont have enough information from the context or tools, answer "I don't know" or "I don't have enough information". +Never make up information or provide false information! +Allow yourself to be very creative and use your imagination. +---------------- +Context from uploaded sources: +{summaries} \ No newline at end of file diff --git a/deployment/docker-compose.yaml b/deployment/docker-compose.yaml index 15d9522f..c4b81a08 100644 --- a/deployment/docker-compose.yaml +++ b/deployment/docker-compose.yaml @@ -12,6 +12,7 @@ services: - backend backend: + user: root build: ../application environment: - API_KEY=$API_KEY @@ -27,13 +28,14 @@ services: - "7091:7091" volumes: - ../application/indexes:/app/application/indexes - - ../application/inputs:/app/application/inputs + - ../application/inputs:/app/inputs - ../application/vectors:/app/application/vectors depends_on: - redis - mongo worker: + user: root build: ../application command: celery -A application.app.celery worker -l INFO -B environment: @@ -45,6 +47,10 @@ services: - MONGO_URI=mongodb://mongo:27017/docsgpt - API_URL=http://backend:7091 - CACHE_REDIS_URL=redis://redis:6379/2 + volumes: + - ../application/indexes:/app/application/indexes + - ../application/inputs:/app/inputs + - ../application/vectors:/app/application/vectors depends_on: - redis - mongo diff --git a/docs/pages/_app.mdx b/docs/pages/_app.mdx index 817cc517..52ff477a 100644 --- a/docs/pages/_app.mdx +++ b/docs/pages/_app.mdx @@ -4,7 +4,7 @@ export default function MyApp({ Component, pageProps }) { return ( <> - + ) } diff --git a/frontend/public/toolIcons/tool_read_webpage.svg b/frontend/public/toolIcons/tool_read_webpage.svg new file mode 100644 index 00000000..41692e95 --- /dev/null +++ b/frontend/public/toolIcons/tool_read_webpage.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/setup.sh b/setup.sh index 479def9b..5cf013fc 100755 --- a/setup.sh +++ b/setup.sh @@ -240,7 +240,7 @@ serve_local_ollama() { echo "LLM_NAME=openai" >> .env echo "MODEL_NAME=$model_name" >> .env echo "VITE_API_STREAMING=true" >> .env - echo "OPENAI_BASE_URL=http://host.docker.internal:11434/v1" >> .env + echo "OPENAI_BASE_URL=http://ollama:11434/v1" >> .env echo "EMBEDDINGS_NAME=huggingface_sentence-transformers/all-mpnet-base-v2" >> .env echo -e "${GREEN}.env file configured for Ollama ($(echo "$docker_compose_file_suffix" | tr '[:lower:]' '[:upper:]')${NC}${GREEN}).${NC}" echo -e "${YELLOW}Note: MODEL_NAME is set to '${BOLD}$model_name${NC}${YELLOW}'. You can change it later in the .env file.${NC}"