Merge pull request #1608 from siiddhantt/feat/api-tool

feat: API Tool

.github/FUNDING.yml

@@ -0,0 +1,3 @@
+# These are supported funding model platforms
+
+github: arc53

.github/dependabot.yml

@@ -8,8 +8,12 @@ updates:
   - package-ecosystem: "pip" # See documentation for possible values
     directory: "/application" # Location of package manifests
     schedule:
-      interval: "weekly"
+      interval: "daily"
   - package-ecosystem: "npm" # See documentation for possible values
     directory: "/frontend" # Location of package manifests
     schedule:
-      interval: "weekly"
+      interval: "daily"
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "daily"

.github/holopin.yml

@@ -1,5 +1,11 @@
-organization: arc53
-defaultSticker: clqmdf0ed34290glbvqh0kzxd
+organization: docsgpt
+defaultSticker: cm1ulwkkl180570cl82rtzympu
 stickers:
-  - id: clqmdf0ed34290glbvqh0kzxd
-    alias: festive
+  - id: cm1ulwkkl180570cl82rtzympu
+    alias: contributor2024
+  - id: cm1ureg8o130450cl8c1po6mil
+    alias: api
+  - id: cm1urhmag148240cl8yvqxkthx
+    alias: lpc
+  - id: cm1urlcpq622090cl2tvu4w71y
+    alias: lexeu

.github/labeler.yml

@@ -1,23 +1,31 @@
 repo:
-  - '*'
+- changed-files:
+  - any-glob-to-any-file: '*'
 
 github:
-  - .github/**/*
+- changed-files:
+  - any-glob-to-any-file: '.github/**/*'
 
 application:
-  - application/**/*
+- changed-files:
+  - any-glob-to-any-file: 'application/**/*'
 
 docs:
-  - docs/**/*
+- changed-files:
+  - any-glob-to-any-file: 'docs/**/*'
 
 extensions:
-  - extensions/**/*
+- changed-files:
+  - any-glob-to-any-file: 'extensions/**/*'
 
 frontend:
-  - frontend/**/*
+- changed-files:
+  - any-glob-to-any-file: 'frontend/**/*'
 
 scripts:
-  - scripts/**/*
+- changed-files:
+  - any-glob-to-any-file: 'scripts/**/*'
 
 tests:
-  - tests/**/*
+- changed-files:
+  - any-glob-to-any-file: 'tests/**/*'

.github/workflows/ci.yml

@@ -1,10 +1,8 @@
 name: Build and push DocsGPT Docker image
 
 on:
-  workflow_dispatch:
-  push:
-    branches:
-      - main
+  release:
+    types: [published]
 
 jobs:
   deploy:
@@ -14,34 +12,36 @@ jobs:
       contents: read
       packages: write
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
 
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
       - name: Login to ghcr.io
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Build and push Docker images to docker.io and ghcr.io
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v6
         with:
           file: './application/Dockerfile'
           platforms: linux/amd64
           context: ./application
           push: true
           tags: |
-            ${{ secrets.DOCKER_USERNAME }}/docsgpt:latest
-            ghcr.io/${{ github.repository_owner }}/docsgpt:latest
+            ${{ secrets.DOCKER_USERNAME }}/docsgpt:${{ github.event.release.tag_name }},${{ secrets.DOCKER_USERNAME }}/docsgpt:latest
+            ghcr.io/${{ github.repository_owner }}/docsgpt:${{ github.event.release.tag_name }},ghcr.io/${{ github.repository_owner }}/docsgpt:latest
+          cache-from: type=registry,ref=${{ secrets.DOCKER_USERNAME }}/docsgpt:latest
+          cache-to: type=inline

.github/workflows/cife.yml

@@ -1,10 +1,8 @@
 name: Build and push DocsGPT-FE Docker image
 
 on:
-  workflow_dispatch:
-  push:
-    branches:
-      - main
+  release:
+    types: [published]
 
 jobs:
   deploy:
@@ -14,22 +12,22 @@ jobs:
       contents: read
       packages: write
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
 
       - name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
 
       - name: Login to ghcr.io
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
@@ -37,12 +35,14 @@ jobs:
 
       # Runs a single command using the runners shell
       - name: Build and push Docker images to docker.io and ghcr.io
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v6
         with:
           file: './frontend/Dockerfile'
           platforms: linux/amd64, linux/arm64
           context: ./frontend
           push: true
           tags: |
-            ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:latest
-            ghcr.io/${{ github.repository_owner }}/docsgpt-fe:latest
+            ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:${{ github.event.release.tag_name }},${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:latest
+            ghcr.io/${{ github.repository_owner }}/docsgpt-fe:${{ github.event.release.tag_name }},ghcr.io/${{ github.repository_owner }}/docsgpt-fe:latest
+          cache-from: type=registry,ref=${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:latest
+          cache-to: type=inline

.github/workflows/docker-develop-build.yml

@@ -0,0 +1,49 @@
+name: Build and push DocsGPT Docker image for development
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+
+jobs:
+  deploy:
+    if: github.repository == 'arc53/DocsGPT'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Login to ghcr.io
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push Docker images to docker.io and ghcr.io
+        uses: docker/build-push-action@v6
+        with:
+          file: './application/Dockerfile'
+          platforms: linux/amd64
+          context: ./application
+          push: true
+          tags: |
+            ${{ secrets.DOCKER_USERNAME }}/docsgpt:develop
+            ghcr.io/${{ github.repository_owner }}/docsgpt:develop
+          cache-from: type=registry,ref=${{ secrets.DOCKER_USERNAME }}/docsgpt:develop
+          cache-to: type=inline

.github/workflows/docker-develop-fe-build.yml

@@ -0,0 +1,49 @@
+name: Build and push DocsGPT FE Docker image for development
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+
+jobs:
+  deploy:
+    if: github.repository == 'arc53/DocsGPT'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Login to ghcr.io
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push Docker images to docker.io and ghcr.io
+        uses: docker/build-push-action@v6
+        with:
+          file: './frontend/Dockerfile'
+          platforms: linux/amd64
+          context: ./frontend
+          push: true
+          tags: |
+            ${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:develop
+            ghcr.io/${{ github.repository_owner }}/docsgpt-fe:develop
+          cache-from: type=registry,ref=${{ secrets.DOCKER_USERNAME }}/docsgpt-fe:develop
+          cache-to: type=inline

.github/workflows/labeler.yml

@@ -10,7 +10,7 @@ jobs:
       pull-requests: write
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/labeler@v4
+      - uses: actions/labeler@v5
         with:
           repo-token: "${{ secrets.GITHUB_TOKEN }}"
           sync-labels: true

.github/workflows/lint.yml

@@ -11,7 +11,7 @@ jobs:
   ruff:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Lint with Ruff
         uses: chartboost/ruff-action@v1

.github/workflows/pytest.yml

@@ -6,11 +6,11 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.11"]
+        python-version: ["3.12"]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
@@ -23,8 +23,8 @@ jobs:
         run: |
           python -m pytest --cov=application --cov-report=xml
       - name: Upload coverage reports to Codecov
-        if: github.event_name == 'pull_request' && matrix.python-version == '3.11'
-        uses: codecov/codecov-action@v3
+        if: github.event_name == 'pull_request' && matrix.python-version == '3.12'
+        uses: codecov/codecov-action@v5
         env:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 

.github/workflows/sync_fork.yaml

@@ -17,7 +17,7 @@ jobs:
     steps:
       # Step 1: run a standard checkout action
       - name: Checkout target repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       # Step 2: run the sync action
       - name: Sync upstream changes

.vscode/launch.json

@@ -11,6 +11,44 @@
       "skipFiles": [
         "<node_internals>/**"
       ]
+    },
+    {
+        "name": "Flask Debugger",
+        "type": "debugpy",
+        "request": "launch",
+        "module": "flask",
+        "env": {
+            "FLASK_APP": "application/app.py",
+            "PYTHONPATH": "${workspaceFolder}",
+            "FLASK_ENV": "development",
+            "FLASK_DEBUG": "1",
+            "FLASK_RUN_PORT": "7091",
+            "FLASK_RUN_HOST": "0.0.0.0"
+
+        },
+        "args": [
+            "run",
+            "--no-debugger"
+        ],
+        "cwd": "${workspaceFolder}",
+    },
+    {
+      "name": "Celery Debugger",
+      "type": "debugpy",
+      "request": "launch",
+      "module": "celery",
+      "env": {
+        "PYTHONPATH": "${workspaceFolder}",
+      },
+      "args": [
+        "-A",
+        "application.app.celery",
+        "worker",
+        "-l",
+        "INFO",
+        "--pool=solo"
+      ],
+      "cwd": "${workspaceFolder}"
     }
   ]
 }

CONTRIBUTING.md

@@ -6,7 +6,7 @@ Thank you for choosing to contribute to DocsGPT! We are all very grateful!
 
 📣 **Discussions** - Engage in conversations, start new topics, or help answer questions.
 
-🐞 **Issues** - This is where we keep track of tasks. It could be bugs,fixes or suggestions for new features.
+🐞 **Issues** - This is where we keep track of tasks. It could be bugs, fixes or suggestions for new features.
 
 🛠️ **Pull requests** - Suggest changes to our repository, either by working on existing issues or adding new features.
 
@@ -21,11 +21,13 @@ Thank you for choosing to contribute to DocsGPT! We are all very grateful!
 - If you're interested in contributing code, here are some important things to know:
 
 - We have a frontend built on React (Vite) and a backend in Python.
-=======
-Before creating issues, please check out how the latest version of our app looks and works by launching it via [Quickstart](https://github.com/arc53/DocsGPT#quickstart) the version on our live demo is slightly modified with login. Your issues should relate to the version that you can launch via [Quickstart](https://github.com/arc53/DocsGPT#quickstart).
+
+  
+Before creating issues, please check out how the latest version of our app looks and works by launching it via [Quickstart](https://github.com/arc53/DocsGPT#quickstart) the version on our live demo is slightly modified with login. Your issues should relate to the version you can launch via [Quickstart](https://github.com/arc53/DocsGPT#quickstart).
 
 ### 👨‍💻 If you're interested in contributing code, here are some important things to know:
 
+For instructions on setting up a development environment, please refer to our [Development Deployment Guide](https://docs.docsgpt.cloud/Deploying/Development-Environment).
 
 Tech Stack Overview:
 
@@ -35,15 +37,14 @@ Tech Stack Overview:
 
 ### 🌐 If you are looking to contribute to frontend (⚛️React, Vite):
 
-- The current frontend is being migrated from [`/application`](https://github.com/arc53/DocsGPT/tree/main/application) to [`/frontend`](https://github.com/arc53/DocsGPT/tree/main/frontend) with a new design, so please contribute to the new one.
-- Check out this [milestone](https://github.com/arc53/DocsGPT/milestone/1) and its issues.
+
 - The updated Figma design can be found [here](https://www.figma.com/file/OXLtrl1EAy885to6S69554/DocsGPT?node-id=0%3A1&t=hjWVuxRg9yi5YkJ9-1).
 
 Please try to follow the guidelines.
 
 ### 🖥 If you are looking to contribute to Backend (🐍 Python):
 
-- Review our issues and contribute to [`/application`](https://github.com/arc53/DocsGPT/tree/main/application) or [`/scripts`](https://github.com/arc53/DocsGPT/tree/main/scripts) (please disregard old [`ingest_rst.py`](https://github.com/arc53/DocsGPT/blob/main/scripts/old/ingest_rst.py) [`ingest_rst_sphinx.py`](https://github.com/arc53/DocsGPT/blob/main/scripts/old/ingest_rst_sphinx.py) files; they will be deprecated soon).
+- Review our issues and contribute to [`/application`](https://github.com/arc53/DocsGPT/tree/main/application) 
 - All new code should be covered with unit tests ([pytest](https://github.com/pytest-dev/pytest)). Please find tests under [`/tests`](https://github.com/arc53/DocsGPT/tree/main/tests) folder.
 - Before submitting your Pull Request, ensure it can be queried after ingesting some test data.
   
@@ -125,4 +126,4 @@ Thank you for considering contributing to DocsGPT! 🙏
 
 ## Questions/collaboration
 Feel free to join our [Discord](https://discord.gg/n5BX8dh8rU). We're very friendly and welcoming to new contributors, so don't hesitate to reach out.
-# Thank you so much for considering to contribute DocsGPT!🙏
+# Thank you so much for considering to contributing DocsGPT!🙏

HACKTOBERFEST.md

@@ -1,37 +0,0 @@
-# **🎉 Join the Hacktoberfest with DocsGPT and win a Free T-shirt and other prizes! 🎉**
-
-Welcome, contributors! We're excited to announce that DocsGPT is participating in Hacktoberfest. Get involved by submitting meaningful pull requests.
-
-All contributors with accepted PRs will receive a cool Holopin! 🤩 (Watch out for a reply in your PR to collect it).
-
-### 🏆 Top 50 contributors will recieve a special T-shirt
-
-### 🏆 [LLM Document analysis by LexEU competition](https://github.com/arc53/DocsGPT/blob/main/lexeu-competition.md): 
-A separate competition is available for those sumbit best new retrieval / workflow method that will analyze a Document using EU laws.
-With 200$, 100$, 50$ prize for 1st, 2nd and 3rd place respectively.
-You can find more information [here](https://github.com/arc53/DocsGPT/blob/main/lexeu-competition.md)
-
-## 📜 Here's How to Contribute:
-```text
-🛠️ Code: This is the golden ticket! Make meaningful contributions through PRs.
-
-🧩 API extention: Build an app utilising DocsGPT API. We prefer submissions that showcase original ideas and turn the API into an AI agent.
-
-Non-Code Contributions:
-
-📚 Wiki: Improve our documentation, Create a guide or change existing documentation.
-
-🖥️ Design: Improve the UI/UX or design a new feature.
-
-📝 Blogging or Content Creation: Write articles or create videos to showcase DocsGPT or highlight your contributions!
-```
-
-### 📝 Guidelines for Pull Requests:
-- Familiarize yourself with the current contributions and our [Roadmap](https://github.com/orgs/arc53/projects/2).
-- Before contributing we highly advise that you check existing [issues](https://github.com/arc53/DocsGPT/issues) or [create](https://github.com/arc53/DocsGPT/issues/new/choose) an issue and wait to get assigned.
-- Once you are finished with your contribution, please fill in this [form](https://airtable.com/appikMaJwdHhC1SDP/pagoblCJ9W29wf6Hf/form).
-- Refer to the [Documentation](https://docs.docsgpt.cloud/).
-- Feel free to join our [Discord](https://discord.gg/n5BX8dh8rU) server. We're here to help newcomers, so don't hesitate to jump in! Join us [here](https://discord.gg/n5BX8dh8rU).
-  
-Thank you very much for considering contributing to DocsGPT during Hacktoberfest! 🙏 Your contributions (not just simple typo) could earn you a stylish new t-shirt and other prizes as a token of our appreciation. 🎁 Join us, and let's code together! 🚀
-

README.md

@@ -3,13 +3,11 @@
 </h1>
 
 <p align="center">
-  <strong>Open-Source Documentation Assistant</strong>
+  <strong>Open-Source RAG Assistant</strong>
 </p>
 
 <p align="left">
-  <strong><a href="https://www.docsgpt.cloud/">DocsGPT</a></strong> is a cutting-edge open-source solution that streamlines the process of finding information in the project documentation. With its integration of the powerful <strong>GPT</strong> models, developers can easily ask questions about a project and receive accurate answers.
-  
-Say goodbye to time-consuming manual searches, and let <strong><a href="https://www.docsgpt.cloud/">DocsGPT</a></strong> help you quickly find the information you need. Try it out and see how it revolutionizes your project documentation experience. Contribute to its development and be a part of the future of AI-powered assistance.
+  <strong><a href="https://www.docsgpt.cloud/">DocsGPT</a></strong> is an open-source genAI tool that helps users get reliable answers from any knowledge source, while avoiding hallucinations. It enables quick and reliable information retrieval, with tooling and agentic system capability built in.
 </p>
 
 <div align="center">
@@ -20,177 +18,122 @@ Say goodbye to time-consuming manual searches, and let <strong><a href="https://
   <a href="https://discord.gg/n5BX8dh8rU">![link to discord](https://img.shields.io/discord/1070046503302877216)</a>
   <a href="https://twitter.com/docsgptai">![X (formerly Twitter) URL](https://img.shields.io/twitter/follow/docsgptai)</a>
 
- 
+  <br>
+
+  [☁️ Cloud Version](https://app.docsgpt.cloud/) • [💬 Discord](https://discord.gg/n5BX8dh8rU) • [📖 Guides](https://docs.docsgpt.cloud/)
+  <br>
+  [👫 Contribute](https://github.com/arc53/DocsGPT/blob/main/CONTRIBUTING.md) • [🏠 Self-host](https://docs.docsgpt.cloud/Guides/How-to-use-different-LLM) • [⚡️ Quickstart](https://github.com/arc53/DocsGPT#quickstart) 
+
 </div>
+<div align="center">
+<img src="https://d3dg1063dc54p9.cloudfront.net/videos/demov7.gif" alt="video-example-of-docs-gpt" width="800" height="450">
+</div>
+<h3 align="left">
+  <strong>Key Features:</strong>
+</h3>
+<ul align="left">
+    <li><strong>🗂️ Wide Format Support:</strong> Reads PDF, DOCX, CSV, XLSX, EPUB, MD, RST, HTML, MDX, JSON, PPTX, and images.</li>
+    <li><strong>🌐 Web & Data Integration:</strong> Ingests from URLs, sitemaps, Reddit, GitHub and web crawlers.</li>
+    <li><strong>✅ Reliable Answers:</strong> Get accurate, hallucination-free responses with source citations viewable in a clean UI.</li>
+    <li><strong>🔗 Actionable Tooling:</strong> Connect to APIs, tools, and other services to enable LLM actions.</li>
+    <li><strong>🧩 Pre-built Integrations:</strong> Use readily available HTML/React chat widgets, search tools, Discord/Telegram bots, and more.</li>
+    <li><strong>🔌 Flexible Deployment:</strong> Works with major LLMs (OpenAI, Google, Anthropic) and local models (Ollama, llama_cpp).</li>
+    <li><strong>🏢 Secure & Scalable:</strong> Run privately and securely with Kubernetes support, designed for enterprise-grade reliability.</li>
+</ul>
 
-### 🎃 [Hacktoberfest Prizes, Rules & Q&A](https://github.com/arc53/DocsGPT/blob/main/HACKTOBERFEST.md) 🎃
+## Roadmap
 
-### Our [Livestream to Dive into Hacktoberfest! Prizes, Rules & Q&A 🎉](https://www.youtube.com/watch?v=5QQaFFu9BC8) on 3rd of October 
+- [x] Full GoogleAI compatibility (Jan 2025)
+- [x] Add tools (Jan 2025)
+- [ ] Anthropic Tool compatibility
+- [ ] Add triggerable actions / tools (webhook)
+- [ ] Add OAuth 2.0 authentication for tools and sources
+- [ ] Manually updating chunks in the app UI
+- [ ] Devcontainer for easy development
+- [ ] Chatbots menu re-design to handle tools, scheduling, and more
+
+You can find our full roadmap [here](https://github.com/orgs/arc53/projects/2). Please don't hesitate to contribute or create issues, it helps us improve DocsGPT!
 
 ### Production Support / Help for Companies:
 
 We're eager to provide personalized assistance when deploying your DocsGPT to a live environment.
 
-- [Book Enterprise / teams Demo :wave:](https://cal.com/arc53/docsgpt-demo-b2b?date=2024-09-27&month=2024-09)
-- [Send Email :email:](mailto:contact@arc53.com?subject=DocsGPT%20support%2Fsolutions)
+[Get a Demo :wave:](https://www.docsgpt.cloud/contact)⁠
 
-![video-example-of-docs-gpt](https://d3dg1063dc54p9.cloudfront.net/videos/demov3.gif)
+[Send Email :email:](mailto:support@docsgpt.cloud?subject=DocsGPT%20support%2Fsolutions)
 
-## Roadmap
-
-You can find our roadmap [here](https://github.com/orgs/arc53/projects/2). Please don't hesitate to contribute or create issues, it helps us improve DocsGPT!
-
-## Our Open-Source Models Optimized for DocsGPT:
-
-| Name                                                                  | Base Model  | Requirements (or similar) |
-| --------------------------------------------------------------------- | ----------- | ------------------------- |
-| [Docsgpt-7b-mistral](https://huggingface.co/Arc53/docsgpt-7b-mistral)   | Mistral-7b   | 1xA10G gpu                |
-| [Docsgpt-14b](https://huggingface.co/Arc53/docsgpt-14b)               | llama-2-14b | 2xA10 gpu's               |
-| [Docsgpt-40b-falcon](https://huggingface.co/Arc53/docsgpt-40b-falcon) | falcon-40b  | 8xA10G gpu's              |
-
-If you don't have enough resources to run it, you can use bitsnbytes to quantize.
-
-## End to End AI Framework for Information Retrieval
-
-![Architecture chart](https://github.com/user-attachments/assets/fc6a7841-ddfc-45e6-b5a0-d05fe648cbe2)
-
-## Useful Links
-
-- :mag: :fire: [Cloud Version](https://app.docsgpt.cloud/)
-
-- :speech_balloon: :tada: [Join our Discord](https://discord.gg/n5BX8dh8rU)
-
-- :books: :sunglasses: [Guides](https://docs.docsgpt.cloud/)
-
-- :couple: [Interested in contributing?](https://github.com/arc53/DocsGPT/blob/main/CONTRIBUTING.md)
-
-- :file_folder: :rocket: [How to use any other documentation](https://docs.docsgpt.cloud/Guides/How-to-train-on-other-documentation)
-
-- :house: :closed_lock_with_key: [How to host it locally (so all data will stay on-premises)](https://docs.docsgpt.cloud/Guides/How-to-use-different-LLM)
-
-## Project Structure
-
-- Application - Flask app (main application).
-
-- Extensions - Chrome extension.
-
-- Scripts - Script that creates similarity search index for other libraries.
-
-- Frontend - Frontend uses <a href="https://vitejs.dev/">Vite</a> and <a href="https://react.dev/">React</a>.
 
 ## QuickStart
 
 > [!Note]
 > Make sure you have [Docker](https://docs.docker.com/engine/install/) installed
 
+
+1. Clone the repository and run the following command:
+  ```bash
+  git clone https://github.com/arc53/DocsGPT.git
+  cd DocsGPT
+  ```
+
 On Mac OS or Linux, write:
 
-`./setup.sh`
+
+2. Run the following command:
+  ```bash
+  ./setup.sh
+  ```
 
 It will install all the dependencies and allow you to download the local model, use OpenAI or use our LLM API.
 
 Otherwise, refer to this Guide for Windows:
 
-1. Download and open this repository with `git clone https://github.com/arc53/DocsGPT.git`
-2. Create a `.env` file in your root directory and set the env variables and `VITE_API_STREAMING` to true or false, depending on whether you want streaming answers or not.
+On windows:
+
+2. Create a `.env` file in your root directory and set the env variables.
    It should look like this inside:
 
    ```
    LLM_NAME=[docsgpt or openai or others] 
-   VITE_API_STREAMING=true
    API_KEY=[if LLM_NAME is openai]
    ```
 
-   See optional environment variables in the [/.env-template](https://github.com/arc53/DocsGPT/blob/main/.env-template) and [/application/.env_sample](https://github.com/arc53/DocsGPT/blob/main/application/.env_sample) files.
+   See optional environment variables in the [/application/.env_sample](https://github.com/arc53/DocsGPT/blob/main/application/.env_sample) file.
 
-3. Run [./run-with-docker-compose.sh](https://github.com/arc53/DocsGPT/blob/main/run-with-docker-compose.sh).
+3. Run the following command:
+
+   ```bash
+   docker compose up --build
+   ```
 4. Navigate to http://localhost:5173/.
 
 To stop, just run `Ctrl + C`.
 
-## Development Environments
-
-### Spin up Mongo and Redis
-
-For development, only two containers are used from [docker-compose.yaml](https://github.com/arc53/DocsGPT/blob/main/docker-compose.yaml) (by deleting all services except for Redis and Mongo).
-See file [docker-compose-dev.yaml](./docker-compose-dev.yaml).
-
-Run
-
-```
-docker compose -f docker-compose-dev.yaml build
-docker compose -f docker-compose-dev.yaml up -d
-```
-
-### Run the Backend
-
 > [!Note]
-> Make sure you have Python 3.10 or 3.11 installed.
-
-1. Export required environment variables or prepare a `.env` file in the project folder:
-   - Copy [.env_sample](https://github.com/arc53/DocsGPT/blob/main/application/.env_sample) and create `.env`.
-
-(check out [`application/core/settings.py`](application/core/settings.py) if you want to see more config options.)
-
-2. (optional) Create a Python virtual environment:
-   You can follow the [Python official documentation](https://docs.python.org/3/tutorial/venv.html) for virtual environments.
-
-a) On Mac OS and Linux
-
-```commandline
-python -m venv venv
-. venv/bin/activate
-```
-
-b) On Windows
-
-```commandline
-python -m venv venv
- venv/Scripts/activate
-```
-
-3. Download embedding model and save it in the `model/` folder:
-You can use the script below, or download it manually from [here](https://d3dg1063dc54p9.cloudfront.net/models/embeddings/mpnet-base-v2.zip), unzip it and save it in the `model/` folder.
-
-```commandline
-wget https://d3dg1063dc54p9.cloudfront.net/models/embeddings/mpnet-base-v2.zip
-unzip mpnet-base-v2.zip -d model
-rm mpnet-base-v2.zip
-```
-
-4. Install dependencies for the backend:
-
-```commandline
-pip install -r application/requirements.txt
-```
-
-5. Run the app using `flask --app application/app.py run --host=0.0.0.0 --port=7091`.
-6. Start worker with `celery -A application.app.celery worker -l INFO`.
-
-### Start Frontend
-
-> [!Note]
-> Make sure you have Node version 16 or higher.
-
-1. Navigate to the [/frontend](https://github.com/arc53/DocsGPT/tree/main/frontend) folder.
-2. Install the required packages `husky` and `vite` (ignore if already installed).
-
-```commandline
-npm install husky -g
-npm install vite -g
-```
-
-3. Install dependencies by running `npm install --include=dev`.
-4. Run the app using `npm run dev`.
+> For development environment setup instructions, please refer to the [Development Environment Guide](https://docs.docsgpt.cloud/Deploying/Development-Environment).
 
 ## Contributing
 
 Please refer to the [CONTRIBUTING.md](CONTRIBUTING.md) file for information about how to get involved. We welcome issues, questions, and pull requests.
 
+## Architecture
+
+![Architecture chart](https://github.com/user-attachments/assets/fc6a7841-ddfc-45e6-b5a0-d05fe648cbe2)
+
+## Project Structure
+
+- Application - Flask app (main application).
+
+- Extensions - Extensions, like react widget or discord bot.
+
+- Frontend - Frontend uses <a href="https://vitejs.dev/">Vite</a> and <a href="https://react.dev/">React</a>.
+
+- Scripts - Miscellaneous scripts.
+
 ## Code Of Conduct
 
 We as members, contributors, and leaders, pledge to make participation in our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation. Please refer to the [CODE_OF_CONDUCT.md](CODE_OF_CONDUCT.md) file for more information about contributing.
 
+
 ## Many Thanks To Our Contributors⚡
 
 <a href="https://github.com/arc53/DocsGPT/graphs/contributors" alt="View Contributors">
@@ -201,4 +144,9 @@ We as members, contributors, and leaders, pledge to make participation in our co
 
 The source code license is [MIT](https://opensource.org/license/mit/), as described in the [LICENSE](LICENSE) file.
 
-Built with [:bird: :link: LangChain](https://github.com/hwchase17/langchain)
+<p>This project is supported by:</p>
+<p>
+  <a href="https://www.digitalocean.com/?utm_medium=opensource&utm_source=DocsGPT">
+    <img src="https://opensource.nyc3.cdn.digitaloceanspaces.com/attribution/assets/SVG/DO_Logo_horizontal_blue.svg" width="201px">
+  </a>
+</p>

application/Dockerfile

@@ -8,14 +8,14 @@ RUN apt-get update && \
     add-apt-repository ppa:deadsnakes/ppa && \
 # Install necessary packages and Python
     apt-get update && \
-    apt-get install -y --no-install-recommends gcc wget unzip libc6-dev python3.11 python3.11-distutils python3.11-venv && \
+    apt-get install -y --no-install-recommends gcc wget unzip libc6-dev python3.12 python3.12-venv && \
     rm -rf /var/lib/apt/lists/* 
 
 # Verify Python installation and setup symlink
-RUN if [ -f /usr/bin/python3.11 ]; then \
-        ln -s /usr/bin/python3.11 /usr/bin/python; \
+RUN if [ -f /usr/bin/python3.12 ]; then \
+        ln -s /usr/bin/python3.12 /usr/bin/python; \
     else \
-        echo "Python 3.11 not found"; exit 1; \
+        echo "Python 3.12 not found"; exit 1; \
     fi
 
 # Download and unzip the model
@@ -33,7 +33,7 @@ RUN apt-get remove --purge -y wget unzip && apt-get autoremove -y && rm -rf /var
 COPY requirements.txt .
 
 # Setup Python virtual environment
-RUN python3.11 -m venv /venv
+RUN python3.12 -m venv /venv
 
 # Activate virtual environment and install Python packages
 ENV PATH="/venv/bin:$PATH"
@@ -50,8 +50,8 @@ RUN apt-get update && \
     apt-get install -y software-properties-common && \
     add-apt-repository ppa:deadsnakes/ppa && \
 # Install Python
-    apt-get update && apt-get install -y --no-install-recommends python3.11 && \
-    ln -s /usr/bin/python3.11 /usr/bin/python && \
+    apt-get update && apt-get install -y --no-install-recommends python3.12 && \
+    ln -s /usr/bin/python3.12 /usr/bin/python && \
     rm -rf /var/lib/apt/lists/*
 
 # Set working directory

application/api/answer/routes.py

@@ -11,18 +11,18 @@ from bson.objectid import ObjectId
 from flask import Blueprint, current_app, make_response, request, Response
 from flask_restx import fields, Namespace, Resource
 
-from pymongo import MongoClient
 
+from application.core.mongo_db import MongoDB
 from application.core.settings import settings
 from application.error import bad_request
 from application.extensions import api
 from application.llm.llm_creator import LLMCreator
 from application.retriever.retriever_creator import RetrieverCreator
-from application.utils import check_required_fields
+from application.utils import check_required_fields, limit_chat_history
 
 logger = logging.getLogger(__name__)
 
-mongo = MongoClient(settings.MONGO_URI)
+mongo = MongoDB.get_client()
 db = mongo["docsgpt"]
 conversations_collection = db["conversations"]
 sources_collection = db["sources"]
@@ -37,9 +37,11 @@ api.add_namespace(answer_ns)
 gpt_model = ""
 # to have some kind of default behaviour
 if settings.LLM_NAME == "openai":
-    gpt_model = "gpt-3.5-turbo"
+    gpt_model = "gpt-4o-mini"
 elif settings.LLM_NAME == "anthropic":
     gpt_model = "claude-2"
+elif settings.LLM_NAME == "groq":
+    gpt_model = "llama3-8b-8192"
 
 if settings.MODEL_NAME:  # in case there is particular model name configured
     gpt_model = settings.MODEL_NAME
@@ -116,8 +118,31 @@ def is_azure_configured():
     )
 
 
-def save_conversation(conversation_id, question, response, source_log_docs, llm):
-    if conversation_id is not None and conversation_id != "None":
+def save_conversation(conversation_id, question, response, source_log_docs, llm,index=None):
+    if conversation_id is not None and index is not None:
+        conversations_collection.update_one(
+            {"_id": ObjectId(conversation_id), f"queries.{index}": {"$exists": True}},
+            {
+                "$set": {
+                    f"queries.{index}.prompt": question,
+                    f"queries.{index}.response": response,
+                    f"queries.{index}.sources": source_log_docs,
+                }
+            }
+        )
+        ##remove following queries from the array
+        conversations_collection.update_one(
+            {"_id": ObjectId(conversation_id), f"queries.{index}": {"$exists": True}},
+            {
+                "$push":{
+                    "queries":{
+                        "$each":[],
+                        "$slice":index+1
+                    }
+                }
+            }
+        )
+    elif conversation_id is not None and conversation_id != "None":
         conversations_collection.update_one(
             {"_id": ObjectId(conversation_id)},
             {
@@ -139,17 +164,17 @@ def save_conversation(conversation_id, question, response, source_log_docs, llm)
                 "role": "assistant",
                 "content": "Summarise following conversation in no more than 3 "
                 "words, respond ONLY with the summary, use the same "
-                "language as the system \n\nUser: "
-                + question
-                + "\n\n"
-                + "AI: "
-                + response,
+                "language as the system",
             },
             {
                 "role": "user",
                 "content": "Summarise following conversation in no more than 3 words, "
                 "respond ONLY with the summary, use the same language as the "
-                "system",
+                "system \n\nUser: "
+                + question
+                + "\n\n"
+                + "AI: "
+                + response,
             },
         ]
 
@@ -184,7 +209,7 @@ def get_prompt(prompt_id):
 
 
 def complete_stream(
-    question, retriever, conversation_id, user_api_key, isNoneDoc=False
+    question, retriever, conversation_id, user_api_key, isNoneDoc=False,index=None
 ):
 
     try:
@@ -215,7 +240,7 @@ def complete_stream(
         )
         if user_api_key is None:
             conversation_id = save_conversation(
-                conversation_id, question, response_full, source_log_docs, llm
+                conversation_id, question, response_full, source_log_docs, llm,index
             )
             # send data.type = "end" to indicate that the stream has ended as json
             data = json.dumps({"type": "id", "id": str(conversation_id)})
@@ -239,6 +264,7 @@ def complete_stream(
         yield f"data: {data}\n\n"
     except Exception as e:
         print("\033[91merr", str(e), file=sys.stderr)
+        traceback.print_exc()
         data = json.dumps(
             {
                 "type": "error",
@@ -267,9 +293,6 @@ class Stream(Resource):
             "prompt_id": fields.String(
                 required=False, default="default", description="Prompt ID"
             ),
-            "selectedDocs": fields.String(
-                required=False, description="Selected documents"
-            ),
             "chunks": fields.Integer(
                 required=False, default=2, description="Number of chunks"
             ),
@@ -282,6 +305,9 @@ class Stream(Resource):
             "isNoneDoc": fields.Boolean(
                 required=False, description="Flag indicating if no document is used"
             ),
+            "index":fields.Integer(
+                required=False, description="The position where query is to be updated"
+            ),
         },
     )
 
@@ -290,23 +316,23 @@ class Stream(Resource):
     def post(self):
         data = request.get_json()
         required_fields = ["question"]
+        if "index" in data:
+            required_fields = ["question","conversation_id"]
         missing_fields = check_required_fields(data, required_fields)
         if missing_fields:
             return missing_fields
 
         try:
             question = data["question"]
-            history = data.get("history", [])
-            history = json.loads(history)
+            history = limit_chat_history(json.loads(data.get("history", [])), gpt_model=gpt_model)
             conversation_id = data.get("conversation_id")
             prompt_id = data.get("prompt_id", "default")
-            if "selectedDocs" in data and data["selectedDocs"] is None:
-                chunks = 0
-            else:
-                chunks = int(data.get("chunks", 2))
+            
+            index=data.get("index",None)
+            chunks = int(data.get("chunks", 2))
             token_limit = data.get("token_limit", settings.DEFAULT_MAX_HISTORY)
             retriever_name = data.get("retriever", "classic")
-
+            
             if "api_key" in data:
                 data_key = get_data_from_api_key(data["api_key"])
                 chunks = int(data_key.get("chunks", 2))
@@ -330,7 +356,8 @@ class Stream(Resource):
             )
 
             prompt = get_prompt(prompt_id)
-
+            if "isNoneDoc" in data and data["isNoneDoc"] is True:
+                chunks = 0
             retriever = RetrieverCreator.create_retriever(
                 retriever_name,
                 question=question,
@@ -342,7 +369,7 @@ class Stream(Resource):
                 gpt_model=gpt_model,
                 user_api_key=user_api_key,
             )
-
+            
             return Response(
                 complete_stream(
                     question=question,
@@ -350,6 +377,7 @@ class Stream(Resource):
                     conversation_id=conversation_id,
                     user_api_key=user_api_key,
                     isNoneDoc=data.get("isNoneDoc"),
+                    index=index,
                 ),
                 mimetype="text/event-stream",
             )
@@ -420,14 +448,14 @@ class Answer(Resource):
     @api.doc(description="Provide an answer based on the question and retriever")
     def post(self):
         data = request.get_json()
-        required_fields = ["question"]
+        required_fields = ["question"]       
         missing_fields = check_required_fields(data, required_fields)
         if missing_fields:
             return missing_fields
 
         try:
             question = data["question"]
-            history = data.get("history", [])
+            history = limit_chat_history(json.loads(data.get("history", [])), gpt_model=gpt_model)
             conversation_id = data.get("conversation_id")
             prompt_id = data.get("prompt_id", "default")
             chunks = int(data.get("chunks", 2))

application/api/internal/routes.py

@@ -1,13 +1,13 @@
 import os
 import datetime
 from flask import Blueprint, request, send_from_directory
-from pymongo import MongoClient
 from werkzeug.utils import secure_filename
 from bson.objectid import ObjectId
 
+from application.core.mongo_db import MongoDB
 from application.core.settings import settings
 
-mongo = MongoClient(settings.MONGO_URI)
+mongo = MongoDB.get_client()
 db = mongo["docsgpt"]
 conversations_collection = db["conversations"]
 sources_collection = db["sources"]
@@ -75,7 +75,7 @@ def upload_index_files():
                     "user": user,
                     "name": job_name,
                     "language": job_name,
-                    "date": datetime.datetime.now().strftime("%d/%m/%Y %H:%M:%S"),
+                    "date": datetime.datetime.now(),
                     "model": settings.EMBEDDINGS_NAME,
                     "type": type,
                     "tokens": tokens,
@@ -92,7 +92,7 @@ def upload_index_files():
                 "user": user,
                 "name": job_name,
                 "language": job_name,
-                "date": datetime.datetime.now().strftime("%d/%m/%Y %H:%M:%S"),
+                "date": datetime.datetime.now(),
                 "model": settings.EMBEDDINGS_NAME,
                 "type": type,
                 "tokens": tokens,

application/api/user/routes.py

@@ -1,4 +1,5 @@
 import datetime
+import math
 import os
 import shutil
 import uuid
@@ -6,19 +7,21 @@ import uuid
 from bson.binary import Binary, UuidRepresentation
 from bson.dbref import DBRef
 from bson.objectid import ObjectId
-from flask import Blueprint, jsonify, make_response, request
-from flask_restx import fields, Namespace, Resource
-from pymongo import MongoClient
+from flask import Blueprint, jsonify, make_response, redirect, request
+from flask_restx import fields, inputs, Namespace, Resource
 from werkzeug.utils import secure_filename
 
 from application.api.user.tasks import ingest, ingest_remote
 
+from application.core.mongo_db import MongoDB
 from application.core.settings import settings
 from application.extensions import api
+from application.tools.tool_manager import ToolManager
+from application.tts.google_tts import GoogleTTS
 from application.utils import check_required_fields
 from application.vectorstore.vector_creator import VectorCreator
 
-mongo = MongoClient(settings.MONGO_URI)
+mongo = MongoDB.get_client()
 db = mongo["docsgpt"]
 conversations_collection = db["conversations"]
 sources_collection = db["sources"]
@@ -28,6 +31,7 @@ api_key_collection = db["api_keys"]
 token_usage_collection = db["token_usage"]
 shared_conversations_collections = db["shared_conversations"]
 user_logs_collection = db["user_logs"]
+user_tools_collection = db["user_tools"]
 
 user = Blueprint("user", __name__)
 user_ns = Namespace("user", description="User related operations", path="/")
@@ -37,6 +41,9 @@ current_dir = os.path.dirname(
     os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 )
 
+tool_config = {}
+tool_manager = ToolManager(config=tool_config)
+
 
 def generate_minute_range(start_date, end_date):
     return {
@@ -174,10 +181,17 @@ class SubmitFeedback(Resource):
             "FeedbackModel",
             {
                 "question": fields.String(
-                    required=True, description="The user question"
+                    required=False, description="The user question"
                 ),
-                "answer": fields.String(required=True, description="The AI answer"),
+                "answer": fields.String(required=False, description="The AI answer"),
                 "feedback": fields.String(required=True, description="User feedback"),
+                "question_index": fields.Integer(
+                    required=True,
+                    description="The question number in that particular conversation",
+                ),
+                "conversation_id": fields.String(
+                    required=True, description="id of the particular conversation"
+                ),
                 "api_key": fields.String(description="Optional API key"),
             },
         )
@@ -187,23 +201,24 @@ class SubmitFeedback(Resource):
     )
     def post(self):
         data = request.get_json()
-        required_fields = ["question", "answer", "feedback"]
+        required_fields = ["feedback", "conversation_id", "question_index"]
         missing_fields = check_required_fields(data, required_fields)
         if missing_fields:
             return missing_fields
 
-        new_doc = {
-            "question": data["question"],
-            "answer": data["answer"],
-            "feedback": data["feedback"],
-            "timestamp": datetime.datetime.now(datetime.timezone.utc),
-        }
-
-        if "api_key" in data:
-            new_doc["api_key"] = data["api_key"]
-
         try:
-            feedback_collection.insert_one(new_doc)
+            conversations_collection.update_one(
+                {
+                    "_id": ObjectId(data["conversation_id"]),
+                    f"queries.{data['question_index']}": {"$exists": True},
+                },
+                {
+                    "$set": {
+                        f"queries.{data['question_index']}.feedback": data["feedback"]
+                    }
+                },
+            )
+
         except Exception as err:
             return make_response(jsonify({"success": False, "error": str(err)}), 400)
 
@@ -246,13 +261,10 @@ class DeleteOldIndexes(Resource):
                 jsonify({"success": False, "message": "Missing required fields"}), 400
             )
 
+        doc = sources_collection.find_one({"_id": ObjectId(source_id), "user": "local"})
+        if not doc:
+            return make_response(jsonify({"status": "not found"}), 404)
         try:
-            doc = sources_collection.find_one(
-                {"_id": ObjectId(source_id), "user": "local"}
-            )
-            if not doc:
-                return make_response(jsonify({"status": "not found"}), 404)
-
             if settings.VECTOR_STORE == "faiss":
                 shutil.rmtree(os.path.join(current_dir, "indexes", str(doc["_id"])))
             else:
@@ -261,12 +273,12 @@ class DeleteOldIndexes(Resource):
                 )
                 vectorstore.delete_index()
 
-            sources_collection.delete_one({"_id": ObjectId(source_id)})
         except FileNotFoundError:
             pass
         except Exception as err:
             return make_response(jsonify({"success": False, "error": str(err)}), 400)
 
+        sources_collection.delete_one({"_id": ObjectId(source_id)})
         return make_response(jsonify({"success": True}), 200)
 
 
@@ -314,7 +326,7 @@ class UploadFile(Resource):
                 for file in files:
                     filename = secure_filename(file.filename)
                     file.save(os.path.join(temp_dir, filename))
-
+                    print(f"Saved file: {filename}")
                 zip_path = shutil.make_archive(
                     base_name=os.path.join(save_dir, job_name),
                     format="zip",
@@ -322,6 +334,29 @@ class UploadFile(Resource):
                 )
                 final_filename = os.path.basename(zip_path)
                 shutil.rmtree(temp_dir)
+                task = ingest.delay(
+                    settings.UPLOAD_FOLDER,
+                    [
+                        ".rst",
+                        ".md",
+                        ".pdf",
+                        ".txt",
+                        ".docx",
+                        ".csv",
+                        ".epub",
+                        ".html",
+                        ".mdx",
+                        ".json",
+                        ".xlsx",
+                        ".pptx",
+                        ".png",
+                        ".jpg",
+                        ".jpeg",
+                    ],
+                    job_name,
+                    final_filename,
+                    user,
+                )
             else:
                 file = files[0]
                 final_filename = secure_filename(file.filename)
@@ -340,14 +375,21 @@ class UploadFile(Resource):
                         ".epub",
                         ".html",
                         ".mdx",
+                        ".json",
+                        ".xlsx",
+                        ".pptx",
+                        ".png",
+                        ".jpg",
+                        ".jpeg",
                     ],
                     job_name,
                     final_filename,
                     user,
                 )
-        except Exception as err:
-            return make_response(jsonify({"success": False, "error": str(err)}), 400)
 
+        except Exception as err:
+            print(f"Error: {err}")
+            return make_response(jsonify({"success": False, "error": str(err)}), 400)
         return make_response(jsonify({"success": True, "task_id": task.id}), 200)
 
 
@@ -363,6 +405,7 @@ class UploadRemote(Resource):
                 ),
                 "name": fields.String(required=True, description="Job name"),
                 "data": fields.String(required=True, description="Data to process"),
+                "repo_url": fields.String(description="GitHub repository URL"),
             },
         )
     )
@@ -377,11 +420,18 @@ class UploadRemote(Resource):
             return missing_fields
 
         try:
+            if "repo_url" in data:
+                source_data = data["repo_url"]
+                loader = "github"
+            else:
+                source_data = data["data"]
+                loader = data["source"]
+
             task = ingest_remote.delay(
-                source_data=data["data"],
+                source_data=source_data,
                 job_name=data["name"],
                 user=data["user"],
-                loader=data["source"],
+                loader=loader,
             )
         except Exception as err:
             return make_response(jsonify({"success": False, "error": str(err)}), 400)
@@ -410,6 +460,11 @@ class TaskStatus(Resource):
 
             task = celery.AsyncResult(task_id)
             task_meta = task.info
+            print(f"Task status: {task.status}")
+            if not isinstance(
+                task_meta, (dict, list, str, int, float, bool, type(None))
+            ):
+                task_meta = str(task_meta)  # Convert to a string representation
         except Exception as err:
             return make_response(jsonify({"success": False, "error": str(err)}), 400)
 
@@ -417,13 +472,86 @@ class TaskStatus(Resource):
 
 
 @user_ns.route("/api/combine")
+class RedirectToSources(Resource):
+    @api.doc(
+        description="Redirects /api/combine to /api/sources for backward compatibility"
+    )
+    def get(self):
+        return redirect("/api/sources", code=301)
+
+
+@user_ns.route("/api/sources/paginated")
+class PaginatedSources(Resource):
+    @api.doc(description="Get document with pagination, sorting and filtering")
+    def get(self):
+        user = "local"
+        sort_field = request.args.get("sort", "date")  # Default to 'date'
+        sort_order = request.args.get("order", "desc")  # Default to 'desc'
+        page = int(request.args.get("page", 1))  # Default to 1
+        rows_per_page = int(request.args.get("rows", 10))  # Default to 10
+        # add .strip() to remove leading and trailing whitespaces
+        search_term = request.args.get(
+            "search", ""
+        ).strip()  # add search for filter documents
+
+        # Prepare query for filtering
+        query = {"user": user}
+        if search_term:
+            query["name"] = {
+                "$regex": search_term,
+                "$options": "i",  # using case-insensitive search
+            }
+
+        total_documents = sources_collection.count_documents(query)
+        total_pages = max(1, math.ceil(total_documents / rows_per_page))
+        page = min(
+            max(1, page), total_pages
+        )  # add this to make sure page inbound is within the range
+        sort_order = 1 if sort_order == "asc" else -1
+        skip = (page - 1) * rows_per_page
+
+        try:
+            documents = (
+                sources_collection.find(query)
+                .sort(sort_field, sort_order)
+                .skip(skip)
+                .limit(rows_per_page)
+            )
+
+            paginated_docs = []
+            for doc in documents:
+                doc_data = {
+                    "id": str(doc["_id"]),
+                    "name": doc.get("name", ""),
+                    "date": doc.get("date", ""),
+                    "model": settings.EMBEDDINGS_NAME,
+                    "location": "local",
+                    "tokens": doc.get("tokens", ""),
+                    "retriever": doc.get("retriever", "classic"),
+                    "syncFrequency": doc.get("sync_frequency", ""),
+                }
+                paginated_docs.append(doc_data)
+
+            response = {
+                "total": total_documents,
+                "totalPages": total_pages,
+                "currentPage": page,
+                "paginated": paginated_docs,
+            }
+            return make_response(jsonify(response), 200)
+
+        except Exception as err:
+            return make_response(jsonify({"success": False, "error": str(err)}), 400)
+
+
+@user_ns.route("/api/sources")
 class CombinedJson(Resource):
     @api.doc(description="Provide JSON file with combined available indexes")
     def get(self):
         user = "local"
         data = [
             {
-                "name": "default",
+                "name": "Default",
                 "date": "default",
                 "model": settings.EMBEDDINGS_NAME,
                 "location": "remote",
@@ -471,6 +599,7 @@ class CombinedJson(Resource):
                         "retriever": "brave_search",
                     }
                 )
+
         except Exception as err:
             return make_response(jsonify({"success": False, "error": str(err)}), 400)
 
@@ -794,7 +923,7 @@ class ShareConversation(Resource):
         if missing_fields:
             return missing_fields
 
-        is_promptable = request.args.get("isPromptable")
+        is_promptable = request.args.get("isPromptable", type=inputs.boolean)
         if is_promptable is None:
             return make_response(
                 jsonify({"success": False, "message": "isPromptable is required"}), 400
@@ -823,7 +952,7 @@ class ShareConversation(Resource):
                 uuid.uuid4(), UuidRepresentation.STANDARD
             )
 
-            if is_promptable.lower() == "true":
+            if is_promptable:
                 prompt_id = data.get("prompt_id", "default")
                 chunks = data.get("chunks", "2")
 
@@ -851,7 +980,7 @@ class ShareConversation(Resource):
                             "conversation_id": DBRef(
                                 "conversations", ObjectId(conversation_id)
                             ),
-                            "isPromptable": is_promptable.lower() == "true",
+                            "isPromptable": is_promptable,
                             "first_n_queries": current_n_queries,
                             "user": user,
                             "api_key": api_uuid,
@@ -875,7 +1004,7 @@ class ShareConversation(Resource):
                                     "$ref": "conversations",
                                     "$id": ObjectId(conversation_id),
                                 },
-                                "isPromptable": is_promptable.lower() == "true",
+                                "isPromptable": is_promptable,
                                 "first_n_queries": current_n_queries,
                                 "user": user,
                                 "api_key": api_uuid,
@@ -910,7 +1039,7 @@ class ShareConversation(Resource):
                                 "$ref": "conversations",
                                 "$id": ObjectId(conversation_id),
                             },
-                            "isPromptable": is_promptable.lower() == "true",
+                            "isPromptable": is_promptable,
                             "first_n_queries": current_n_queries,
                             "user": user,
                             "api_key": api_uuid,
@@ -931,7 +1060,7 @@ class ShareConversation(Resource):
                     "conversation_id": DBRef(
                         "conversations", ObjectId(conversation_id)
                     ),
-                    "isPromptable": is_promptable.lower() == "false",
+                    "isPromptable": is_promptable,
                     "first_n_queries": current_n_queries,
                     "user": user,
                 }
@@ -954,7 +1083,7 @@ class ShareConversation(Resource):
                             "$ref": "conversations",
                             "$id": ObjectId(conversation_id),
                         },
-                        "isPromptable": is_promptable.lower() == "false",
+                        "isPromptable": is_promptable,
                         "first_n_queries": current_n_queries,
                         "user": user,
                     }
@@ -1349,90 +1478,17 @@ class GetFeedbackAnalytics(Resource):
             )
         except Exception as err:
             return make_response(jsonify({"success": False, "error": str(err)}), 400)
+        
         end_date = datetime.datetime.now(datetime.timezone.utc)
 
         if filter_option == "last_hour":
             start_date = end_date - datetime.timedelta(hours=1)
             group_format = "%Y-%m-%d %H:%M:00"
-            group_stage_1 = {
-                "$group": {
-                    "_id": {
-                        "minute": {
-                            "$dateToString": {
-                                "format": group_format,
-                                "date": "$timestamp",
-                            }
-                        },
-                        "feedback": "$feedback",
-                    },
-                    "count": {"$sum": 1},
-                }
-            }
-            group_stage_2 = {
-                "$group": {
-                    "_id": "$_id.minute",
-                    "likes": {
-                        "$sum": {
-                            "$cond": [
-                                {"$eq": ["$_id.feedback", "LIKE"]},
-                                "$count",
-                                0,
-                            ]
-                        }
-                    },
-                    "dislikes": {
-                        "$sum": {
-                            "$cond": [
-                                {"$eq": ["$_id.feedback", "DISLIKE"]},
-                                "$count",
-                                0,
-                            ]
-                        }
-                    },
-                }
-            }
-
+            date_field = {"$dateToString": {"format": group_format, "date": "$date"}}
         elif filter_option == "last_24_hour":
             start_date = end_date - datetime.timedelta(hours=24)
             group_format = "%Y-%m-%d %H:00"
-            group_stage_1 = {
-                "$group": {
-                    "_id": {
-                        "hour": {
-                            "$dateToString": {
-                                "format": group_format,
-                                "date": "$timestamp",
-                            }
-                        },
-                        "feedback": "$feedback",
-                    },
-                    "count": {"$sum": 1},
-                }
-            }
-            group_stage_2 = {
-                "$group": {
-                    "_id": "$_id.hour",
-                    "likes": {
-                        "$sum": {
-                            "$cond": [
-                                {"$eq": ["$_id.feedback", "LIKE"]},
-                                "$count",
-                                0,
-                            ]
-                        }
-                    },
-                    "dislikes": {
-                        "$sum": {
-                            "$cond": [
-                                {"$eq": ["$_id.feedback", "DISLIKE"]},
-                                "$count",
-                                0,
-                            ]
-                        }
-                    },
-                }
-            }
-
+            date_field = {"$dateToString": {"format": group_format, "date": "$date"}}
         else:
             if filter_option in ["last_7_days", "last_15_days", "last_30_days"]:
                 filter_days = (
@@ -1450,61 +1506,59 @@ class GetFeedbackAnalytics(Resource):
                 hour=23, minute=59, second=59, microsecond=999999
             )
             group_format = "%Y-%m-%d"
-            group_stage_1 = {
-                "$group": {
-                    "_id": {
-                        "day": {
-                            "$dateToString": {
-                                "format": group_format,
-                                "date": "$timestamp",
-                            }
-                        },
-                        "feedback": "$feedback",
-                    },
-                    "count": {"$sum": 1},
-                }
-            }
-            group_stage_2 = {
-                "$group": {
-                    "_id": "$_id.day",
-                    "likes": {
-                        "$sum": {
-                            "$cond": [
-                                {"$eq": ["$_id.feedback", "LIKE"]},
-                                "$count",
-                                0,
-                            ]
-                        }
-                    },
-                    "dislikes": {
-                        "$sum": {
-                            "$cond": [
-                                {"$eq": ["$_id.feedback", "DISLIKE"]},
-                                "$count",
-                                0,
-                            ]
-                        }
-                    },
-                }
-            }
+            date_field = {"$dateToString": {"format": group_format, "date": "$date"}}
 
         try:
             match_stage = {
                 "$match": {
-                    "timestamp": {"$gte": start_date, "$lte": end_date},
+                    "date": {"$gte": start_date, "$lte": end_date},
+                    "queries": {"$exists": True, "$ne": []},
                 }
             }
             if api_key:
                 match_stage["$match"]["api_key"] = api_key
 
-            feedback_data = feedback_collection.aggregate(
-                [
-                    match_stage,
-                    group_stage_1,
-                    group_stage_2,
-                    {"$sort": {"_id": 1}},
-                ]
-            )
+            # Unwind the queries array to process each query separately
+            pipeline = [
+                match_stage,
+                {"$unwind": "$queries"},
+                {"$match": {"queries.feedback": {"$exists": True}}},
+                {
+                    "$group": {
+                        "_id": {
+                            "time": date_field,
+                            "feedback": "$queries.feedback"
+                        },
+                        "count": {"$sum": 1}
+                    }
+                },
+                {
+                    "$group": {
+                        "_id": "$_id.time",
+                        "positive": {
+                            "$sum": {
+                                "$cond": [
+                                    {"$eq": ["$_id.feedback", "LIKE"]},
+                                    "$count",
+                                    0
+                                ]
+                            }
+                        },
+                        "negative": {
+                            "$sum": {
+                                "$cond": [
+                                    {"$eq": ["$_id.feedback", "DISLIKE"]},
+                                    "$count",
+                                    0
+                                ]
+                            }
+                        }
+                    }
+                },
+                {"$sort": {"_id": 1}}
+            ]
+
+            feedback_data = conversations_collection.aggregate(pipeline)
 
             if filter_option == "last_hour":
                 intervals = generate_minute_range(start_date, end_date)
@@ -1519,8 +1573,8 @@ class GetFeedbackAnalytics(Resource):
 
             for entry in feedback_data:
                 daily_feedback[entry["_id"]] = {
-                    "positive": entry["likes"],
-                    "negative": entry["dislikes"],
+                    "positive": entry["positive"],
+                    "negative": entry["negative"]
                 }
 
         except Exception as err:
@@ -1653,3 +1707,327 @@ class ManageSync(Resource):
             return make_response(jsonify({"success": False, "error": str(err)}), 400)
 
         return make_response(jsonify({"success": True}), 200)
+
+
+@user_ns.route("/api/tts")
+class TextToSpeech(Resource):
+    tts_model = api.model(
+        "TextToSpeechModel",
+        {
+            "text": fields.String(
+                required=True, description="Text to be synthesized as audio"
+            ),
+        },
+    )
+
+    @api.expect(tts_model)
+    @api.doc(description="Synthesize audio speech from text")
+    def post(self):
+        data = request.get_json()
+        text = data["text"]
+        try:
+            tts_instance = GoogleTTS()
+            audio_base64, detected_language = tts_instance.text_to_speech(text)
+            return make_response(
+                jsonify(
+                    {
+                        "success": True,
+                        "audio_base64": audio_base64,
+                        "lang": detected_language,
+                    }
+                ),
+                200,
+            )
+        except Exception as err:
+            return make_response(jsonify({"success": False, "error": str(err)}), 400)
+
+
+@user_ns.route("/api/available_tools")
+class AvailableTools(Resource):
+    @api.doc(description="Get available tools for a user")
+    def get(self):
+        try:
+            tools_metadata = []
+            for tool_name, tool_instance in tool_manager.tools.items():
+                doc = tool_instance.__doc__.strip()
+                lines = doc.split("\n", 1)
+                name = lines[0].strip()
+                description = lines[1].strip() if len(lines) > 1 else ""
+                tools_metadata.append(
+                    {
+                        "name": tool_name,
+                        "displayName": name,
+                        "description": description,
+                        "configRequirements": tool_instance.get_config_requirements(),
+                        "actions": tool_instance.get_actions_metadata(),
+                    }
+                )
+        except Exception as err:
+            return make_response(jsonify({"success": False, "error": str(err)}), 400)
+
+        return make_response(jsonify({"success": True, "data": tools_metadata}), 200)
+
+
+@user_ns.route("/api/get_tools")
+class GetTools(Resource):
+    @api.doc(description="Get tools created by a user")
+    def get(self):
+        try:
+            user = "local"
+            tools = user_tools_collection.find({"user": user})
+            user_tools = []
+            for tool in tools:
+                tool["id"] = str(tool["_id"])
+                tool.pop("_id")
+                user_tools.append(tool)
+        except Exception as err:
+            return make_response(jsonify({"success": False, "error": str(err)}), 400)
+
+        return make_response(jsonify({"success": True, "tools": user_tools}), 200)
+
+
+@user_ns.route("/api/create_tool")
+class CreateTool(Resource):
+    @api.expect(
+        api.model(
+            "CreateToolModel",
+            {
+                "name": fields.String(required=True, description="Name of the tool"),
+                "displayName": fields.String(
+                    required=True, description="Display name for the tool"
+                ),
+                "description": fields.String(
+                    required=True, description="Tool description"
+                ),
+                "config": fields.Raw(
+                    required=True, description="Configuration of the tool"
+                ),
+                "actions": fields.List(
+                    fields.Raw,
+                    required=True,
+                    description="Actions the tool can perform",
+                ),
+                "status": fields.Boolean(
+                    required=True, description="Status of the tool"
+                ),
+            },
+        )
+    )
+    @api.doc(description="Create a new tool")
+    def post(self):
+        data = request.get_json()
+        required_fields = [
+            "name",
+            "displayName",
+            "description",
+            "actions",
+            "config",
+            "status",
+        ]
+        missing_fields = check_required_fields(data, required_fields)
+        if missing_fields:
+            return missing_fields
+
+        user = "local"
+        transformed_actions = []
+        for action in data["actions"]:
+            action["active"] = True
+            if "parameters" in action:
+                if "properties" in action["parameters"]:
+                    for param_name, param_details in action["parameters"][
+                        "properties"
+                    ].items():
+                        param_details["filled_by_llm"] = True
+                        param_details["value"] = ""
+            transformed_actions.append(action)
+        try:
+            new_tool = {
+                "user": user,
+                "name": data["name"],
+                "displayName": data["displayName"],
+                "description": data["description"],
+                "actions": transformed_actions,
+                "config": data["config"],
+                "status": data["status"],
+            }
+            resp = user_tools_collection.insert_one(new_tool)
+            new_id = str(resp.inserted_id)
+        except Exception as err:
+            return make_response(jsonify({"success": False, "error": str(err)}), 400)
+
+        return make_response(jsonify({"id": new_id}), 200)
+
+
+@user_ns.route("/api/update_tool")
+class UpdateTool(Resource):
+    @api.expect(
+        api.model(
+            "UpdateToolModel",
+            {
+                "id": fields.String(required=True, description="Tool ID"),
+                "name": fields.String(description="Name of the tool"),
+                "displayName": fields.String(description="Display name for the tool"),
+                "description": fields.String(description="Tool description"),
+                "config": fields.Raw(description="Configuration of the tool"),
+                "actions": fields.List(
+                    fields.Raw, description="Actions the tool can perform"
+                ),
+                "status": fields.Boolean(description="Status of the tool"),
+            },
+        )
+    )
+    @api.doc(description="Update a tool by ID")
+    def post(self):
+        data = request.get_json()
+        required_fields = ["id"]
+        missing_fields = check_required_fields(data, required_fields)
+        if missing_fields:
+            return missing_fields
+
+        try:
+            update_data = {}
+            if "name" in data:
+                update_data["name"] = data["name"]
+            if "displayName" in data:
+                update_data["displayName"] = data["displayName"]
+            if "description" in data:
+                update_data["description"] = data["description"]
+            if "actions" in data:
+                update_data["actions"] = data["actions"]
+            if "config" in data:
+                update_data["config"] = data["config"]
+            if "status" in data:
+                update_data["status"] = data["status"]
+
+            user_tools_collection.update_one(
+                {"_id": ObjectId(data["id"]), "user": "local"},
+                {"$set": update_data},
+            )
+        except Exception as err:
+            return make_response(jsonify({"success": False, "error": str(err)}), 400)
+
+        return make_response(jsonify({"success": True}), 200)
+
+
+@user_ns.route("/api/update_tool_config")
+class UpdateToolConfig(Resource):
+    @api.expect(
+        api.model(
+            "UpdateToolConfigModel",
+            {
+                "id": fields.String(required=True, description="Tool ID"),
+                "config": fields.Raw(
+                    required=True, description="Configuration of the tool"
+                ),
+            },
+        )
+    )
+    @api.doc(description="Update the configuration of a tool")
+    def post(self):
+        data = request.get_json()
+        required_fields = ["id", "config"]
+        missing_fields = check_required_fields(data, required_fields)
+        if missing_fields:
+            return missing_fields
+
+        try:
+            user_tools_collection.update_one(
+                {"_id": ObjectId(data["id"])},
+                {"$set": {"config": data["config"]}},
+            )
+        except Exception as err:
+            return make_response(jsonify({"success": False, "error": str(err)}), 400)
+
+        return make_response(jsonify({"success": True}), 200)
+
+
+@user_ns.route("/api/update_tool_actions")
+class UpdateToolActions(Resource):
+    @api.expect(
+        api.model(
+            "UpdateToolActionsModel",
+            {
+                "id": fields.String(required=True, description="Tool ID"),
+                "actions": fields.List(
+                    fields.Raw,
+                    required=True,
+                    description="Actions the tool can perform",
+                ),
+            },
+        )
+    )
+    @api.doc(description="Update the actions of a tool")
+    def post(self):
+        data = request.get_json()
+        required_fields = ["id", "actions"]
+        missing_fields = check_required_fields(data, required_fields)
+        if missing_fields:
+            return missing_fields
+
+        try:
+            user_tools_collection.update_one(
+                {"_id": ObjectId(data["id"])},
+                {"$set": {"actions": data["actions"]}},
+            )
+        except Exception as err:
+            return make_response(jsonify({"success": False, "error": str(err)}), 400)
+
+        return make_response(jsonify({"success": True}), 200)
+
+
+@user_ns.route("/api/update_tool_status")
+class UpdateToolStatus(Resource):
+    @api.expect(
+        api.model(
+            "UpdateToolStatusModel",
+            {
+                "id": fields.String(required=True, description="Tool ID"),
+                "status": fields.Boolean(
+                    required=True, description="Status of the tool"
+                ),
+            },
+        )
+    )
+    @api.doc(description="Update the status of a tool")
+    def post(self):
+        data = request.get_json()
+        required_fields = ["id", "status"]
+        missing_fields = check_required_fields(data, required_fields)
+        if missing_fields:
+            return missing_fields
+
+        try:
+            user_tools_collection.update_one(
+                {"_id": ObjectId(data["id"])},
+                {"$set": {"status": data["status"]}},
+            )
+        except Exception as err:
+            return make_response(jsonify({"success": False, "error": str(err)}), 400)
+
+        return make_response(jsonify({"success": True}), 200)
+
+
+@user_ns.route("/api/delete_tool")
+class DeleteTool(Resource):
+    @api.expect(
+        api.model(
+            "DeleteToolModel",
+            {"id": fields.String(required=True, description="Tool ID")},
+        )
+    )
+    @api.doc(description="Delete a tool by ID")
+    def post(self):
+        data = request.get_json()
+        required_fields = ["id"]
+        missing_fields = check_required_fields(data, required_fields)
+        if missing_fields:
+            return missing_fields
+
+        try:
+            result = user_tools_collection.delete_one({"_id": ObjectId(data["id"])})
+            if result.deleted_count == 0:
+                return {"success": False, "message": "Tool not found"}, 404
+        except Exception as err:
+            return {"success": False, "error": str(err)}, 400
+
+        return {"success": True}, 200

application/cache.py

@@ -0,0 +1,103 @@
+import json
+import logging
+import time
+from threading import Lock
+
+import redis
+
+from application.core.settings import settings
+from application.utils import get_hash
+
+logger = logging.getLogger(__name__)
+
+_redis_instance = None
+_instance_lock = Lock()
+
+
+def get_redis_instance():
+    global _redis_instance
+    if _redis_instance is None:
+        with _instance_lock:
+            if _redis_instance is None:
+                try:
+                    _redis_instance = redis.Redis.from_url(
+                        settings.CACHE_REDIS_URL, socket_connect_timeout=2
+                    )
+                except redis.ConnectionError as e:
+                    logger.error(f"Redis connection error: {e}")
+                    _redis_instance = None
+    return _redis_instance
+
+
+def gen_cache_key(messages, model="docgpt", tools=None):
+    if not all(isinstance(msg, dict) for msg in messages):
+        raise ValueError("All messages must be dictionaries.")
+    messages_str = json.dumps(messages)
+    tools_str = json.dumps(str(tools)) if tools else ""
+    combined = f"{model}_{messages_str}_{tools_str}"
+    cache_key = get_hash(combined)
+    return cache_key
+
+
+def gen_cache(func):
+    def wrapper(self, model, messages, stream, tools=None, *args, **kwargs):
+        try:
+            cache_key = gen_cache_key(messages, model, tools)
+            redis_client = get_redis_instance()
+            if redis_client:
+                try:
+                    cached_response = redis_client.get(cache_key)
+                    if cached_response:
+                        return cached_response.decode("utf-8")
+                except redis.ConnectionError as e:
+                    logger.error(f"Redis connection error: {e}")
+
+            result = func(self, model, messages, stream, tools, *args, **kwargs)
+            if redis_client and isinstance(result, str):
+                try:
+                    redis_client.set(cache_key, result, ex=1800)
+                except redis.ConnectionError as e:
+                    logger.error(f"Redis connection error: {e}")
+
+            return result
+        except ValueError as e:
+            logger.error(e)
+            return "Error: No user message found in the conversation to generate a cache key."
+
+    return wrapper
+
+
+def stream_cache(func):
+    def wrapper(self, model, messages, stream, tools=None, *args, **kwargs):
+        cache_key = gen_cache_key(messages, model, tools)
+        logger.info(f"Stream cache key: {cache_key}")
+
+        redis_client = get_redis_instance()
+        if redis_client:
+            try:
+                cached_response = redis_client.get(cache_key)
+                if cached_response:
+                    logger.info(f"Cache hit for stream key: {cache_key}")
+                    cached_response = json.loads(cached_response.decode("utf-8"))
+                    for chunk in cached_response:
+                        yield chunk
+                        time.sleep(0.03)
+                    return
+            except redis.ConnectionError as e:
+                logger.error(f"Redis connection error: {e}")
+
+        result = func(self, model, messages, stream, tools=tools, *args, **kwargs)
+        stream_cache_data = []
+
+        for chunk in result:
+            stream_cache_data.append(chunk)
+            yield chunk
+
+        if redis_client:
+            try:
+                redis_client.set(cache_key, json.dumps(stream_cache_data), ex=1800)
+                logger.info(f"Stream cache saved for key: {cache_key}")
+            except redis.ConnectionError as e:
+                logger.error(f"Redis connection error: {e}")
+
+    return wrapper

application/celery_init.py

@@ -2,14 +2,22 @@ from celery import Celery
 from application.core.settings import settings
 from celery.signals import setup_logging
 
+
 def make_celery(app_name=__name__):
-    celery = Celery(app_name, broker=settings.CELERY_BROKER_URL, backend=settings.CELERY_RESULT_BACKEND)
+    celery = Celery(
+        app_name,
+        broker=settings.CELERY_BROKER_URL,
+        backend=settings.CELERY_RESULT_BACKEND,
+    )
     celery.conf.update(settings)
     return celery
 
+
 @setup_logging.connect
 def config_loggers(*args, **kwargs):
     from application.core.logging_config import setup_logging
+
     setup_logging()
 
+
 celery = make_celery()

application/core/mongo_db.py

@@ -0,0 +1,24 @@
+from application.core.settings import settings
+from pymongo import MongoClient
+
+
+class MongoDB:
+    _client = None
+
+    @classmethod
+    def get_client(cls):
+        """
+        Get the MongoDB client instance, creating it if necessary.
+        """
+        if cls._client is None:
+            cls._client = MongoClient(settings.MONGO_URI)
+        return cls._client
+
+    @classmethod
+    def close_client(cls):
+        """
+        Close the MongoDB client connection.
+        """
+        if cls._client is not None:
+            cls._client.close()
+            cls._client = None

application/core/settings.py

@@ -16,11 +16,15 @@ class Settings(BaseSettings):
     MONGO_URI: str = "mongodb://localhost:27017/docsgpt"
     MODEL_PATH: str = os.path.join(current_dir, "models/docsgpt-7b-f16.gguf")
     DEFAULT_MAX_HISTORY: int = 150
-    MODEL_TOKEN_LIMITS: dict = {"gpt-3.5-turbo": 4096, "claude-2": 1e5}
+    MODEL_TOKEN_LIMITS: dict = {"gpt-4o-mini": 128000, "gpt-3.5-turbo": 4096, "claude-2": 1e5}
     UPLOAD_FOLDER: str = "inputs"
-    VECTOR_STORE: str = "faiss"  # "faiss" or "elasticsearch" or "qdrant" or "milvus"
+    PARSE_PDF_AS_IMAGE: bool = False
+    VECTOR_STORE: str = "faiss" #  "faiss" or "elasticsearch" or "qdrant" or "milvus" or "lancedb"
     RETRIEVERS_ENABLED: list = ["classic_rag", "duckduck_search"] # also brave_search
 
+    # LLM Cache
+    CACHE_REDIS_URL: str = "redis://localhost:6379/2"
+
     API_URL: str = "http://localhost:7091"  # backend url for celery worker
 
     API_KEY: Optional[str] = None  # LLM api key
@@ -67,6 +71,9 @@ class Settings(BaseSettings):
     MILVUS_URI: Optional[str] = "./milvus_local.db"   # milvus lite version as default
     MILVUS_TOKEN: Optional[str] = ""
 
+    # LanceDB vectorstore config
+    LANCEDB_PATH: str = "/tmp/lancedb"  # Path where LanceDB stores its local data
+    LANCEDB_TABLE_NAME: Optional[str] = "docsgpts"  # Name of the table to use for storing vectors
     BRAVE_SEARCH_API_KEY: Optional[str] = None
 
     FLASK_DEBUG_MODE: bool = False

application/llm/anthropic.py

@@ -17,7 +17,7 @@ class AnthropicLLM(BaseLLM):
         self.AI_PROMPT = AI_PROMPT
 
     def _raw_gen(
-        self, baseself, model, messages, stream=False, max_tokens=300, **kwargs
+        self, baseself, model, messages, stream=False, tools=None, max_tokens=300, **kwargs
     ):
         context = messages[0]["content"]
         user_question = messages[-1]["content"]
@@ -34,7 +34,7 @@ class AnthropicLLM(BaseLLM):
         return completion.completion
 
     def _raw_gen_stream(
-        self, baseself, model, messages, stream=True, max_tokens=300, **kwargs
+        self, baseself, model, messages, stream=True, tools=None, max_tokens=300, **kwargs
     ):
         context = messages[0]["content"]
         user_question = messages[-1]["content"]

application/llm/base.py

@@ -1,4 +1,6 @@
 from abc import ABC, abstractmethod
+
+from application.cache import gen_cache, stream_cache
 from application.usage import gen_token_usage, stream_token_usage
 
 
@@ -6,23 +8,49 @@ class BaseLLM(ABC):
     def __init__(self):
         self.token_usage = {"prompt_tokens": 0, "generated_tokens": 0}
 
-    def _apply_decorator(self, method, decorator, *args, **kwargs):
-        return decorator(method, *args, **kwargs)
+    def _apply_decorator(self, method, decorators, *args, **kwargs):
+        for decorator in decorators:
+            method = decorator(method)
+        return method(self, *args, **kwargs)
 
     @abstractmethod
-    def _raw_gen(self, model, messages, stream, *args, **kwargs):
+    def _raw_gen(self, model, messages, stream, tools, *args, **kwargs):
         pass
 
-    def gen(self, model, messages, stream=False, *args, **kwargs):
-        return self._apply_decorator(self._raw_gen, gen_token_usage)(
-            self, model=model, messages=messages, stream=stream, *args, **kwargs
+    def gen(self, model, messages, stream=False, tools=None, *args, **kwargs):
+        decorators = [gen_token_usage, gen_cache]
+        return self._apply_decorator(
+            self._raw_gen,
+            decorators=decorators,
+            model=model,
+            messages=messages,
+            stream=stream,
+            tools=tools,
+            *args,
+            **kwargs
         )
 
     @abstractmethod
     def _raw_gen_stream(self, model, messages, stream, *args, **kwargs):
         pass
 
-    def gen_stream(self, model, messages, stream=True, *args, **kwargs):
-        return self._apply_decorator(self._raw_gen_stream, stream_token_usage)(
-            self, model=model, messages=messages, stream=stream, *args, **kwargs
+    def gen_stream(self, model, messages, stream=True, tools=None, *args, **kwargs):
+        decorators = [stream_cache, stream_token_usage]
+        return self._apply_decorator(
+            self._raw_gen_stream,
+            decorators=decorators,
+            model=model,
+            messages=messages,
+            stream=stream,
+            tools=tools,
+            *args,
+            **kwargs
         )
+
+    def supports_tools(self):
+        return hasattr(self, "_supports_tools") and callable(
+            getattr(self, "_supports_tools")
+        )
+
+    def _supports_tools(self):
+        raise NotImplementedError("Subclass must implement _supports_tools method")

application/llm/docsgpt_provider.py

@@ -9,35 +9,25 @@ class DocsGPTAPILLM(BaseLLM):
         super().__init__(*args, **kwargs)
         self.api_key = api_key
         self.user_api_key = user_api_key
-        self.endpoint = "https://llm.docsgpt.co.uk"
+        self.endpoint = "https://llm.arc53.com"
 
     def _raw_gen(self, baseself, model, messages, stream=False, *args, **kwargs):
-        context = messages[0]["content"]
-        user_question = messages[-1]["content"]
-        prompt = f"### Instruction \n {user_question} \n ### Context \n {context} \n ### Answer \n"
-
         response = requests.post(
-            f"{self.endpoint}/answer", json={"prompt": prompt, "max_new_tokens": 30}
+            f"{self.endpoint}/answer", json={"messages": messages, "max_new_tokens": 30}
         )
         response_clean = response.json()["a"].replace("###", "")
 
         return response_clean
 
     def _raw_gen_stream(self, baseself, model, messages, stream=True, *args, **kwargs):
-        context = messages[0]["content"]
-        user_question = messages[-1]["content"]
-        prompt = f"### Instruction \n {user_question} \n ### Context \n {context} \n ### Answer \n"
-
-        # send prompt to endpoint /stream
         response = requests.post(
             f"{self.endpoint}/stream",
-            json={"prompt": prompt, "max_new_tokens": 256},
+            json={"messages": messages, "max_new_tokens": 256},
             stream=True,
         )
 
         for line in response.iter_lines():
             if line:
-                # data = json.loads(line)
                 data_str = line.decode("utf-8")
                 if data_str.startswith("data: "):
                     data = json.loads(data_str[6:])

application/llm/google_ai.py

@@ -0,0 +1,159 @@
+from google import genai
+from google.genai import types
+
+from application.llm.base import BaseLLM
+
+
+class GoogleLLM(BaseLLM):
+    def __init__(self, api_key=None, user_api_key=None, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.api_key = api_key
+        self.user_api_key = user_api_key
+
+    def _clean_messages_google(self, messages):
+        cleaned_messages = []
+        for message in messages:
+            role = message.get("role")
+            content = message.get("content")
+
+            if role == "assistant":
+                role = "model"
+
+            parts = []
+            if role and content is not None:
+                if isinstance(content, str):
+                    parts = [types.Part.from_text(content)]
+                elif isinstance(content, list):
+                    for item in content:
+                        if "text" in item:
+                            parts.append(types.Part.from_text(item["text"]))
+                        elif "function_call" in item:
+                            parts.append(
+                                types.Part.from_function_call(
+                                    name=item["function_call"]["name"],
+                                    args=item["function_call"]["args"],
+                                )
+                            )
+                        elif "function_response" in item:
+                            parts.append(
+                                types.Part.from_function_response(
+                                    name=item["function_response"]["name"],
+                                    response=item["function_response"]["response"],
+                                )
+                            )
+                        else:
+                            raise ValueError(
+                                f"Unexpected content dictionary format:{item}"
+                            )
+                else:
+                    raise ValueError(f"Unexpected content type: {type(content)}")
+
+                cleaned_messages.append(types.Content(role=role, parts=parts))
+
+        return cleaned_messages
+
+    def _clean_tools_format(self, tools_list):
+        genai_tools = []
+        for tool_data in tools_list:
+            if tool_data["type"] == "function":
+                function = tool_data["function"]
+                parameters = function["parameters"]
+                properties = parameters.get("properties", {})
+
+                if properties:
+                    genai_function = dict(
+                        name=function["name"],
+                        description=function["description"],
+                        parameters={
+                            "type": "OBJECT",
+                            "properties": {
+                                k: {
+                                    **v,
+                                    "type": v["type"].upper() if v["type"] else None,
+                                }
+                                for k, v in properties.items()
+                            },
+                            "required": (
+                                parameters["required"]
+                                if "required" in parameters
+                                else []
+                            ),
+                        },
+                    )
+                else:
+                    genai_function = dict(
+                        name=function["name"],
+                        description=function["description"],
+                    )
+
+                genai_tool = types.Tool(function_declarations=[genai_function])
+                genai_tools.append(genai_tool)
+
+        return genai_tools
+
+    def _raw_gen(
+        self,
+        baseself,
+        model,
+        messages,
+        stream=False,
+        tools=None,
+        formatting="openai",
+        **kwargs,
+    ):
+        client = genai.Client(api_key=self.api_key)
+        if formatting == "openai":
+            messages = self._clean_messages_google(messages)
+        config = types.GenerateContentConfig()
+        if messages[0].role == "system":
+            config.system_instruction = messages[0].parts[0].text
+            messages = messages[1:]
+
+        if tools:
+            cleaned_tools = self._clean_tools_format(tools)
+            config.tools = cleaned_tools
+            response = client.models.generate_content(
+                model=model,
+                contents=messages,
+                config=config,
+            )
+            return response
+        else:
+            response = client.models.generate_content(
+                model=model, contents=messages, config=config
+            )
+            return response.text
+
+    def _raw_gen_stream(
+        self,
+        baseself,
+        model,
+        messages,
+        stream=True,
+        tools=None,
+        formatting="openai",
+        **kwargs,
+    ):
+        client = genai.Client(api_key=self.api_key)
+        if formatting == "openai":
+            messages = self._clean_messages_google(messages)
+        config = types.GenerateContentConfig()
+        if messages[0].role == "system":
+            config.system_instruction = messages[0].parts[0].text
+            messages = messages[1:]
+
+        if tools:
+            cleaned_tools = self._clean_tools_format(tools)
+            config.tools = cleaned_tools
+
+        response = client.models.generate_content_stream(
+            model=model,
+            contents=messages,
+            config=config,
+        )
+        for chunk in response:
+            if chunk.text is not None:
+                yield chunk.text
+
+    def _supports_tools(self):
+        return True

application/llm/groq.py

@@ -0,0 +1,32 @@
+from application.llm.base import BaseLLM
+from openai import OpenAI
+
+
+class GroqLLM(BaseLLM):
+    def __init__(self, api_key=None, user_api_key=None, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.client = OpenAI(api_key=api_key, base_url="https://api.groq.com/openai/v1")
+        self.api_key = api_key
+        self.user_api_key = user_api_key
+
+    def _raw_gen(self, baseself, model, messages, stream=False, tools=None, **kwargs):
+        if tools:
+            response = self.client.chat.completions.create(
+                model=model, messages=messages, stream=stream, tools=tools, **kwargs
+            )
+            return response.choices[0]
+        else:
+            response = self.client.chat.completions.create(
+                model=model, messages=messages, stream=stream, **kwargs
+            )
+            return response.choices[0].message.content
+
+    def _raw_gen_stream(
+        self, baseself, model, messages, stream=True, tools=None, **kwargs
+    ):
+        response = self.client.chat.completions.create(
+            model=model, messages=messages, stream=stream, **kwargs
+        )
+        for line in response:
+            if line.choices[0].delta.content is not None:
+                yield line.choices[0].delta.content

application/llm/llm_creator.py

@@ -1,3 +1,4 @@
+from application.llm.groq import GroqLLM
 from application.llm.openai import OpenAILLM, AzureOpenAILLM
 from application.llm.sagemaker import SagemakerAPILLM
 from application.llm.huggingface import HuggingFaceLLM
@@ -5,6 +6,7 @@ from application.llm.llama_cpp import LlamaCpp
 from application.llm.anthropic import AnthropicLLM
 from application.llm.docsgpt_provider import DocsGPTAPILLM
 from application.llm.premai import PremAILLM
+from application.llm.google_ai import GoogleLLM
 
 
 class LLMCreator:
@@ -17,6 +19,8 @@ class LLMCreator:
         "anthropic": AnthropicLLM,
         "docsgpt": DocsGPTAPILLM,
         "premai": PremAILLM,
+        "groq": GroqLLM,
+        "google": GoogleLLM
     }
 
     @classmethod

application/llm/openai.py

@@ -1,6 +1,5 @@
-from application.llm.base import BaseLLM
 from application.core.settings import settings
-
+from application.llm.base import BaseLLM
 
 
 class OpenAILLM(BaseLLM):
@@ -10,10 +9,7 @@ class OpenAILLM(BaseLLM):
 
         super().__init__(*args, **kwargs)
         if settings.OPENAI_BASE_URL:
-            self.client = OpenAI(
-                api_key=api_key,
-                base_url=settings.OPENAI_BASE_URL
-            )
+            self.client = OpenAI(api_key=api_key, base_url=settings.OPENAI_BASE_URL)
         else:
             self.client = OpenAI(api_key=api_key)
         self.api_key = api_key
@@ -25,14 +21,20 @@ class OpenAILLM(BaseLLM):
         model,
         messages,
         stream=False,
+        tools=None,
         engine=settings.AZURE_DEPLOYMENT_NAME,
-        **kwargs
-    ):  
-        response = self.client.chat.completions.create(
-            model=model, messages=messages, stream=stream, **kwargs
-        )
-
-        return response.choices[0].message.content
+        **kwargs,
+    ):
+        if tools:
+            response = self.client.chat.completions.create(
+                model=model, messages=messages, stream=stream, tools=tools, **kwargs
+            )
+            return response.choices[0]
+        else:
+            response = self.client.chat.completions.create(
+                model=model, messages=messages, stream=stream, **kwargs
+            )
+            return response.choices[0].message.content
 
     def _raw_gen_stream(
         self,
@@ -40,19 +42,21 @@ class OpenAILLM(BaseLLM):
         model,
         messages,
         stream=True,
+        tools=None,
         engine=settings.AZURE_DEPLOYMENT_NAME,
-        **kwargs
-    ):  
+        **kwargs,
+    ):
         response = self.client.chat.completions.create(
             model=model, messages=messages, stream=stream, **kwargs
         )
 
         for line in response:
-            # import sys
-            # print(line.choices[0].delta.content, file=sys.stderr)
             if line.choices[0].delta.content is not None:
                 yield line.choices[0].delta.content
 
+    def _supports_tools(self):
+        return True
+
 
 class AzureOpenAILLM(OpenAILLM):
 

application/llm/sagemaker.py

@@ -76,7 +76,7 @@ class SagemakerAPILLM(BaseLLM):
         self.endpoint = settings.SAGEMAKER_ENDPOINT
         self.runtime = runtime
 
-    def _raw_gen(self, baseself, model, messages, stream=False, **kwargs):
+    def _raw_gen(self, baseself, model, messages, stream=False, tools=None, **kwargs):
         context = messages[0]["content"]
         user_question = messages[-1]["content"]
         prompt = f"### Instruction \n {user_question} \n ### Context \n {context} \n ### Answer \n"
@@ -105,7 +105,7 @@ class SagemakerAPILLM(BaseLLM):
         print(result[0]["generated_text"], file=sys.stderr)
         return result[0]["generated_text"][len(prompt) :]
 
-    def _raw_gen_stream(self, baseself, model, messages, stream=True, **kwargs):
+    def _raw_gen_stream(self, baseself, model, messages, stream=True, tools=None, **kwargs):
         context = messages[0]["content"]
         user_question = messages[-1]["content"]
         prompt = f"### Instruction \n {user_question} \n ### Context \n {context} \n ### Answer \n"

application/parser/chunking.py

@@ -0,0 +1,118 @@
+import re
+from typing import List, Tuple
+import logging
+from application.parser.schema.base import Document
+from application.utils import get_encoding
+
+logger = logging.getLogger(__name__)
+
+class Chunker:
+    def __init__(
+        self,
+        chunking_strategy: str = "classic_chunk",
+        max_tokens: int = 2000,
+        min_tokens: int = 150,
+        duplicate_headers: bool = False,
+    ):
+        if chunking_strategy not in ["classic_chunk"]:
+            raise ValueError(f"Unsupported chunking strategy: {chunking_strategy}")
+        self.chunking_strategy = chunking_strategy
+        self.max_tokens = max_tokens
+        self.min_tokens = min_tokens
+        self.duplicate_headers = duplicate_headers
+        self.encoding = get_encoding()
+
+    def separate_header_and_body(self, text: str) -> Tuple[str, str]:
+        header_pattern = r"^(.*?\n){3}"
+        match = re.match(header_pattern, text)
+        if match:
+            header = match.group(0)
+            body = text[len(header):]
+        else:
+            header, body = "", text  # No header, treat entire text as body
+        return header, body
+
+    def combine_documents(self, doc: Document, next_doc: Document) -> Document:
+        combined_text = doc.text + " " + next_doc.text
+        combined_token_count = len(self.encoding.encode(combined_text))
+        new_doc = Document(
+            text=combined_text,
+            doc_id=doc.doc_id,
+            embedding=doc.embedding,
+            extra_info={**(doc.extra_info or {}), "token_count": combined_token_count}
+        )
+        return new_doc
+    
+    def split_document(self, doc: Document) -> List[Document]:
+        split_docs = []
+        header, body = self.separate_header_and_body(doc.text)
+        header_tokens = self.encoding.encode(header) if header else []
+        body_tokens = self.encoding.encode(body)
+
+        current_position = 0
+        part_index = 0
+        while current_position < len(body_tokens):
+            end_position = current_position + self.max_tokens - len(header_tokens)
+            chunk_tokens = (header_tokens + body_tokens[current_position:end_position]
+                            if self.duplicate_headers or part_index == 0 else body_tokens[current_position:end_position])
+            chunk_text = self.encoding.decode(chunk_tokens)
+            new_doc = Document(
+                text=chunk_text,
+                doc_id=f"{doc.doc_id}-{part_index}",
+                embedding=doc.embedding,
+                extra_info={**(doc.extra_info or {}), "token_count": len(chunk_tokens)}
+            )
+            split_docs.append(new_doc)
+            current_position = end_position
+            part_index += 1
+            header_tokens = []
+        return split_docs
+
+    def classic_chunk(self, documents: List[Document]) -> List[Document]:
+        processed_docs = []
+        i = 0
+        while i < len(documents):
+            doc = documents[i]
+            tokens = self.encoding.encode(doc.text)
+            token_count = len(tokens)
+
+            if self.min_tokens <= token_count <= self.max_tokens:
+                doc.extra_info = doc.extra_info or {}
+                doc.extra_info["token_count"] = token_count
+                processed_docs.append(doc)
+                i += 1
+            elif token_count < self.min_tokens:
+                if i + 1 < len(documents):
+                    next_doc = documents[i + 1]
+                    next_tokens = self.encoding.encode(next_doc.text)
+                    if token_count + len(next_tokens) <= self.max_tokens:
+                        # Combine small documents
+                        combined_doc = self.combine_documents(doc, next_doc)
+                        processed_docs.append(combined_doc)
+                        i += 2
+                    else:
+                        # Keep the small document as is if adding next_doc would exceed max_tokens
+                        doc.extra_info = doc.extra_info or {}
+                        doc.extra_info["token_count"] = token_count
+                        processed_docs.append(doc)
+                        i += 1
+                else:
+                    # No next document to combine with; add the small document as is
+                    doc.extra_info = doc.extra_info or {}
+                    doc.extra_info["token_count"] = token_count
+                    processed_docs.append(doc)
+                    i += 1
+            else:
+                # Split large documents
+                processed_docs.extend(self.split_document(doc))
+                i += 1
+        return processed_docs
+
+    def chunk(
+        self,
+        documents: List[Document]
+    ) -> List[Document]:
+        if self.chunking_strategy == "classic_chunk":
+            return self.classic_chunk(documents)
+        else:
+            raise ValueError("Unsupported chunking strategy")

application/parser/embedding_pipeline.py

@@ -0,0 +1,86 @@
+import os
+import logging
+from retry import retry
+from tqdm import tqdm
+from application.core.settings import settings
+from application.vectorstore.vector_creator import VectorCreator
+
+
+@retry(tries=10, delay=60)
+def add_text_to_store_with_retry(store, doc, source_id):
+    """
+    Add a document's text and metadata to the vector store with retry logic.
+    Args:
+        store: The vector store object.
+        doc: The document to be added.
+        source_id: Unique identifier for the source.
+    """
+    try:
+        doc.metadata["source_id"] = str(source_id)
+        store.add_texts([doc.page_content], metadatas=[doc.metadata])
+    except Exception as e:
+        logging.error(f"Failed to add document with retry: {e}")
+        raise
+
+
+def embed_and_store_documents(docs, folder_name, source_id, task_status):
+    """
+    Embeds documents and stores them in a vector store.
+
+    Args:
+        docs (list): List of documents to be embedded and stored.
+        folder_name (str): Directory to save the vector store.
+        source_id (str): Unique identifier for the source.
+        task_status: Task state manager for progress updates.
+
+    Returns:
+        None
+    """
+    # Ensure the folder exists
+    if not os.path.exists(folder_name):
+        os.makedirs(folder_name)
+
+    # Initialize vector store
+    if settings.VECTOR_STORE == "faiss":
+        docs_init = [docs.pop(0)]
+        store = VectorCreator.create_vectorstore(
+            settings.VECTOR_STORE,
+            docs_init=docs_init,
+            source_id=folder_name,
+            embeddings_key=os.getenv("EMBEDDINGS_KEY"),
+        )
+    else:
+        store = VectorCreator.create_vectorstore(
+            settings.VECTOR_STORE,
+            source_id=source_id,
+            embeddings_key=os.getenv("EMBEDDINGS_KEY"),
+        )
+        store.delete_index()
+
+    total_docs = len(docs)
+
+    # Process and embed documents
+    for idx, doc in tqdm(
+        enumerate(docs),
+        desc="Embedding 🦖",
+        unit="docs",
+        total=total_docs,
+        bar_format="{l_bar}{bar}| Time Left: {remaining}",
+    ):
+        try:
+            # Update task status for progress tracking
+            progress = int(((idx + 1) / total_docs) * 100)
+            task_status.update_state(state="PROGRESS", meta={"current": progress})
+
+            # Add document to vector store
+            add_text_to_store_with_retry(store, doc, source_id)
+        except Exception as e:
+            logging.error(f"Error embedding document {idx}: {e}")
+            logging.info(f"Saving progress at document {idx} out of {total_docs}")
+            store.save_local(folder_name)
+            break
+
+    # Save the vector store
+    if settings.VECTOR_STORE == "faiss":
+        store.save_local(folder_name)
+    logging.info("Vector store saved successfully.")

application/parser/file/bulk.py

@@ -10,18 +10,27 @@ from application.parser.file.epub_parser import EpubParser
 from application.parser.file.html_parser import HTMLParser
 from application.parser.file.markdown_parser import MarkdownParser
 from application.parser.file.rst_parser import RstParser
-from application.parser.file.tabular_parser import PandasCSVParser
+from application.parser.file.tabular_parser import PandasCSVParser,ExcelParser
+from application.parser.file.json_parser import JSONParser
+from application.parser.file.pptx_parser import PPTXParser
+from application.parser.file.image_parser import ImageParser
 from application.parser.schema.base import Document
 
 DEFAULT_FILE_EXTRACTOR: Dict[str, BaseParser] = {
     ".pdf": PDFParser(),
     ".docx": DocxParser(),
     ".csv": PandasCSVParser(),
+    ".xlsx":ExcelParser(),
     ".epub": EpubParser(),
     ".md": MarkdownParser(),
     ".rst": RstParser(),
     ".html": HTMLParser(),
     ".mdx": MarkdownParser(),
+    ".json":JSONParser(),
+    ".pptx":PPTXParser(),
+    ".png": ImageParser(),
+    ".jpg": ImageParser(),
+    ".jpeg": ImageParser(),
 }
 
 

application/parser/file/docs_parser.py

@@ -7,7 +7,8 @@ from pathlib import Path
 from typing import Dict
 
 from application.parser.file.base_parser import BaseParser
-
+from application.core.settings import settings
+import requests
 
 class PDFParser(BaseParser):
     """PDF parser."""
@@ -18,6 +19,15 @@ class PDFParser(BaseParser):
 
     def parse_file(self, file: Path, errors: str = "ignore") -> str:
         """Parse file."""
+        if settings.PARSE_PDF_AS_IMAGE:
+            doc2md_service = "https://llm.arc53.com/doc2md"
+            # alternatively you can use local vision capable LLM
+            with open(file, "rb") as file_loaded:
+                files = {'file': file_loaded}
+                response = requests.post(doc2md_service, files=files)   
+                data = response.json()["markdown"] 
+            return data
+
         try:
             import PyPDF2
         except ImportError:

application/parser/file/image_parser.py

@@ -0,0 +1,27 @@
+"""Image parser.
+
+Contains parser for .png, .jpg, .jpeg files.
+
+"""
+from pathlib import Path
+import requests
+from typing import Dict, Union
+
+from application.parser.file.base_parser import BaseParser
+
+
+class ImageParser(BaseParser):
+    """Image parser."""
+
+    def _init_parser(self) -> Dict:
+        """Init parser."""
+        return {}
+
+    def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, list[str]]:
+        doc2md_service = "https://llm.arc53.com/doc2md"
+        # alternatively you can use local vision capable LLM
+        with open(file, "rb") as file_loaded:
+            files = {'file': file_loaded}
+            response = requests.post(doc2md_service, files=files)   
+            data = response.json()["markdown"] 
+        return data

application/parser/file/json_parser.py

@@ -0,0 +1,57 @@
+import json
+from typing import Any, Dict, List, Union
+from pathlib import Path
+
+from application.parser.file.base_parser import BaseParser
+
+class JSONParser(BaseParser):
+    r"""JSON (.json) parser.
+
+    Parses JSON files into a list of strings or a concatenated document.
+    It handles both JSON objects (dictionaries) and arrays (lists).
+
+    Args:
+        concat_rows (bool): Whether to concatenate all rows into one document.
+            If set to False, a Document will be created for each item in the JSON.
+            True by default.
+
+        row_joiner (str): Separator to use for joining each row.
+            Only used when `concat_rows=True`.
+            Set to "\n" by default.
+
+        json_config (dict): Options for parsing JSON. Can be used to specify options like
+        custom decoding or formatting. Set to empty dict by default.
+
+    """
+
+    def __init__(
+            self,
+            *args: Any,
+            concat_rows: bool = True,
+            row_joiner: str = "\n",
+            json_config: dict = {},
+            **kwargs: Any
+    ) -> None:
+        """Init params."""
+        super().__init__(*args, **kwargs)
+        self._concat_rows = concat_rows
+        self._row_joiner = row_joiner
+        self._json_config = json_config
+
+    def _init_parser(self) -> Dict:
+        """Init parser."""
+        return {}
+
+    def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, List[str]]:
+        """Parse JSON file."""
+        
+        with open(file, 'r', encoding='utf-8') as f:
+                data = json.load(f, **self._json_config)
+
+        if isinstance(data, dict):
+            data = [data]
+
+        if self._concat_rows:
+            return self._row_joiner.join([str(item) for item in data])
+        else:
+            return data

application/parser/file/pptx_parser.py

@@ -0,0 +1,75 @@
+"""PPT parser.
+Contains parsers for presentation (.pptx) files to extract slide text.
+"""
+from pathlib import Path
+from typing import Any, Dict, List, Union
+
+from application.parser.file.base_parser import BaseParser
+
+class PPTXParser(BaseParser):
+    r"""PPTX (.pptx) parser for extracting text from PowerPoint slides.
+    Args:
+        concat_slides (bool): Specifies whether to concatenate all slide text into one document.
+            - If True, slide texts will be joined together as a single string.
+            - If False, each slide's text will be stored as a separate entry in a list.
+            Set to True by default.
+        slide_separator (str): Separator used to join slides' text content.
+            Only used when `concat_slides=True`. Default is "\n".
+        Refer to https://python-pptx.readthedocs.io/en/latest/ for more information.
+    """
+
+    def __init__(
+        self,
+        *args: Any,
+        concat_slides: bool = True,
+        slide_separator: str = "\n",
+        **kwargs: Any
+    ) -> None:
+        """Init params."""
+        super().__init__(*args, **kwargs)
+        self._concat_slides = concat_slides
+        self._slide_separator = slide_separator
+
+    def _init_parser(self) -> Dict:
+        """Init parser."""
+        return {}
+
+    def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, List[str]]:
+        r"""
+        Parse a .pptx file and extract text from each slide.
+        Args:
+            file (Path): Path to the .pptx file.
+            errors (str): Error handling policy ('ignore' by default).
+        Returns:
+            Union[str, List[str]]: Concatenated text if concat_slides is True,
+            otherwise a list of slide texts.
+        """
+
+        try:
+            from pptx import Presentation
+        except ImportError:
+            raise ImportError("pptx module is required to read .PPTX files.")
+
+        try:
+            presentation = Presentation(file)
+            slide_texts=[]
+
+            # Iterate over each slide in the presentation
+            for slide in presentation.slides:
+                slide_text=""
+
+                # Iterate over each shape in the slide
+                for shape in slide.shapes:
+                    # Check if the shape has a 'text' attribute and append that to the slide_text
+                    if hasattr(shape,"text"):
+                        slide_text+=shape.text
+
+                slide_texts.append(slide_text.strip())
+
+            if self._concat_slides:
+                return self._slide_separator.join(slide_texts)
+            else:
+                return slide_texts
+
+        except Exception as e:
+            raise e

application/parser/file/rst_parser.py

@@ -91,6 +91,25 @@ class RstParser(BaseParser):
             ]
         return rst_tups
 
+    def chunk_by_token_count(self, text: str, max_tokens: int = 100) -> List[str]:
+        """Chunk text by token count."""
+    
+        avg_token_length = 5
+    
+        chunk_size = max_tokens * avg_token_length
+
+        chunks = []
+        for i in range(0, len(text), chunk_size):
+            chunk = text[i:i+chunk_size]
+            if i + chunk_size < len(text):
+                last_space = chunk.rfind(' ')
+                if last_space != -1:
+                    chunk = chunk[:last_space]
+            
+            chunks.append(chunk.strip())
+        
+        return chunks
+    
     def remove_images(self, content: str) -> str:
         pattern = r"\.\. image:: (.*)"
         content = re.sub(pattern, "", content)
@@ -136,7 +155,7 @@ class RstParser(BaseParser):
         return {}
 
     def parse_tups(
-            self, filepath: Path, errors: str = "ignore"
+            self, filepath: Path, errors: str = "ignore",max_tokens: Optional[int] = 1000
     ) -> List[Tuple[Optional[str], str]]:
         """Parse file into tuples."""
         with open(filepath, "r") as f:
@@ -156,6 +175,15 @@ class RstParser(BaseParser):
             rst_tups = self.remove_whitespaces_excess(rst_tups)
         if self._remove_characters_excess:
             rst_tups = self.remove_characters_excess(rst_tups)
+
+        # Apply chunking if max_tokens is provided
+        if max_tokens is not None:
+            chunked_tups = []
+            for header, text in rst_tups:
+                chunks = self.chunk_by_token_count(text, max_tokens)
+                for idx, chunk in enumerate(chunks):
+                    chunked_tups.append((f"{header} - Chunk {idx + 1}", chunk))
+            return chunked_tups    
         return rst_tups
 
     def parse_file(

application/parser/file/tabular_parser.py

@@ -113,3 +113,68 @@ class PandasCSVParser(BaseParser):
             return (self._row_joiner).join(text_list)
         else:
             return text_list
+
+
+class ExcelParser(BaseParser):
+    r"""Excel (.xlsx) parser.
+
+    Parses Excel files using Pandas `read_excel` function.
+    If special parameters are required, use the `pandas_config` dict.
+
+    Args:
+        concat_rows (bool): whether to concatenate all rows into one document.
+            If set to False, a Document will be created for each row.
+            True by default.
+
+        col_joiner (str): Separator to use for joining cols per row.
+            Set to ", " by default.
+
+        row_joiner (str): Separator to use for joining each row.
+            Only used when `concat_rows=True`.
+            Set to "\n" by default.
+
+        pandas_config (dict): Options for the `pandas.read_excel` function call.
+            Refer to https://pandas.pydata.org/docs/reference/api/pandas.read_excel.html
+            for more information.
+            Set to empty dict by default, this means pandas will try to figure
+            out the table structure on its own.
+
+    """
+
+    def __init__(
+            self,
+            *args: Any,
+            concat_rows: bool = True,
+            col_joiner: str = ", ",
+            row_joiner: str = "\n",
+            pandas_config: dict = {},
+            **kwargs: Any
+    ) -> None:
+        """Init params."""
+        super().__init__(*args, **kwargs)
+        self._concat_rows = concat_rows
+        self._col_joiner = col_joiner
+        self._row_joiner = row_joiner
+        self._pandas_config = pandas_config
+
+    def _init_parser(self) -> Dict:
+        """Init parser."""
+        return {}
+
+    def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, List[str]]:
+        """Parse file."""
+        try:
+            import pandas as pd
+        except ImportError:
+            raise ValueError("pandas module is required to read Excel files.")
+
+        df = pd.read_excel(file, **self._pandas_config)
+
+        text_list = df.apply(
+            lambda row: (self._col_joiner).join(row.astype(str).tolist()), axis=1
+        ).tolist()
+
+        if self._concat_rows:
+            return (self._row_joiner).join(text_list)
+        else:
+            return text_list

application/parser/java2doc.py

@@ -1,66 +0,0 @@
-import os
-
-import javalang
-
-
-def find_files(directory):
-    files_list = []
-    for root, dirs, files in os.walk(directory):
-        for file in files:
-            if file.endswith('.java'):
-                files_list.append(os.path.join(root, file))
-    return files_list
-
-
-def extract_functions(file_path):
-    with open(file_path, "r") as file:
-        java_code = file.read()
-        methods = {}
-        tree = javalang.parse.parse(java_code)
-        for _, node in tree.filter(javalang.tree.MethodDeclaration):
-            method_name = node.name
-            start_line = node.position.line - 1
-            end_line = start_line
-            brace_count = 0
-            for line in java_code.splitlines()[start_line:]:
-                end_line += 1
-                brace_count += line.count("{") - line.count("}")
-                if brace_count == 0:
-                    break
-            method_source_code = "\n".join(java_code.splitlines()[start_line:end_line])
-            methods[method_name] = method_source_code
-    return methods
-
-
-def extract_classes(file_path):
-    with open(file_path, 'r') as file:
-        source_code = file.read()
-        classes = {}
-        tree = javalang.parse.parse(source_code)
-        for class_decl in tree.types:
-            class_name = class_decl.name
-            declarations = []
-            methods = []
-            for field_decl in class_decl.fields:
-                field_name = field_decl.declarators[0].name
-                field_type = field_decl.type.name
-                declarations.append(f"{field_type} {field_name}")
-            for method_decl in class_decl.methods:
-                methods.append(method_decl.name)
-            class_string = "Declarations: " + ", ".join(declarations) + "\n  Method name: " + ", ".join(methods)
-            classes[class_name] = class_string
-    return classes
-
-
-def extract_functions_and_classes(directory):
-    files = find_files(directory)
-    functions_dict = {}
-    classes_dict = {}
-    for file in files:
-        functions = extract_functions(file)
-        if functions:
-            functions_dict[file] = functions
-        classes = extract_classes(file)
-        if classes:
-            classes_dict[file] = classes
-    return functions_dict, classes_dict

application/parser/js2doc.py

@@ -1,70 +0,0 @@
-import os
-
-import escodegen
-import esprima
-
-
-def find_files(directory):
-    files_list = []
-    for root, dirs, files in os.walk(directory):
-        for file in files:
-            if file.endswith('.js'):
-                files_list.append(os.path.join(root, file))
-    return files_list
-
-
-def extract_functions(file_path):
-    with open(file_path, 'r') as file:
-        source_code = file.read()
-        functions = {}
-        tree = esprima.parseScript(source_code)
-        for node in tree.body:
-            if node.type == 'FunctionDeclaration':
-                func_name = node.id.name if node.id else '<anonymous>'
-                functions[func_name] = escodegen.generate(node)
-            elif node.type == 'VariableDeclaration':
-                for declaration in node.declarations:
-                    if declaration.init and declaration.init.type == 'FunctionExpression':
-                        func_name = declaration.id.name if declaration.id else '<anonymous>'
-                        functions[func_name] = escodegen.generate(declaration.init)
-            elif node.type == 'ClassDeclaration':
-                for subnode in node.body.body:
-                    if subnode.type == 'MethodDefinition':
-                        func_name = subnode.key.name
-                        functions[func_name] = escodegen.generate(subnode.value)
-                    elif subnode.type == 'VariableDeclaration':
-                        for declaration in subnode.declarations:
-                            if declaration.init and declaration.init.type == 'FunctionExpression':
-                                func_name = declaration.id.name if declaration.id else '<anonymous>'
-                                functions[func_name] = escodegen.generate(declaration.init)
-        return functions
-
-
-def extract_classes(file_path):
-    with open(file_path, 'r') as file:
-        source_code = file.read()
-        classes = {}
-        tree = esprima.parseScript(source_code)
-        for node in tree.body:
-            if node.type == 'ClassDeclaration':
-                class_name = node.id.name
-                function_names = []
-                for subnode in node.body.body:
-                    if subnode.type == 'MethodDefinition':
-                        function_names.append(subnode.key.name)
-                classes[class_name] = ", ".join(function_names)
-    return classes
-
-
-def extract_functions_and_classes(directory):
-    files = find_files(directory)
-    functions_dict = {}
-    classes_dict = {}
-    for file in files:
-        functions = extract_functions(file)
-        if functions:
-            functions_dict[file] = functions
-        classes = extract_classes(file)
-        if classes:
-            classes_dict[file] = classes
-    return functions_dict, classes_dict

application/parser/open_ai_func.py

@@ -1,75 +0,0 @@
-import os
-
-from retry import retry
-
-from application.core.settings import settings
-
-from application.vectorstore.vector_creator import VectorCreator
-
-
-# from langchain_community.embeddings import HuggingFaceEmbeddings
-# from langchain_community.embeddings import HuggingFaceInstructEmbeddings
-# from langchain_community.embeddings import CohereEmbeddings
-
-
-@retry(tries=10, delay=60)
-def store_add_texts_with_retry(store, i, id):
-    # add source_id to the metadata
-    i.metadata["source_id"] = str(id)
-    store.add_texts([i.page_content], metadatas=[i.metadata])
-    # store_pine.add_texts([i.page_content], metadatas=[i.metadata])
-
-
-def call_openai_api(docs, folder_name, id, task_status):
-    # Function to create a vector store from the documents and save it to disk
-
-    if not os.path.exists(f"{folder_name}"):
-        os.makedirs(f"{folder_name}")
-
-    from tqdm import tqdm
-
-    c1 = 0
-    if settings.VECTOR_STORE == "faiss":
-        docs_init = [docs[0]]
-        docs.pop(0)
-
-        store = VectorCreator.create_vectorstore(
-            settings.VECTOR_STORE,
-            docs_init=docs_init,
-            source_id=f"{folder_name}",
-            embeddings_key=os.getenv("EMBEDDINGS_KEY"),
-        )
-    else:
-        store = VectorCreator.create_vectorstore(
-            settings.VECTOR_STORE,
-            source_id=str(id),
-            embeddings_key=os.getenv("EMBEDDINGS_KEY"),
-        )
-        store.delete_index()
-    # Uncomment for MPNet embeddings
-    # model_name = "sentence-transformers/all-mpnet-base-v2"
-    # hf = HuggingFaceEmbeddings(model_name=model_name)
-    # store = FAISS.from_documents(docs_test, hf)
-    s1 = len(docs)
-    for i in tqdm(
-        docs,
-        desc="Embedding 🦖",
-        unit="docs",
-        total=len(docs),
-        bar_format="{l_bar}{bar}| Time Left: {remaining}",
-    ):
-        try:
-            task_status.update_state(
-                state="PROGRESS", meta={"current": int((c1 / s1) * 100)}
-            )
-            store_add_texts_with_retry(store, i, id)
-        except Exception as e:
-            print(e)
-            print("Error on ", i)
-            print("Saving progress")
-            print(f"stopped at {c1} out of {len(docs)}")
-            store.save_local(f"{folder_name}")
-            break
-        c1 += 1
-    if settings.VECTOR_STORE == "faiss":
-        store.save_local(f"{folder_name}")

application/parser/py2doc.py

@@ -1,121 +0,0 @@
-import ast
-import os
-from pathlib import Path
-
-import tiktoken
-from langchain.llms import OpenAI
-from langchain.prompts import PromptTemplate
-
-
-def find_files(directory):
-    files_list = []
-    for root, dirs, files in os.walk(directory):
-        for file in files:
-            if file.endswith('.py'):
-                files_list.append(os.path.join(root, file))
-    return files_list
-
-
-def extract_functions(file_path):
-    with open(file_path, 'r') as file:
-        source_code = file.read()
-        functions = {}
-        tree = ast.parse(source_code)
-        for node in ast.walk(tree):
-            if isinstance(node, ast.FunctionDef):
-                func_name = node.name
-                func_def = ast.get_source_segment(source_code, node)
-                functions[func_name] = func_def
-    return functions
-
-
-def extract_classes(file_path):
-    with open(file_path, 'r') as file:
-        source_code = file.read()
-        classes = {}
-        tree = ast.parse(source_code)
-        for node in ast.walk(tree):
-            if isinstance(node, ast.ClassDef):
-                class_name = node.name
-                function_names = []
-                for subnode in ast.walk(node):
-                    if isinstance(subnode, ast.FunctionDef):
-                        function_names.append(subnode.name)
-                classes[class_name] = ", ".join(function_names)
-    return classes
-
-
-def extract_functions_and_classes(directory):
-    files = find_files(directory)
-    functions_dict = {}
-    classes_dict = {}
-    for file in files:
-        functions = extract_functions(file)
-        if functions:
-            functions_dict[file] = functions
-        classes = extract_classes(file)
-        if classes:
-            classes_dict[file] = classes
-    return functions_dict, classes_dict
-
-
-def parse_functions(functions_dict, formats, dir):
-    c1 = len(functions_dict)
-    for i, (source, functions) in enumerate(functions_dict.items(), start=1):
-        print(f"Processing file {i}/{c1}")
-        source_w = source.replace(dir + "/", "").replace("." + formats, ".md")
-        subfolders = "/".join(source_w.split("/")[:-1])
-        Path(f"outputs/{subfolders}").mkdir(parents=True, exist_ok=True)
-        for j, (name, function) in enumerate(functions.items(), start=1):
-            print(f"Processing function {j}/{len(functions)}")
-            prompt = PromptTemplate(
-                input_variables=["code"],
-                template="Code: \n{code}, \nDocumentation: ",
-            )
-            llm = OpenAI(temperature=0)
-            response = llm(prompt.format(code=function))
-            mode = "a" if Path(f"outputs/{source_w}").exists() else "w"
-            with open(f"outputs/{source_w}", mode) as f:
-                f.write(
-                    f"\n\n# Function name: {name} \n\nFunction: \n```\n{function}\n```, \nDocumentation: \n{response}")
-
-
-def parse_classes(classes_dict, formats, dir):
-    c1 = len(classes_dict)
-    for i, (source, classes) in enumerate(classes_dict.items()):
-        print(f"Processing file {i + 1}/{c1}")
-        source_w = source.replace(dir + "/", "").replace("." + formats, ".md")
-        subfolders = "/".join(source_w.split("/")[:-1])
-        Path(f"outputs/{subfolders}").mkdir(parents=True, exist_ok=True)
-        for name, function_names in classes.items():
-            print(f"Processing Class {i + 1}/{c1}")
-            prompt = PromptTemplate(
-                input_variables=["class_name", "functions_names"],
-                template="Class name: {class_name} \nFunctions: {functions_names}, \nDocumentation: ",
-            )
-            llm = OpenAI(temperature=0)
-            response = llm(prompt.format(class_name=name, functions_names=function_names))
-
-            with open(f"outputs/{source_w}", "a" if Path(f"outputs/{source_w}").exists() else "w") as f:
-                f.write(f"\n\n# Class name: {name} \n\nFunctions: \n{function_names}, \nDocumentation: \n{response}")
-
-
-def transform_to_docs(functions_dict, classes_dict, formats, dir):
-    docs_content = ''.join([str(key) + str(value) for key, value in functions_dict.items()])
-    docs_content += ''.join([str(key) + str(value) for key, value in classes_dict.items()])
-
-    num_tokens = len(tiktoken.get_encoding("cl100k_base").encode(docs_content))
-    total_price = ((num_tokens / 1000) * 0.02)
-
-    print(f"Number of Tokens = {num_tokens:,d}")
-    print(f"Approx Cost = ${total_price:,.2f}")
-
-    user_input = input("Price Okay? (Y/N)\n").lower()
-    if user_input == "y" or user_input == "":
-        if not Path("outputs").exists():
-            Path("outputs").mkdir()
-        parse_functions(functions_dict, formats, dir)
-        parse_classes(classes_dict, formats, dir)
-        print("All done!")
-    else:
-        print("The API was not called. No money was spent.")

application/parser/remote/crawler_loader.py

@@ -2,16 +2,16 @@ import requests
 from urllib.parse import urlparse, urljoin
 from bs4 import BeautifulSoup
 from application.parser.remote.base import BaseRemote
+from application.parser.schema.base import Document
+from langchain_community.document_loaders import WebBaseLoader
 
 class CrawlerLoader(BaseRemote):
     def __init__(self, limit=10):
-        from langchain_community.document_loaders import WebBaseLoader
         self.loader = WebBaseLoader  # Initialize the document loader
         self.limit = limit  # Set the limit for the number of pages to scrape
 
     def load_data(self, inputs):
         url = inputs
-        # Check if the input is a list and if it is, use the first element
         if isinstance(url, list) and url:
             url = url[0]
 
@@ -19,24 +19,29 @@ class CrawlerLoader(BaseRemote):
         if not urlparse(url).scheme:
             url = "http://" + url
 
-        visited_urls = set()  # Keep track of URLs that have been visited
-        base_url = urlparse(url).scheme + "://" + urlparse(url).hostname  # Extract the base URL
-        urls_to_visit = [url]  # List of URLs to be visited, starting with the initial URL
-        loaded_content = []  # Store the loaded content from each URL
+        visited_urls = set()
+        base_url = urlparse(url).scheme + "://" + urlparse(url).hostname
+        urls_to_visit = [url]
+        loaded_content = []
 
-        # Continue crawling until there are no more URLs to visit
         while urls_to_visit:
-            current_url = urls_to_visit.pop(0)  # Get the next URL to visit
-            visited_urls.add(current_url)  # Mark the URL as visited
+            current_url = urls_to_visit.pop(0)
+            visited_urls.add(current_url)
 
-            # Try to load and process the content from the current URL
             try:
-                response = requests.get(current_url)  # Fetch the content of the current URL
-                response.raise_for_status()  # Raise an exception for HTTP errors
-                loader = self.loader([current_url])  # Initialize the document loader for the current URL
-                loaded_content.extend(loader.load())  # Load the content and add it to the loaded_content list
+                response = requests.get(current_url)
+                response.raise_for_status()
+                loader = self.loader([current_url])
+                docs = loader.load()
+                # Convert the loaded documents to your Document schema
+                for doc in docs:
+                    loaded_content.append(
+                        Document(
+                            doc.page_content,
+                            extra_info=doc.metadata
+                        )
+                    )
             except Exception as e:
-                # Print an error message if loading or processing fails and continue with the next URL
                 print(f"Error processing URL {current_url}: {e}")
                 continue
 
@@ -45,15 +50,15 @@ class CrawlerLoader(BaseRemote):
             all_links = [
                 urljoin(current_url, a['href'])
                 for a in soup.find_all('a', href=True)
-                if base_url in urljoin(current_url, a['href'])  # Ensure links are from the same domain
+                if base_url in urljoin(current_url, a['href'])
             ]
 
             # Add new links to the list of URLs to visit if they haven't been visited yet
             urls_to_visit.extend([link for link in all_links if link not in visited_urls])
-            urls_to_visit = list(set(urls_to_visit))  # Remove duplicate URLs
+            urls_to_visit = list(set(urls_to_visit))
 
             # Stop crawling if the limit of pages to scrape is reached
             if self.limit is not None and len(visited_urls) >= self.limit:
                 break
 
-        return loaded_content  # Return the loaded content from all visited URLs
+        return loaded_content

application/parser/remote/crawler_markdown.py

@@ -0,0 +1,139 @@
+import requests
+from urllib.parse import urlparse, urljoin
+from bs4 import BeautifulSoup
+from application.parser.remote.base import BaseRemote
+import re
+from markdownify import markdownify
+from application.parser.schema.base import Document
+import tldextract
+
+class CrawlerLoader(BaseRemote):
+    def __init__(self, limit=10, allow_subdomains=False):
+        """
+        Given a URL crawl web pages up to `self.limit`,
+        convert HTML content to Markdown, and returning a list of Document objects.
+
+        :param limit: The maximum number of pages to crawl.
+        :param allow_subdomains: If True, crawl pages on subdomains of the base domain.
+        """
+        self.limit = limit
+        self.allow_subdomains = allow_subdomains
+        self.session = requests.Session()
+
+    def load_data(self, inputs):
+        url = inputs
+        if isinstance(url, list) and url:
+            url = url[0]
+
+        # Ensure the URL has a scheme (if not, default to http)
+        if not urlparse(url).scheme:
+            url = "http://" + url
+
+        # Keep track of visited URLs to avoid revisiting the same page
+        visited_urls = set()
+
+        # Determine the base domain for link filtering using tldextract
+        base_domain = self._get_base_domain(url)
+        urls_to_visit = {url}
+        documents = []
+
+        while urls_to_visit:
+            current_url = urls_to_visit.pop()
+
+            # Skip if already visited
+            if current_url in visited_urls:
+                continue
+            visited_urls.add(current_url)
+
+            # Fetch the page content
+            html_content = self._fetch_page(current_url)
+            if html_content is None:
+                continue
+
+            # Convert the HTML to Markdown for cleaner text formatting
+            title, language, processed_markdown = self._process_html_to_markdown(html_content, current_url)
+            if processed_markdown:
+                # Create a Document for each visited page
+                documents.append(
+                    Document(
+                        processed_markdown,  # content
+                        None,  # doc_id
+                        None,  # embedding
+                        {"source": current_url, "title": title, "language": language} # extra_info
+                    )
+                )
+
+            # Extract links and filter them according to domain rules
+            new_links = self._extract_links(html_content, current_url)
+            filtered_links = self._filter_links(new_links, base_domain)
+
+            # Add any new, not-yet-visited links to the queue
+            urls_to_visit.update(link for link in filtered_links if link not in visited_urls)
+
+            # If we've reached the limit, stop crawling
+            if self.limit is not None and len(visited_urls) >= self.limit:
+                break
+
+        return documents
+
+    def _fetch_page(self, url):
+        try:
+            response = self.session.get(url, timeout=10)
+            response.raise_for_status()
+            return response.text
+        except requests.exceptions.RequestException as e:
+            print(f"Error fetching URL {url}: {e}")
+            return None
+
+    def _process_html_to_markdown(self, html_content, current_url):
+        soup = BeautifulSoup(html_content, 'html.parser')
+        title_tag = soup.find('title')
+        title = title_tag.text.strip() if title_tag else "No Title"
+
+        # Extract language
+        language_tag = soup.find('html')
+        language = language_tag.get('lang', 'en') if language_tag else "en"
+
+        markdownified = markdownify(html_content, heading_style="ATX", newline_style="BACKSLASH")
+        # Reduce sequences of more than two newlines to exactly three
+        markdownified = re.sub(r'\n{3,}', '\n\n\n', markdownified)
+        return title, language, markdownified
+
+    def _extract_links(self, html_content, current_url):
+        soup = BeautifulSoup(html_content, 'html.parser')
+        links = []
+        for a in soup.find_all('a', href=True):
+            full_url = urljoin(current_url, a['href'])
+            links.append((full_url, a.text.strip()))
+        return links
+
+    def _get_base_domain(self, url):
+        extracted = tldextract.extract(url)
+        # Reconstruct the domain as domain.suffix
+        base_domain = f"{extracted.domain}.{extracted.suffix}"
+        return base_domain
+
+    def _filter_links(self, links, base_domain):
+        """
+        Filter the extracted links to only include those that match the crawling criteria:
+        - If allow_subdomains is True, allow any link whose domain ends with the base_domain.
+        - If allow_subdomains is False, only allow exact matches of the base_domain.
+        """
+        filtered = []
+        for link, _ in links:
+            parsed_link = urlparse(link)
+            if not parsed_link.netloc:
+                continue
+
+            extracted = tldextract.extract(parsed_link.netloc)
+            link_base = f"{extracted.domain}.{extracted.suffix}"
+
+            if self.allow_subdomains:
+                # For subdomains: sub.example.com ends with example.com
+                if link_base == base_domain or link_base.endswith("." + base_domain):
+                    filtered.append(link)
+            else:
+                # Exact domain match
+                if link_base == base_domain:
+                    filtered.append(link)
+        return filtered

application/parser/remote/github_loader.py

@@ -0,0 +1,58 @@
+import base64
+import requests
+from typing import List
+from application.parser.remote.base import BaseRemote
+from langchain_core.documents import Document
+import mimetypes
+
+class GitHubLoader(BaseRemote):
+    def __init__(self):
+        self.access_token = None
+        self.headers = {
+            "Authorization": f"token {self.access_token}"
+        } if self.access_token else {}
+        return
+
+    def fetch_file_content(self, repo_url: str, file_path: str) -> str:
+        url = f"https://api.github.com/repos/{repo_url}/contents/{file_path}"
+        response = requests.get(url, headers=self.headers)
+
+        if response.status_code == 200:
+            content = response.json()
+            mime_type, _ = mimetypes.guess_type(file_path)  # Guess the MIME type based on the file extension
+
+            if content.get("encoding") == "base64":
+                if mime_type and mime_type.startswith("text"):  # Handle only text files
+                    try:
+                        decoded_content = base64.b64decode(content["content"]).decode("utf-8")
+                        return f"Filename: {file_path}\n\n{decoded_content}"
+                    except Exception as e:
+                        raise e
+                else:
+                    return f"Filename: {file_path} is a binary file and was skipped."
+            else:
+                return f"Filename: {file_path}\n\n{content['content']}"
+        else:
+            response.raise_for_status()
+
+    def fetch_repo_files(self, repo_url: str, path: str = "") -> List[str]:
+        url = f"https://api.github.com/repos/{repo_url}/contents/{path}"
+        response = requests.get(url, headers={**self.headers, "Accept": "application/vnd.github.v3.raw"})
+        contents = response.json()
+        files = []
+        for item in contents:
+            if item["type"] == "file":
+                files.append(item["path"])
+            elif item["type"] == "dir":
+                files.extend(self.fetch_repo_files(repo_url, item["path"]))
+        return files
+
+    def load_data(self, repo_url: str) -> List[Document]:
+        repo_name = repo_url.split("github.com/")[-1]
+        files = self.fetch_repo_files(repo_name)
+        documents = []
+        for file_path in files:
+            content = self.fetch_file_content(repo_name, file_path)
+            documents.append(Document(page_content=content, metadata={"title": file_path, 
+            "source": f"https://github.com/{repo_name}/blob/main/{file_path}"}))
+        return documents

application/parser/remote/reddit_loader.py

@@ -1,10 +1,19 @@
 from application.parser.remote.base import BaseRemote
 from langchain_community.document_loaders import RedditPostsLoader
+import json
 
 
 class RedditPostsLoaderRemote(BaseRemote):
     def load_data(self, inputs):
-        data = eval(inputs)
+        try:
+            data = json.loads(inputs)
+        except json.JSONDecodeError as e:
+            raise ValueError(f"Invalid JSON input: {e}")
+
+        required_fields = ["client_id", "client_secret", "user_agent", "search_queries"]
+        missing_fields = [field for field in required_fields if field not in data]
+        if missing_fields:
+            raise ValueError(f"Missing required fields: {', '.join(missing_fields)}")
         client_id = data.get("client_id")
         client_secret = data.get("client_secret")
         user_agent = data.get("user_agent")

application/parser/remote/remote_creator.py

@@ -2,6 +2,7 @@ from application.parser.remote.sitemap_loader import SitemapLoader
 from application.parser.remote.crawler_loader import CrawlerLoader
 from application.parser.remote.web_loader import WebLoader
 from application.parser.remote.reddit_loader import RedditPostsLoaderRemote
+from application.parser.remote.github_loader import GitHubLoader
 
 
 class RemoteCreator:
@@ -10,6 +11,7 @@ class RemoteCreator:
         "sitemap": SitemapLoader,
         "crawler": CrawlerLoader,
         "reddit": RedditPostsLoaderRemote,
+        "github": GitHubLoader,
     }
 
     @classmethod

application/parser/remote/web_loader.py

@@ -1,5 +1,7 @@
 from application.parser.remote.base import BaseRemote
+from application.parser.schema.base import Document
 from langchain_community.document_loaders import WebBaseLoader
+from urllib.parse import urlparse
 
 headers = {
     "User-Agent": "Mozilla/5.0",
@@ -23,10 +25,20 @@ class WebLoader(BaseRemote):
             urls = [urls]
         documents = []
         for url in urls:
+            # Check if the URL scheme is provided, if not, assume http
+            if not urlparse(url).scheme:
+                url = "http://" + url
             try:
                 loader = self.loader([url], header_template=headers)
-                documents.extend(loader.load())
+                loaded_docs = loader.load()
+                for doc in loaded_docs:
+                    documents.append(
+                        Document(
+                            doc.page_content,
+                            extra_info=doc.metadata,
+                        )
+                    )
             except Exception as e:
                 print(f"Error processing URL {url}: {e}")
                 continue
-        return documents
+        return documents

application/parser/token_func.py

@@ -1,79 +0,0 @@
-import re
-from math import ceil
-from typing import List
-
-import tiktoken
-from application.parser.schema.base import Document
-
-
-def separate_header_and_body(text):
-    header_pattern = r"^(.*?\n){3}"
-    match = re.match(header_pattern, text)
-    header = match.group(0)
-    body = text[len(header):]
-    return header, body
-
-
-def group_documents(documents: List[Document], min_tokens: int, max_tokens: int) -> List[Document]:
-    docs = []
-    current_group = None
-
-    for doc in documents:
-        doc_len = len(tiktoken.get_encoding("cl100k_base").encode(doc.text))
-
-        # Check if current group is empty or if the document can be added based on token count and matching metadata
-        if (current_group is None or 
-            (len(tiktoken.get_encoding("cl100k_base").encode(current_group.text)) + doc_len < max_tokens and 
-             doc_len < min_tokens and 
-             current_group.extra_info == doc.extra_info)):
-            if current_group is None:
-                current_group = doc  # Use the document directly to retain its metadata
-            else:
-                current_group.text += " " + doc.text  # Append text to the current group
-        else:
-            docs.append(current_group)
-            current_group = doc  # Start a new group with the current document
-
-    if current_group is not None:
-        docs.append(current_group)
-
-    return docs
-
-
-def split_documents(documents: List[Document], max_tokens: int) -> List[Document]:
-    docs = []
-    for doc in documents:
-        token_length = len(tiktoken.get_encoding("cl100k_base").encode(doc.text))
-        if token_length <= max_tokens:
-            docs.append(doc)
-        else:
-            header, body = separate_header_and_body(doc.text)
-            if len(tiktoken.get_encoding("cl100k_base").encode(header)) > max_tokens:
-                body = doc.text
-                header = ""
-            num_body_parts = ceil(token_length / max_tokens)
-            part_length = ceil(len(body) / num_body_parts)
-            body_parts = [body[i:i + part_length] for i in range(0, len(body), part_length)]
-            for i, body_part in enumerate(body_parts):
-                new_doc = Document(text=header + body_part.strip(),
-                                   doc_id=f"{doc.doc_id}-{i}",
-                                   embedding=doc.embedding,
-                                   extra_info=doc.extra_info)
-                docs.append(new_doc)
-    return docs
-
-
-def group_split(documents: List[Document], max_tokens: int = 2000, min_tokens: int = 150, token_check: bool = True):
-    if not token_check:
-        return documents
-    print("Grouping small documents")
-    try:
-        documents = group_documents(documents=documents, min_tokens=min_tokens, max_tokens=max_tokens)
-    except Exception:
-        print("Grouping failed, try running without token_check")
-    print("Separating large documents")
-    try:
-        documents = split_documents(documents=documents, max_tokens=max_tokens)
-    except Exception:
-        print("Grouping failed, try running without token_check")
-    return documents

application/requirements.txt

@@ -1,24 +1,27 @@
-anthropic==0.34.2
-boto3==1.34.153
+anthropic==0.40.0
+boto3==1.35.97
 beautifulsoup4==4.12.3
-celery==5.3.6
+celery==5.4.0
 dataclasses-json==0.6.7
 docx2txt==0.8
-duckduckgo-search==6.2.6
+duckduckgo-search==6.3.0
 ebooklib==0.18
-elastic-transport==8.15.0
-elasticsearch==8.15.1
+elastic-transport==8.17.0
+elasticsearch==8.17.0
 escodegen==1.0.11
 esprima==4.0.1
 esutils==1.0.1
-Flask==3.0.3
-faiss-cpu==1.8.0.post1
+Flask==3.1.0
+faiss-cpu==1.9.0.post1
 flask-restx==1.3.0
+google-genai==0.5.0
+google-generativeai==0.8.3
+gTTS==2.5.4
 gunicorn==23.0.0
 html2text==2024.2.26
 javalang==0.13.0
-jinja2==3.1.4
-jiter==0.5.0
+jinja2==3.1.5
+jiter==0.8.2
 jmespath==1.0.1
 joblib==1.4.2
 jsonpatch==1.33
@@ -27,60 +30,66 @@ jsonschema==4.23.0
 jsonschema-spec==0.2.4
 jsonschema-specifications==2023.7.1
 kombu==5.4.2
-langchain==0.3.0
-langchain-community==0.3.0
-langchain-core==0.3.2
-langchain-openai==0.2.0
-langchain-text-splitters==0.3.0
-langsmith==0.1.125
+langchain==0.3.14
+langchain-community==0.3.14
+langchain-core==0.3.29
+langchain-openai==0.3.0
+langchain-text-splitters==0.3.5
+langsmith==0.2.10
 lazy-object-proxy==1.10.0
 lxml==5.3.0
-markupsafe==2.1.5
-marshmallow==3.22.0
+markupsafe==3.0.2
+marshmallow==3.24.1
 mpmath==1.3.0
 multidict==6.1.0
 mypy-extensions==1.0.0
-networkx==3.3
-numpy==1.26.4
-openai==1.46.1
+networkx==3.4.2
+numpy==2.2.1
+openai==1.59.5
 openapi-schema-validator==0.6.2
 openapi-spec-validator==0.6.0
-openapi3-parser==1.1.18
-orjson==3.10.7
+openapi3-parser==1.1.19
+orjson==3.10.14
 packaging==24.1
 pandas==2.2.3
-pathable==0.4.3
-pillow==10.4.0
+openpyxl==3.1.5
+pathable==0.4.4
+pillow==11.1.0
 portalocker==2.10.1
 prance==23.6.21.0
-primp==0.6.2
-prompt-toolkit==3.0.47
-protobuf==5.28.2
+primp==0.10.0
+prompt-toolkit==3.0.48
+protobuf==5.29.3
+psycopg2-binary==2.9.10
 py==1.11.0
-pydantic==2.9.2
-pydantic-core==2.23.4
-pydantic-settings==2.4.0
-pymongo==4.8.0
+pydantic==2.10.4
+pydantic-core==2.27.2
+pydantic-settings==2.7.1
+pymongo==4.10.1
 pypdf2==3.0.1
 python-dateutil==2.9.0.post0
 python-dotenv==1.0.1
-qdrant-client==1.11.0
-redis==5.0.1
+python-pptx==1.0.2
+qdrant-client==1.12.2
+redis==5.2.1
 referencing==0.30.2
-regex==2024.9.11
+regex==2024.11.6
 requests==2.32.3
 retry==0.9.2
-sentence-transformers==3.0.1
-tiktoken==0.7.0
-tokenizers==0.19.1
-torch==2.4.1
-tqdm==4.66.5
-transformers==4.44.2
+sentence-transformers==3.3.1
+tiktoken==0.8.0
+tokenizers==0.21.0
+torch==2.5.1
+tqdm==4.67.1
+transformers==4.48.0
 typing-extensions==4.12.2
 typing-inspect==0.9.0
 tzdata==2024.2
-urllib3==2.2.3
+urllib3==2.3.0
 vine==5.1.0
 wcwidth==0.2.13
-werkzeug==3.0.4
-yarl==1.11.1
+werkzeug==3.1.3
+yarl==1.18.3
+markdownify==0.14.1
+tldextract==5.1.3
+websockets==14.1

application/retriever/brave_search.py

@@ -2,7 +2,6 @@ import json
 from application.retriever.base import BaseRetriever
 from application.core.settings import settings
 from application.llm.llm_creator import LLMCreator
-from application.utils import num_tokens_from_string
 from langchain_community.tools import BraveSearch
 
 
@@ -72,23 +71,13 @@ class BraveRetSearch(BaseRetriever):
         for doc in docs:
             yield {"source": doc}
 
-        if len(self.chat_history) > 1:
-            tokens_current_history = 0
-            # count tokens in history
-            self.chat_history.reverse()
+        if len(self.chat_history) > 0:
             for i in self.chat_history:
                 if "prompt" in i and "response" in i:
-                    tokens_batch = num_tokens_from_string(i["prompt"]) + num_tokens_from_string(
-                        i["response"]
+                    messages_combine.append({"role": "user", "content": i["prompt"]})
+                    messages_combine.append(
+                        {"role": "assistant", "content": i["response"]}
                     )
-                    if tokens_current_history + tokens_batch < self.token_limit:
-                        tokens_current_history += tokens_batch
-                        messages_combine.append(
-                            {"role": "user", "content": i["prompt"]}
-                        )
-                        messages_combine.append(
-                            {"role": "system", "content": i["response"]}
-                        )
         messages_combine.append({"role": "user", "content": self.question})
 
         llm = LLMCreator.create_llm(

application/retriever/classic_rag.py

@@ -1,9 +1,8 @@
-from application.retriever.base import BaseRetriever
 from application.core.settings import settings
-from application.vectorstore.vector_creator import VectorCreator
-from application.llm.llm_creator import LLMCreator
+from application.retriever.base import BaseRetriever
+from application.tools.agent import Agent
 
-from application.utils import num_tokens_from_string
+from application.vectorstore.vector_creator import VectorCreator
 
 
 class ClassicRAG(BaseRetriever):
@@ -20,7 +19,7 @@ class ClassicRAG(BaseRetriever):
         user_api_key=None,
     ):
         self.question = question
-        self.vectorstore = source['active_docs'] if 'active_docs' in source else None
+        self.vectorstore = source["active_docs"] if "active_docs" in source else None
         self.chat_history = chat_history
         self.prompt = prompt
         self.chunks = chunks
@@ -45,7 +44,6 @@ class ClassicRAG(BaseRetriever):
                 settings.VECTOR_STORE, self.vectorstore, settings.EMBEDDINGS_KEY
             )
             docs_temp = docsearch.search(self.question, k=self.chunks)
-            print(docs_temp)
             docs = [
                 {
                     "title": i.metadata.get(
@@ -60,8 +58,6 @@ class ClassicRAG(BaseRetriever):
                 }
                 for i in docs_temp
             ]
-        if settings.LLM_NAME == "llama.cpp":
-            docs = [docs[0]]
 
         return docs
 
@@ -75,36 +71,31 @@ class ClassicRAG(BaseRetriever):
         for doc in docs:
             yield {"source": doc}
 
-        if len(self.chat_history) > 1:
-            tokens_current_history = 0
-            # count tokens in history
-            self.chat_history.reverse()
+        if len(self.chat_history) > 0:
             for i in self.chat_history:
                 if "prompt" in i and "response" in i:
-                    tokens_batch = num_tokens_from_string(i["prompt"]) + num_tokens_from_string(
-                        i["response"]
+                    messages_combine.append({"role": "user", "content": i["prompt"]})
+                    messages_combine.append(
+                        {"role": "assistant", "content": i["response"]}
                     )
-                    if tokens_current_history + tokens_batch < self.token_limit:
-                        tokens_current_history += tokens_batch
-                        messages_combine.append(
-                            {"role": "user", "content": i["prompt"]}
-                        )
-                        messages_combine.append(
-                            {"role": "system", "content": i["response"]}
-                        )
         messages_combine.append({"role": "user", "content": self.question})
-
-        llm = LLMCreator.create_llm(
-            settings.LLM_NAME, api_key=settings.API_KEY, user_api_key=self.user_api_key
+        # llm = LLMCreator.create_llm(
+        #     settings.LLM_NAME, api_key=settings.API_KEY, user_api_key=self.user_api_key
+        # )
+        # completion = llm.gen_stream(model=self.gpt_model, messages=messages_combine)
+        agent = Agent(
+            llm_name=settings.LLM_NAME,
+            gpt_model=self.gpt_model,
+            api_key=settings.API_KEY,
+            user_api_key=self.user_api_key,
         )
-
-        completion = llm.gen_stream(model=self.gpt_model, messages=messages_combine)
+        completion = agent.gen(messages_combine)
         for line in completion:
             yield {"answer": str(line)}
 
     def search(self):
         return self._get_data()
-    
+
     def get_params(self):
         return {
             "question": self.question,
@@ -114,5 +105,5 @@ class ClassicRAG(BaseRetriever):
             "chunks": self.chunks,
             "token_limit": self.token_limit,
             "gpt_model": self.gpt_model,
-            "user_api_key": self.user_api_key
+            "user_api_key": self.user_api_key,
         }

application/retriever/duckduck_search.py

@@ -1,7 +1,6 @@
 from application.retriever.base import BaseRetriever
 from application.core.settings import settings
 from application.llm.llm_creator import LLMCreator
-from application.utils import num_tokens_from_string
 from langchain_community.tools import DuckDuckGoSearchResults
 from langchain_community.utilities import DuckDuckGoSearchAPIWrapper
 
@@ -89,22 +88,12 @@ class DuckDuckSearch(BaseRetriever):
         for doc in docs:
             yield {"source": doc}
 
-        if len(self.chat_history) > 1:
-            tokens_current_history = 0
-            # count tokens in history
-            self.chat_history.reverse()
+        if len(self.chat_history) > 0:      
             for i in self.chat_history:
-                if "prompt" in i and "response" in i:
-                    tokens_batch = num_tokens_from_string(i["prompt"]) + num_tokens_from_string(
-                        i["response"]
-                    )
-                    if tokens_current_history + tokens_batch < self.token_limit:
-                        tokens_current_history += tokens_batch
+                    if "prompt" in i and "response" in i:
+                        messages_combine.append({"role": "user", "content": i["prompt"]})
                         messages_combine.append(
-                            {"role": "user", "content": i["prompt"]}
-                        )
-                        messages_combine.append(
-                            {"role": "system", "content": i["response"]}
+                            {"role": "assistant", "content": i["response"]}
                         )
         messages_combine.append({"role": "user", "content": self.question})
 

application/tools/agent.py

@@ -0,0 +1,160 @@
+from application.core.mongo_db import MongoDB
+from application.llm.llm_creator import LLMCreator
+from application.tools.llm_handler import get_llm_handler
+from application.tools.tool_action_parser import ToolActionParser
+from application.tools.tool_manager import ToolManager
+
+
+class Agent:
+    def __init__(self, llm_name, gpt_model, api_key, user_api_key=None):
+        # Initialize the LLM with the provided parameters
+        self.llm = LLMCreator.create_llm(
+            llm_name, api_key=api_key, user_api_key=user_api_key
+        )
+        self.llm_handler = get_llm_handler(llm_name)
+        self.gpt_model = gpt_model
+        # Static tool configuration (to be replaced later)
+        self.tools = []
+        self.tool_config = {}
+
+    def _get_user_tools(self, user="local"):
+        mongo = MongoDB.get_client()
+        db = mongo["docsgpt"]
+        user_tools_collection = db["user_tools"]
+        user_tools = user_tools_collection.find({"user": user, "status": True})
+        user_tools = list(user_tools)
+        tools_by_id = {str(tool["_id"]): tool for tool in user_tools}
+        return tools_by_id
+
+    def _build_tool_parameters(self, action):
+        params = {"type": "object", "properties": {}, "required": []}
+        for param_type in ["query_params", "headers", "body", "parameters"]:
+            if param_type in action and action[param_type].get("properties"):
+                for k, v in action[param_type]["properties"].items():
+                    if v.get("filled_by_llm", True):
+                        params["properties"][k] = {
+                            key: value
+                            for key, value in v.items()
+                            if key != "filled_by_llm" and key != "value"
+                        }
+
+                        params["required"].append(k)
+        return params
+
+    def _prepare_tools(self, tools_dict):
+        self.tools = [
+            {
+                "type": "function",
+                "function": {
+                    "name": f"{action['name']}_{tool_id}",
+                    "description": action["description"],
+                    "parameters": self._build_tool_parameters(action),
+                },
+            }
+            for tool_id, tool in tools_dict.items()
+            for action in (
+                tool["config"]["actions"].values()
+                if tool["name"] == "api_tool"
+                else tool["actions"]
+            )
+            if action.get("active", True)
+        ]
+
+    def _execute_tool_action(self, tools_dict, call):
+        parser = ToolActionParser(self.llm.__class__.__name__)
+        tool_id, action_name, call_args = parser.parse_args(call)
+
+        tool_data = tools_dict[tool_id]
+        action_data = (
+            tool_data["config"]["actions"][action_name]
+            if tool_data["name"] == "api_tool"
+            else next(
+                action
+                for action in tool_data["actions"]
+                if action["name"] == action_name
+            )
+        )
+
+        query_params, headers, body, parameters = {}, {}, {}, {}
+        param_types = {
+            "query_params": query_params,
+            "headers": headers,
+            "body": body,
+            "parameters": parameters,
+        }
+
+        for param_type, target_dict in param_types.items():
+            if param_type in action_data and action_data[param_type].get("properties"):
+                for param, details in action_data[param_type]["properties"].items():
+                    if param not in call_args and "value" in details:
+                        target_dict[param] = details["value"]
+
+        for param, value in call_args.items():
+            for param_type, target_dict in param_types.items():
+                if param_type in action_data and param in action_data[param_type].get(
+                    "properties", {}
+                ):
+                    target_dict[param] = value
+
+        tm = ToolManager(config={})
+        tool = tm.load_tool(
+            tool_data["name"],
+            tool_config=(
+                {
+                    "url": tool_data["config"]["actions"][action_name]["url"],
+                    "method": tool_data["config"]["actions"][action_name]["method"],
+                    "headers": headers,
+                    "query_params": query_params,
+                }
+                if tool_data["name"] == "api_tool"
+                else tool_data["config"]
+            ),
+        )
+        if tool_data["name"] == "api_tool":
+            print(
+                f"Executing api: {action_name} with query_params: {query_params}, headers: {headers}, body: {body}"
+            )
+            result = tool.execute_action(action_name, **body)
+        else:
+            print(f"Executing tool: {action_name} with args: {call_args}")
+            result = tool.execute_action(action_name, **parameters)
+        call_id = getattr(call, "id", None)
+        return result, call_id
+
+    def _simple_tool_agent(self, messages):
+        tools_dict = self._get_user_tools()
+        self._prepare_tools(tools_dict)
+
+        resp = self.llm.gen(model=self.gpt_model, messages=messages, tools=self.tools)
+
+        if isinstance(resp, str):
+            yield resp
+            return
+        if hasattr(resp, "message") and hasattr(resp.message, "content"):
+            yield resp.message.content
+            return
+
+        resp = self.llm_handler.handle_response(self, resp, tools_dict, messages)
+
+        if isinstance(resp, str):
+            yield resp
+        elif hasattr(resp, "message") and hasattr(resp.message, "content"):
+            yield resp.message.content
+        else:
+            completion = self.llm.gen_stream(
+                model=self.gpt_model, messages=messages, tools=self.tools
+            )
+            for line in completion:
+                yield line
+
+        return
+
+    def gen(self, messages):
+        if self.llm.supports_tools():
+            resp = self._simple_tool_agent(messages)
+            for line in resp:
+                yield line
+        else:
+            resp = self.llm.gen_stream(model=self.gpt_model, messages=messages)
+            for line in resp:
+                yield line

application/tools/base.py

@@ -0,0 +1,21 @@
+from abc import ABC, abstractmethod
+
+
+class Tool(ABC):
+    @abstractmethod
+    def execute_action(self, action_name: str, **kwargs):
+        pass
+
+    @abstractmethod
+    def get_actions_metadata(self):
+        """
+        Returns a list of JSON objects describing the actions supported by the tool.
+        """
+        pass
+
+    @abstractmethod
+    def get_config_requirements(self):
+        """
+        Returns a dictionary describing the configuration requirements for the tool.
+        """
+        pass

application/tools/implementations/api_tool.py

@@ -0,0 +1,54 @@
+import json
+
+import requests
+from application.tools.base import Tool
+
+
+class APITool(Tool):
+    """
+    API Tool
+    A flexible tool for performing various API actions (e.g., sending messages, retrieving data) via custom user-specified APIs
+    """
+
+    def __init__(self, config):
+        self.config = config
+        self.url = config.get("url", "")
+        self.method = config.get("method", "GET")
+        self.headers = config.get("headers", {"Content-Type": "application/json"})
+        self.query_params = config.get("query_params", {})
+
+    def execute_action(self, action_name, **kwargs):
+        return self._make_api_call(
+            self.url, self.method, self.headers, self.query_params, kwargs
+        )
+
+    def _make_api_call(self, url, method, headers, query_params, body):
+        if query_params:
+            url = f"{url}?{requests.compat.urlencode(query_params)}"
+        if isinstance(body, dict):
+            body = json.dumps(body)
+        try:
+            print(f"Making API call: {method} {url} with body: {body}")
+            response = requests.request(method, url, headers=headers, data=body)
+            response.raise_for_status()
+            try:
+                data = response.json()
+            except ValueError:
+                data = None
+
+            return {
+                "status_code": response.status_code,
+                "data": data,
+                "message": "API call successful.",
+            }
+        except requests.exceptions.RequestException as e:
+            return {
+                "status_code": response.status_code if response else None,
+                "message": f"API call failed: {str(e)}",
+            }
+
+    def get_actions_metadata(self):
+        return []
+
+    def get_config_requirements(self):
+        return {}

application/tools/implementations/cryptoprice.py

@@ -0,0 +1,77 @@
+import requests
+from application.tools.base import Tool
+
+
+class CryptoPriceTool(Tool):
+    """
+    CryptoPrice
+    A tool for retrieving cryptocurrency prices using the CryptoCompare public API
+    """
+
+    def __init__(self, config):
+        self.config = config
+
+    def execute_action(self, action_name, **kwargs):
+        actions = {"cryptoprice_get": self._get_price}
+
+        if action_name in actions:
+            return actions[action_name](**kwargs)
+        else:
+            raise ValueError(f"Unknown action: {action_name}")
+
+    def _get_price(self, symbol, currency):
+        """
+        Fetches the current price of a given cryptocurrency symbol in the specified currency.
+        Example:
+            symbol = "BTC"
+            currency = "USD"
+            returns price in USD.
+        """
+        url = f"https://min-api.cryptocompare.com/data/price?fsym={symbol.upper()}&tsyms={currency.upper()}"
+        response = requests.get(url)
+        if response.status_code == 200:
+            data = response.json()
+            # data will be like {"USD": <price>} if the call is successful
+            if currency.upper() in data:
+                return {
+                    "status_code": response.status_code,
+                    "price": data[currency.upper()],
+                    "message": f"Price of {symbol.upper()} in {currency.upper()} retrieved successfully.",
+                }
+            else:
+                return {
+                    "status_code": response.status_code,
+                    "message": f"Couldn't find price for {symbol.upper()} in {currency.upper()}.",
+                }
+        else:
+            return {
+                "status_code": response.status_code,
+                "message": "Failed to retrieve price.",
+            }
+
+    def get_actions_metadata(self):
+        return [
+            {
+                "name": "cryptoprice_get",
+                "description": "Retrieve the price of a specified cryptocurrency in a given currency",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "symbol": {
+                            "type": "string",
+                            "description": "The cryptocurrency symbol (e.g. BTC)",
+                        },
+                        "currency": {
+                            "type": "string",
+                            "description": "The currency in which you want the price (e.g. USD)",
+                        },
+                    },
+                    "required": ["symbol", "currency"],
+                    "additionalProperties": False,
+                },
+            }
+        ]
+
+    def get_config_requirements(self):
+        # No specific configuration needed for this tool as it just queries a public endpoint
+        return {}

application/tools/implementations/postgres.py

@@ -0,0 +1,163 @@
+import psycopg2
+from application.tools.base import Tool
+
+class PostgresTool(Tool):
+    """
+    PostgreSQL Database Tool
+    A tool for connecting to a PostgreSQL database using a connection string,
+    executing SQL queries, and retrieving schema information.
+    """
+
+    def __init__(self, config):
+        self.config = config
+        self.connection_string = config.get("token", "")
+
+    def execute_action(self, action_name, **kwargs):
+        actions = {
+            "postgres_execute_sql": self._execute_sql,
+            "postgres_get_schema": self._get_schema,
+        }
+
+        if action_name in actions:
+            return actions[action_name](**kwargs)
+        else:
+            raise ValueError(f"Unknown action: {action_name}")
+
+    def _execute_sql(self, sql_query):
+        """
+        Executes an SQL query against the PostgreSQL database using a connection string.
+        """
+        conn = None  # Initialize conn to None for error handling
+        try:
+            conn = psycopg2.connect(self.connection_string)
+            cur = conn.cursor()
+            cur.execute(sql_query)
+            conn.commit()
+
+            if sql_query.strip().lower().startswith("select"):
+                column_names = [desc[0] for desc in cur.description] if cur.description else []
+                results = []
+                rows = cur.fetchall()
+                for row in rows:
+                    results.append(dict(zip(column_names, row)))
+                response_data = {"data": results, "column_names": column_names}
+            else:
+                row_count = cur.rowcount
+                response_data = {"message": f"Query executed successfully, {row_count} rows affected."}
+
+            cur.close()
+            return {
+                "status_code": 200,
+                "message": "SQL query executed successfully.",
+                "response_data": response_data,
+            }
+
+        except psycopg2.Error as e:
+            error_message = f"Database error: {e}"
+            print(f"Database error: {e}")
+            return {
+                "status_code": 500,
+                "message": "Failed to execute SQL query.",
+                "error": error_message,
+            }
+        finally:
+            if conn:  # Ensure connection is closed even if errors occur
+                conn.close()
+
+    def _get_schema(self, db_name):
+        """
+        Retrieves the schema of the PostgreSQL database using a connection string.
+        """
+        conn = None # Initialize conn to None for error handling
+        try:
+            conn = psycopg2.connect(self.connection_string)
+            cur = conn.cursor()
+
+            cur.execute("""
+                SELECT
+                    table_name,
+                    column_name,
+                    data_type,
+                    column_default,
+                    is_nullable
+                FROM
+                    information_schema.columns
+                WHERE
+                    table_schema = 'public'
+                ORDER BY
+                    table_name,
+                    ordinal_position;
+            """)
+
+            schema_data = {}
+            for row in cur.fetchall():
+                table_name, column_name, data_type, column_default, is_nullable = row
+                if table_name not in schema_data:
+                    schema_data[table_name] = []
+                schema_data[table_name].append({
+                    "column_name": column_name,
+                    "data_type": data_type,
+                    "column_default": column_default,
+                    "is_nullable": is_nullable
+                })
+
+            cur.close()
+            return {
+                "status_code": 200,
+                "message": "Database schema retrieved successfully.",
+                "schema": schema_data,
+            }
+
+        except psycopg2.Error as e:
+            error_message = f"Database error: {e}"
+            print(f"Database error: {e}")
+            return {
+                "status_code": 500,
+                "message": "Failed to retrieve database schema.",
+                "error": error_message,
+            }
+        finally:
+            if conn: # Ensure connection is closed even if errors occur
+                conn.close()
+
+    def get_actions_metadata(self):
+        return [
+            {
+                "name": "postgres_execute_sql",
+                "description": "Execute an SQL query against the PostgreSQL database and return the results. Use this tool to interact with the database, e.g., retrieve specific data or perform updates. Only SELECT queries will return data, other queries will return execution status.",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "sql_query": {
+                            "type": "string",
+                            "description": "The SQL query to execute.",
+                        },
+                    },
+                    "required": ["sql_query"],
+                    "additionalProperties": False,
+                },
+            },
+            {
+                "name": "postgres_get_schema",
+                "description": "Retrieve the schema of the PostgreSQL database, including tables and their columns. Use this to understand the database structure before executing queries. db_name is 'default' if not provided.",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "db_name": {
+                            "type": "string",
+                            "description": "The name of the database to retrieve the schema for.",
+                        },
+                    },
+                    "required": ["db_name"],
+                    "additionalProperties": False,
+                },
+            },
+        ]
+
+    def get_config_requirements(self):
+        return {
+            "token": {
+                "type": "string",
+                "description": "PostgreSQL database connection string (e.g., 'postgresql://user:password@host:port/dbname')",
+            },
+        }

application/tools/implementations/telegram.py

@@ -0,0 +1,86 @@
+import requests
+from application.tools.base import Tool
+
+
+class TelegramTool(Tool):
+    """
+    Telegram Bot
+    A flexible Telegram tool for performing various actions (e.g., sending messages, images).
+    Requires a bot token and chat ID for configuration
+    """
+
+    def __init__(self, config):
+        self.config = config
+        self.token = config.get("token", "")
+
+    def execute_action(self, action_name, **kwargs):
+        actions = {
+            "telegram_send_message": self._send_message,
+            "telegram_send_image": self._send_image,
+        }
+
+        if action_name in actions:
+            return actions[action_name](**kwargs)
+        else:
+            raise ValueError(f"Unknown action: {action_name}")
+
+    def _send_message(self, text, chat_id):
+        print(f"Sending message: {text}")
+        url = f"https://api.telegram.org/bot{self.token}/sendMessage"
+        payload = {"chat_id": chat_id, "text": text}
+        response = requests.post(url, data=payload)
+        return {"status_code": response.status_code, "message": "Message sent"}
+
+    def _send_image(self, image_url, chat_id):
+        print(f"Sending image: {image_url}")
+        url = f"https://api.telegram.org/bot{self.token}/sendPhoto"
+        payload = {"chat_id": chat_id, "photo": image_url}
+        response = requests.post(url, data=payload)
+        return {"status_code": response.status_code, "message": "Image sent"}
+
+    def get_actions_metadata(self):
+        return [
+            {
+                "name": "telegram_send_message",
+                "description": "Send a notification to Telegram chat",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "text": {
+                            "type": "string",
+                            "description": "Text to send in the notification",
+                        },
+                        "chat_id": {
+                            "type": "string",
+                            "description": "Chat ID to send the notification to",
+                        },
+                    },
+                    "required": ["text"],
+                    "additionalProperties": False,
+                },
+            },
+            {
+                "name": "telegram_send_image",
+                "description": "Send an image to the Telegram chat",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "image_url": {
+                            "type": "string",
+                            "description": "URL of the image to send",
+                        },
+                        "chat_id": {
+                            "type": "string",
+                            "description": "Chat ID to send the image to",
+                        },
+                    },
+                    "required": ["image_url"],
+                    "additionalProperties": False,
+                },
+            },
+        ]
+
+    def get_config_requirements(self):
+        return {
+            "token": {"type": "string", "description": "Bot token for authentication"},
+        }

application/tools/llm_handler.py

@@ -0,0 +1,97 @@
+import json
+from abc import ABC, abstractmethod
+
+
+class LLMHandler(ABC):
+    @abstractmethod
+    def handle_response(self, agent, resp, tools_dict, messages, **kwargs):
+        pass
+
+
+class OpenAILLMHandler(LLMHandler):
+    def handle_response(self, agent, resp, tools_dict, messages):
+        while resp.finish_reason == "tool_calls":
+            message = json.loads(resp.model_dump_json())["message"]
+            keys_to_remove = {"audio", "function_call", "refusal"}
+            filtered_data = {
+                k: v for k, v in message.items() if k not in keys_to_remove
+            }
+            messages.append(filtered_data)
+
+            tool_calls = resp.message.tool_calls
+            for call in tool_calls:
+                try:
+                    tool_response, call_id = agent._execute_tool_action(
+                        tools_dict, call
+                    )
+                    messages.append(
+                        {
+                            "role": "tool",
+                            "content": str(tool_response),
+                            "tool_call_id": call_id,
+                        }
+                    )
+                except Exception as e:
+                    messages.append(
+                        {
+                            "role": "tool",
+                            "content": f"Error executing tool: {str(e)}",
+                            "tool_call_id": call_id,
+                        }
+                    )
+            resp = agent.llm.gen(
+                model=agent.gpt_model, messages=messages, tools=agent.tools
+            )
+        return resp
+
+
+class GoogleLLMHandler(LLMHandler):
+    def handle_response(self, agent, resp, tools_dict, messages):
+        from google.genai import types
+
+        while True:
+            response = agent.llm.gen(
+                model=agent.gpt_model, messages=messages, tools=agent.tools
+            )
+            if response.candidates and response.candidates[0].content.parts:
+                tool_call_found = False
+                for part in response.candidates[0].content.parts:
+                    if part.function_call:
+                        tool_call_found = True
+                        tool_response, call_id = agent._execute_tool_action(
+                            tools_dict, part.function_call
+                        )
+                        function_response_part = types.Part.from_function_response(
+                            name=part.function_call.name,
+                            response={"result": tool_response},
+                        )
+
+                        messages.append(
+                            {"role": "model", "content": [part.to_json_dict()]}
+                        )
+                        messages.append(
+                            {
+                                "role": "tool",
+                                "content": [function_response_part.to_json_dict()],
+                            }
+                        )
+
+                if (
+                    not tool_call_found
+                    and response.candidates[0].content.parts
+                    and response.candidates[0].content.parts[0].text
+                ):
+                    return response.candidates[0].content.parts[0].text
+                elif not tool_call_found:
+                    return response.candidates[0].content.parts
+
+            else:
+                return response
+
+
+def get_llm_handler(llm_type):
+    handlers = {
+        "openai": OpenAILLMHandler(),
+        "google": GoogleLLMHandler(),
+    }
+    return handlers.get(llm_type, OpenAILLMHandler())

application/tools/tool_action_parser.py

@@ -0,0 +1,26 @@
+import json
+
+
+class ToolActionParser:
+    def __init__(self, llm_type):
+        self.llm_type = llm_type
+        self.parsers = {
+            "OpenAILLM": self._parse_openai_llm,
+            "GoogleLLM": self._parse_google_llm,
+        }
+
+    def parse_args(self, call):
+        parser = self.parsers.get(self.llm_type, self._parse_openai_llm)
+        return parser(call)
+
+    def _parse_openai_llm(self, call):
+        call_args = json.loads(call.function.arguments)
+        tool_id = call.function.name.split("_")[-1]
+        action_name = call.function.name.rsplit("_", 1)[0]
+        return tool_id, action_name, call_args
+
+    def _parse_google_llm(self, call):
+        call_args = call.args
+        tool_id = call.name.split("_")[-1]
+        action_name = call.name.rsplit("_", 1)[0]
+        return tool_id, action_name, call_args

application/tools/tool_manager.py

@@ -0,0 +1,46 @@
+import importlib
+import inspect
+import os
+import pkgutil
+
+from application.tools.base import Tool
+
+
+class ToolManager:
+    def __init__(self, config):
+        self.config = config
+        self.tools = {}
+        self.load_tools()
+
+    def load_tools(self):
+        tools_dir = os.path.join(os.path.dirname(__file__), "implementations")
+        for finder, name, ispkg in pkgutil.iter_modules([tools_dir]):
+            if name == "base" or name.startswith("__"):
+                continue
+            module = importlib.import_module(
+                f"application.tools.implementations.{name}"
+            )
+            for member_name, obj in inspect.getmembers(module, inspect.isclass):
+                if issubclass(obj, Tool) and obj is not Tool:
+                    tool_config = self.config.get(name, {})
+                    self.tools[name] = obj(tool_config)
+
+    def load_tool(self, tool_name, tool_config):
+        self.config[tool_name] = tool_config
+        module = importlib.import_module(
+            f"application.tools.implementations.{tool_name}"
+        )
+        for member_name, obj in inspect.getmembers(module, inspect.isclass):
+            if issubclass(obj, Tool) and obj is not Tool:
+                return obj(tool_config)
+
+    def execute_action(self, tool_name, action_name, **kwargs):
+        if tool_name not in self.tools:
+            raise ValueError(f"Tool '{tool_name}' not loaded")
+        return self.tools[tool_name].execute_action(action_name, **kwargs)
+
+    def get_all_actions_metadata(self):
+        metadata = []
+        for tool in self.tools.values():
+            metadata.extend(tool.get_actions_metadata())
+        return metadata

application/tts/base.py

@@ -0,0 +1,10 @@
+from abc import ABC, abstractmethod
+
+
+class BaseTTS(ABC):
+    def __init__(self):
+        pass
+
+    @abstractmethod
+    def text_to_speech(self, *args, **kwargs):
+        pass

application/tts/elevenlabs.py

@@ -0,0 +1,84 @@
+import asyncio
+import websockets
+import json
+import base64
+from io import BytesIO
+from application.tts.base import BaseTTS
+
+
+class ElevenlabsTTS(BaseTTS):
+    def __init__(self):        
+        self.api_key = 'ELEVENLABS_API_KEY'# here you should put your api key
+        self.model = "eleven_flash_v2_5"
+        self.voice = "VOICE_ID" # this is the hash code for the voice not the name!
+        self.write_audio = 1
+
+    def text_to_speech(self, text):
+        asyncio.run(self._text_to_speech_websocket(text))
+
+    async def _text_to_speech_websocket(self, text):
+        uri = f"wss://api.elevenlabs.io/v1/text-to-speech/{self.voice}/stream-input?model_id={self.model}"
+        websocket = await websockets.connect(uri)
+        payload = {
+            "text": " ",
+            "voice_settings": {
+                "stability": 0.5,
+                "similarity_boost": 0.8,
+            },
+            "xi_api_key": self.api_key,
+        }
+
+        await websocket.send(json.dumps(payload))
+        
+        async def listen():
+            while 1:
+                try:
+                    msg = await websocket.recv()
+                    data = json.loads(msg)
+
+                    if data.get("audio"):
+                        print("audio received")
+                        yield base64.b64decode(data["audio"])
+                    elif data.get("isFinal"):
+                        break
+                except websockets.exceptions.ConnectionClosed:
+                    print("websocket closed")
+                    break
+        listen_task =  asyncio.create_task(self.stream(listen()))
+        
+        await websocket.send(json.dumps({"text": text}))
+        # this is to signal the end of the text, either use this or flush
+        await websocket.send(json.dumps({"text": ""})) 
+
+        await listen_task
+    
+    async def stream(self, audio_stream):
+        if self.write_audio:
+            audio_bytes = BytesIO()
+            async for chunk in audio_stream:
+                if chunk:
+                    audio_bytes.write(chunk)
+            with open("output_audio.mp3", "wb") as f:
+                f.write(audio_bytes.getvalue())
+        
+        else:
+            async for chunk in audio_stream:
+                pass # depends on the streamer!
+
+
+def test_elevenlabs_websocket():
+    """
+    Tests the ElevenlabsTTS text_to_speech method with a sample prompt.
+    Prints out the base64-encoded result and writes it to 'output_audio.mp3'.
+    """
+    # Instantiate your TTS class
+    tts = ElevenlabsTTS()
+
+    # Call the method with some sample text
+    tts.text_to_speech("Hello from ElevenLabs WebSocket!")
+
+    print("Saved audio to output_audio.mp3.")
+
+
+if __name__ == "__main__":
+    test_elevenlabs_websocket()

application/tts/google_tts.py

@@ -0,0 +1,19 @@
+import io
+import base64
+from gtts import gTTS
+from application.tts.base import BaseTTS
+
+
+class GoogleTTS(BaseTTS):
+    def __init__(self):
+        pass
+
+
+    def text_to_speech(self, text):
+        lang = "en"
+        audio_fp = io.BytesIO()
+        tts = gTTS(text=text, lang=lang, slow=False)
+        tts.write_to_fp(audio_fp)
+        audio_fp.seek(0)
+        audio_base64 = base64.b64encode(audio_fp.read()).decode("utf-8")
+        return audio_base64, lang

application/usage.py

@@ -1,10 +1,9 @@
 import sys
-from pymongo import MongoClient
 from datetime import datetime
-from application.core.settings import settings
-from application.utils import num_tokens_from_string
+from application.core.mongo_db import MongoDB
+from application.utils import num_tokens_from_string, num_tokens_from_object_or_list
 
-mongo = MongoClient(settings.MONGO_URI)
+mongo = MongoDB.get_client()
 db = mongo["docsgpt"]
 usage_collection = db["token_usage"]
 
@@ -22,11 +21,16 @@ def update_token_usage(user_api_key, token_usage):
 
 
 def gen_token_usage(func):
-    def wrapper(self, model, messages, stream, **kwargs):
+    def wrapper(self, model, messages, stream, tools, **kwargs):
         for message in messages:
-            self.token_usage["prompt_tokens"] += num_tokens_from_string(message["content"])
-        result = func(self, model, messages, stream, **kwargs)
-        self.token_usage["generated_tokens"] += num_tokens_from_string(result)
+            if message["content"]:
+                self.token_usage["prompt_tokens"] += num_tokens_from_string(message["content"])
+        result = func(self, model, messages, stream, tools, **kwargs)
+        # check if result is a string
+        if isinstance(result, str):
+            self.token_usage["generated_tokens"] += num_tokens_from_string(result)
+        else:
+            self.token_usage["generated_tokens"] += num_tokens_from_object_or_list(result)
         update_token_usage(self.user_api_key, self.token_usage)
         return result
 
@@ -34,11 +38,11 @@ def gen_token_usage(func):
 
 
 def stream_token_usage(func):
-    def wrapper(self, model, messages, stream, **kwargs):
+    def wrapper(self, model, messages, stream, tools, **kwargs):
         for message in messages:
             self.token_usage["prompt_tokens"] += num_tokens_from_string(message["content"])
         batch = []
-        result = func(self, model, messages, stream, **kwargs)
+        result = func(self, model, messages, stream, tools, **kwargs)
         for r in result:
             batch.append(r)
             yield r

application/utils.py

@@ -1,6 +1,8 @@
 import tiktoken
+import hashlib
 from flask import jsonify, make_response
 
+
 _encoding = None
 
 
@@ -13,9 +15,21 @@ def get_encoding():
 
 def num_tokens_from_string(string: str) -> int:
     encoding = get_encoding()
-    num_tokens = len(encoding.encode(string))
-    return num_tokens
+    if isinstance(string, str):
+        num_tokens = len(encoding.encode(string))
+        return num_tokens
+    else:
+        return 0
 
+def num_tokens_from_object_or_list(thing):
+    if isinstance(thing, list):
+        return sum([num_tokens_from_object_or_list(x) for x in thing])
+    elif isinstance(thing, dict):
+        return sum([num_tokens_from_object_or_list(x) for x in thing.values()])
+    elif isinstance(thing, str):
+        return num_tokens_from_string(thing)
+    else:
+        return 0
 
 def count_tokens_docs(docs):
     docs_content = ""
@@ -39,3 +53,45 @@ def check_required_fields(data, required_fields):
             400,
         )
     return None
+
+
+def get_hash(data):
+    return hashlib.md5(data.encode()).hexdigest()
+
+def limit_chat_history(history, max_token_limit=None, gpt_model="docsgpt"):
+    """
+    Limits chat history based on token count.
+    Returns a list of messages that fit within the token limit.
+    """
+    from application.core.settings import settings
+
+    max_token_limit = (
+            max_token_limit
+            if max_token_limit and 
+            max_token_limit < settings.MODEL_TOKEN_LIMITS.get(
+                gpt_model, settings.DEFAULT_MAX_HISTORY
+            )
+            else settings.MODEL_TOKEN_LIMITS.get(
+                gpt_model, settings.DEFAULT_MAX_HISTORY
+            )
+        )
+    
+
+    if not history:
+        return []
+        
+    tokens_current_history = 0
+    trimmed_history = []
+    
+    for message in reversed(history):
+        if "prompt" in message and "response" in message:
+            tokens_batch = num_tokens_from_string(message["prompt"]) + num_tokens_from_string(
+                message["response"]
+            )
+            if tokens_current_history + tokens_batch < max_token_limit:
+                tokens_current_history += tokens_batch
+                trimmed_history.insert(0, message)
+            else:
+                break
+
+    return trimmed_history

application/vectorstore/faiss.py

@@ -3,30 +3,27 @@ from application.vectorstore.base import BaseVectorStore
 from application.core.settings import settings
 import os
 
-def get_vectorstore(path):
+def get_vectorstore(path: str) -> str:
     if path:
-        vectorstore = "indexes/"+path
-        vectorstore = os.path.join("application", vectorstore)
+        vectorstore = os.path.join("application", "indexes", path)
     else:
         vectorstore = os.path.join("application")
-
     return vectorstore
 
 class FaissStore(BaseVectorStore):
-
-    def __init__(self, source_id, embeddings_key, docs_init=None):
+    def __init__(self, source_id: str, embeddings_key: str, docs_init=None):
         super().__init__()
         self.path = get_vectorstore(source_id)
         embeddings = self._get_embeddings(settings.EMBEDDINGS_NAME, embeddings_key)
-        if docs_init:
-            self.docsearch = FAISS.from_documents(
-                docs_init, embeddings
-            )
-        else:
-            self.docsearch = FAISS.load_local(
-                self.path, embeddings, 
-                allow_dangerous_deserialization=True
-            )
+
+        try:
+            if docs_init:
+                self.docsearch = FAISS.from_documents(docs_init, embeddings)
+            else:
+                self.docsearch = FAISS.load_local(self.path, embeddings, allow_dangerous_deserialization=True)
+        except Exception:
+            raise
+
         self.assert_embedding_dimensions(embeddings)
 
     def search(self, *args, **kwargs):
@@ -42,16 +39,12 @@ class FaissStore(BaseVectorStore):
         return self.docsearch.delete(*args, **kwargs)
 
     def assert_embedding_dimensions(self, embeddings):
-        """
-        Check that the word embedding dimension of the docsearch index matches
-        the dimension of the word embeddings used 
-        """
+        """Check that the word embedding dimension of the docsearch index matches the dimension of the word embeddings used."""
         if settings.EMBEDDINGS_NAME == "huggingface_sentence-transformers/all-mpnet-base-v2":
-            try:
-                word_embedding_dimension = embeddings.dimension
-            except AttributeError as e:
-                raise AttributeError("'dimension' attribute not found in embeddings instance. Make sure the embeddings object is properly initialized.") from e
+            word_embedding_dimension = getattr(embeddings, 'dimension', None)
+            if word_embedding_dimension is None:
+                raise AttributeError("'dimension' attribute not found in embeddings instance.")
+            
             docsearch_index_dimension = self.docsearch.index.d
             if word_embedding_dimension != docsearch_index_dimension:
-                raise ValueError(f"Embedding dimension mismatch: embeddings.dimension ({word_embedding_dimension}) " +
-                                 f"!= docsearch index dimension ({docsearch_index_dimension})")
+                raise ValueError(f"Embedding dimension mismatch: embeddings.dimension ({word_embedding_dimension}) != docsearch index dimension ({docsearch_index_dimension})")

application/vectorstore/lancedb.py

@@ -0,0 +1,119 @@
+from typing import List, Optional
+import importlib
+from application.vectorstore.base import BaseVectorStore
+from application.core.settings import settings
+
+class LanceDBVectorStore(BaseVectorStore):
+    """Class for LanceDB Vector Store integration."""
+
+    def __init__(self, path: str = settings.LANCEDB_PATH,
+                 table_name_prefix: str = settings.LANCEDB_TABLE_NAME,
+                 source_id: str = None,
+                 embeddings_key: str = "embeddings"):
+        """Initialize the LanceDB vector store."""
+        super().__init__()
+        self.path = path
+        self.table_name = f"{table_name_prefix}_{source_id}" if source_id else table_name_prefix
+        self.embeddings_key = embeddings_key
+        self._lance_db = None
+        self.docsearch = None
+        self._pa = None  # PyArrow (pa) will be lazy loaded
+
+    @property
+    def pa(self):
+        """Lazy load pyarrow module."""
+        if self._pa is None:
+            self._pa = importlib.import_module("pyarrow")
+        return self._pa
+
+    @property
+    def lancedb(self):
+        """Lazy load lancedb module."""
+        if not hasattr(self, "_lancedb_module"):
+            self._lancedb_module = importlib.import_module("lancedb")
+        return self._lancedb_module
+
+    @property
+    def lance_db(self):
+        """Lazy load the LanceDB connection."""
+        if self._lance_db is None:
+            self._lance_db = self.lancedb.connect(self.path)
+        return self._lance_db
+
+    @property
+    def table(self):
+        """Lazy load the LanceDB table."""
+        if self.docsearch is None:
+            if self.table_name in self.lance_db.table_names():
+                self.docsearch = self.lance_db.open_table(self.table_name)
+            else:
+                self.docsearch = None
+        return self.docsearch
+
+    def ensure_table_exists(self):
+        """Ensure the table exists before performing operations."""
+        if self.table is None:
+            embeddings = self._get_embeddings(settings.EMBEDDINGS_NAME, self.embeddings_key)
+            schema = self.pa.schema([
+                self.pa.field("vector", self.pa.list_(self.pa.float32(), list_size=embeddings.dimension)),
+                self.pa.field("text", self.pa.string()),
+                self.pa.field("metadata", self.pa.struct([
+                    self.pa.field("key", self.pa.string()),
+                    self.pa.field("value", self.pa.string())
+                ]))
+            ])
+            self.docsearch = self.lance_db.create_table(self.table_name, schema=schema)
+
+    def add_texts(self, texts: List[str], metadatas: Optional[List[dict]] = None, source_id: str = None):
+        """Add texts with metadata and their embeddings to the LanceDB table."""
+        embeddings = self._get_embeddings(settings.EMBEDDINGS_NAME, self.embeddings_key).embed_documents(texts)
+        vectors = []
+        for embedding, text, metadata in zip(embeddings, texts, metadatas or [{}] * len(texts)):
+            if source_id:
+                metadata["source_id"] = source_id
+            metadata_struct = [{"key": k, "value": str(v)} for k, v in metadata.items()]
+            vectors.append({
+                "vector": embedding,
+                "text": text,
+                "metadata": metadata_struct
+            })
+        self.ensure_table_exists()
+        self.docsearch.add(vectors)
+
+    def search(self, query: str, k: int = 2, *args, **kwargs):
+        """Search LanceDB for the top k most similar vectors."""
+        self.ensure_table_exists()
+        query_embedding = self._get_embeddings(settings.EMBEDDINGS_NAME, self.embeddings_key).embed_query(query)
+        results = self.docsearch.search(query_embedding).limit(k).to_list()
+        return [(result["_distance"], result["text"], result["metadata"]) for result in results]
+
+    def delete_index(self):
+        """Delete the entire LanceDB index (table)."""
+        if self.table:
+            self.lance_db.drop_table(self.table_name)
+
+    def assert_embedding_dimensions(self, embeddings):
+        """Ensure that embedding dimensions match the table index dimensions."""
+        word_embedding_dimension = embeddings.dimension
+        if self.table:
+            table_index_dimension = len(self.docsearch.schema["vector"].type.value_type)
+            if word_embedding_dimension != table_index_dimension:
+                raise ValueError(
+                    f"Embedding dimension mismatch: embeddings.dimension ({word_embedding_dimension}) "
+                    f"!= table index dimension ({table_index_dimension})"
+                )
+
+    def filter_documents(self, filter_condition: dict) -> List[dict]:
+        """Filter documents based on certain conditions."""
+        self.ensure_table_exists()
+
+        # Ensure source_id exists in the filter condition
+        if 'source_id' not in filter_condition:
+            raise ValueError("filter_condition must contain 'source_id'")
+
+        source_id = filter_condition["source_id"]
+
+        # Use LanceDB's native filtering if supported, otherwise filter manually
+        filtered_data = self.docsearch.filter(lambda x: x.metadata and x.metadata.get("source_id") == source_id).to_list()
+
+        return filtered_data

application/vectorstore/milvus.py

@@ -7,7 +7,7 @@ from application.vectorstore.base import BaseVectorStore
 
 
 class MilvusStore(BaseVectorStore):
-    def __init__(self, path: str = "", embeddings_key: str = "embeddings"):
+    def __init__(self, source_id: str = "", embeddings_key: str = "embeddings"):
         super().__init__()
         from langchain_milvus import Milvus
 
@@ -20,10 +20,11 @@ class MilvusStore(BaseVectorStore):
             collection_name=settings.MILVUS_COLLECTION_NAME,
             connection_args=connection_args,
         )
-        self._path = path
+        self._source_id = source_id
 
     def search(self, question, k=2, *args, **kwargs):
-        return self._docsearch.similarity_search(query=question, k=k, filter={"path": self._path} *args, **kwargs)
+        expr = f"source_id == '{self._source_id}'"
+        return self._docsearch.similarity_search(query=question, k=k, expr=expr, *args, **kwargs)
 
     def add_texts(self, texts: List[str], metadatas: Optional[List[dict]], *args, **kwargs):
         ids = [str(uuid4()) for _ in range(len(texts))]

application/worker.py

@@ -8,36 +8,37 @@ from urllib.parse import urljoin
 
 import requests
 from bson.objectid import ObjectId
-from pymongo import MongoClient
 
+from application.core.mongo_db import MongoDB
 from application.core.settings import settings
 from application.parser.file.bulk import SimpleDirectoryReader
-from application.parser.open_ai_func import call_openai_api
+from application.parser.embedding_pipeline import embed_and_store_documents
 from application.parser.remote.remote_creator import RemoteCreator
 from application.parser.schema.base import Document
-from application.parser.token_func import group_split
+from application.parser.chunking import Chunker
 from application.utils import count_tokens_docs
 
-mongo = MongoClient(settings.MONGO_URI)
+mongo = MongoDB.get_client()
 db = mongo["docsgpt"]
 sources_collection = db["sources"]
 
+# Constants
+MIN_TOKENS = 150
+MAX_TOKENS = 1250
+RECURSION_DEPTH = 2
 
 # Define a function to extract metadata from a given filename.
 def metadata_from_filename(title):
     return {"title": title}
 
-
 # Define a function to generate a random string of a given length.
 def generate_random_string(length):
     return "".join([string.ascii_letters[i % 52] for i in range(length)])
 
-
 current_dir = os.path.dirname(
     os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 )
 
-
 def extract_zip_recursive(zip_path, extract_to, current_depth=0, max_depth=5):
     """
     Recursively extract zip files with a limit on recursion depth.
@@ -52,9 +53,13 @@ def extract_zip_recursive(zip_path, extract_to, current_depth=0, max_depth=5):
         logging.warning(f"Reached maximum recursion depth of {max_depth}")
         return
 
-    with zipfile.ZipFile(zip_path, "r") as zip_ref:
-        zip_ref.extractall(extract_to)
-    os.remove(zip_path)  # Remove the zip file after extracting
+    try:
+        with zipfile.ZipFile(zip_path, "r") as zip_ref:
+            zip_ref.extractall(extract_to)
+        os.remove(zip_path)  # Remove the zip file after extracting
+    except Exception as e:
+        logging.error(f"Error extracting zip file {zip_path}: {e}")
+        return
 
     # Check for nested zip files and extract them
     for root, dirs, files in os.walk(extract_to):
@@ -64,6 +69,38 @@ def extract_zip_recursive(zip_path, extract_to, current_depth=0, max_depth=5):
                 file_path = os.path.join(root, file)
                 extract_zip_recursive(file_path, root, current_depth + 1, max_depth)
 
+def download_file(url, params, dest_path):
+    try:
+        response = requests.get(url, params=params)
+        response.raise_for_status()
+        with open(dest_path, "wb") as f:
+            f.write(response.content)
+    except requests.RequestException as e:
+        logging.error(f"Error downloading file: {e}")
+        raise
+
+def upload_index(full_path, file_data):
+    try:
+        if settings.VECTOR_STORE == "faiss":
+            files = {
+                "file_faiss": open(full_path + "/index.faiss", "rb"),
+                "file_pkl": open(full_path + "/index.pkl", "rb"),
+            }
+            response = requests.post(
+                urljoin(settings.API_URL, "/api/upload_index"), files=files, data=file_data
+            )
+        else:
+            response = requests.post(
+                urljoin(settings.API_URL, "/api/upload_index"), data=file_data
+            )
+        response.raise_for_status()
+    except requests.RequestException as e:
+        logging.error(f"Error uploading index: {e}")
+        raise
+    finally:
+        if settings.VECTOR_STORE == "faiss":
+            for file in files.values():
+                file.close()
 
 # Define the main function for ingesting and processing documents.
 def ingest_worker(
@@ -84,39 +121,24 @@ def ingest_worker(
     Returns:
         dict: Information about the completed ingestion task, including input parameters and a "limited" flag.
     """
-    # directory = 'inputs' or 'temp'
-    # formats = [".rst", ".md"]
     input_files = None
     recursive = True
     limit = None
     exclude = True
-    # name_job = 'job1'
-    # filename = 'install.rst'
-    # user = 'local'
     sample = False
-    token_check = True
-    min_tokens = 150
-    max_tokens = 1250
-    recursion_depth = 2
     full_path = os.path.join(directory, user, name_job)
 
     logging.info(f"Ingest file: {full_path}", extra={"user": user, "job": name_job})
-    # check if API_URL env variable is set
     file_data = {"name": name_job, "file": filename, "user": user}
-    response = requests.get(
-        urljoin(settings.API_URL, "/api/download"), params=file_data
-    )
-    file = response.content
 
     if not os.path.exists(full_path):
         os.makedirs(full_path)
-    with open(os.path.join(full_path, filename), "wb") as f:
-        f.write(file)
+    download_file(urljoin(settings.API_URL, "/api/download"), file_data, os.path.join(full_path, filename))
 
     # check if file is .zip and extract it
     if filename.endswith(".zip"):
         extract_zip_recursive(
-            os.path.join(full_path, filename), full_path, 0, recursion_depth
+            os.path.join(full_path, filename), full_path, 0, RECURSION_DEPTH
         )
 
     self.update_state(state="PROGRESS", meta={"current": 1})
@@ -130,17 +152,19 @@ def ingest_worker(
         exclude_hidden=exclude,
         file_metadata=metadata_from_filename,
     ).load_data()
-    raw_docs = group_split(
-        documents=raw_docs,
-        min_tokens=min_tokens,
-        max_tokens=max_tokens,
-        token_check=token_check,
+
+    chunker = Chunker(
+        chunking_strategy="classic_chunk",
+        max_tokens=MAX_TOKENS,
+        min_tokens=MIN_TOKENS,
+        duplicate_headers=False
     )
+    raw_docs = chunker.chunk(documents=raw_docs)
 
     docs = [Document.to_langchain_format(raw_doc) for raw_doc in raw_docs]
     id = ObjectId()
 
-    call_openai_api(docs, full_path, id, self)
+    embed_and_store_documents(docs, full_path, id, self)
     tokens = count_tokens_docs(docs)
     self.update_state(state="PROGRESS", meta={"current": 100})
 
@@ -148,28 +172,13 @@ def ingest_worker(
         for i in range(min(5, len(raw_docs))):
             logging.info(f"Sample document {i}: {raw_docs[i]}")
 
-    # get files from outputs/inputs/index.faiss and outputs/inputs/index.pkl
-    # and send them to the server (provide user and name in form)
-    file_data = {
-        "name": name_job,
-        "user": user,
+    file_data.update({
         "tokens": tokens,
         "retriever": retriever,
         "id": str(id),
         "type": "local",
-    }
-    if settings.VECTOR_STORE == "faiss":
-        files = {
-            "file_faiss": open(full_path + "/index.faiss", "rb"),
-            "file_pkl": open(full_path + "/index.pkl", "rb"),
-        }
-        response = requests.post(
-            urljoin(settings.API_URL, "/api/upload_index"), files=files, data=file_data
-        )
-    else:
-        response = requests.post(
-            urljoin(settings.API_URL, "/api/upload_index"), data=file_data
-        )
+    })
+    upload_index(full_path, file_data)
 
     # delete local
     shutil.rmtree(full_path)
@@ -183,7 +192,6 @@ def ingest_worker(
         "limited": False,
     }
 
-
 def remote_worker(
     self,
     source_data,
@@ -195,69 +203,63 @@ def remote_worker(
     sync_frequency="never",
     operation_mode="upload",
     doc_id=None,
-):
-    token_check = True
-    min_tokens = 150
-    max_tokens = 1250
-    full_path = directory + "/" + user + "/" + name_job
-
+):  
+    full_path = os.path.join(directory, user, name_job)
     if not os.path.exists(full_path):
         os.makedirs(full_path)
+
     self.update_state(state="PROGRESS", meta={"current": 1})
-    logging.info(
-        f"Remote job: {full_path}",
-        extra={"user": user, "job": name_job, source_data: source_data},
-    )
+    try:
+        logging.info("Initializing remote loader with type: %s", loader)
+        remote_loader = RemoteCreator.create_loader(loader)
+        raw_docs = remote_loader.load_data(source_data)
 
-    remote_loader = RemoteCreator.create_loader(loader)
-    raw_docs = remote_loader.load_data(source_data)
-
-    docs = group_split(
-        documents=raw_docs,
-        min_tokens=min_tokens,
-        max_tokens=max_tokens,
-        token_check=token_check,
-    )
-    # docs = [Document.to_langchain_format(raw_doc) for raw_doc in raw_docs]
-    tokens = count_tokens_docs(docs)
-    if operation_mode == "upload":
-        id = ObjectId()
-        call_openai_api(docs, full_path, id, self)
-    elif operation_mode == "sync":
-        if not doc_id or not ObjectId.is_valid(doc_id):
-            raise ValueError("doc_id must be provided for sync operation.")
-        id = ObjectId(doc_id)
-        call_openai_api(docs, full_path, id, self)
-    self.update_state(state="PROGRESS", meta={"current": 100})
-
-    # Proceed with uploading and cleaning as in the original function
-    file_data = {
-        "name": name_job,
-        "user": user,
-        "tokens": tokens,
-        "retriever": retriever,
-        "id": str(id),
-        "type": loader,
-        "remote_data": source_data,
-        "sync_frequency": sync_frequency,
-    }
-    if settings.VECTOR_STORE == "faiss":
-        files = {
-            "file_faiss": open(full_path + "/index.faiss", "rb"),
-            "file_pkl": open(full_path + "/index.pkl", "rb"),
-        }
-
-        requests.post(
-            urljoin(settings.API_URL, "/api/upload_index"), files=files, data=file_data
+        chunker = Chunker(
+            chunking_strategy="classic_chunk",
+            max_tokens=MAX_TOKENS,
+            min_tokens=MIN_TOKENS,
+            duplicate_headers=False
         )
-    else:
-        requests.post(urljoin(settings.API_URL, "/api/upload_index"), data=file_data)
+        docs = chunker.chunk(documents=raw_docs)
+        docs = [Document.to_langchain_format(raw_doc) for raw_doc in raw_docs]
+        tokens = count_tokens_docs(docs)
+        logging.info("Total tokens calculated: %d", tokens)
 
-    shutil.rmtree(full_path)
+        if operation_mode == "upload":
+            id = ObjectId()
+            embed_and_store_documents(docs, full_path, id, self)
+        elif operation_mode == "sync":
+            if not doc_id or not ObjectId.is_valid(doc_id):
+                logging.error("Invalid doc_id provided for sync operation: %s", doc_id)
+                raise ValueError("doc_id must be provided for sync operation.")
+            id = ObjectId(doc_id)
+            embed_and_store_documents(docs, full_path, id, self)
 
+        self.update_state(state="PROGRESS", meta={"current": 100})
+
+        file_data = {
+            "name": name_job,
+            "user": user,
+            "tokens": tokens,
+            "retriever": retriever,
+            "id": str(id),
+            "type": loader,
+            "remote_data": source_data,
+            "sync_frequency": sync_frequency,
+        }
+        upload_index(full_path, file_data)
+
+    except Exception as e:
+        logging.error("Error in remote_worker task: %s", str(e), exc_info=True)
+        raise
+
+    finally:
+        if os.path.exists(full_path):
+            shutil.rmtree(full_path)
+
+    logging.info("remote_worker task completed successfully")
     return {"urls": source_data, "name_job": name_job, "user": user, "limited": False}
 
-
 def sync(
     self,
     source_data,
@@ -283,10 +285,10 @@ def sync(
             doc_id,
         )
     except Exception as e:
+        logging.error(f"Error during sync: {e}")
         return {"status": "error", "error": str(e)}
     return {"status": "success"}
 
-
 def sync_worker(self, frequency):
     sync_counts = Counter()
     sources = sources_collection.find()

docker-compose.yaml

@@ -20,6 +20,7 @@ services:
       - CELERY_BROKER_URL=redis://redis:6379/0
       - CELERY_RESULT_BACKEND=redis://redis:6379/1
       - MONGO_URI=mongodb://mongo:27017/docsgpt
+      - CACHE_REDIS_URL=redis://redis:6379/2
     ports:
       - "7091:7091"
     volumes:
@@ -41,6 +42,7 @@ services:
       - CELERY_RESULT_BACKEND=redis://redis:6379/1
       - MONGO_URI=mongodb://mongo:27017/docsgpt
       - API_URL=http://backend:7091
+      - CACHE_REDIS_URL=redis://redis:6379/2
     depends_on:
       - redis
       - mongo

docs/README.md

@@ -46,6 +46,6 @@ yarn install
 yarn dev
 ```
 
-- Now, you should be able to view the docs on your local environment by visiting `http://localhost:5000`. You can explore the different markdown files and make changes as you see fit.
+- Now, you should be able to view the docs on your local environment by visiting `http://localhost:3000`. You can explore the different markdown files and make changes as you see fit.
 
 - **Footnotes:** This guide assumes you have Node.js and npm installed. The guide involves running a local server using yarn, and viewing the documentation offline. If you encounter any issues, it may be worth verifying your Node.js and npm installations and whether you have installed yarn correctly.

docs/package.json

@@ -7,8 +7,8 @@
   "license": "MIT",
   "dependencies": {
     "@vercel/analytics": "^1.1.1",
-    "docsgpt": "^0.4.1",
-    "next": "^14.2.12",
+    "docsgpt-react": "^0.4.9",
+    "next": "^14.2.22",
     "nextra": "^2.13.2",
     "nextra-theme-docs": "^2.13.2",
     "react": "^18.2.0",

docs/pages/Deploying/Development-Environment.md

@@ -0,0 +1,78 @@
+## Development Environments
+
+### Spin up Mongo and Redis
+
+For development, only two containers are used from [docker-compose.yaml](https://github.com/arc53/DocsGPT/blob/main/docker-compose.yaml) (by deleting all services except for Redis and Mongo).
+See file [docker-compose-dev.yaml](https://github.com/arc53/DocsGPT/blob/main/docker-compose-dev.yaml).
+
+Run
+
+```
+docker compose -f docker-compose-dev.yaml build
+docker compose -f docker-compose-dev.yaml up -d
+```
+
+### Run the Backend
+
+> [!Note]
+> Make sure you have Python 3.12 installed.
+
+1. Export required environment variables or prepare a `.env` file in the project folder:
+   - Copy [.env-template](https://github.com/arc53/DocsGPT/blob/main/application/.env-template) and create `.env`.
+
+(check out [`application/core/settings.py`](application/core/settings.py) if you want to see more config options.)
+
+2. (optional) Create a Python virtual environment:
+   You can follow the [Python official documentation](https://docs.python.org/3/tutorial/venv.html) for virtual environments.
+
+a) On Mac OS and Linux
+
+```commandline
+python -m venv venv
+. venv/bin/activate
+```
+
+b) On Windows
+
+```commandline
+python -m venv venv
+ venv/Scripts/activate
+```
+
+3. Download embedding model and save it in the `model/` folder:
+You can use the script below, or download it manually from [here](https://d3dg1063dc54p9.cloudfront.net/models/embeddings/mpnet-base-v2.zip), unzip it and save it in the `model/` folder.
+
+```commandline
+wget https://d3dg1063dc54p9.cloudfront.net/models/embeddings/mpnet-base-v2.zip
+unzip mpnet-base-v2.zip -d model
+rm mpnet-base-v2.zip
+```
+
+4. Install dependencies for the backend:
+
+```commandline
+pip install -r application/requirements.txt
+```
+
+5. Run the app using `flask --app application/app.py run --host=0.0.0.0 --port=7091`.
+6. Start worker with `celery -A application.app.celery worker -l INFO`.
+
+> [!Note]
+> You can also launch the in a debugger mode in vscode by accessing SHIFT + CMD + D or SHIFT + Windows + D on windows and selecting Flask or Celery.
+
+
+### Start Frontend
+
+> [!Note]
+> Make sure you have Node version 16 or higher.
+
+1. Navigate to the [/frontend](https://github.com/arc53/DocsGPT/tree/main/frontend) folder.
+2. Install the required packages `husky` and `vite` (ignore if already installed).
+
+```commandline
+npm install husky -g
+npm install vite -g
+```
+
+3. Install dependencies by running `npm install --include=dev`.
+4. Run the app using `npm run dev`.

docs/pages/Deploying/Quickstart.md

@@ -15,11 +15,21 @@ If you prefer to follow manual steps, refer to this guide:
 1. Open and download this repository with 
    ```bash
    git clone https://github.com/arc53/DocsGPT.git
+   cd DocsGPT
    ```
-2. Create a `.env` file in your root directory and set your `API_KEY` with your [OpenAI API key](https://platform.openai.com/account/api-keys). (optional in case you want to use OpenAI)
+2. Create a `.env` file in your root directory and set the env variables.
+   It should look like this inside:
+
+   ```
+   LLM_NAME=[docsgpt or openai or others] 
+   API_KEY=[if LLM_NAME is openai]
+   ```
+
+   See optional environment variables in the [/application/.env_sample](https://github.com/arc53/DocsGPT/blob/main/application/.env_sample) file.
+   
 3. Run the following commands:
    ```bash
-   docker-compose build && docker-compose up
+   docker compose up
    ```
 4. Navigate to http://localhost:5173/.
 
@@ -27,43 +37,28 @@ To stop, simply press **Ctrl + C**.
 
 **For WINDOWS:**
 
-To run the setup on Windows, you have two options: using the Windows Subsystem for Linux (WSL) or using Git Bash or Command Prompt.
-
-**Option 1: Using Windows Subsystem for Linux (WSL):**
-
-1. Install WSL if you haven't already. You can follow the official Microsoft documentation for installation: (https://learn.microsoft.com/en-us/windows/wsl/install).
-2. After setting up WSL, open the WSL terminal.
-3. Clone the repository and create the `.env` file:
+1. Open and download this repository with 
    ```bash
    git clone https://github.com/arc53/DocsGPT.git
    cd DocsGPT
-   echo "API_KEY=Yourkey" > .env
-   echo "VITE_API_STREAMING=true" >> .env
    ```
-4. Run the following command to start the setup with Docker Compose:
-   ```bash
-   ./run-with-docker-compose.sh
-   ```
-6. Open your web browser and navigate to http://localhost:5173/.
-7. To stop the setup, just press **Ctrl + C** in the WSL terminal
 
-**Option 2: Using Git Bash or Command Prompt (CMD):**
+2. Create a `.env` file in your root directory and set the env variables.
+   It should look like this inside:
 
-1. Install Git for Windows if you haven't already. Download it from the official website: (https://gitforwindows.org/).
-2. Open Git Bash or Command Prompt.
-3. Clone the repository and create the `.env` file:
-   ```bash
-   git clone https://github.com/arc53/DocsGPT.git
-   cd DocsGPT
-   echo "API_KEY=Yourkey" > .env
-   echo "VITE_API_STREAMING=true" >> .env
    ```
-4. Run the following command to start the setup with Docker Compose:
-   ```bash
-   ./run-with-docker-compose.sh
+   LLM_NAME=[docsgpt or openai or others] 
+   API_KEY=[if LLM_NAME is openai]
    ```
-5. Open your web browser and navigate to http://localhost:5173/.
-6. To stop the setup, just press **Ctrl + C** in the Git Bash or Command Prompt terminal.
 
-These steps should help you set up and run the project on Windows using either WSL or Git Bash/Command Prompt. 
+   See optional environment variables in the [/application/.env_sample](https://github.com/arc53/DocsGPT/blob/main/application/.env_sample) file.
+
+3. Run the following command:
+
+   ```bash
+   docker-compose up
+   ```
+4. Navigate to http://localhost:5173/.
+5. To stop the setup, just press **Ctrl + C** in the WSL terminal
+
 **Important:** Ensure that Docker is installed and properly configured on your Windows system for these steps to work.

docs/pages/Deploying/_meta.json

@@ -7,6 +7,10 @@
     "title": "⚡️Quickstart",
     "href": "/Deploying/Quickstart"
   },
+  "Development-Environment": {
+    "title": "🛠️Development Environment",
+    "href": "/Deploying/Development-Environment"
+  },
   "Railway-Deploying": {
     "title": "🚂Deploying on Railway",
     "href": "/Deploying/Railway-Deploying"

docs/pages/Extensions/react-widget.md

@@ -29,18 +29,30 @@ Now, you can use the widget in your component like this :
   buttonBg = "#222327"
 />
 ```
-To tailor the widget to your needs, you can configure the following props in your component:
-1. `apiHost` — The URL of your DocsGPT API.
-2. `theme` — Allows to select your specific theme (dark or light).
-3. `apiKey` — Usually, it's empty.
-4. `avatar`: Specifies the URL of the avatar or image representing the chatbot.
-5. `title`: Sets the title text displayed in the chatbot interface.
-6. `description`: Provides a brief description of the chatbot's purpose or functionality.
-7. `heroTitle`: Displays a welcome title when users interact with the chatbot.
-8. `heroDescription`: Provide additional introductory text or information about the chatbot's capabilities.
-9. `buttonIcon`: Specifies the url of the icon image for the widget.
-10. `buttonBg`: Allows to specify the Background color of the widget.
-11. `size`: Sets the size of the widget ( small, medium).
+### Props Table for DocsGPT Widget
+
+| **Prop**          | **Type**         | **Default Value**                                           | **Description**                                                                                     |
+|--------------------|------------------|-------------------------------------------------------------|-----------------------------------------------------------------------------------------------------|
+| **`apiHost`**      | `string`         | `"https://gptcloud.arc53.com"`                                     | The URL of your DocsGPT API for vector search and chatbot queries.                                 |
+| **`apiKey`**       | `string`         | `""`                                                        | Your API key for authentication. Can be left empty if authentication is not required.              |
+| **`avatar`**       | `string`         | `"https://d3dg1063dc54p9.cloudfront.net/cute-docsgpt.png"`   | Specifies the URL of the avatar or image representing the chatbot.                                 |
+| **`title`**        | `string`         | `"Get AI assistance"`                                       | Sets the title text displayed in the chatbot interface.                                            |
+| **`description`**  | `string`         | `"DocsGPT's AI Chatbot is here to help"`                     | Provides a brief description of the chatbot's purpose or functionality.                            |
+| **`heroTitle`**    | `string`         | `"Welcome to DocsGPT !"`                                    | Displays a welcome title when users interact with the chatbot.                                     |
+| **`heroDescription`** | `string`     | `"This chatbot is built with DocsGPT and utilises GenAI, please review important information using sources."` | Provides additional introductory text or information about the chatbot's capabilities.             |
+| **`theme`**        | `"dark" \| "light"` | `"dark"`                                                  | Allows you to select the theme for the chatbot interface. Accepts `"dark"` or `"light"`.           |
+| **`buttonIcon`**   | `string`         | `"https://your-icon"`                                        | Specifies the URL of the icon image for the widget's launch button.                                |
+| **`buttonBg`**     | `string`         | `"#222327"`                                                 | Sets the background color of the widget's launch button.                                           |
+| **`size`**         | `"small" \| "medium"` | `"medium"`                                              | Sets the size of the widget. Options are `"small"` or `"medium"`.                                  |
+
+---
+
+### Notes
+- **Customizing Props:** All properties can be overridden when embedding the widget. For example, you can provide a unique avatar, title, or color scheme to better align with your brand.
+- **Default Theme:** The widget defaults to the dark theme unless explicitly set to `"light"`. 
+- **API Key:** If the `apiKey` is not required for your application, leave it empty.
+
+This table provides a clear overview of the customization options available for tailoring the DocsGPT widget to fit your application.
 
 
 ### How to use DocsGPTWidget with [Nextra](https://nextra.site/) (Next.js + MDX)
@@ -121,5 +133,80 @@ To link the widget to your api and your documents you can pass parameters to the
 </html>
 ```
 
+# SearchBar
+
+The `SearchBar` component is an interactive search bar designed to provide search results based on **vector similarity search**. It also includes the capability to open the AI Chatbot, enabling users to query.
+
+---
+
+### Importing the Component
+```tsx
+import { SearchBar } from "docsgpt-react";
+```
+
+---
+
+### Usage Example
+```tsx
+<SearchBar 
+    apiKey="your-api-key"
+    apiHost="https://gptcloud.arc53.com"
+    theme="light"
+    placeholder="Search or Ask AI..."
+    width="300px"
+/>
+```
+
+---
+
+## HTML embedding for Search bar
+
+```html
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <title>SearchBar Embedding</title>
+  <script src="https://unpkg.com/docsgpt/dist/modern/main.js"></script> <!-- The bundled JavaScript file -->
+</head>
+<body>
+  <!-- Element where the SearchBar will render -->
+  <div id="search-bar-container"></div>
+
+  <script>
+    // Render the SearchBar into the specified element
+    renderSearchBar('search-bar-container', {
+      apiKey: 'your-api-key-here',
+      apiHost: 'https://your-api-host.com',
+      theme: 'light',
+      placeholder: 'Search here...',
+      width: '300px'
+    });
+  </script>
+</body>
+</html>
+
+```
+
+### Props
+
+| **Prop**       | **Type**  | **Default Value**                   | **Description**                                                                                  |
+|-----------------|-----------|-------------------------------------|--------------------------------------------------------------------------------------------------|
+| **`apiKey`**    | `string`  | `"74039c6d-bff7-44ce-ae55-2973cbf13837"` | Your API key generated from the app. Used for authenticating requests.                         |
+| **`apiHost`**   | `string`  | `"https://gptcloud.arc53.com"`       | The base URL of the server hosting the vector similarity search and chatbot services.           |
+| **`theme`**     | `"dark" \| "light"` | `"dark"`                            | The theme of the search bar. Accepts `"dark"` or `"light"`.                                     |
+| **`placeholder`** | `string` | `"Search or Ask AI..."`             | Placeholder text displayed in the search input field.                                           |
+| **`width`**     | `string`  | `"256px"`                          | Width of the search bar. Accepts any valid CSS width value (e.g., `"300px"`, `"100%"`, `"20rem"`). |
+
+
+Feel free to reach out if you need help customizing or extending the `SearchBar`!
+
+## Our github
+
+[DocsGPT](https://github.com/arc53/DocsGPT)
+
+You can find the source code in the extensions/react-widget folder.
+
 For more information about React, refer to this [link here](https://react.dev/learn)
 

docs/pages/Guides/How-to-train-on-other-documentation.mdx

@@ -28,15 +28,15 @@ Navigate to the sidebar where you will find `Source Docs` option,here you will f
 
 
 ### Step 2
-Click on the `Upload icon` just beside the source docs options,now borwse and upload the document which you want to train on or select the `remote` option if you have to insert the link of the documentation.
+Click on the `Upload icon` just beside the source docs options,now browse and upload the document which you want to train on or select the `remote` option if you have to insert the link of the documentation.
 
 
 ### Step 3
-Now you will be able to see the name of the file uploaded under the Uploaded Files ,now click on `Train`,once you click on train it might take some time to train on the document. You will be able to see the `Training progress` and once the training is completed you can click the `finish` button and there you go your docuemnt is uploaded.
+Now you will be able to see the name of the file uploaded under the Uploaded Files ,now click on `Train`,once you click on train it might take some time to train on the document. You will be able to see the `Training progress` and once the training is completed you can click the `finish` button and there you go your document is uploaded.
 
 
 ### Step 4
-Go to `New chat`  and from the side bar select the document  you uploaded  under the `Source Docs` and go ahead with your chat, now you can ask qestions regarding the document you uploaded and you will get the effective answer based on it.
+Go to `New chat`  and from the side bar select the document  you uploaded  under the `Source Docs` and go ahead with your chat, now you can ask questions regarding the document you uploaded and you will get the effective answer based on it.
 
 </Steps>
 

docs/pages/Guides/How-to-use-different-LLM.mdx

@@ -33,7 +33,7 @@ For open source you  have to edit .env file with LLM_NAME with their desired LLM
 All the supported LLM providers are here application/llm and you  can check what env variable are needed for each
 List of latest supported LLMs are https://github.com/arc53/DocsGPT/blob/main/application/llm/llm_creator.py
 ### Step 3
-Visit application/llm and select the file of your selected llm and there you will find the speicifc requirements needed to be filled in order to use it,i.e API key of that llm.
+Visit application/llm and select the file of your selected llm and there you will find the specific requirements needed to be filled in order to use it,i.e API key of that llm.
 </Steps>
 
 ### For OpenAI-Compatible Endpoints:

docs/pages/_app.mdx

@@ -1,10 +1,10 @@
-import { DocsGPTWidget } from "docsgpt";
+import { DocsGPTWidget } from "docsgpt-react";
 
 export default function MyApp({ Component, pageProps }) {
   return (
     <>
       <Component {...pageProps} />
-        <DocsGPTWidget apiKey="d61a020c-ac8f-4f23-bb98-458e4da3c240" theme="dark" />
+        <DocsGPTWidget apiKey="d61a020c-ac8f-4f23-bb98-458e4da3c240" theme="dark" size="medium" />
     </>
   )
 }

docs/pages/index.mdx

@@ -25,7 +25,10 @@ DocsGPT 🦖 is an innovative open-source tool designed to simplify the retrieva
 
 
 
-<Image src="/homevideo.gif" alt="homedemo" width={800} height={500}/>
+<video controls width={1920} height={1080} muted autoPlay loop playsInline>
+  <source src="https://d3dg1063dc54p9.cloudfront.net/videos/demov4.mp4" type="video/mp4" />
+  Your browser does not support the video tag.
+</video>
 
 
 Try it yourself: [https://www.docsgpt.cloud/](https://www.docsgpt.cloud/)

docs/theme.config.jsx

@@ -51,6 +51,9 @@ const config = {
   footer: {
     text: `MIT ${new Date().getFullYear()} © DocsGPT`,
   },
+  editLink: {
+    content: 'Edit this page on GitHub',
+  },
   logo() {
     return (
       <div className="flex items-center gap-2">

extensions/discord/bot.py

@@ -1,25 +1,60 @@
 import os
 import re
-
+import logging
+import aiohttp
 import discord
-import requests
 from discord.ext import commands
 import dotenv
 
 dotenv.load_dotenv()
 
-# Replace 'YOUR_BOT_TOKEN' with your bot's token
+# Enable logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+# Bot configuration
 TOKEN = os.getenv("DISCORD_TOKEN")
-PREFIX = '@DocsGPT'
-BASE_API_URL = 'http://localhost:7091'
+PREFIX = '!'  # Command prefix
+BASE_API_URL = os.getenv("API_BASE", "https://gptcloud.arc53.com")
+API_URL = BASE_API_URL + "/api/answer"
+API_KEY = os.getenv("API_KEY")
 
 intents = discord.Intents.default()
 intents.message_content = True
 
 bot = commands.Bot(command_prefix=PREFIX, intents=intents)
 
+# Store conversation history per user
+conversation_histories = {}
+
+def chunk_string(text, max_length=2000):
+    """Splits a string into chunks of a specified maximum length."""
+    # Create list to store the split strings
+    chunks = []
+    # Loop through the text, create substrings with max_length
+    while len(text) > max_length:
+        # Find last space within the limit
+        idx = text.rfind(' ', 0, max_length)
+        # Ensure we don't have an empty part
+        if idx == -1:
+            # If no spaces, just take chunk
+            chunks.append(text[:max_length])
+            text = text[max_length:]
+        else:
+            # Push whatever we've got up to the last space
+            chunks.append(text[:idx])
+            text = text[idx+1:]
+    # Catches the remaining part
+    chunks.append(text)
+    return chunks
+
+def escape_markdown(text):
+    """Escapes Discord markdown characters."""
+    escape_chars = r'\*_$$$$()~>#+-=|{}.!'
+    return re.sub(f'([{re.escape(escape_chars)}])', r'\\\1', text)
 
 def split_string(input_str):
+    """Splits the input string to detect bot mentions."""
     pattern = r'^<@!?{0}>\s*'.format(bot.user.id)
     match = re.match(pattern, input_str)
     if match:
@@ -27,42 +62,97 @@ def split_string(input_str):
         return str(bot.user.id), content
     return None, input_str
 
-
 @bot.event
 async def on_ready():
     print(f'{bot.user.name} has connected to Discord!')
 
-
-async def fetch_answer(question):
-    data = {
-        'sender': 'discord',
-        'question': question,
-        'history': ''
+async def generate_answer(question, messages, conversation_id):
+    """Generates an answer using the external API."""
+    payload = {
+        "question": question,
+        "api_key": API_KEY,
+        "history": messages,
+        "conversation_id": conversation_id
     }
-    headers = {"Content-Type": "application/json",
-               "Accept": "application/json"}
-    response = requests.post(BASE_API_URL + '/api/answer', json=data, headers=headers)
-    if response.status_code == 200:
-        return response.json()['answer']
-    return 'Sorry, I could not fetch the answer.'
+    headers = {
+        "Content-Type": "application/json; charset=utf-8"
+    }
+    timeout = aiohttp.ClientTimeout(total=60)
+    async with aiohttp.ClientSession(timeout=timeout) as session:
+        async with session.post(API_URL, json=payload, headers=headers) as resp:
+            if resp.status == 200:
+                data = await resp.json()
+                conversation_id = data.get("conversation_id")
+                answer = data.get("answer", "Sorry, I couldn't find an answer.")
+                return {"answer": answer, "conversation_id": conversation_id}
+            else:
+                return {"answer": "Sorry, I couldn't find an answer.", "conversation_id": None}
 
+@bot.command(name="start")
+async def start(ctx):
+    """Handles the /start command."""
+    await ctx.send(f"Hi {ctx.author.mention}! How can I assist you today?")
+
+@bot.command(name="custom_help")
+async def custom_help_command(ctx):
+    """Handles the /custom_help command."""
+    help_text = (
+        "Here are the available commands:\n"
+        "`!start` - Begin a new conversation with the bot\n"
+        "`!help` - Display this help message\n\n"
+        "You can also mention me or send a direct message to ask a question!"
+    )
+    await ctx.send(help_text)
 
 @bot.event
 async def on_message(message):
     if message.author == bot.user:
         return
 
-    content = message.content.strip()
-    prefix, content = split_string(content)
-    if prefix is None:
-        return
-
-    part_prefix = str(bot.user.id)
-    if part_prefix == prefix:
-        answer = await fetch_answer(content)
-        await message.channel.send(answer)
-
+    # Process commands first
     await bot.process_commands(message)
 
+    # Check if the message is in a DM channel
+    if isinstance(message.channel, discord.DMChannel):
+        content = message.content.strip()
+    else:
+        # In guild channels, check if the message mentions the bot at the start
+        content = message.content.strip()
+        prefix, content = split_string(content)
+        if prefix is None:
+            return
+        part_prefix = str(bot.user.id)
+        if part_prefix != prefix:
+            return  # Bot not mentioned at the start, so do not process
 
-bot.run(TOKEN)
+    # Now process the message
+    user_id = message.author.id
+    if user_id not in conversation_histories:
+        conversation_histories[user_id] = {
+            "history": [],
+            "conversation_id": None
+        }
+
+    conversation = conversation_histories[user_id]
+    conversation["history"].append({"prompt": content})
+
+    # Generate the answer
+    response_doc = await generate_answer(
+        content,
+        conversation["history"],
+        conversation["conversation_id"]
+    )
+    answer = response_doc["answer"]
+    conversation_id = response_doc["conversation_id"]
+
+    answer_chunks = chunk_string(answer)
+    for chunk in answer_chunks:
+        await message.channel.send(chunk)
+
+    conversation["history"][-1]["response"] = answer
+    conversation["conversation_id"] = conversation_id
+
+    # Keep conversation history to last 10 exchanges
+    conversation["history"] = conversation["history"][-10:]
+
+bot.run(TOKEN)

extensions/react-widget/README.md

@@ -13,7 +13,7 @@ npm install  docsgpt
 ### React
 
 ```javascript
-    import { DocsGPTWidget } from "docsgpt";
+    import { DocsGPTWidget } from "docsgpt-react";
 
     const App = () => {
       return <DocsGPTWidget />;
@@ -23,11 +23,11 @@ npm install  docsgpt
 To link the widget to your api and your documents you can pass parameters to the <DocsGPTWidget /> component.
 
 ```javascript
-    import { DocsGPTWidget } from "docsgpt";
+    import { DocsGPTWidget } from "docsgpt-react";
 
     const App = () => {
       return <DocsGPTWidget
-               apiHost="https://your-docsgpt-api.com"
+               apiHost="https://gptcloud.arc53.com"
                apiKey=""
                avatar = "https://d3dg1063dc54p9.cloudfront.net/cute-docsgpt.png"
                title = "Get AI assistance"
@@ -101,6 +101,75 @@ To link the widget to your api and your documents you can pass parameters to the
     </html>
 ```
 
+# SearchBar
+
+The `SearchBar` component is an interactive search bar designed to provide search results based on **vector similarity search**. It also includes the capability to open the AI Chatbot, enabling users to query.
+
+---
+
+### Importing the Component
+```tsx
+import { SearchBar } from "docsgpt-react";
+```
+
+---
+
+### Usage Example
+```tsx
+<SearchBar 
+    apiKey="your-api-key"
+    apiHost="https://gptcloud.arc53.com"
+    theme="light"
+    placeholder="Search or Ask AI..."
+    width="300px"
+/>
+```
+
+---
+
+## HTML embedding for Search bar
+
+```html
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <title>SearchBar Embedding</title>
+  <script src="https://unpkg.com/docsgpt/dist/modern/main.js"></script> <!-- The bundled JavaScript file -->
+</head>
+<body>
+  <!-- Element where the SearchBar will render -->
+  <div id="search-bar-container"></div>
+
+  <script>
+    // Render the SearchBar into the specified element
+    renderSearchBar('search-bar-container', {
+      apiKey: 'your-api-key-here',
+      apiHost: 'https://your-api-host.com',
+      theme: 'light',
+      placeholder: 'Search here...',
+      width: '300px'
+    });
+  </script>
+</body>
+</html>
+
+```
+
+### Props
+
+| **Prop**       | **Type**  | **Default Value**                   | **Description**                                                                                  |
+|-----------------|-----------|-------------------------------------|--------------------------------------------------------------------------------------------------|
+| **`apiKey`**    | `string`  | `"74039c6d-bff7-44ce-ae55-2973cbf13837"` | Your API key generated from the app. Used for authenticating requests.                         |
+| **`apiHost`**   | `string`  | `"https://gptcloud.arc53.com"`       | The base URL of the server hosting the vector similarity search and chatbot services.           |
+| **`theme`**     | `"dark" \| "light"` | `"dark"`                            | The theme of the search bar. Accepts `"dark"` or `"light"`.                                     |
+| **`placeholder`** | `string` | `"Search or Ask AI..."`             | Placeholder text displayed in the search input field.                                           |
+| **`width`**     | `string`  | `"256px"`                          | Width of the search bar. Accepts any valid CSS width value (e.g., `"300px"`, `"100%"`, `"20rem"`). |
+
+
+Feel free to reach out if you need help customizing or extending the `SearchBar`!
+
 ## Our github
 
 [DocsGPT](https://github.com/arc53/DocsGPT)

extensions/react-widget/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "docsgpt",
-  "version": "0.4.2",
+  "version": "0.4.9",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "docsgpt",
-      "version": "0.4.2",
+      "version": "0.4.9",
       "license": "Apache-2.0",
       "dependencies": {
         "@babel/plugin-transform-flow-strip-types": "^7.23.3",
@@ -4598,6 +4598,28 @@
         "@types/trusted-types": "*"
       }
     },
+    "node_modules/@types/eslint": {
+      "version": "9.6.1",
+      "resolved": "https://registry.npmjs.org/@types/eslint/-/eslint-9.6.1.tgz",
+      "integrity": "sha512-FXx2pKgId/WyYo2jXw63kk7/+TY7u7AziEJxJAnSFzHlqTAS3Ync6SvgYAN/k4/PQpnnVuzoMuVnByKK2qp0ag==",
+      "dev": true,
+      "peer": true,
+      "dependencies": {
+        "@types/estree": "*",
+        "@types/json-schema": "*"
+      }
+    },
+    "node_modules/@types/eslint-scope": {
+      "version": "3.7.7",
+      "resolved": "https://registry.npmjs.org/@types/eslint-scope/-/eslint-scope-3.7.7.tgz",
+      "integrity": "sha512-MzMFlSLBqNF2gcHWO0G1vP/YQyfvrxZ0bF+u7mzUdZ1/xK4A4sru+nraZz5i3iEIk1l1uyicaDVTB4QbbEkAYg==",
+      "dev": true,
+      "peer": true,
+      "dependencies": {
+        "@types/eslint": "*",
+        "@types/estree": "*"
+      }
+    },
     "node_modules/@types/estree": {
       "version": "1.0.6",
       "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.6.tgz",
@@ -4637,13 +4659,13 @@
       "license": "MIT"
     },
     "node_modules/@types/node": {
-      "version": "22.5.5",
-      "resolved": "https://registry.npmjs.org/@types/node/-/node-22.5.5.tgz",
-      "integrity": "sha512-Xjs4y5UPO/CLdzpgR6GirZJx36yScjh73+2NlLlkFRSoQN8B0DpfXPdZGnvVmLRLOsqDpOfTNv7D9trgGhmOIA==",
+      "version": "22.10.1",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-22.10.1.tgz",
+      "integrity": "sha512-qKgsUwfHZV2WCWLAnVP1JqnpE6Im6h3Y0+fYgMTasNQ7V++CBX5OT1as0g0f+OyubbFqhf6XVNIsmN4IIhEgGQ==",
       "dev": true,
       "peer": true,
       "dependencies": {
-        "undici-types": "~6.19.2"
+        "undici-types": "~6.20.0"
       }
     },
     "node_modules/@types/parse-json": {
@@ -4688,73 +4710,73 @@
       "dev": true
     },
     "node_modules/@webassemblyjs/ast": {
-      "version": "1.12.1",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.12.1.tgz",
-      "integrity": "sha512-EKfMUOPRRUTy5UII4qJDGPpqfwjOmZ5jeGFwid9mnoqIFK+e0vqoi1qH56JpmZSzEL53jKnNzScdmftJyG5xWg==",
+      "version": "1.14.1",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.14.1.tgz",
+      "integrity": "sha512-nuBEDgQfm1ccRp/8bCQrx1frohyufl4JlbMMZ4P1wpeOfDhF6FQkxZJ1b/e+PLwr6X1Nhw6OLme5usuBWYBvuQ==",
       "dev": true,
       "peer": true,
       "dependencies": {
-        "@webassemblyjs/helper-numbers": "1.11.6",
-        "@webassemblyjs/helper-wasm-bytecode": "1.11.6"
+        "@webassemblyjs/helper-numbers": "1.13.2",
+        "@webassemblyjs/helper-wasm-bytecode": "1.13.2"
       }
     },
     "node_modules/@webassemblyjs/floating-point-hex-parser": {
-      "version": "1.11.6",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/floating-point-hex-parser/-/floating-point-hex-parser-1.11.6.tgz",
-      "integrity": "sha512-ejAj9hfRJ2XMsNHk/v6Fu2dGS+i4UaXBXGemOfQ/JfQ6mdQg/WXtwleQRLLS4OvfDhv8rYnVwH27YJLMyYsxhw==",
+      "version": "1.13.2",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/floating-point-hex-parser/-/floating-point-hex-parser-1.13.2.tgz",
+      "integrity": "sha512-6oXyTOzbKxGH4steLbLNOu71Oj+C8Lg34n6CqRvqfS2O71BxY6ByfMDRhBytzknj9yGUPVJ1qIKhRlAwO1AovA==",
       "dev": true,
       "peer": true
     },
     "node_modules/@webassemblyjs/helper-api-error": {
-      "version": "1.11.6",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-api-error/-/helper-api-error-1.11.6.tgz",
-      "integrity": "sha512-o0YkoP4pVu4rN8aTJgAyj9hC2Sv5UlkzCHhxqWj8butaLvnpdc2jOwh4ewE6CX0txSfLn/UYaV/pheS2Txg//Q==",
+      "version": "1.13.2",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-api-error/-/helper-api-error-1.13.2.tgz",
+      "integrity": "sha512-U56GMYxy4ZQCbDZd6JuvvNV/WFildOjsaWD3Tzzvmw/mas3cXzRJPMjP83JqEsgSbyrmaGjBfDtV7KDXV9UzFQ==",
       "dev": true,
       "peer": true
     },
     "node_modules/@webassemblyjs/helper-buffer": {
-      "version": "1.12.1",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-buffer/-/helper-buffer-1.12.1.tgz",
-      "integrity": "sha512-nzJwQw99DNDKr9BVCOZcLuJJUlqkJh+kVzVl6Fmq/tI5ZtEyWT1KZMyOXltXLZJmDtvLCDgwsyrkohEtopTXCw==",
+      "version": "1.14.1",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-buffer/-/helper-buffer-1.14.1.tgz",
+      "integrity": "sha512-jyH7wtcHiKssDtFPRB+iQdxlDf96m0E39yb0k5uJVhFGleZFoNw1c4aeIcVUPPbXUVJ94wwnMOAqUHyzoEPVMA==",
       "dev": true,
       "peer": true
     },
     "node_modules/@webassemblyjs/helper-numbers": {
-      "version": "1.11.6",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-numbers/-/helper-numbers-1.11.6.tgz",
-      "integrity": "sha512-vUIhZ8LZoIWHBohiEObxVm6hwP034jwmc9kuq5GdHZH0wiLVLIPcMCdpJzG4C11cHoQ25TFIQj9kaVADVX7N3g==",
+      "version": "1.13.2",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-numbers/-/helper-numbers-1.13.2.tgz",
+      "integrity": "sha512-FE8aCmS5Q6eQYcV3gI35O4J789wlQA+7JrqTTpJqn5emA4U2hvwJmvFRC0HODS+3Ye6WioDklgd6scJ3+PLnEA==",
       "dev": true,
       "peer": true,
       "dependencies": {
-        "@webassemblyjs/floating-point-hex-parser": "1.11.6",
-        "@webassemblyjs/helper-api-error": "1.11.6",
+        "@webassemblyjs/floating-point-hex-parser": "1.13.2",
+        "@webassemblyjs/helper-api-error": "1.13.2",
         "@xtuc/long": "4.2.2"
       }
     },
     "node_modules/@webassemblyjs/helper-wasm-bytecode": {
-      "version": "1.11.6",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-bytecode/-/helper-wasm-bytecode-1.11.6.tgz",
-      "integrity": "sha512-sFFHKwcmBprO9e7Icf0+gddyWYDViL8bpPjJJl0WHxCdETktXdmtWLGVzoHbqUcY4Be1LkNfwTmXOJUFZYSJdA==",
+      "version": "1.13.2",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-bytecode/-/helper-wasm-bytecode-1.13.2.tgz",
+      "integrity": "sha512-3QbLKy93F0EAIXLh0ogEVR6rOubA9AoZ+WRYhNbFyuB70j3dRdwH9g+qXhLAO0kiYGlg3TxDV+I4rQTr/YNXkA==",
       "dev": true,
       "peer": true
     },
     "node_modules/@webassemblyjs/helper-wasm-section": {
-      "version": "1.12.1",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-section/-/helper-wasm-section-1.12.1.tgz",
-      "integrity": "sha512-Jif4vfB6FJlUlSbgEMHUyk1j234GTNG9dBJ4XJdOySoj518Xj0oGsNi59cUQF4RRMS9ouBUxDDdyBVfPTypa5g==",
+      "version": "1.14.1",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/helper-wasm-section/-/helper-wasm-section-1.14.1.tgz",
+      "integrity": "sha512-ds5mXEqTJ6oxRoqjhWDU83OgzAYjwsCV8Lo/N+oRsNDmx/ZDpqalmrtgOMkHwxsG0iI//3BwWAErYRHtgn0dZw==",
       "dev": true,
       "peer": true,
       "dependencies": {
-        "@webassemblyjs/ast": "1.12.1",
-        "@webassemblyjs/helper-buffer": "1.12.1",
-        "@webassemblyjs/helper-wasm-bytecode": "1.11.6",
-        "@webassemblyjs/wasm-gen": "1.12.1"
+        "@webassemblyjs/ast": "1.14.1",
+        "@webassemblyjs/helper-buffer": "1.14.1",
+        "@webassemblyjs/helper-wasm-bytecode": "1.13.2",
+        "@webassemblyjs/wasm-gen": "1.14.1"
       }
     },
     "node_modules/@webassemblyjs/ieee754": {
-      "version": "1.11.6",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/ieee754/-/ieee754-1.11.6.tgz",
-      "integrity": "sha512-LM4p2csPNvbij6U1f19v6WR56QZ8JcHg3QIJTlSwzFcmx6WSORicYj6I63f9yU1kEUtrpG+kjkiIAkevHpDXrg==",
+      "version": "1.13.2",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/ieee754/-/ieee754-1.13.2.tgz",
+      "integrity": "sha512-4LtOzh58S/5lX4ITKxnAK2USuNEvpdVV9AlgGQb8rJDHaLeHciwG4zlGr0j/SNWlr7x3vO1lDEsuePvtcDNCkw==",
       "dev": true,
       "peer": true,
       "dependencies": {
@@ -4762,9 +4784,9 @@
       }
     },
     "node_modules/@webassemblyjs/leb128": {
-      "version": "1.11.6",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/leb128/-/leb128-1.11.6.tgz",
-      "integrity": "sha512-m7a0FhE67DQXgouf1tbN5XQcdWoNgaAuoULHIfGFIEVKA6tu/edls6XnIlkmS6FrXAquJRPni3ZZKjw6FSPjPQ==",
+      "version": "1.13.2",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/leb128/-/leb128-1.13.2.tgz",
+      "integrity": "sha512-Lde1oNoIdzVzdkNEAWZ1dZ5orIbff80YPdHx20mrHwHrVNNTjNr8E3xz9BdpcGqRQbAEa+fkrCb+fRFTl/6sQw==",
       "dev": true,
       "peer": true,
       "dependencies": {
@@ -4772,79 +4794,79 @@
       }
     },
     "node_modules/@webassemblyjs/utf8": {
-      "version": "1.11.6",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/utf8/-/utf8-1.11.6.tgz",
-      "integrity": "sha512-vtXf2wTQ3+up9Zsg8sa2yWiQpzSsMyXj0qViVP6xKGCUT8p8YJ6HqI7l5eCnWx1T/FYdsv07HQs2wTFbbof/RA==",
+      "version": "1.13.2",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/utf8/-/utf8-1.13.2.tgz",
+      "integrity": "sha512-3NQWGjKTASY1xV5m7Hr0iPeXD9+RDobLll3T9d2AO+g3my8xy5peVyjSag4I50mR1bBSN/Ct12lo+R9tJk0NZQ==",
       "dev": true,
       "peer": true
     },
     "node_modules/@webassemblyjs/wasm-edit": {
-      "version": "1.12.1",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-edit/-/wasm-edit-1.12.1.tgz",
-      "integrity": "sha512-1DuwbVvADvS5mGnXbE+c9NfA8QRcZ6iKquqjjmR10k6o+zzsRVesil54DKexiowcFCPdr/Q0qaMgB01+SQ1u6g==",
+      "version": "1.14.1",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-edit/-/wasm-edit-1.14.1.tgz",
+      "integrity": "sha512-RNJUIQH/J8iA/1NzlE4N7KtyZNHi3w7at7hDjvRNm5rcUXa00z1vRz3glZoULfJ5mpvYhLybmVcwcjGrC1pRrQ==",
       "dev": true,
       "peer": true,
       "dependencies": {
-        "@webassemblyjs/ast": "1.12.1",
-        "@webassemblyjs/helper-buffer": "1.12.1",
-        "@webassemblyjs/helper-wasm-bytecode": "1.11.6",
-        "@webassemblyjs/helper-wasm-section": "1.12.1",
-        "@webassemblyjs/wasm-gen": "1.12.1",
-        "@webassemblyjs/wasm-opt": "1.12.1",
-        "@webassemblyjs/wasm-parser": "1.12.1",
-        "@webassemblyjs/wast-printer": "1.12.1"
+        "@webassemblyjs/ast": "1.14.1",
+        "@webassemblyjs/helper-buffer": "1.14.1",
+        "@webassemblyjs/helper-wasm-bytecode": "1.13.2",
+        "@webassemblyjs/helper-wasm-section": "1.14.1",
+        "@webassemblyjs/wasm-gen": "1.14.1",
+        "@webassemblyjs/wasm-opt": "1.14.1",
+        "@webassemblyjs/wasm-parser": "1.14.1",
+        "@webassemblyjs/wast-printer": "1.14.1"
       }
     },
     "node_modules/@webassemblyjs/wasm-gen": {
-      "version": "1.12.1",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-gen/-/wasm-gen-1.12.1.tgz",
-      "integrity": "sha512-TDq4Ojh9fcohAw6OIMXqiIcTq5KUXTGRkVxbSo1hQnSy6lAM5GSdfwWeSxpAo0YzgsgF182E/U0mDNhuA0tW7w==",
+      "version": "1.14.1",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-gen/-/wasm-gen-1.14.1.tgz",
+      "integrity": "sha512-AmomSIjP8ZbfGQhumkNvgC33AY7qtMCXnN6bL2u2Js4gVCg8fp735aEiMSBbDR7UQIj90n4wKAFUSEd0QN2Ukg==",
       "dev": true,
       "peer": true,
       "dependencies": {
-        "@webassemblyjs/ast": "1.12.1",
-        "@webassemblyjs/helper-wasm-bytecode": "1.11.6",
-        "@webassemblyjs/ieee754": "1.11.6",
-        "@webassemblyjs/leb128": "1.11.6",
-        "@webassemblyjs/utf8": "1.11.6"
+        "@webassemblyjs/ast": "1.14.1",
+        "@webassemblyjs/helper-wasm-bytecode": "1.13.2",
+        "@webassemblyjs/ieee754": "1.13.2",
+        "@webassemblyjs/leb128": "1.13.2",
+        "@webassemblyjs/utf8": "1.13.2"
       }
     },
     "node_modules/@webassemblyjs/wasm-opt": {
-      "version": "1.12.1",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-opt/-/wasm-opt-1.12.1.tgz",
-      "integrity": "sha512-Jg99j/2gG2iaz3hijw857AVYekZe2SAskcqlWIZXjji5WStnOpVoat3gQfT/Q5tb2djnCjBtMocY/Su1GfxPBg==",
+      "version": "1.14.1",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-opt/-/wasm-opt-1.14.1.tgz",
+      "integrity": "sha512-PTcKLUNvBqnY2U6E5bdOQcSM+oVP/PmrDY9NzowJjislEjwP/C4an2303MCVS2Mg9d3AJpIGdUFIQQWbPds0Sw==",
       "dev": true,
       "peer": true,
       "dependencies": {
-        "@webassemblyjs/ast": "1.12.1",
-        "@webassemblyjs/helper-buffer": "1.12.1",
-        "@webassemblyjs/wasm-gen": "1.12.1",
-        "@webassemblyjs/wasm-parser": "1.12.1"
+        "@webassemblyjs/ast": "1.14.1",
+        "@webassemblyjs/helper-buffer": "1.14.1",
+        "@webassemblyjs/wasm-gen": "1.14.1",
+        "@webassemblyjs/wasm-parser": "1.14.1"
       }
     },
     "node_modules/@webassemblyjs/wasm-parser": {
-      "version": "1.12.1",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-parser/-/wasm-parser-1.12.1.tgz",
-      "integrity": "sha512-xikIi7c2FHXysxXe3COrVUPSheuBtpcfhbpFj4gmu7KRLYOzANztwUU0IbsqvMqzuNK2+glRGWCEqZo1WCLyAQ==",
+      "version": "1.14.1",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/wasm-parser/-/wasm-parser-1.14.1.tgz",
+      "integrity": "sha512-JLBl+KZ0R5qB7mCnud/yyX08jWFw5MsoalJ1pQ4EdFlgj9VdXKGuENGsiCIjegI1W7p91rUlcB/LB5yRJKNTcQ==",
       "dev": true,
       "peer": true,
       "dependencies": {
-        "@webassemblyjs/ast": "1.12.1",
-        "@webassemblyjs/helper-api-error": "1.11.6",
-        "@webassemblyjs/helper-wasm-bytecode": "1.11.6",
-        "@webassemblyjs/ieee754": "1.11.6",
-        "@webassemblyjs/leb128": "1.11.6",
-        "@webassemblyjs/utf8": "1.11.6"
+        "@webassemblyjs/ast": "1.14.1",
+        "@webassemblyjs/helper-api-error": "1.13.2",
+        "@webassemblyjs/helper-wasm-bytecode": "1.13.2",
+        "@webassemblyjs/ieee754": "1.13.2",
+        "@webassemblyjs/leb128": "1.13.2",
+        "@webassemblyjs/utf8": "1.13.2"
       }
     },
     "node_modules/@webassemblyjs/wast-printer": {
-      "version": "1.12.1",
-      "resolved": "https://registry.npmjs.org/@webassemblyjs/wast-printer/-/wast-printer-1.12.1.tgz",
-      "integrity": "sha512-+X4WAlOisVWQMikjbcvY2e0rwPsKQ9F688lksZhBcPycBBuii3O7m8FACbDMWDojpAqvjIncrG8J0XHKyQfVeA==",
+      "version": "1.14.1",
+      "resolved": "https://registry.npmjs.org/@webassemblyjs/wast-printer/-/wast-printer-1.14.1.tgz",
+      "integrity": "sha512-kPSSXE6De1XOR820C90RIo2ogvZG+c3KiHzqUoO/F34Y2shGzesfqv7o57xrxovZJH/MetF5UjroJ/R/3isoiw==",
       "dev": true,
       "peer": true,
       "dependencies": {
-        "@webassemblyjs/ast": "1.12.1",
+        "@webassemblyjs/ast": "1.14.1",
         "@xtuc/long": "4.2.2"
       }
     },
@@ -4868,9 +4890,9 @@
       "integrity": "sha512-JMJ5soJWP18htbbxJjG7bG6yuI6pRhgJ0scHHTfkUjf6wjP912xZWvM+A4sJK3gqd9E8fcPbDnOefbA9Th/FIQ=="
     },
     "node_modules/acorn": {
-      "version": "8.12.1",
-      "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.12.1.tgz",
-      "integrity": "sha512-tcpGyI9zbizT9JbV6oYE477V6mTlXvvi0T0G3SNIYE2apm/G5huBa1+K89VGeovbg+jycCrfhl3ADxErOuO6Jg==",
+      "version": "8.14.0",
+      "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.14.0.tgz",
+      "integrity": "sha512-cl669nCJTZBsL97OF4kUQm5g5hC2uihk0NxY3WENAC0TYdILVkAyHymAntgxGkl7K+t0cXIrH5siy5S4XkFycA==",
       "dev": true,
       "peer": true,
       "bin": {
@@ -4880,16 +4902,6 @@
         "node": ">=0.4.0"
       }
     },
-    "node_modules/acorn-import-attributes": {
-      "version": "1.9.5",
-      "resolved": "https://registry.npmjs.org/acorn-import-attributes/-/acorn-import-attributes-1.9.5.tgz",
-      "integrity": "sha512-n02Vykv5uA3eHGM/Z2dQrcD56kL8TyDb2p1+0P83PClMnC/nc+anbQRhIOWnSq4Ke/KvDPrY3C9hDtC/A3eHnQ==",
-      "dev": true,
-      "peer": true,
-      "peerDependencies": {
-        "acorn": "^8"
-      }
-    },
     "node_modules/ajv": {
       "version": "6.12.6",
       "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz",
@@ -5023,9 +5035,9 @@
       }
     },
     "node_modules/browserslist": {
-      "version": "4.23.0",
-      "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.23.0.tgz",
-      "integrity": "sha512-QW8HiM1shhT2GuzkvklfjcKDiWFXHOeFCIA/huJPwHsslwcydgk7X+z2zXpEijP98UCY7HbubZt5J2Zgvf0CaQ==",
+      "version": "4.24.2",
+      "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.24.2.tgz",
+      "integrity": "sha512-ZIc+Q62revdMcqC6aChtW4jz3My3klmCO1fEmINZY/8J3EpBg5/A/D0AKmBveUh6pgoeycoMkVMko84tuYS+Gg==",
       "funding": [
         {
           "type": "opencollective",
@@ -5041,10 +5053,10 @@
         }
       ],
       "dependencies": {
-        "caniuse-lite": "^1.0.30001587",
-        "electron-to-chromium": "^1.4.668",
-        "node-releases": "^2.0.14",
-        "update-browserslist-db": "^1.0.13"
+        "caniuse-lite": "^1.0.30001669",
+        "electron-to-chromium": "^1.5.41",
+        "node-releases": "^2.0.18",
+        "update-browserslist-db": "^1.1.1"
       },
       "bin": {
         "browserslist": "cli.js"
@@ -5088,9 +5100,9 @@
       }
     },
     "node_modules/caniuse-lite": {
-      "version": "1.0.30001625",
-      "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001625.tgz",
-      "integrity": "sha512-4KE9N2gcRH+HQhpeiRZXd+1niLB/XNLAhSy4z7fI8EzcbcPoAqjNInxVHTiTwWfTIV4w096XG8OtCOCQQKPv3w==",
+      "version": "1.0.30001680",
+      "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001680.tgz",
+      "integrity": "sha512-rPQy70G6AGUMnbwS1z6Xg+RkHYPAi18ihs47GH0jcxIG7wArmPgY3XbS2sRdBbxJljp3thdT8BIqv9ccCypiPA==",
       "funding": [
         {
           "type": "opencollective",
@@ -5104,7 +5116,8 @@
           "type": "github",
           "url": "https://github.com/sponsors/ai"
         }
-      ]
+      ],
+      "license": "CC-BY-4.0"
     },
     "node_modules/chalk": {
       "version": "2.4.2",
@@ -5369,9 +5382,9 @@
       "integrity": "sha512-YXQl1DSa4/PQyRfgrv6aoNjhasp/p4qs9FjJ4q4cQk+8m4r6k4ZSiEyytKG8f8W9gi8WsQtIObNmKd+tMzNTmA=="
     },
     "node_modules/electron-to-chromium": {
-      "version": "1.4.788",
-      "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.788.tgz",
-      "integrity": "sha512-ubp5+Ev/VV8KuRoWnfP2QF2Bg+O2ZFdb49DiiNbz2VmgkIqrnyYaqIOqj8A6K/3p1xV0QcU5hBQ1+BmB6ot1OA=="
+      "version": "1.5.72",
+      "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.72.tgz",
+      "integrity": "sha512-ZpSAUOZ2Izby7qnZluSrAlGgGQzucmFbN0n64dYzocYxnxV5ufurpj3VgEe4cUp7ir9LmeLxNYo8bVnlM8bQHw=="
     },
     "node_modules/emojis-list": {
       "version": "3.0.0",
@@ -5432,9 +5445,9 @@
       "peer": true
     },
     "node_modules/escalade": {
-      "version": "3.1.2",
-      "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.2.tgz",
-      "integrity": "sha512-ErCHMCae19vR8vQGe50xIsVomy19rg6gFu3+r3jkEO46suLMWBksvVyoGgQV+jOfl84ZSOSlmv6Gxa89PmTGmA==",
+      "version": "3.2.0",
+      "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz",
+      "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==",
       "engines": {
         "node": ">=6"
       }
@@ -6358,9 +6371,9 @@
       }
     },
     "node_modules/nanoid": {
-      "version": "3.3.7",
-      "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.7.tgz",
-      "integrity": "sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g==",
+      "version": "3.3.8",
+      "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.8.tgz",
+      "integrity": "sha512-WNLf5Sd8oZxOm+TzppcYk8gVOgP+l58xNy58D0nbUnOxOWRWvlcCV4kUF7ltmI6PsrLl/BgKEyS4mqsGChFN0w==",
       "funding": [
         {
           "type": "github",
@@ -6411,9 +6424,9 @@
       }
     },
     "node_modules/node-releases": {
-      "version": "2.0.14",
-      "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.14.tgz",
-      "integrity": "sha512-y10wOWt8yZpqXmOgRo77WaHEmhYQYGNA6y421PKsKYWEK8aW+cqAphborZDhqfyKrbZEN92CN1X2KbafY2s7Yw=="
+      "version": "2.0.19",
+      "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.19.tgz",
+      "integrity": "sha512-xxOWJsBKtzAq7DY0J+DTzuz58K8e7sJbdgwkbMWQe8UYB6ekmsQ45q0M/tJDsGaZmbC+l7n57UV8Hl5tHxO9uw=="
     },
     "node_modules/npm": {
       "version": "10.8.1",
@@ -9020,9 +9033,9 @@
       }
     },
     "node_modules/picocolors": {
-      "version": "1.0.1",
-      "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.1.tgz",
-      "integrity": "sha512-anP1Z8qwhkbmu7MFP5iTt+wQKXgwzf7zTyGlcdzabySa9vd0Xt392U0rVmz9poOaBj0uHJKyyo9/upk0HrEQew=="
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz",
+      "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA=="
     },
     "node_modules/picomatch": {
       "version": "2.3.1",
@@ -9483,9 +9496,9 @@
       }
     },
     "node_modules/terser": {
-      "version": "5.33.0",
-      "resolved": "https://registry.npmjs.org/terser/-/terser-5.33.0.tgz",
-      "integrity": "sha512-JuPVaB7s1gdFKPKTelwUyRq5Sid2A3Gko2S0PncwdBq7kN9Ti9HPWDQ06MPsEDGsZeVESjKEnyGy68quBk1w6g==",
+      "version": "5.37.0",
+      "resolved": "https://registry.npmjs.org/terser/-/terser-5.37.0.tgz",
+      "integrity": "sha512-B8wRRkmre4ERucLM/uXx4MOV5cbnOlVAqUst+1+iLKPI0dOgFO28f84ptoQt9HEI537PMzfYa/d+GEPKTRXmYA==",
       "dev": true,
       "peer": true,
       "dependencies": {
@@ -9623,9 +9636,9 @@
       "license": "MIT"
     },
     "node_modules/undici-types": {
-      "version": "6.19.8",
-      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.19.8.tgz",
-      "integrity": "sha512-ve2KP6f/JnbPBFyobGHuerC9g1FYGn/F8n1LWTwNxCEzd6IfqTwUQcNXgEtmmQ6DlRrC1hrSrBnCZPokRrDHjw==",
+      "version": "6.20.0",
+      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.20.0.tgz",
+      "integrity": "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==",
       "dev": true,
       "peer": true
     },
@@ -9670,9 +9683,9 @@
       }
     },
     "node_modules/update-browserslist-db": {
-      "version": "1.0.16",
-      "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.0.16.tgz",
-      "integrity": "sha512-KVbTxlBYlckhF5wgfyZXTWnMn7MMZjMu9XG8bPlliUOP9ThaF4QnhP8qrjrH7DRzHfSk0oQv1wToW+iA5GajEQ==",
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.1.1.tgz",
+      "integrity": "sha512-R8UzCaa9Az+38REPiJ1tXlImTJXlVfgHZsglwBD/k6nj76ctsH1E3q4doGrukiLQd3sGQYu56r5+lo5r94l29A==",
       "funding": [
         {
           "type": "opencollective",
@@ -9688,8 +9701,8 @@
         }
       ],
       "dependencies": {
-        "escalade": "^3.1.2",
-        "picocolors": "^1.0.1"
+        "escalade": "^3.2.0",
+        "picocolors": "^1.1.0"
       },
       "bin": {
         "update-browserslist-db": "cli.js"
@@ -9735,19 +9748,19 @@
       "integrity": "sha512-DEAoo25RfSYMuTGc9vPJzZcZullwIqRDSI9LOy+fkCJPi6hykCnfKaXTuPBDuXAUcqHXyOgFtHNp/kB2FjYHbw=="
     },
     "node_modules/webpack": {
-      "version": "5.94.0",
-      "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.94.0.tgz",
-      "integrity": "sha512-KcsGn50VT+06JH/iunZJedYGUJS5FGjow8wb9c0v5n1Om8O1g4L6LjtfxwlXIATopoQu+vOXXa7gYisWxCoPyg==",
+      "version": "5.97.1",
+      "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.97.1.tgz",
+      "integrity": "sha512-EksG6gFY3L1eFMROS/7Wzgrii5mBAFe4rIr3r2BTfo7bcc+DWwFZ4OJ/miOuHJO/A85HwyI4eQ0F6IKXesO7Fg==",
       "dev": true,
       "peer": true,
       "dependencies": {
-        "@types/estree": "^1.0.5",
-        "@webassemblyjs/ast": "^1.12.1",
-        "@webassemblyjs/wasm-edit": "^1.12.1",
-        "@webassemblyjs/wasm-parser": "^1.12.1",
-        "acorn": "^8.7.1",
-        "acorn-import-attributes": "^1.9.5",
-        "browserslist": "^4.21.10",
+        "@types/eslint-scope": "^3.7.7",
+        "@types/estree": "^1.0.6",
+        "@webassemblyjs/ast": "^1.14.1",
+        "@webassemblyjs/wasm-edit": "^1.14.1",
+        "@webassemblyjs/wasm-parser": "^1.14.1",
+        "acorn": "^8.14.0",
+        "browserslist": "^4.24.0",
         "chrome-trace-event": "^1.0.2",
         "enhanced-resolve": "^5.17.1",
         "es-module-lexer": "^1.2.1",

extensions/react-widget/package.json

@@ -1,6 +1,6 @@
 {
   "name": "docsgpt",
-  "version": "0.4.2",
+  "version": "0.4.9",
   "private": false,
   "description": "DocsGPT 🦖 is an innovative open-source tool designed to simplify the retrieval of information from project documentation using advanced GPT models 🤖.",
   "source": "./src/index.html",
@@ -30,9 +30,10 @@
     "styled-components": "^5"
   },
   "scripts": {
-    "build": "parcel build src/main.tsx --public-url ./",
+    "build": "parcel build src/browser.tsx --public-url ./",
     "build:react": "parcel build src/index.ts",
-    "dev": "parcel src/index.html -p 3000",
+    "serve": "parcel serve -p 3000",
+    "dev": "parcel -p 3000",
     "test": "jest",
     "lint": "eslint",
     "check": "tsc --noEmit",

extensions/react-widget/publish.sh

@@ -1,43 +1,85 @@
 #!/bin/bash
-## chmod +x publish.sh - to upgrade ownership
 set -e
-cat package.json >> package_copy.json
-cat package-lock.json >> package-lock_copy.json
+
+# Create backup of original files
+cp package.json package_original.json
+cp package-lock.json package-lock_original.json
+
+# Store the latest version after publishing
+LATEST_VERSION=""
+
 publish_package() {
-  PACKAGE_NAME=$1
-  BUILD_COMMAND=$2
-  # Update package name in package.json
-  jq --arg name "$PACKAGE_NAME" '.name=$name' package.json > temp.json && mv temp.json package.json
+    PACKAGE_NAME=$1
+    BUILD_COMMAND=$2
+    IS_REACT=$3
 
-  # Remove 'target' key if the package name is 'docsgpt-react'
-  if [ "$PACKAGE_NAME" = "docsgpt-react" ]; then
-    jq 'del(.targets)' package.json > temp.json && mv temp.json package.json
-  fi
+    echo "Preparing to publish ${PACKAGE_NAME}..."
+    
+    # Restore original package.json state before each publish
+    cp package_original.json package.json
+    cp package-lock_original.json package-lock.json
 
-  if [ -d "dist" ]; then
-    echo "Deleting existing dist directory..."
-    rm -rf dist
-  fi
+    # Update package name in package.json
+    jq --arg name "$PACKAGE_NAME" '.name=$name' package.json > temp.json && mv temp.json package.json
 
-  npm version patch
+    # Handle targets based on package type
+    if [ "$IS_REACT" = "true" ]; then
+        echo "Removing targets for React library build..."
+        jq 'del(.targets)' package.json > temp.json && mv temp.json package.json
+    fi
 
-  npm run "$BUILD_COMMAND"
+    # Clean dist directory
+    if [ -d "dist" ]; then
+        echo "Cleaning dist directory..."
+        rm -rf dist
+    fi
 
-  # Publish to npm
-  npm publish
-  # Clean up
-  mv package_copy.json package.json
-  mv package-lock_copy.json package-lock.json
-  echo "Published ${PACKAGE_NAME}"
+    # update version and store it
+    LATEST_VERSION=$(npm version patch)
+    echo "New version: ${LATEST_VERSION}"
+
+    # Build package
+    npm run "$BUILD_COMMAND"
+
+    # Replace npm publish with npm pack for testing
+    npm publish
+
+    echo "Successfully packaged ${PACKAGE_NAME}"
+
+    # Log the bundle size
+    TARBALL="${PACKAGE_NAME}-${LATEST_VERSION#v}.tgz"
+    if [ -f "$TARBALL" ]; then
+        BUNDLE_SIZE=$(du -h "$TARBALL" | cut -f1)
+        echo "Bundle size for ${PACKAGE_NAME}: ${BUNDLE_SIZE}"
+    else
+        echo "Error: ${TARBALL} not found."
+        exit 1
+    fi
 }
 
-# Publish docsgpt package
-publish_package "docsgpt" "build"
+# First publish docsgpt (HTML bundle)
+publish_package "docsgpt" "build" "false"
 
-# Publish docsgpt-react package
-publish_package "docsgpt-react" "build:react"
+# Then publish docsgpt-react (React library)
+publish_package "docsgpt-react" "build:react" "true"
 
+# Restore original state but keep the updated version
+cp package_original.json package.json
+cp package-lock_original.json package-lock.json
 
-rm -rf package_copy.json
-rm -rf package-lock_copy.json
-echo "---Process completed---"
+# Update the version in the final package.json
+jq --arg version "${LATEST_VERSION#v}" '.version=$version' package.json > temp.json && mv temp.json package.json
+
+# Run npm install to update package-lock.json with the new version
+npm install --package-lock-only
+
+# Cleanup backup files
+rm -f package_original.json
+rm -f package-lock_original.json
+rm -f temp.json
+
+echo "---Process completed---"
+echo "Final version in package.json: $(jq -r '.version' package.json)"
+echo "Final version in package-lock.json: $(jq -r '.version' package-lock.json)"
+echo "Generated test packages:"
+ls *.tgz

extensions/react-widget/src/App.tsx

@@ -1,11 +1,11 @@
 import React from "react"
 import {DocsGPTWidget} from "./components/DocsGPTWidget"
-const App = () => {
+import {SearchBar} from "./components/SearchBar"
+export const App = () => {
   return (
     <div>
+      <SearchBar/>
       <DocsGPTWidget/>
     </div>
   )
-}
-
-export default App
+}

extensions/react-widget/src/browser.tsx

@@ -0,0 +1,22 @@
+//exports browser ready methods
+
+import { createRoot } from "react-dom/client";
+
+import { DocsGPTWidget } from './components/DocsGPTWidget';
+import { SearchBar } from './components/SearchBar';
+import React from "react";
+if (typeof window !== 'undefined') {
+  const renderWidget = (elementId: string, props = {}) => {
+    const root = createRoot(document.getElementById(elementId) as HTMLElement);
+    root.render(<DocsGPTWidget {...props} />);
+  };
+  const renderSearchBar = (elementId: string, props = {}) => {
+    const root = createRoot(document.getElementById(elementId) as HTMLElement);
+    root.render(<SearchBar {...props} />);
+  };
+  (window as any).renderDocsGPTWidget = renderWidget;
+
+  (window as any).renderSearchBar = renderSearchBar;
+}
+
+export { DocsGPTWidget, SearchBar };

extensions/react-widget/src/components/DocsGPTWidget.tsx

@@ -1,14 +1,15 @@
 "use client";
 import React, { useRef } from 'react'
 import DOMPurify from 'dompurify';
-import styled, { keyframes, createGlobalStyle } from 'styled-components';
+import styled, { keyframes, css } from 'styled-components';
 import { PaperPlaneIcon, RocketIcon, ExclamationTriangleIcon, Cross2Icon } from '@radix-ui/react-icons';
-import { FEEDBACK, MESSAGE_TYPE, Query, Status, WidgetProps } from '../types/index';
+import { FEEDBACK, MESSAGE_TYPE, Query, Status, WidgetCoreProps, WidgetProps } from '../types/index';
 import { fetchAnswerStreaming, sendFeedback } from '../requests/streamingApi';
 import { ThemeProvider } from 'styled-components';
 import Like from "../assets/like.svg"
 import Dislike from "../assets/dislike.svg"
 import MarkdownIt from 'markdown-it';
+
 const themes = {
   dark: {
     bg: '#222327',
@@ -35,41 +36,99 @@ const themes = {
     }
   }
 }
-const GlobalStyles = createGlobalStyle`
-.response pre {
-    padding: 8px;
-    width: 90%;
-    font-size: 12px;
-    border-radius: 6px;
-    overflow-x: auto;
-    background-color: #1B1C1F;
-    color: #fff !important;
-}
-.response h1{
-  font-size: 20px;
-}
-.response h2{
-  font-size: 18px;
-}
-.response h3{
-  font-size: 16px;
-}
-.response p{
-  margin:0px;
-}
-.response code:not(pre code){
-  border-radius: 6px;
-  padding: 1px 3px 1px 3px;
-  font-size: 12px;
-  display: inline-block;
-  background-color: #646464;
-  color: #fff !important;
-}
-.response code {
-  white-space: pre-wrap !important;
-  line-break: loose !important;
-}
-`;
+
+const sizesConfig = {
+  small: { size: 'small', width: '320px', height: '400px' },
+  medium: { size: 'medium', width: '400px', height: '80vh' },
+  large: { size: 'large', width: '666px', height: '75vh' },
+  getCustom: (custom: { width: string; height: string; maxWidth?: string; maxHeight?: string }) => ({
+    size: 'custom',
+    width: custom.width,
+    height: custom.height,
+    maxWidth: custom.maxWidth || '968px',
+    maxHeight: custom.maxHeight || '70vh',
+  }),
+};
+const createBox = keyframes`
+   0% {
+        transform: scale(0.6);
+      }
+      90% {
+        transform: scale(1.02);
+      }
+      100% {
+        transform: scale(1);
+      }
+`
+const closeBox = keyframes`
+  0% {
+        transform: scale(1); 
+      }
+      10% {
+        transform: scale(1.02); 
+      }
+      100% {
+        transform: scale(0.6);
+      }
+`
+
+const openContainer = keyframes`
+      0% {
+        width: 200px;
+        height: 100px;
+      }
+      100% {
+        width: ${(props) => props.theme.dimensions.width};
+        height: ${(props) => props.theme.dimensions.height};
+        border-radius: 12px;
+      }`
+const closeContainer = keyframes`
+  0% {
+        width: ${(props) => props.theme.dimensions.width};
+        height: ${(props) => props.theme.dimensions.height};
+        border-radius: 12px;
+      }
+      100% {
+        width: 200px;
+        height: 100px;
+      }
+`
+const fadeIn = keyframes`
+  from {
+        opacity: 0;
+        width: ${(props) => props.theme.dimensions.width};
+        height: ${(props) => props.theme.dimensions.height};
+        transform: scale(0.9);
+      }
+      to {
+        opacity: 1;
+        transform: scale(1);
+        width: ${(props) => props.theme.dimensions.width};
+        height: ${(props) => props.theme.dimensions.height};
+      }
+`
+
+const fadeOut = keyframes`
+  from {
+        opacity: 1;
+        width: ${(props) => props.theme.dimensions.width};
+        height: ${(props) => props.theme.dimensions.height};
+      }
+      to {
+        opacity: 0;
+        transform: scale(0.9);
+        width: ${(props) => props.theme.dimensions.width};
+        height: ${(props) => props.theme.dimensions.height};
+      }
+`
+const scaleAnimation = keyframes`
+  from {
+      transform: scale(1.2);
+      }
+      to {
+      transform: scale(1);
+      }
+`
 const Overlay = styled.div`
   position: fixed;
   top: 0;
@@ -80,54 +139,85 @@ const Overlay = styled.div`
   z-index: 999;
   transition: opacity 0.5s;
 `
-const WidgetContainer = styled.div<{ modal: boolean }>`
-    display: block;
+
+
+const WidgetContainer = styled.div<{ modal?: boolean, isOpen?: boolean }>`
+    all: initial;
     position: fixed;
     right: ${props => props.modal ? '50%' : '10px'};
     bottom: ${props => props.modal ? '50%' : '10px'};
-    z-index: 1000;
-    display: flex;
-    ${props => props.modal &&
-    "transform : translate(50%,50%);"
-  }
-    flex-direction: column;
+    z-index: 1001;
+    transform-origin:100% 100%;
+    display: block;
+    &.modal{
+      transform : translate(50%,50%);
+    }
+    &.open {
+        animation: css ${createBox} 250ms cubic-bezier(0.25, 0.1, 0.25, 1) forwards;
+    }
+    &.close {
+      animation: css ${closeBox} 250ms cubic-bezier(0.25, 0.1, 0.25, 1) forwards;
+    }
     align-items: center;
     text-align: left;
-    @media only screen and (max-width: 768px) {
-    max-height: 100vh !important;
-    overflow: auto;
-    }
 `;
-const StyledContainer = styled.div`
-    display: flex;
+
+const StyledContainer = styled.div<{ isOpen: boolean }>`
+    all: initial;
+    max-height: ${(props) => props.theme.dimensions.maxHeight};
+    max-width: ${(props) => props.theme.dimensions.maxWidth};
+    width: ${(props) => props.theme.dimensions.width};
+    height: ${(props) => props.theme.dimensions.height} ;
     position: relative;
     flex-direction: column;
-    justify-content: center;
+    justify-content: space-between;
     bottom: 0;
     left: 0;
-    border-radius: 0.75rem;
-    background-color: ${props => props.theme.primary.bg};
+    background-color: ${(props) => props.theme.primary.bg};
     font-family: sans-serif;
-    box-shadow: 0 1px 2px rgba(0, 0, 0, 0.05), 0 2px 4px rgba(0, 0, 0, 0.1);
-    transition: visibility 0.3s, opacity 0.3s;
-`;
-const FloatingButton = styled.div<{ bgcolor: string }>`
-    position: fixed;
     display: flex;
+    border-radius: 12px;
+    box-shadow: 0 1px 2px rgba(0, 0, 0, 0.05), 0 2px 4px rgba(0, 0, 0, 0.1);
+    padding: 26px 26px 0px 26px;
+    animation: ${({ isOpen, theme }) =>
+    theme.dimensions.size === 'large'
+      ? isOpen
+        ? css`${fadeIn} 150ms ease-in forwards`
+        : css` ${fadeOut} 150ms ease-in forwards`
+      : isOpen
+        ? css`${openContainer} 150ms ease-in forwards`
+        : css`${closeContainer} 250ms ease-in forwards`};
+    @media only screen and (max-width: 768px) {
+      max-height: 100vh;
+      max-width: 80vw;
+      overflow: auto;
+    }
+`;
+
+const FloatingButton = styled.div<{ bgcolor: string, hidden: boolean, isAnimatingButton: boolean }>`
+    position: fixed;
+    display: ${props => props.hidden ? "none" : "flex"};
     z-index: 500;
     justify-content: center;
+    gap: 8px;
+    padding: 14px;
     align-items: center;
-    bottom: 1rem;
-    right: 1rem;
-    width: 5rem;
-    height: 5rem;
+    bottom: 16px;
+    color: white;
+    font-family: sans-serif;
+    right: 16px;
+    font-weight: 500;
     border-radius: 9999px;
     background: ${props => props.bgcolor};
     box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1);
     cursor: pointer;
+    animation: ${props => props.isAnimatingButton ? css`${scaleAnimation} 200ms forwards` : 'none'};
     &:hover {
-        transform: scale(1.1);
-        transition: transform 0.2s ease-in-out;
+      transform: scale(1.1);
+      transition: transform 0.2s ease-in-out;
+    }
+    &:not(:hover) {
+      transition: transform 0.2s ease-in-out;
     }
 `;
 const CancelButton = styled.button`
@@ -135,7 +225,7 @@ const CancelButton = styled.button`
     position: absolute;
     top: 0;
     right: 0;
-    margin: 0.5rem;
+    margin: 8px;
     width: 30px;
     padding: 0;
     background-color: transparent;
@@ -154,52 +244,37 @@ const CancelButton = styled.button`
 
 const Header = styled.div`
     display: flex;
-    align-items: center;
-    padding-inline: 0.75rem;
-    padding-top: 1rem;
-    padding-bottom: 0.5rem;
-`;
-
-const IconWrapper = styled.div`
-    padding: 0.5rem;
+    align-items: flex-start;
 `;
 
 const ContentWrapper = styled.div`
-    flex: 1;
-    margin-left: 0.5rem;
+    display: flex;
+    flex-direction: column;
+    gap:2px; 
+    margin-left: 8px;
 `;
 
 const Title = styled.h3`
-    font-size: 1rem;
+    font-size: 14px;
     font-weight: normal;
     color: ${props => props.theme.primary.text};
-    margin-top: 0;
-    margin-bottom: 0.25rem;
+    margin: 0;
 `;
 
 const Description = styled.p`
-    font-size: 0.85rem;
+    font-size: 13.75px;
     color: ${props => props.theme.secondary.text};
-    margin-top: 0;
+    margin: 0 ;
+    padding: 0 ;
 `;
 
-const Conversation = styled.div<{ size: string }>`
-    min-height: 250px;
-    max-width: 968px;
-    height: ${props => props.size === 'large' ? '75vh' : props.size === 'medium' ? '70vh' : '320px'};
-    width: ${props => props.size === 'large' ? '60vw' : props.size === 'medium' ? '28vw' : '400px'};
-    padding-inline: 0.5rem;
-    border-radius: 0.375rem;
-    text-align: left;
-    overflow-y: auto;
-    scrollbar-width: thin;
-    scrollbar-color: #4a4a4a transparent; /* thumb color track color */
-    @media only screen and (max-width: 768px) {
-    width: 90vw !important;
-    }
-    @media only screen and (min-width:768px ) and (max-width: 1280px) {
-    width:${props => props.size === 'large' ? '90vw' : props.size === 'medium' ? '60vw' : '400px'} !important;
-    }
+const Conversation = styled.div`
+  height: 70%;
+  border-radius: 6px;
+  text-align: left;
+  overflow-y: auto;
+  scrollbar-width: thin;
+  scrollbar-color: #4a4a4a transparent; /* thumb color track color */
 `;
 const Feedback = styled.div`
   background-color: transparent;
@@ -215,9 +290,9 @@ const MessageBubble = styled.div<{ type: MESSAGE_TYPE }>`
     position: relative;
     width: 100%;;
     float: right;
-    margin: 0rem;
+    margin: 0px;
     &:hover ${Feedback} * {
-    visibility: visible !important;
+    visibility: visible ;
   }
 `;
 const Message = styled.div<{ type: MESSAGE_TYPE }>`
@@ -232,19 +307,66 @@ const Message = styled.div<{ type: MESSAGE_TYPE }>`
     margin: 4px;
     display: block;
     line-height: 1.5;
-    padding: 0.75rem;
-    border-radius: 0.375rem;
+    padding: 12px;
+    border-radius: 6px;
 `;
+const Markdown = styled.div`
+ pre {
+      padding: 8px;
+      width: 90%;
+      font-size: 12px;
+      border-radius: 6px;
+      overflow-x: auto;
+      background-color: #1B1C1F;
+      color: #fff ;
+    }
+
+    h1 {
+      font-size: 16px;
+    }
+
+    h2 {
+      font-size: 14px;
+    }
+
+    h3 {
+      font-size: 14px;
+    }
+
+    p {
+      margin: 0px;
+    }
+
+    code:not(pre code) {
+      border-radius: 6px;
+      padding: 1px 3px;
+      font-size: 12px;
+      display: inline-block;
+      background-color: #646464;
+      color: #fff ;
+    }
+
+    code {
+      white-space: pre-wrap ;
+      overflow-wrap: break-word;
+      word-break: break-all;
+    }
+
+    ul{
+      padding:0px;
+      list-style-position: inside;
+    }
+`
 const ErrorAlert = styled.div`
   color: #b91c1c;
   border:0.1px solid #b91c1c;
   display: flex;
   padding:4px;
-  margin:0.7rem;
+  margin:11.2px;
   opacity: 90%;
   max-width: 70%;
   font-weight: 400;
-  border-radius: 0.375rem;
+  border-radius: 6px;
   justify-content: space-evenly;
 `
 //dot loading animation
@@ -265,10 +387,9 @@ const DotAnimation = styled.div`
 const Delay = styled(DotAnimation) <{ delay: number }>`
   animation-delay: ${props => props.delay + 'ms'};
 `;
-const PromptContainer = styled.form<{ size: string }>`
+const PromptContainer = styled.form`
   background-color: transparent;
-  height: ${props => props.size == 'large' ? '60px' : '40px'};
-  margin: 16px;
+  height: ${props => props.theme.dimensions.size == 'large' ? '60px' : '40px'};
   display: flex;
   justify-content: space-evenly;
 `;
@@ -282,16 +403,18 @@ const StyledInput = styled.input`
   color: ${props => props.theme.text};
   outline: none;
 `;
-const StyledButton = styled.button<{ size: string }>`
+const StyledButton = styled.button`
   display: flex;
   justify-content: center;
   align-items: center;
   background-image: linear-gradient(to bottom right, #5AF0EC, #E80D9D);
+  background-color: rgba(0, 0, 0, 0.3);
   border-radius: 6px;
-  min-width: ${props => props.size === 'large' ? '60px' : '36px'};
-  height: ${props => props.size === 'large' ? '60px' : '36px'};
+  min-width: ${props => props.theme.dimensions.size === 'large' ? '60px' : '40px'};
+  height: ${props => props.theme.dimensions.size === 'large' ? '60px' : '40px'};
   margin-left:8px;
   padding: 0px;
+  
   border: none;
   cursor: pointer;
   outline: none;
@@ -299,62 +422,107 @@ const StyledButton = styled.button<{ size: string }>`
     opacity: 90%;
   }
   &:disabled {
-    opacity: 60%;
+    background-image: linear-gradient(to bottom right, #2d938f, #b31877);
   }`;
 const HeroContainer = styled.div`
-  position: absolute;
-  top: 50%;
-  left: 50%;
-  display: flex;
-  justify-content: center;
-  align-items: middle;
-  transform: translate(-50%, -50%);
-  width: 80%;
+  position: relative;
+  width: 90%;
   max-width: 500px;
   background-image: linear-gradient(to bottom right, #5AF0EC, #ff1bf4);
   border-radius: 10px;
-  margin: 0 auto;
+  margin: 16px auto;
   padding: 2px;
 `;
 const HeroWrapper = styled.div`
+  display: flex;
+  flex-direction: column;
+  justify-content: flex-start;
+  gap: 8px;
+  align-items: middle;
   background-color: ${props => props.theme.primary.bg};
   border-radius: 10px; 
   font-weight: normal;
-  padding: 6px;
-  display: flex;
-  justify-content: space-between;
+  padding: 12px;
 `
 const HeroTitle = styled.h3`
   color: ${props => props.theme.text};
-  margin-bottom: 5px;
-  padding: 2px;
+  font-size: 16px;
+  margin:0px ;
+  padding: 0px;
 `;
 const HeroDescription = styled.p`
   color: ${props => props.theme.text};
-  font-size: 14px;
+  font-size: 12px;
   line-height: 1.5;
+  margin: 0px;
+  padding: 0px;
 `;
+const Hyperlink = styled.a`
+  color: #9971EC;
+  text-decoration: none;
+`;
+const Tagline = styled.div`
+  text-align: center;
+  display: block;
+  color: ${props => props.theme.secondary.text};
+  padding: 12px ;
+  font-size: 12px;
+`;
+
+
 
 const Hero = ({ title, description, theme }: { title: string, description: string, theme: string }) => {
   return (
-    <>
-      <HeroContainer>
-        <HeroWrapper>
-          <IconWrapper style={{ marginTop: '12px' }}>
-            <RocketIcon color={theme === 'light' ? 'black' : 'white'} width={20} height={20} />
-          </IconWrapper>
-          <div>
-            <HeroTitle>{title}</HeroTitle>
-            <HeroDescription>
-              {description}
-            </HeroDescription>
-          </div>
-        </HeroWrapper>
-      </HeroContainer>
-    </>
+    <HeroContainer>
+      <HeroWrapper>
+        <RocketIcon color={theme === 'light' ? 'black' : 'white'} width={24} height={24} />
+        <HeroTitle>{title}</HeroTitle>
+        <HeroDescription>{description}</HeroDescription>
+      </HeroWrapper>
+    </HeroContainer>
   );
 };
-export const DocsGPTWidget = ({
+export const DocsGPTWidget = (props: WidgetProps) => {
+
+  const {
+    buttonIcon = 'https://d3dg1063dc54p9.cloudfront.net/widget/chat.svg',
+    buttonText = 'Ask a question',
+    buttonBg = 'linear-gradient(to bottom right, #5AF0EC, #E80D9D)',
+    defaultOpen = false,
+    ...coreProps
+  } = props
+
+  const [open, setOpen] = React.useState<boolean>(defaultOpen);
+  const [isAnimatingButton, setIsAnimatingButton] = React.useState(false);
+  const [isFloatingButtonVisible, setIsFloatingButtonVisible] = React.useState(true);
+
+  React.useEffect(() => {
+    if (isFloatingButtonVisible)
+      setTimeout(() => setIsAnimatingButton(true), 250);
+    return () => {
+      setIsAnimatingButton(false)
+    }
+  }, [isFloatingButtonVisible])
+
+  const handleClose = () => {
+    setIsFloatingButtonVisible(true);
+    setOpen(false);
+  };
+  const handleOpen = () => {
+    setOpen(true);
+    setIsFloatingButtonVisible(false);
+  }
+  return (
+    <>
+      <FloatingButton bgcolor={buttonBg} onClick={handleOpen} hidden={!isFloatingButtonVisible} isAnimatingButton={isAnimatingButton}>
+        <img width={24} src={buttonIcon} />
+        <span>{buttonText}</span>
+      </FloatingButton>
+      <WidgetCore isOpen={open} handleClose={handleClose} {...coreProps} />
+    </>
+  )
+}
+export const WidgetCore = ({
   apiHost = 'https://gptcloud.arc53.com',
   apiKey = '82962c9a-aa77-4152-94e5-a4f84fd44c6a',
   avatar = 'https://d3dg1063dc54p9.cloudfront.net/cute-docsgpt.png',
@@ -364,20 +532,37 @@ export const DocsGPTWidget = ({
   heroDescription = 'This chatbot is built with DocsGPT and utilises GenAI, please review important information using sources.',
   size = 'small',
   theme = 'dark',
-  buttonIcon = 'https://d3dg1063dc54p9.cloudfront.net/widget/message.svg',
-  buttonBg = 'linear-gradient(to bottom right, #5AF0EC, #E80D9D)',
-  collectFeedback = true
-}: WidgetProps) => {
-  const [prompt, setPrompt] = React.useState('');
+  collectFeedback = true,
+  isOpen = false,
+  prefilledQuery = "",
+  handleClose
+}: WidgetCoreProps) => {
+  const [prompt, setPrompt] = React.useState<string>("");
+  const [mounted, setMounted] = React.useState(false);
   const [status, setStatus] = React.useState<Status>('idle');
-  const [queries, setQueries] = React.useState<Query[]>([])
-  const [conversationId, setConversationId] = React.useState<string | null>(null)
-  const [open, setOpen] = React.useState<boolean>(false)
+  const [queries, setQueries] = React.useState<Query[]>([]);
+  const [conversationId, setConversationId] = React.useState<string | null>(null);
   const [eventInterrupt, setEventInterrupt] = React.useState<boolean>(false); //click or scroll by user while autoScrolling
-  const isBubbleHovered = useRef<boolean>(false)
+
+  const isBubbleHovered = useRef<boolean>(false);
   const endMessageRef = React.useRef<HTMLDivElement | null>(null);
   const md = new MarkdownIt();
 
+  React.useEffect(() => {
+    if (isOpen) {
+      setMounted(true); // Mount the component
+      appendQuery(prefilledQuery)
+    } else {
+      // Wait for animations before unmounting
+      const timeout = setTimeout(() => {
+        setMounted(false)
+      }, 250);
+      return () => clearTimeout(timeout);
+    }
+  }, [isOpen]);
+
+
+
   const handleUserInterrupt = () => {
     (status === 'loading') && setEventInterrupt(true);
   }
@@ -417,7 +602,7 @@ export const DocsGPTWidget = ({
             });
           }
         })
-        .catch(err => console.log("Connection failed",err))
+        .catch(err => console.log("Connection failed", err))
     }
     else {
       delete query.feedback;
@@ -453,8 +638,11 @@ export const DocsGPTWidget = ({
               setQueries(updatedQueries);
               setStatus('idle')
             }
+            else if (data.type === 'source') {
+              // handle the case where data type === 'source'
+            }
             else {
-              const result = data.answer;
+              const result = data.answer ? data.answer : ''; //Fallback to an empty string if data.answer is undefined
               const streamingResponse = queries[queries.length - 1].response ? queries[queries.length - 1].response : '';
               const updatedQueries = [...queries];
               updatedQueries[updatedQueries.length - 1].response = streamingResponse + result;
@@ -474,126 +662,138 @@ export const DocsGPTWidget = ({
   }
   // submit handler
   const handleSubmit = async (e: React.FormEvent<HTMLFormElement>) => {
-    e.preventDefault()
+    e.preventDefault();
+    await appendQuery(prompt)
+  }
+
+  const appendQuery = async (userQuery:string) => {
+    console.log(userQuery)
+    if(!userQuery)
+      return;
+
     setEventInterrupt(false);
-    queries.push({ prompt })
-    setPrompt('')
-    await stream(prompt)
+    queries.push({ prompt:userQuery});
+    setPrompt('');
+    await stream(userQuery);
   }
   const handleImageError = (event: React.SyntheticEvent<HTMLImageElement, Event>) => {
     event.currentTarget.src = "https://d3dg1063dc54p9.cloudfront.net/cute-docsgpt.png";
   };
-  return (
-    <ThemeProvider theme={themes[theme]}>
-      {open && size === 'large' &&
-        <Overlay onClick={() => {
-          setOpen(false)
-        }} />
-      }
-      <FloatingButton bgcolor={buttonBg} onClick={() => setOpen(!open)} hidden={open}>
-        <img style={{ maxHeight: '4rem', maxWidth: '4rem' }} src={buttonIcon} />
-      </FloatingButton>
-      <WidgetContainer modal={size == 'large'}>
-        <GlobalStyles />
-        {open && <StyledContainer>
-          <div>
-            <CancelButton onClick={() => setOpen(false)}>
-              <Cross2Icon width={24} height={24} color={theme === 'light' ? 'black' : 'white'} />
-            </CancelButton>
-            <Header>
-              <IconWrapper>
-                <img style={{ maxWidth: "42px", maxHeight: "42px" }} onError={handleImageError} src={avatar} alt='docs-gpt' />
-              </IconWrapper>
-              <ContentWrapper>
-                <Title>{title}</Title>
-                <Description>{description}</Description>
-              </ContentWrapper>
-            </Header>
-          </div>
-          <Conversation size={size} onWheel={handleUserInterrupt} onTouchMove={handleUserInterrupt}>
-            {
-              queries.length > 0 ? queries?.map((query, index) => {
-                return (
-                  <React.Fragment key={index}>
-                    {
-                      query.prompt && <MessageBubble type='QUESTION'>
-                        <Message
-                          type='QUESTION'
-                          ref={(!(query.response || query.error) && index === queries.length - 1) ? endMessageRef : null}>
-                          {query.prompt}
-                        </Message>
-                      </MessageBubble>
-                    }
-                    {
-                      query.response ? <MessageBubble onMouseOver={() => { isBubbleHovered.current = true }} type='ANSWER'>
-                        <Message
-                          type='ANSWER'
-                          ref={(index === queries.length - 1) ? endMessageRef : null}
-                        >
-                          <div
-                            className="response"
-                            dangerouslySetInnerHTML={{ __html: DOMPurify.sanitize(md.render(query.response)) }}
-                          />
-                        </Message>
 
-                        {collectFeedback &&
-                          <Feedback>
-                            <Like
-                              style={{
-                                stroke: query.feedback == 'LIKE' ? '#8860DB' : '#c0c0c0',
-                                visibility: query.feedback == 'LIKE' ? 'visible' : 'hidden'
-                              }}
-                              fill='none'
-                              onClick={() => handleFeedback("LIKE", index)} />
-                            <Dislike
-                              style={{
-                                stroke: query.feedback == 'DISLIKE' ? '#ed8085' : '#c0c0c0',
-                                visibility: query.feedback == 'DISLIKE' ? 'visible' : 'hidden'
-                              }}
-                              fill='none'
-                              onClick={() => handleFeedback("DISLIKE", index)} />
-                          </Feedback>}
-                      </MessageBubble>
-                        : <div>
-                          {
-                            query.error ? <ErrorAlert>
-                              <IconWrapper>
-                                <ExclamationTriangleIcon style={{ marginTop: '4px' }} width={22} height={22} color='#b91c1c' />
-                              </IconWrapper>
-                              <div>
-                                <h5 style={{ margin: 2 }}>Network Error</h5>
-                                <span style={{ margin: 2, fontSize: '13px' }}>{query.error}</span>
-                              </div>
-                            </ErrorAlert>
-                              : <MessageBubble type='ANSWER'>
-                                <Message type='ANSWER' style={{ fontWeight: 600 }}>
-                                  <DotAnimation>.</DotAnimation>
-                                  <Delay delay={200}>.</Delay>
-                                  <Delay delay={400}>.</Delay>
-                                </Message>
-                              </MessageBubble>
-                          }
-                        </div>
-                    }
-                  </React.Fragment>)
-              })
-                : <Hero title={heroTitle} description={heroDescription} theme={theme} />
-            }
-          </Conversation>
-          <PromptContainer
-            size={size}
-            onSubmit={handleSubmit}>
-            <StyledInput
-              value={prompt} onChange={(event) => setPrompt(event.target.value)}
-              type='text' placeholder="What do you want to do?" />
-            <StyledButton
-              size={size}
-              disabled={prompt.trim().length == 0 || status !== 'idle'}>
-              <PaperPlaneIcon width={15} height={15} color='white' />
-            </StyledButton>
-          </PromptContainer>
-        </StyledContainer>}
-      </WidgetContainer>
+  const dimensions =
+    typeof size === 'object' && 'custom' in size
+      ? sizesConfig.getCustom(size.custom)
+      : sizesConfig[size];
+  if (!mounted) return null;
+  return (
+    <ThemeProvider theme={{ ...themes[theme], dimensions }}>
+      {isOpen && size === 'large' &&
+        <Overlay onClick={handleClose} />
+      }
+      {(
+        <WidgetContainer className={`${size !== 'large' ? (isOpen ? "open" : "close") : "modal"}`} modal={size === 'large'}>
+          <StyledContainer isOpen={isOpen}>
+            <div>
+              <CancelButton onClick={handleClose}>
+                <Cross2Icon width={24} height={24} color={theme === 'light' ? 'black' : 'white'} />
+              </CancelButton>
+              <Header>
+                <img style={{ transform: 'translateY(-5px)', maxWidth: "42px", maxHeight: "42px" }} onError={handleImageError} src={avatar} alt='docs-gpt' />
+                <ContentWrapper>
+                  <Title>{title}</Title>
+                  <Description>{description}</Description>
+                </ContentWrapper>
+              </Header>
+            </div>
+            <Conversation onWheel={handleUserInterrupt} onTouchMove={handleUserInterrupt}>
+              {
+                queries.length > 0 ? queries?.map((query, index) => {
+                  return (
+                    <React.Fragment key={index}>
+                      {
+                        query.prompt && <MessageBubble type='QUESTION'>
+                          <Message
+                            type='QUESTION'
+                            ref={(!(query.response || query.error) && index === queries.length - 1) ? endMessageRef : null}>
+                            {query.prompt}
+                          </Message>
+                        </MessageBubble>
+                      }
+                      {
+                        query.response ? <MessageBubble onMouseOver={() => { isBubbleHovered.current = true }} type='ANSWER'>
+                          <Message
+                            type='ANSWER'
+                            ref={(index === queries.length - 1) ? endMessageRef : null}
+                          >
+                            <Markdown
+                              dangerouslySetInnerHTML={{ __html: DOMPurify.sanitize(md.render(query.response)) }}
+                            />
+                          </Message>
+
+                          {collectFeedback &&
+                            <Feedback>
+                              <Like
+                                style={{
+                                  stroke: query.feedback == 'LIKE' ? '#8860DB' : '#c0c0c0',
+                                  visibility: query.feedback == 'LIKE' ? 'visible' : 'hidden'
+                                }}
+                                fill='none'
+                                onClick={() => handleFeedback("LIKE", index)} />
+                              <Dislike
+                                style={{
+                                  stroke: query.feedback == 'DISLIKE' ? '#ed8085' : '#c0c0c0',
+                                  visibility: query.feedback == 'DISLIKE' ? 'visible' : 'hidden'
+                                }}
+                                fill='none'
+                                onClick={() => handleFeedback("DISLIKE", index)} />
+                            </Feedback>}
+                        </MessageBubble>
+                          : <div>
+                            {
+                              query.error ? <ErrorAlert>
+
+                                <ExclamationTriangleIcon width={22} height={22} color='#b91c1c' />
+                                <div>
+                                  <h5 style={{ margin: 2 }}>Network Error</h5>
+                                  <span style={{ margin: 2, fontSize: '13px' }}>{query.error}</span>
+                                </div>
+                              </ErrorAlert>
+                                : <MessageBubble type='ANSWER'>
+                                  <Message type='ANSWER' style={{ fontWeight: 600 }}>
+                                    <DotAnimation>.</DotAnimation>
+                                    <Delay delay={200}>.</Delay>
+                                    <Delay delay={400}>.</Delay>
+                                  </Message>
+                                </MessageBubble>
+                            }
+                          </div>
+                      }
+                    </React.Fragment>)
+                })
+                  : <Hero title={heroTitle} description={heroDescription} theme={theme} />
+              }
+            </Conversation>
+            <div>
+              <PromptContainer
+                onSubmit={handleSubmit}>
+                <StyledInput
+                  autoFocus
+                  value={prompt} onChange={(event) => setPrompt(event.target.value)}
+                  type='text' placeholder="Ask your question" />
+                <StyledButton
+                  disabled={prompt.trim().length == 0 || status !== 'idle'}>
+                  <PaperPlaneIcon width={18} height={18} color='white' />
+                </StyledButton>
+              </PromptContainer>
+              <Tagline>
+                Powered by&nbsp;
+                <Hyperlink target='_blank' href='https://www.docsgpt.cloud/'>DocsGPT</Hyperlink>
+              </Tagline>
+            </div>
+          </StyledContainer>
+        </WidgetContainer>
+      )
+      }
     </ThemeProvider>
   )
 }

extensions/react-widget/src/components/SearchBar.tsx

@@ -0,0 +1,572 @@
+import React from 'react';
+import styled, { ThemeProvider, createGlobalStyle } from 'styled-components';
+import { WidgetCore } from './DocsGPTWidget';
+import { SearchBarProps } from '@/types';
+import { getSearchResults } from '../requests/searchAPI';
+import { Result } from '@/types';
+import MarkdownIt from 'markdown-it';
+import { getOS, processMarkdownString } from '../utils/helper';
+import DOMPurify from 'dompurify';
+import { 
+    CodeIcon, 
+    TextAlignLeftIcon,
+    HeadingIcon,
+    ReaderIcon, 
+    ListBulletIcon, 
+    QuoteIcon 
+} from '@radix-ui/react-icons';
+const themes = {
+    dark: {
+        bg: '#202124',
+        text: '#EDEDED',
+        primary: {
+            text: "#FAFAFA",
+            bg: '#111111'
+        },
+        secondary: {
+            text: "#A1A1AA",
+            bg: "#38383b"
+        }
+    },
+    light: {
+        bg: '#EAEAEA',
+        text: '#171717',
+        primary: {
+            text: "#222327",
+            bg: "#fff"
+        },
+        secondary: {
+            text: "#A1A1AA",
+            bg: "#F6F6F6"
+        }
+    }
+}
+
+const GlobalStyle = createGlobalStyle`
+  .highlight {
+    color:#007EE6;
+  }
+`;
+
+const loadGeistFont = () => {
+  const link = document.createElement('link');
+  link.href = 'https://fonts.googleapis.com/css2?family=Geist:wght@100..900&display=swap'; 
+  link.rel = 'stylesheet';
+  document.head.appendChild(link);
+};
+
+const Main = styled.div`
+    all: initial;
+    font-family: 'Geist', sans-serif;
+`
+const SearchButton = styled.button<{ inputWidth: string }>`
+    padding: 6px 6px;
+    font-family: inherit;
+    width: ${({ inputWidth }) => inputWidth};
+    border-radius: 8px;
+    display: inline;
+    color: ${props => props.theme.secondary.text};
+    outline: none;
+    border: none;
+    background-color: ${props => props.theme.secondary.bg};
+    -webkit-appearance: none;
+    -moz-appearance: none;
+    appearance: none;
+    transition: background-color 128ms linear;
+    text-align: left;
+    cursor: pointer;
+`
+
+const Container = styled.div`
+    position: relative;
+    display: inline-block;
+`
+const SearchResults = styled.div`
+    position: fixed;
+    display: flex;
+    flex-direction: column;
+    background-color: ${props => props.theme.primary.bg};
+    border: 1px solid ${props => props.theme.bg};
+    border-radius: 15px;
+    padding: 8px 0px 8px 0px;
+    width: 792px;
+    max-width: 90vw;
+    height: 396px;
+    z-index: 100;
+    left: 50%;
+    top: 50%;
+    transform: translate(-50%, -50%);
+    color: ${props => props.theme.primary.text};
+    box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1), 0 2px 4px rgba(0, 0, 0, 0.1);
+    backdrop-filter: blur(16px);
+    box-sizing: border-box;
+
+    @media only screen and (max-width: 768px) {
+        height: 80vh;
+        width: 90vw;
+    }
+`;
+
+const SearchResultsScroll = styled.div`
+    flex: 1;
+    overflow-y: auto;
+    overflow-x: hidden;
+    scrollbar-gutter: stable;
+    scrollbar-width: thin;
+    scrollbar-color: #383838 transparent;
+    padding: 0 16px;
+`;
+
+const IconTitleWrapper = styled.div`
+    display: flex;
+    align-items: center;
+    gap: 8px;
+
+    .element-icon{
+        margin: 4px;
+    }
+`;
+
+const Title = styled.h3`
+    font-size: 15px;
+    font-weight: 400;
+    color: ${props => props.theme.primary.text};
+    margin: 0;
+    overflow-wrap: break-word;
+    white-space: normal;
+    overflow: hidden;
+    text-overflow: ellipsis;
+`;
+const ContentWrapper = styled.div`
+    display: flex;
+    flex-direction: column;
+    gap: 12px; 
+`;
+const Content = styled.div`
+    display: flex;
+    margin-left: 8px;
+    flex-direction: column;
+    gap: 8px;
+    padding: 4px 0px 0px 12px;
+    font-size: 15px;
+    color: ${props => props.theme.primary.text};
+    line-height: 1.6;
+    border-left: 2px solid #585858;
+    overflow: hidden;
+`
+const ContentSegment = styled.div`
+    display: flex;
+    align-items: flex-start;
+    gap: 8px;
+    padding-right: 16px;
+    overflow-wrap: break-word;
+    white-space: normal;
+    overflow: hidden; 
+    text-overflow: ellipsis;
+`
+
+const ResultWrapper = styled.div`
+    display: flex;
+    align-items: flex-start;
+    width: 100%;
+    box-sizing: border-box;
+    padding: 8px 16px;
+    cursor: pointer;
+    background-color: ${props => props.theme.primary.bg};
+    font-family: 'Geist', sans-serif;
+    transition: background-color 0.2s;
+    border-radius: 8px;
+
+    word-wrap: break-word;
+    overflow-wrap: break-word;
+    word-break: break-word;
+    white-space: normal;
+    overflow: hidden;
+    text-overflow: ellipsis;
+
+    &:hover {
+        background-color: ${props => props.theme.bg};
+    }
+`
+const Markdown = styled.div`
+line-height:18px;
+font-size: 11px;
+white-space: pre-wrap;
+ pre {
+      padding: 8px;
+      width: 90%;
+      font-size: 11px;
+      border-radius: 6px;
+      overflow-x: auto;
+      background-color: #1B1C1F;
+      color: #fff ;
+    }
+
+    h1,h2 {
+      font-size: 14px;
+      font-weight: 600;
+      color: ${(props) => props.theme.text};
+      opacity: 0.8;
+    }
+
+
+    h3 {
+      font-size: 12px;
+    }
+
+    p {
+      margin: 0px;
+      line-height: 1.35rem;
+      font-size: 11px;
+    }
+
+    code:not(pre code) {
+      border-radius: 6px;
+      padding: 2px 2px;
+      margin: 2px;
+      font-size: 9px;
+      display: inline;
+      background-color: #646464;
+      color: #fff ;
+    }
+    img{
+        max-width: 50%;
+    }
+    code {
+      overflow-x: auto;
+    }
+    a{
+        color: #007ee6;
+    }
+`
+const Toolkit = styled.kbd`
+    position: absolute;
+    right: 4px;
+    top: 50%;
+    transform: translateY(-50%);
+    background-color: ${(props) => props.theme.primary.bg};
+    color: ${(props) => props.theme.secondary.text};
+    font-weight: 600;
+    font-size: 10px;
+    padding: 3px 6px;
+    border: 1px solid ${(props) => props.theme.secondary.text};
+    border-radius: 4px;
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    z-index: 1;
+    pointer-events: none;
+`
+const Loader = styled.div`
+  margin: 2rem auto;
+  border: 4px solid ${props => props.theme.secondary.text};
+  border-top: 4px solid ${props => props.theme.primary.bg};
+  border-radius: 50%;
+  width: 12px;
+  height: 12px;
+  animation: spin 1s linear infinite;
+
+  @keyframes spin {
+    0% {
+      transform: rotate(0deg);
+    }
+    100% {
+      transform: rotate(360deg);
+    }
+  }
+`;
+
+const NoResults = styled.div`
+  margin-top: 2rem;
+  text-align: center;
+  font-size: 14px;
+  color: #888;
+`;
+const AskAIButton = styled.button`
+    display: flex;
+    align-items: center;
+    justify-content: flex-start;
+    gap: 12px;
+    width: calc(100% - 32px);
+    margin: 0 16px 16px 16px;
+    box-sizing: border-box;
+    height: 50px;
+    padding: 8px 24px;
+    border: none;
+    border-radius: 6px;
+    background-color: ${props => props.theme.bg};
+    color: ${props => props.theme.text}; 
+    cursor: pointer;
+    transition: background-color 0.2s, box-shadow 0.2s;
+    font-size: 16px;
+
+    &:hover {
+        opacity: 0.8;
+    }
+`
+const SearchHeader = styled.div`
+    display: flex;
+    align-items: center;
+    gap: 8px;
+    margin-bottom: 12px;
+    padding-bottom: 12px;
+    border-bottom: 1px solid ${props => props.theme.bg};
+`
+
+const TextField = styled.input`
+    width: calc(100% - 32px);
+    margin: 0 16px;
+    padding: 12px 16px;
+    border: none;
+    background-color: transparent;
+    color: ${props => props.theme.text}; 
+    font-size: 20px;
+    font-weight: 400; 
+    outline: none;
+    
+    &:focus {
+        border-color: none;
+    }
+`
+
+const EscapeInstruction = styled.kbd`
+    display: flex;
+    align-items: center;
+    justify-content: center;
+    margin: 12px 16px 0;
+    padding: 4px 8px;
+    border-radius: 4px;
+    background-color: transparent;
+    border: 1px solid ${props => props.theme.secondary.text};
+    color: ${props => props.theme.text};
+    font-size: 12px;
+    font-family: 'Geist', sans-serif;
+    white-space: nowrap;
+    cursor: pointer;
+    width: fit-content;
+    &:hover {
+        background-color: rgba(255, 255, 255, 0.1);
+    }
+`
+export const SearchBar = ({
+    apiKey = "74039c6d-bff7-44ce-ae55-2973cbf13837",
+    apiHost = "https://gptcloud.arc53.com",
+    theme = "dark",
+    placeholder = "Search or Ask AI...",
+    width = "256px",
+    buttonText = "Search here"
+}: SearchBarProps) => {
+    const [input, setInput] = React.useState<string>("");
+    const [loading, setLoading] = React.useState<boolean>(false);
+    const [isWidgetOpen, setIsWidgetOpen] = React.useState<boolean>(false);
+    const inputRef = React.useRef<HTMLInputElement>(null);
+    const containerRef = React.useRef<HTMLInputElement>(null);
+    const [isResultVisible, setIsResultVisible] = React.useState<boolean>(false);
+    const [results, setResults] = React.useState<Result[]>([]);
+    const debounceTimeout = React.useRef<ReturnType<typeof setTimeout> | null>(null);
+    const abortControllerRef = React.useRef<AbortController | null>(null);
+    const browserOS = getOS();
+    const isTouch = 'ontouchstart' in window;
+    
+    const getKeyboardInstruction = () => {
+        if (isResultVisible) return "Enter";
+        return browserOS === 'mac' ? '⌘ + K' : 'Ctrl + K';
+    };
+
+    React.useEffect(() => {
+        loadGeistFont()
+        const handleClickOutside = (event: MouseEvent) => {
+            if (containerRef.current && !containerRef.current.contains(event.target as Node)) {
+                setIsResultVisible(false);
+            }
+        };
+
+        const handleKeyDown = (event: KeyboardEvent) => {
+            if (
+                ((browserOS === 'win' || browserOS === 'linux') && event.ctrlKey && event.key === 'k') ||
+                (browserOS === 'mac' && event.metaKey && event.key === 'k')
+            ) {
+                event.preventDefault();
+                inputRef.current?.focus();
+                setIsResultVisible(true);
+            } else if (event.key === 'Escape') {
+                setIsResultVisible(false);
+            }
+        };
+
+     
+        document.addEventListener('mousedown', handleClickOutside);
+        document.addEventListener('keydown', handleKeyDown);
+        return () => {
+            document.removeEventListener('mousedown', handleClickOutside);
+            document.removeEventListener('keydown', handleKeyDown);
+        };
+    }, []);
+
+    React.useEffect(() => {
+        if (!input) {
+            setResults([]);
+            return;
+        }
+        setLoading(true);
+        if (debounceTimeout.current) {
+            clearTimeout(debounceTimeout.current);
+        }
+
+        if (abortControllerRef.current) {
+            abortControllerRef.current.abort();
+        }
+        const abortController = new AbortController();
+        abortControllerRef.current = abortController;
+
+        debounceTimeout.current = setTimeout(() => {
+            getSearchResults(input, apiKey, apiHost, abortController.signal)
+                .then((data) => setResults(data))
+                .catch((err) => !abortController.signal.aborted && console.log(err))
+                .finally(() => setLoading(false));
+        }, 500);
+
+        return () => {
+            abortController.abort();
+            clearTimeout(debounceTimeout.current ?? undefined);
+        };
+    }, [input])
+
+    const handleKeyDown = (event: React.KeyboardEvent<HTMLInputElement>) => {
+        if (event.key === 'Enter') {
+            event.preventDefault();
+            openWidget();
+        }
+    };
+
+    const openWidget = () => {
+        setIsWidgetOpen(true);
+        setIsResultVisible(false);
+    };
+
+    const handleClose = () => {
+        setIsWidgetOpen(false);
+        setIsResultVisible(true);
+    };
+
+    return (
+        <ThemeProvider theme={{ ...themes[theme] }}>
+            <Main>
+                <GlobalStyle />
+                <Container ref={containerRef}>
+                    <SearchButton
+                        onClick={() => setIsResultVisible(true)}
+                        inputWidth={width}
+                    >
+                        {buttonText}
+                    </SearchButton>
+                    {
+                        isResultVisible && (
+                            <SearchResults>
+                                <SearchHeader>
+                                    <TextField
+                                        ref={inputRef}
+                                        value={input}
+                                        onChange={(e) => setInput(e.target.value)}
+                                        onKeyDown={(e) => handleKeyDown(e)}
+                                        placeholder={placeholder}
+                                        autoFocus
+                                    />
+                                    <EscapeInstruction onClick={() => setIsResultVisible(false)}>
+                                        Esc
+                                    </EscapeInstruction>
+                                </SearchHeader>
+                                <AskAIButton onClick={openWidget}>
+                                    <img 
+                                        src="https://d3dg1063dc54p9.cloudfront.net/cute-docsgpt.png" 
+                                        alt="DocsGPT"
+                                        width={24}
+                                        height={24}
+                                    />
+                                    <span>Ask the AI</span>
+                                </AskAIButton>
+                                <SearchResultsScroll>
+                                    {!loading ? (
+                                        results.length > 0 ? (
+                                            results.map((res, key) => {
+                                                const containsSource = res.source !== 'local';
+                                                const processedResults = processMarkdownString(res.text, input);
+                                                if (processedResults)
+                                                    return (
+                                                        <ResultWrapper
+                                                            key={key}
+                                                            onClick={() => {
+                                                                if (!containsSource) return;
+                                                                window.open(res.source, '_blank', 'noopener, noreferrer');
+                                                            }}
+                                                        >
+                                                            <div style={{ flex: 1 }}>
+                                                                <ContentWrapper>
+                                                                    <IconTitleWrapper>
+                                                                        <ReaderIcon className="title-icon" />
+                                                                        <Title>{res.title}</Title>
+                                                                    </IconTitleWrapper>
+                                                                    <Content>
+                                                                        {processedResults.map((element, index) => (
+                                                                            <ContentSegment key={index}>
+                                                                                <IconTitleWrapper>
+                                                                                    {element.tag === 'code' && <CodeIcon className="element-icon" />}
+                                                                                    {(element.tag === 'bulletList' || element.tag === 'numberedList') && <ListBulletIcon className="element-icon" />}
+                                                                                    {element.tag === 'text' && <TextAlignLeftIcon className="element-icon" />}
+                                                                                    {element.tag === 'heading' && <HeadingIcon className="element-icon" />}
+                                                                                    {element.tag === 'blockquote' && <QuoteIcon className="element-icon" />}
+                                                                                </IconTitleWrapper>
+                                                                                <div
+                                                                                    style={{ flex: 1 }}
+                                                                                    dangerouslySetInnerHTML={{
+                                                                                        __html: DOMPurify.sanitize(element.content),
+                                                                                    }}
+                                                                                />
+                                                                            </ContentSegment>
+                                                                        ))}
+                                                                    </Content>
+                                                                </ContentWrapper>
+                                                            </div>
+                                                        </ResultWrapper>
+                                                    );
+                                                return null;
+                                            })
+                                        ) : (
+                                            <NoResults>No results found</NoResults>
+                                        )
+                                    ) : (
+                                        <Loader />
+                                    )}
+                                </SearchResultsScroll>
+                            </SearchResults>
+                        )
+                    }
+                    {
+                        isTouch ?
+
+                            <Toolkit
+                                onClick={() => {
+                                    setIsWidgetOpen(true)
+                                }}
+                                title={"Tap to Ask the AI"}>
+                                Tap
+                            </Toolkit>
+                            :
+                            <Toolkit
+                                title={getKeyboardInstruction() === "Enter" ? "Press Enter to Ask AI" : ""}>
+                                {getKeyboardInstruction()}
+                            </Toolkit>
+                    }
+                </Container>
+                <WidgetCore
+                    theme={theme}
+                    apiHost={apiHost}
+                    apiKey={apiKey}
+                    prefilledQuery={input}
+                    isOpen={isWidgetOpen}
+                    handleClose={handleClose} size={"large"}
+                />
+            </Main>
+        </ThemeProvider>
+    )
+}

extensions/react-widget/src/index.html

@@ -9,11 +9,11 @@
   <body>
     <div id="app"></div>
     <script type="module" src="main.tsx"></script>
-    <script type="module" src="../dist/main.js"></script>
-    <script type="module">
+   <!--  <script type="module">
       window.onload = function() {
         renderDocsGPTWidget('app');
+        renderSearchBar('app')
       }
-    </script>
+    </script> -->
   </body>
 </html>