Merge branch 'arc53:main' into issues/1262

2025-11-29 08:33:20 +00:00 · 2024-10-08 23:00:29 +05:30
parent 8308bd0039 bb82b9a9d3
commit 68f497517d
32 changed files with 500 additions and 177 deletions
--- a/HACKTOBERFEST.md
+++ b/HACKTOBERFEST.md
@@ -7,7 +7,7 @@ All contributors with accepted PRs will receive a cool Holopin! 🤩 (Watch out
 ### 🏆 Top 50 contributors will recieve a special T-shirt

 ### 🏆 [LLM Document analysis by LexEU competition](https://github.com/arc53/DocsGPT/blob/main/lexeu-competition.md): 
-A separate competition is available for those who sumbit new retrieval / workflow method that will analyze a Document using EU laws.
+A separate competition is available for those who submit new retrieval / workflow method that will analyze a Document using EU laws.
 With 200$, 100$, 50$ prize for 1st, 2nd and 3rd place respectively.
 You can find more information [here](https://github.com/arc53/DocsGPT/blob/main/lexeu-competition.md)

--- a/application/api/answer/routes.py
+++ b/application/api/answer/routes.py
@@ -40,6 +40,8 @@ if settings.LLM_NAME == "openai":
    gpt_model = "gpt-3.5-turbo"
 elif settings.LLM_NAME == "anthropic":
    gpt_model = "claude-2"
+elif settings.LLM_NAME == "groq":
+    gpt_model = "llama3-8b-8192"

 if settings.MODEL_NAME:  # in case there is particular model name configured
    gpt_model = settings.MODEL_NAME
--- a/application/api/user/routes.py
+++ b/application/api/user/routes.py
@@ -363,6 +363,7 @@ class UploadRemote(Resource):
                ),
                "name": fields.String(required=True, description="Job name"),
                "data": fields.String(required=True, description="Data to process"),
+                "repo_url": fields.String(description="GitHub repository URL"),
            },
        )
    )
@@ -377,11 +378,18 @@ class UploadRemote(Resource):
            return missing_fields

        try:
+            if "repo_url" in data:
+                source_data = data["repo_url"]
+                loader = "github"
+            else:
+                source_data = data["data"]
+                loader = data["source"]
+
            task = ingest_remote.delay(
-                source_data=data["data"],
+                source_data=source_data,
                job_name=data["name"],
                user=data["user"],
-                loader=data["source"],
+                loader=loader,
            )
        except Exception as err:
            return make_response(jsonify({"success": False, "error": str(err)}), 400)
--- a/application/llm/groq.py
+++ b/application/llm/groq.py
@@ -0,0 +1,45 @@
+from application.llm.base import BaseLLM
+
+
+
+class GroqLLM(BaseLLM):
+
+    def __init__(self, api_key=None, user_api_key=None, *args, **kwargs):
+        from openai import OpenAI
+
+        super().__init__(*args, **kwargs)
+        self.client = OpenAI(api_key=api_key, base_url="https://api.groq.com/openai/v1")
+        self.api_key = api_key
+        self.user_api_key = user_api_key
+
+    def _raw_gen(
+        self,
+        baseself,
+        model,
+        messages,
+        stream=False,
+        **kwargs
+    ):  
+        response = self.client.chat.completions.create(
+            model=model, messages=messages, stream=stream, **kwargs
+        )
+
+        return response.choices[0].message.content
+
+    def _raw_gen_stream(
+        self,
+        baseself,
+        model,
+        messages,
+        stream=True,
+        **kwargs
+    ):  
+        response = self.client.chat.completions.create(
+            model=model, messages=messages, stream=stream, **kwargs
+        )
+
+        for line in response:
+            # import sys
+            # print(line.choices[0].delta.content, file=sys.stderr)
+            if line.choices[0].delta.content is not None:
+                yield line.choices[0].delta.content
--- a/application/llm/llm_creator.py
+++ b/application/llm/llm_creator.py
@@ -1,3 +1,4 @@
+from application.llm.groq import GroqLLM
 from application.llm.openai import OpenAILLM, AzureOpenAILLM
 from application.llm.sagemaker import SagemakerAPILLM
 from application.llm.huggingface import HuggingFaceLLM
@@ -17,6 +18,7 @@ class LLMCreator:
        "anthropic": AnthropicLLM,
        "docsgpt": DocsGPTAPILLM,
        "premai": PremAILLM,
+        "groq": GroqLLM
    }

    @classmethod
--- a/application/parser/file/bulk.py
+++ b/application/parser/file/bulk.py
@@ -10,13 +10,14 @@ from application.parser.file.epub_parser import EpubParser
 from application.parser.file.html_parser import HTMLParser
 from application.parser.file.markdown_parser import MarkdownParser
 from application.parser.file.rst_parser import RstParser
-from application.parser.file.tabular_parser import PandasCSVParser
+from application.parser.file.tabular_parser import PandasCSVParser,ExcelParser
 from application.parser.schema.base import Document

 DEFAULT_FILE_EXTRACTOR: Dict[str, BaseParser] = {
    ".pdf": PDFParser(),
    ".docx": DocxParser(),
    ".csv": PandasCSVParser(),
+    ".xlsx":ExcelParser(),
    ".epub": EpubParser(),
    ".md": MarkdownParser(),
    ".rst": RstParser(),
--- a/application/parser/file/tabular_parser.py
+++ b/application/parser/file/tabular_parser.py
@@ -113,3 +113,68 @@ class PandasCSVParser(BaseParser):
            return (self._row_joiner).join(text_list)
        else:
            return text_list
+
+
+class ExcelParser(BaseParser):
+    r"""Excel (.xlsx) parser.
+
+    Parses Excel files using Pandas `read_excel` function.
+    If special parameters are required, use the `pandas_config` dict.
+
+    Args:
+        concat_rows (bool): whether to concatenate all rows into one document.
+            If set to False, a Document will be created for each row.
+            True by default.
+
+        col_joiner (str): Separator to use for joining cols per row.
+            Set to ", " by default.
+
+        row_joiner (str): Separator to use for joining each row.
+            Only used when `concat_rows=True`.
+            Set to "\n" by default.
+
+        pandas_config (dict): Options for the `pandas.read_excel` function call.
+            Refer to https://pandas.pydata.org/docs/reference/api/pandas.read_excel.html
+            for more information.
+            Set to empty dict by default, this means pandas will try to figure
+            out the table structure on its own.
+
+    """
+
+    def __init__(
+            self,
+            *args: Any,
+            concat_rows: bool = True,
+            col_joiner: str = ", ",
+            row_joiner: str = "\n",
+            pandas_config: dict = {},
+            **kwargs: Any
+    ) -> None:
+        """Init params."""
+        super().__init__(*args, **kwargs)
+        self._concat_rows = concat_rows
+        self._col_joiner = col_joiner
+        self._row_joiner = row_joiner
+        self._pandas_config = pandas_config
+
+    def _init_parser(self) -> Dict:
+        """Init parser."""
+        return {}
+
+    def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, List[str]]:
+        """Parse file."""
+        try:
+            import pandas as pd
+        except ImportError:
+            raise ValueError("pandas module is required to read Excel files.")
+
+        df = pd.read_excel(file, **self._pandas_config)
+
+        text_list = df.apply(
+            lambda row: (self._col_joiner).join(row.astype(str).tolist()), axis=1
+        ).tolist()
+
+        if self._concat_rows:
+            return (self._row_joiner).join(text_list)
+        else:
+            return text_list
--- a/application/parser/remote/github_loader.py
+++ b/application/parser/remote/github_loader.py
@@ -0,0 +1,53 @@
+import base64
+import requests
+from typing import List
+from application.parser.remote.base import BaseRemote
+from langchain_core.documents import Document
+
+class GitHubLoader(BaseRemote):
+    def __init__(self):
+        self.access_token = None
+        self.headers = {
+            "Authorization": f"token {self.access_token}"
+        } if self.access_token else {}
+        return
+
+    def fetch_file_content(self, repo_url: str, file_path: str) -> str:
+        url = f"https://api.github.com/repos/{repo_url}/contents/{file_path}"
+        response = requests.get(url, headers=self.headers)
+
+        if response.status_code == 200:
+            content = response.json()
+            if content.get("encoding") == "base64":
+                try:
+                    decoded_content = base64.b64decode(content["content"]).decode("utf-8")
+                    return f"Filename: {file_path}\n\n{decoded_content}"
+                except Exception as e:
+                    print(f"Error decoding content for {file_path}: {e}")
+                    raise
+            else:
+                return f"Filename: {file_path}\n\n{content['content']}"
+        else:
+            response.raise_for_status()
+
+    def fetch_repo_files(self, repo_url: str, path: str = "") -> List[str]:
+        url = f"https://api.github.com/repos/{repo_url}/contents/{path}"
+        response = requests.get(url, headers={**self.headers, "Accept": "application/vnd.github.v3.raw"})
+        contents = response.json()
+        files = []
+        for item in contents:
+            if item["type"] == "file":
+                files.append(item["path"])
+            elif item["type"] == "dir":
+                files.extend(self.fetch_repo_files(repo_url, item["path"]))
+        return files
+
+    def load_data(self, repo_url: str) -> List[Document]:
+        repo_name = repo_url.split("github.com/")[-1]
+        files = self.fetch_repo_files(repo_name)
+        documents = []
+        for file_path in files:
+            content = self.fetch_file_content(repo_name, file_path)
+            documents.append(Document(page_content=content, metadata={"title": file_path, 
+            "source": f"https://github.com/{repo_name}/blob/main/{file_path}"}))
+        return documents
--- a/application/parser/remote/remote_creator.py
+++ b/application/parser/remote/remote_creator.py
@@ -2,6 +2,7 @@ from application.parser.remote.sitemap_loader import SitemapLoader
 from application.parser.remote.crawler_loader import CrawlerLoader
 from application.parser.remote.web_loader import WebLoader
 from application.parser.remote.reddit_loader import RedditPostsLoaderRemote
+from application.parser.remote.github_loader import GitHubLoader


 class RemoteCreator:
@@ -10,6 +11,7 @@ class RemoteCreator:
        "sitemap": SitemapLoader,
        "crawler": CrawlerLoader,
        "reddit": RedditPostsLoaderRemote,
+        "github": GitHubLoader,
    }

    @classmethod
--- a/application/requirements.txt
+++ b/application/requirements.txt
@@ -49,6 +49,7 @@ openapi3-parser==1.1.18
 orjson==3.10.7
 packaging==24.1
 pandas==2.2.3
+openpyxl==3.1.5
 pathable==0.4.3
 pillow==10.4.0
 portalocker==2.10.1
--- a/application/vectorstore/faiss.py
+++ b/application/vectorstore/faiss.py
@@ -22,7 +22,7 @@ class FaissStore(BaseVectorStore):
            else:
                self.docsearch = FAISS.load_local(self.path, embeddings, allow_dangerous_deserialization=True)
        except Exception:
-            raise  # Just re-raise the exception without assigning to e
+            raise

        self.assert_embedding_dimensions(embeddings)

--- a/application/worker.py
+++ b/application/worker.py
@@ -131,10 +131,10 @@ def ingest_worker(

    logging.info(f"Ingest file: {full_path}", extra={"user": user, "job": name_job})
    file_data = {"name": name_job, "file": filename, "user": user}
-    download_file(urljoin(settings.API_URL, "/api/download"), file_data, os.path.join(full_path, filename))

    if not os.path.exists(full_path):
        os.makedirs(full_path)
+    download_file(urljoin(settings.API_URL, "/api/download"), file_data, os.path.join(full_path, filename))

    # check if file is .zip and extract it
    if filename.endswith(".zip"):
--- a/frontend/package-lock.json
+++ b/frontend/package-lock.json
@@ -9,7 +9,6 @@
      "version": "0.0.0",
      "dependencies": {
        "@reduxjs/toolkit": "^2.2.7",
-        "@vercel/analytics": "^1.3.1",
        "chart.js": "^4.4.4",
        "i18next": "^23.15.1",
        "i18next-browser-languagedetector": "^8.0.0",
@@ -2089,26 +2088,6 @@
      "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.2.0.tgz",
      "integrity": "sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ=="
    },
-    "node_modules/@vercel/analytics": {
-      "version": "1.3.1",
-      "resolved": "https://registry.npmjs.org/@vercel/analytics/-/analytics-1.3.1.tgz",
-      "integrity": "sha512-xhSlYgAuJ6Q4WQGkzYTLmXwhYl39sWjoMA3nHxfkvG+WdBT25c563a7QhwwKivEOZtPJXifYHR1m2ihoisbWyA==",
-      "dependencies": {
-        "server-only": "^0.0.1"
-      },
-      "peerDependencies": {
-        "next": ">= 13",
-        "react": "^18 || ^19"
-      },
-      "peerDependenciesMeta": {
-        "next": {
-          "optional": true
-        },
-        "react": {
-          "optional": true
-        }
-      }
-    },
    "node_modules/@vitejs/plugin-react": {
      "version": "4.3.1",
      "resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-4.3.1.tgz",
@@ -8451,11 +8430,6 @@
        "semver": "bin/semver.js"
      }
    },
-    "node_modules/server-only": {
-      "version": "0.0.1",
-      "resolved": "https://registry.npmjs.org/server-only/-/server-only-0.0.1.tgz",
-      "integrity": "sha512-qepMx2JxAa5jjfzxG79yPPq+8BuFToHd1hm7kI+Z4zAq1ftQiP7HcxMhDDItrbtwVeLg/cY2JnKnrcFkmiswNA=="
-    },
    "node_modules/set-function-length": {
      "version": "1.2.2",
      "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz",
--- a/frontend/package.json
+++ b/frontend/package.json
@@ -20,7 +20,6 @@
  },
  "dependencies": {
    "@reduxjs/toolkit": "^2.2.7",
-    "@vercel/analytics": "^1.3.1",
    "chart.js": "^4.4.4",
    "i18next": "^23.15.1",
    "i18next-browser-languagedetector": "^8.0.0",
--- a/frontend/src/Hero.tsx
+++ b/frontend/src/Hero.tsx
@@ -37,7 +37,7 @@ export default function Hero({
              <Fragment key={key}>
                <button
                  onClick={() => handleQuestion({ question: demo.query })}
-                  className="w-full rounded-full border-2 border-silver px-6 py-4 text-left hover:border-gray-4000 dark:hover:border-gray-3000 xl:min-w-[24vw]"
+                  className="w-full rounded-full border border-silver px-6 py-4 text-left hover:border-gray-4000 dark:hover:border-gray-3000 xl:min-w-[24vw]"
                >
                  <p className="mb-1 font-semibold text-black dark:text-silver">
                    {demo.header}
--- a/frontend/src/Navigation.tsx
+++ b/frontend/src/Navigation.tsx
@@ -119,6 +119,7 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
      .delete(id, {})
      .then(() => {
        fetchConversations();
+        resetConversation();
      })
      .catch((error) => console.error(error));
  };
@@ -155,6 +156,15 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
      });
  };

+  const resetConversation = () => {
+    dispatch(setConversation([]));
+    dispatch(
+      updateConversationId({
+        query: { conversationId: null },
+      }),
+    );
+  };
+
  async function updateConversationName(updatedConversation: {
    name: string;
    id: string;
@@ -199,24 +209,28 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
          <img
            src={Expand}
            alt="menu toggle"
-            className={`${
-              !navOpen ? 'rotate-180' : 'rotate-0'
-            } m-auto transition-all duration-200`}
+            className={`${!navOpen ? 'rotate-180' : 'rotate-0'
+              } m-auto transition-all duration-200`}
          />
        </button>
      )}
      <div
        ref={navRef}
-        className={`${
-          !navOpen && '-ml-96 md:-ml-[18rem]'
-        } duration-20 fixed top-0 z-20 flex h-full w-72 flex-col border-r-[1px] border-b-0 bg-white transition-all dark:border-r-purple-taupe dark:bg-chinese-black dark:text-white`}
+        className={`${!navOpen && '-ml-96 md:-ml-[18rem]'
+          } duration-20 fixed top-0 z-20 flex h-full w-72 flex-col border-r-[1px] border-b-0 bg-white transition-all dark:border-r-purple-taupe dark:bg-chinese-black dark:text-white`}
      >
        <div
          className={'visible mt-2 flex h-[6vh] w-full justify-between md:h-12'}
        >
-          <div className="my-auto mx-4 flex cursor-pointer gap-1.5">
-            <img className="mb-2 h-10" src={DocsGPT3} alt="" />
-            <p className="my-auto text-2xl font-semibold">DocsGPT</p>
+          <div className="my-auto mx-4 flex cursor-pointer gap-1.5" onClick={() => {
+            if (isMobile) {
+              setNavOpen(!navOpen);
+            }
+          }}>
+            <a href="/" className="flex gap-1.5">
+              <img className="mb-2 h-10" src={DocsGPT3} alt="" />
+              <p className="my-auto text-2xl font-semibold">DocsGPT</p>
+            </a>
          </div>
          <button
            className="float-right mr-5"
@@ -227,25 +241,21 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
            <img
              src={Expand}
              alt="menu toggle"
-              className={`${
-                !navOpen ? 'rotate-180' : 'rotate-0'
-              } m-auto transition-all duration-200`}
+              className={`${!navOpen ? 'rotate-180' : 'rotate-0'
+                } m-auto transition-all duration-200`}
            />
          </button>
        </div>
        <NavLink
          to={'/'}
          onClick={() => {
-            dispatch(setConversation([]));
-            dispatch(
-              updateConversationId({
-                query: { conversationId: null },
-              }),
-            );
+            if (isMobile) {
+              setNavOpen(!navOpen);
+            }
+            resetConversation();
          }}
          className={({ isActive }) =>
-            `${
-              isActive ? 'bg-gray-3000 dark:bg-transparent' : ''
+            `${isActive ? 'bg-gray-3000 dark:bg-transparent' : ''
            } group sticky mx-4 mt-4 flex cursor-pointer gap-2.5 rounded-3xl border border-silver p-3 hover:border-rainy-gray hover:bg-gray-3000 dark:border-purple-taupe dark:text-white dark:hover:bg-transparent`
          }
        >
@@ -270,6 +280,7 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
                    key={conversation.id}
                    conversation={conversation}
                    selectConversation={(id) => handleConversationClick(id)}
+                    onCoversationClick={() => {if (isMobile) { setNavOpen(false) }}}
                    onDeleteConversation={(id) => handleDeleteConversation(id)}
                    onSave={(conversation) =>
                      updateConversationName(conversation)
@@ -293,21 +304,36 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
                isDocsListOpen={isDocsListOpen}
                setIsDocsListOpen={setIsDocsListOpen}
                handleDeleteClick={handleDeleteClick}
+                handlePostDocumentSelect={(option?: string) => {
+                  if (isMobile) {
+                    setNavOpen(!navOpen)
+                  }
+                }}
              />
              <img
                className="mt-2 h-9 w-9 hover:cursor-pointer"
                src={UploadIcon}
-                onClick={() => setUploadModalState('ACTIVE')}
-              ></img>
+                onClick={() => {
+                  setUploadModalState('ACTIVE')
+                  if (isMobile) {
+                    setNavOpen(!navOpen);
+                  }
+                }
+                }></img>
            </div>
            <p className="ml-5 mt-3 text-sm font-semibold">{t('sourceDocs')}</p>
          </div>
          <div className="flex flex-col gap-2 border-b-[1px] py-2 dark:border-b-purple-taupe">
            <NavLink
+              onClick={() => {
+                if (isMobile) {
+                  setNavOpen(!navOpen);
+                }
+                resetConversation();
+              }}
              to="/settings"
              className={({ isActive }) =>
-                `my-auto mx-4 flex h-9 cursor-pointer gap-4 rounded-3xl hover:bg-gray-100 dark:hover:bg-[#28292E] ${
-                  isActive ? 'bg-gray-3000 dark:bg-transparent' : ''
+                `my-auto mx-4 flex h-9 cursor-pointer gap-4 rounded-3xl hover:bg-gray-100 dark:hover:bg-[#28292E] ${isActive ? 'bg-gray-3000 dark:bg-transparent' : ''
                }`
              }
            >
@@ -323,10 +349,15 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
          </div>
          <div className="flex justify-between gap-2 border-b-[1.5px] py-2 dark:border-b-purple-taupe">
            <NavLink
+              onClick={() => {
+                if (isMobile) {
+                  setNavOpen(!navOpen);
+                }
+                resetConversation();
+              }}
              to="/about"
              className={({ isActive }) =>
-                `my-auto mx-4 flex h-9 cursor-pointer gap-4 rounded-3xl hover:bg-gray-100 dark:hover:bg-[#28292E] ${
-                  isActive ? 'bg-gray-3000 dark:bg-[#28292E]' : ''
+                `my-auto mx-4 flex h-9 cursor-pointer gap-4 rounded-3xl hover:bg-gray-100 dark:hover:bg-[#28292E] ${isActive ? 'bg-gray-3000 dark:bg-[#28292E]' : ''
                }`
              }
            >
--- a/frontend/src/components/SettingsBar.tsx
+++ b/frontend/src/components/SettingsBar.tsx
@@ -0,0 +1,96 @@
+import React, { useCallback, useRef, useState } from 'react';
+import ArrowLeft from '../assets/arrow-left.svg';
+import ArrowRight from '../assets/arrow-right.svg';
+import { useTranslation } from 'react-i18next';
+
+type HiddenGradientType = 'left' | 'right' | undefined;
+
+const useTabs = () => {
+  const { t } = useTranslation();
+  const tabs = [
+    t('settings.general.label'),
+    t('settings.documents.label'),
+    t('settings.apiKeys.label'),
+    t('settings.analytics.label'),
+    t('settings.logs.label'),
+  ];
+  return tabs;
+};
+
+interface SettingsBarProps {
+  setActiveTab: React.Dispatch<React.SetStateAction<string>>;
+  activeTab: string;
+}
+
+const SettingsBar = ({ setActiveTab, activeTab }: SettingsBarProps) => {
+  const [hiddenGradient, setHiddenGradient] =
+    useState<HiddenGradientType>('left');
+  const containerRef = useRef<null | HTMLDivElement>(null);
+  const tabs = useTabs();
+  const scrollTabs = useCallback(
+    (direction: number) => {
+      if (containerRef.current) {
+        const container = containerRef.current;
+        container.scrollLeft += direction * 100; // Adjust the scroll amount as needed
+        if (container.scrollLeft === 0) {
+          setHiddenGradient('left');
+        } else if (
+          container.scrollLeft + container.offsetWidth ===
+          container.scrollWidth
+        ) {
+          setHiddenGradient('right');
+        } else {
+          setHiddenGradient(undefined);
+        }
+      }
+    },
+    [containerRef.current],
+  );
+  return (
+    <div className="relative mt-6 flex flex-row items-center space-x-1 md:space-x-0 overflow-auto">
+      <div
+        className={`${hiddenGradient === 'left' ? 'hidden' : ''} md:hidden absolute inset-y-0 left-6 w-14 bg-gradient-to-r from-white dark:from-raisin-black pointer-events-none`}
+      ></div>
+      <div
+        className={`${hiddenGradient === 'right' ? 'hidden' : ''} md:hidden absolute inset-y-0 right-6 w-14 bg-gradient-to-l from-white dark:from-raisin-black pointer-events-none`}
+      ></div>
+
+      <div className="md:hidden z-10">
+        <button
+          onClick={() => scrollTabs(-1)}
+          className="flex h-6 w-6 items-center rounded-full justify-center transition-all hover:bg-gray-100"
+        >
+          <img src={ArrowLeft} alt="left-arrow" className="h-3" />
+        </button>
+      </div>
+      <div
+        ref={containerRef}
+        className="flex flex-nowrap overflow-x-auto no-scrollbar md:space-x-4 scroll-smooth snap-x"
+      >
+        {tabs.map((tab, index) => (
+          <button
+            key={index}
+            onClick={() => setActiveTab(tab)}
+            className={`snap-start h-9 rounded-3xl px-4 font-bold ${
+              activeTab === tab
+                ? 'bg-purple-3000 text-purple-30 dark:bg-dark-charcoal'
+                : 'text-gray-6000'
+            }`}
+          >
+            {tab}
+          </button>
+        ))}
+      </div>
+      <div className="md:hidden z-10">
+        <button
+          onClick={() => scrollTabs(1)}
+          className="flex h-6 w-6 rounded-full items-center justify-center hover:bg-gray-100"
+        >
+          <img src={ArrowRight} alt="right-arrow" className="h-3" />
+        </button>
+      </div>
+    </div>
+  );
+};
+
+export default SettingsBar;
--- a/frontend/src/components/SourceDropdown.tsx
+++ b/frontend/src/components/SourceDropdown.tsx
@@ -11,6 +11,7 @@ type Props = {
  isDocsListOpen: boolean;
  setIsDocsListOpen: React.Dispatch<React.SetStateAction<boolean>>;
  handleDeleteClick: any;
+  handlePostDocumentSelect: any;
 };

 function SourceDropdown({
@@ -20,6 +21,7 @@ function SourceDropdown({
  setIsDocsListOpen,
  isDocsListOpen,
  handleDeleteClick,
+  handlePostDocumentSelect, // Callback function fired after a document is selected
 }: Props) {
  const dispatch = useDispatch();
  const { t } = useTranslation();
@@ -85,6 +87,7 @@ function SourceDropdown({
                    onClick={() => {
                      dispatch(setSelectedDocs(option));
                      setIsDocsListOpen(false);
+                      handlePostDocumentSelect(option);
                    }}
                  >
                    <span
@@ -118,7 +121,9 @@ function SourceDropdown({
            className="flex cursor-pointer items-center justify-between hover:bg-gray-100 dark:text-bright-gray dark:hover:bg-purple-taupe"
            onClick={handleEmptyDocumentSelect}
          >
-            <span className="ml-4 flex-1 overflow-hidden overflow-ellipsis whitespace-nowrap py-3">
+            <span className="ml-4 flex-1 overflow-hidden overflow-ellipsis whitespace-nowrap py-3" onClick = {() => {
+              handlePostDocumentSelect(null);
+            }}>
              {t('none')}
            </span>
          </div>
--- a/frontend/src/conversation/Conversation.tsx
+++ b/frontend/src/conversation/Conversation.tsx
@@ -54,10 +54,6 @@ export default function Conversation() {
    }
  }, []);

-  useEffect(() => {
-    fetchStream.current && fetchStream.current.abort();
-  }, [conversationId]);
-
  useEffect(() => {
    if (queries.length) {
      queries[queries.length - 1].error && setLastQueryReturnedErr(true);
--- a/frontend/src/conversation/ConversationBubble.tsx
+++ b/frontend/src/conversation/ConversationBubble.tsx
@@ -59,7 +59,12 @@ const ConversationBubble = forwardRef<
        className={`flex flex-row-reverse self-end flex-wrap ${className}`}
      >
        <Avatar className="mt-2 text-2xl" avatar="🧑‍💻"></Avatar>
-        <div className="ml-10 mr-2 flex items-center rounded-[28px] bg-purple-30 py-[14px] px-[19px] text-white max-w-full whitespace-pre-wrap leading-normal break-normal">
+        <div
+          style={{
+            wordBreak: 'break-word',
+          }}
+          className="ml-10 mr-2 flex items-center rounded-[28px] bg-purple-30 py-[14px] px-[19px] text-white max-w-full whitespace-pre-wrap leading-normal"
+        >
          {message}
        </div>
      </div>
--- a/frontend/src/conversation/ConversationTile.tsx
+++ b/frontend/src/conversation/ConversationTile.tsx
@@ -22,6 +22,7 @@ interface ConversationProps {
 interface ConversationTileProps {
  conversation: ConversationProps;
  selectConversation: (arg1: string) => void;
+  onCoversationClick: () => void; //Callback to handle click on conversation tile regardless of selected or not
  onDeleteConversation: (arg1: string) => void;
  onSave: ({ name, id }: ConversationProps) => void;
 }
@@ -29,6 +30,7 @@ interface ConversationTileProps {
 export default function ConversationTile({
  conversation,
  selectConversation,
+  onCoversationClick,
  onDeleteConversation,
  onSave,
 }: ConversationTileProps) {
@@ -90,6 +92,7 @@ export default function ConversationTile({
          setIsHovered(false);
        }}
        onClick={() => {
+          onCoversationClick();
          conversationId !== conversation.id &&
            selectConversation(conversation.id);
        }}
@@ -158,7 +161,12 @@ export default function ConversationTile({
              </button>
            )}
            {isOpen && (
-              <div className="flex-start absolute z-30 flex w-32 translate-x-1 translate-y-5 flex-col rounded-xl bg-stone-100 text-sm text-black shadow-xl dark:bg-chinese-black dark:text-chinese-silver md:w-36">
+              <div
+                className="flex-start absolute z-30 flex w-32 translate-x-1 translate-y-5 flex-col rounded-xl bg-stone-100 text-sm text-black shadow-xl dark:bg-chinese-black dark:text-chinese-silver md:w-36"
+                style={{
+                  top: `${(tileRef.current?.getBoundingClientRect().top ?? 0) + window.scrollY + 8}px`,
+                }}
+              >
                <button
                  onClick={(event: SyntheticEvent) => {
                    event.stopPropagation();
--- a/frontend/src/conversation/conversationSlice.ts
+++ b/frontend/src/conversation/conversationSlice.ts
@@ -151,6 +151,7 @@ export const conversationSlice = createSlice({
      state,
      action: PayloadAction<{ index: number; query: Partial<Query> }>,
    ) {
+      if (state.status === 'idle') return;
      const { index, query } = action.payload;
      if (query.response != undefined) {
        state.queries[index].response =
@@ -167,6 +168,7 @@ export const conversationSlice = createSlice({
      action: PayloadAction<{ query: Partial<Query> }>,
    ) {
      state.conversationId = action.payload.query.conversationId ?? null;
+      state.status = 'idle';
    },
    updateStreamingSource(
      state,
--- a/frontend/src/index.css
+++ b/frontend/src/index.css
@@ -47,6 +47,28 @@ body.dark {
  }
 }

+@layer components {
+  .table-default {
+    @apply block w-max table-auto content-center justify-center rounded-xl border border-silver dark:border-silver/40 text-center dark:text-bright-gray;
+  }
+
+  .table-default th {
+    @apply border-r border-silver dark:border-silver/40 p-4 w-[244px];
+  }
+
+  .table-default th:last-child {
+    @apply w-[auto] border-r-0;
+  }
+
+  .table-default td {
+    @apply border-r border-t border-silver dark:border-silver/40 px-4 py-2;
+  }
+
+  .table-default td:last-child {
+    @apply border-r-0;
+  }
+}
+
 /*! normalize.css v8.0.1 | MIT License | github.com/necolas/normalize.css */

 /* Document
--- a/frontend/src/locale/en.json
+++ b/frontend/src/locale/en.json
@@ -79,12 +79,13 @@
      "remote": "Remote",
      "name": "Name",
      "choose": "Choose Files",
-      "info": "Please upload .pdf, .txt, .rst, .csv, .docx, .md, .zip limited to 25mb",
+      "info": "Please upload .pdf, .txt, .rst, .csv, .xlsx, .docx, .md, .zip limited to 25mb",
      "uploadedFiles": "Uploaded Files",
      "cancel": "Cancel",
      "train": "Train",
      "link": "Link",
      "urlLink": "URL Link",
+      "repoUrl": "Repository URL",
      "reddit": {
        "id": "Client ID",
        "secret": "Client Secret",
--- a/frontend/src/locale/es.json
+++ b/frontend/src/locale/es.json
@@ -85,6 +85,7 @@
      "train": "Entrenar",
      "link": "Enlace",
      "urlLink": "Enlace URL",
+      "repoUrl": "URL del Repositorio",
      "reddit": {
        "id": "ID de Cliente",
        "secret": "Secreto de Cliente",
--- a/frontend/src/locale/jp.json
+++ b/frontend/src/locale/jp.json
@@ -85,6 +85,7 @@
      "train": "トレーニング",
      "link": "リンク",
      "urlLink": "URLリンク",
+      "repoUrl": "リポジトリURL",
      "reddit": {
        "id": "クライアントID",
        "secret": "クライアントシークレット",
--- a/frontend/src/locale/zh.json
+++ b/frontend/src/locale/zh.json
@@ -85,6 +85,7 @@
      "train": "训练",
      "link": "链接",
      "urlLink": "URL 链接",
+      "repoUrl": "存储库 URL",
      "reddit": {
        "id": "客户端 ID",
        "secret": "客户端密钥",
--- a/frontend/src/settings/APIKeys.tsx
+++ b/frontend/src/settings/APIKeys.tsx
@@ -100,35 +100,29 @@ export default function APIKeys() {
        )}
        <div className="mt-[27px] w-full">
          <div className="w-full overflow-x-auto">
-            <table className="block w-max table-auto content-center justify-center rounded-xl border text-center dark:border-chinese-silver dark:text-bright-gray">
+            <table className="table-default">
              <thead>
                <tr>
-                  <th className="w-[244px] border-r p-4">
-                    {t('settings.apiKeys.name')}
-                  </th>
-                  <th className="w-[244px] border-r px-4 py-2">
-                    {t('settings.apiKeys.sourceDoc')}
-                  </th>
-                  <th className="w-[244px] border-r px-4 py-2">
-                    {t('settings.apiKeys.key')}
-                  </th>
-                  <th className="px-4 py-2"></th>
+                  <th>{t('settings.apiKeys.name')}</th>
+                  <th>{t('settings.apiKeys.sourceDoc')}</th>
+                  <th>{t('settings.apiKeys.key')}</th>
+                  <th></th>
                </tr>
              </thead>
              <tbody>
                {!apiKeys?.length && (
                  <tr>
-                    <td colSpan={4} className="border-t p-4">
+                    <td colSpan={4} className="!p-4">
                      {t('settings.apiKeys.noData')}
                    </td>
                  </tr>
                )}
                {apiKeys?.map((element, index) => (
                  <tr key={index}>
-                    <td className="border-r border-t p-4">{element.name}</td>
-                    <td className="border-r border-t p-4">{element.source}</td>
-                    <td className="border-r border-t p-4">{element.key}</td>
-                    <td className="border-t p-4">
+                    <td>{element.name}</td>
+                    <td>{element.source}</td>
+                    <td>{element.key}</td>
+                    <td>
                      <img
                        src={Trash}
                        alt="Delete"
--- a/frontend/src/settings/Documents.tsx
+++ b/frontend/src/settings/Documents.tsx
@@ -55,28 +55,20 @@ const Documents: React.FC<DocumentsProps> = ({
    <div className="mt-8">
      <div className="flex flex-col relative">
        <div className="z-10 w-full overflow-x-auto">
-          <table className="block w-max table-auto content-center justify-center rounded-xl border text-center dark:border-chinese-silver dark:text-bright-gray">
+          <table className="table-default">
            <thead>
              <tr>
-                <th className="border-r p-4 md:w-[244px]">
-                  {t('settings.documents.name')}
-                </th>
-                <th className="w-[244px] border-r px-4 py-2">
-                  {t('settings.documents.date')}
-                </th>
-                <th className="w-[244px] border-r px-4 py-2">
-                  {t('settings.documents.tokenUsage')}
-                </th>
-                <th className="w-[244px] border-r px-4 py-2">
-                  {t('settings.documents.type')}
-                </th>
-                <th className="px-4 py-2"></th>
+                <th>{t('settings.documents.name')}</th>
+                <th>{t('settings.documents.date')}</th>
+                <th>{t('settings.documents.tokenUsage')}</th>
+                <th>{t('settings.documents.type')}</th>
+                <th></th>
              </tr>
            </thead>
            <tbody>
              {!documents?.length && (
                <tr>
-                  <td colSpan={5} className="border-t p-4">
+                  <td colSpan={5} className="!p-4">
                    {t('settings.documents.noData')}
                  </td>
                </tr>
@@ -84,19 +76,15 @@ const Documents: React.FC<DocumentsProps> = ({
              {documents &&
                documents.map((document, index) => (
                  <tr key={index}>
-                    <td className="border-r border-t px-4 py-2">
-                      {document.name}
-                    </td>
-                    <td className="border-r border-t px-4 py-2">
-                      {document.date}
-                    </td>
-                    <td className="border-r border-t px-4 py-2">
+                    <td>{document.name}</td>
+                    <td>{document.date}</td>
+                    <td>
                      {document.tokens ? formatTokens(+document.tokens) : ''}
                    </td>
-                    <td className="border-r border-t px-4 py-2">
+                    <td>
                      {document.type === 'remote' ? 'Pre-loaded' : 'Private'}
                    </td>
-                    <td className="border-t px-4 py-2">
+                    <td>
                      <div className="flex flex-row items-center">
                        {document.type !== 'remote' && (
                          <img
--- a/frontend/src/settings/index.tsx
+++ b/frontend/src/settings/index.tsx
@@ -3,8 +3,7 @@ import { useTranslation } from 'react-i18next';
 import { useDispatch, useSelector } from 'react-redux';

 import userService from '../api/services/userService';
-import ArrowLeft from '../assets/arrow-left.svg';
-import ArrowRight from '../assets/arrow-right.svg';
+import SettingsBar from '../components/SettingsBar';
 import i18n from '../locale/i18n';
 import { Doc } from '../models/misc';
 import {
@@ -21,13 +20,6 @@ import Widgets from './Widgets';
 export default function Settings() {
  const dispatch = useDispatch();
  const { t } = useTranslation();
-  const tabs = [
-    t('settings.general.label'),
-    t('settings.documents.label'),
-    t('settings.apiKeys.label'),
-    t('settings.analytics.label'),
-    t('settings.logs.label'),
-  ];
  const [activeTab, setActiveTab] = React.useState(t('settings.general.label'));
  const [widgetScreenshot, setWidgetScreenshot] = React.useState<File | null>(
    null,
@@ -61,39 +53,7 @@ export default function Settings() {
      <p className="text-2xl font-bold text-eerie-black dark:text-bright-gray">
        {t('settings.label')}
      </p>
-      <div className="mt-6 flex flex-row items-center space-x-4 overflow-auto md:space-x-8 ">
-        <div className="md:hidden">
-          <button
-            onClick={() => scrollTabs(-1)}
-            className="flex h-8 w-8 items-center justify-center rounded-full border-2 border-purple-30 transition-all hover:bg-gray-100"
-          >
-            <img src={ArrowLeft} alt="left-arrow" className="h-6 w-6" />
-          </button>
-        </div>
-        <div className="flex flex-nowrap space-x-4 overflow-x-auto no-scrollbar md:space-x-8">
-          {tabs.map((tab, index) => (
-            <button
-              key={index}
-              onClick={() => setActiveTab(tab)}
-              className={`h-9 rounded-3xl px-4 font-bold ${
-                activeTab === tab
-                  ? 'bg-purple-3000 text-purple-30 dark:bg-dark-charcoal'
-                  : 'text-gray-6000'
-              }`}
-            >
-              {tab}
-            </button>
-          ))}
-        </div>
-        <div className="md:hidden">
-          <button
-            onClick={() => scrollTabs(1)}
-            className="flex h-8 w-8 items-center justify-center rounded-full border-2 border-purple-30 hover:bg-gray-100"
-          >
-            <img src={ArrowRight} alt="right-arrow" className="h-6 w-6" />
-          </button>
-        </div>
-      </div>
+      <SettingsBar activeTab={activeTab} setActiveTab={setActiveTab} />
      {renderActiveTab()}

      {/* {activeTab === 'Widgets' && (
@@ -105,13 +65,6 @@ export default function Settings() {
    </div>
  );

-  function scrollTabs(direction: number) {
-    const container = document.querySelector('.flex-nowrap');
-    if (container) {
-      container.scrollLeft += direction * 100; // Adjust the scroll amount as needed
-    }
-  }
-
  function renderActiveTab() {
    switch (activeTab) {
      case t('settings.general.label'):
--- a/frontend/src/upload/Upload.tsx
+++ b/frontend/src/upload/Upload.tsx
@@ -24,6 +24,7 @@ function Upload({
  const [docName, setDocName] = useState('');
  const [urlName, setUrlName] = useState('');
  const [url, setUrl] = useState('');
+  const [repoUrl, setRepoUrl] = useState(''); // P3f93
  const [redditData, setRedditData] = useState({
    client_id: '',
    client_secret: '',
@@ -48,6 +49,7 @@ function Upload({
    // { label: 'Sitemap', value: 'sitemap' },
    { label: 'Link', value: 'url' },
    { label: 'Reddit', value: 'reddit' },
+    { label: 'GitHub', value: 'github' }, // P3f93
  ];

  const [urlType, setUrlType] = useState<{ label: string; value: string }>({
@@ -238,6 +240,9 @@ function Upload({
      formData.set('name', 'other');
      formData.set('data', JSON.stringify(redditData));
    }
+    if (urlType.value === 'github') {
+      formData.append('repo_url', repoUrl); // Pdeac
+    }
    const apiHost = import.meta.env.VITE_API_HOST;
    const xhr = new XMLHttpRequest();
    xhr.upload.addEventListener('progress', (event) => {
@@ -270,6 +275,7 @@ function Upload({
      'application/vnd.openxmlformats-officedocument.wordprocessingml.document':
        ['.docx'],
      'text/csv': ['.csv'],
+      'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': ['.xlsx'],
    },
  });

@@ -376,7 +382,7 @@ function Upload({
              size="w-full"
              rounded="3xl"
            />
-            {urlType.label !== 'Reddit' ? (
+            {urlType.label !== 'Reddit' && urlType.label !== 'GitHub' ? (
              <>
                <Input
                  placeholder={`Enter ${t('modals.uploadDoc.name')}`}
@@ -403,6 +409,33 @@ function Upload({
                  </span>
                </div>
              </>
+            ) : urlType.label === 'GitHub' ? ( // P3f93
+              <>
+                <Input
+                  placeholder={`Enter ${t('modals.uploadDoc.name')}`}
+                  type="text"
+                  value={urlName}
+                  onChange={(e) => setUrlName(e.target.value)}
+                  borderVariant="thin"
+                ></Input>
+                <div className="relative bottom-12 left-2 mt-[-20px]">
+                  <span className="bg-white px-2 text-xs text-gray-4000 dark:bg-outer-space dark:text-silver">
+                    {t('modals.uploadDoc.name')}
+                  </span>
+                </div>
+                <Input
+                  placeholder={t('modals.uploadDoc.repoUrl')}
+                  type="text"
+                  value={repoUrl}
+                  onChange={(e) => setRepoUrl(e.target.value)}
+                  borderVariant="thin"
+                ></Input>
+                <div className="relative bottom-12 left-2 mt-[-20px]">
+                  <span className="bg-white px-2 text-xs text-gray-4000 dark:bg-outer-space dark:text-silver">
+                    {t('modals.uploadDoc.repoUrl')}
+                  </span>
+                </div>
+              </>
            ) : (
              <div className="flex flex-col gap-1 mt-2">
                <div>
--- a/scripts/migrate_to_v1_vectorstore.py
+++ b/scripts/migrate_to_v1_vectorstore.py
@@ -1,13 +1,35 @@
 import pymongo
 import os
+import shutil
+import logging
+from tqdm import tqdm
+
+# Configure logging
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+logger = logging.getLogger()
+
+# Configuration
+MONGO_URI = "mongodb://localhost:27017/"
+MONGO_ATLAS_URI = "mongodb+srv://<username>:<password>@<cluster>/<dbname>?retryWrites=true&w=majority"
+DB_NAME = "docsgpt"
+
+def backup_collection(collection, backup_collection_name):
+    logger.info(f"Backing up collection {collection.name} to {backup_collection_name}")
+    collection.aggregate([{"$out": backup_collection_name}])
+    logger.info("Backup completed")

 def migrate_to_v1_vectorstore_mongo():
-    client = pymongo.MongoClient("mongodb://localhost:27017/")
-    db = client["docsgpt"]
+    client = pymongo.MongoClient(MONGO_URI)
+    db = client[DB_NAME]
    vectors_collection = db["vectors"]
    sources_collection = db["sources"]

-    for vector in vectors_collection.find():
+    # Backup collections before migration
+    backup_collection(vectors_collection, "vectors_backup")
+    backup_collection(sources_collection, "sources_backup")
+
+    vectors = list(vectors_collection.find())
+    for vector in tqdm(vectors, desc="Updating vectors"):
        if "location" in vector:
            del vector["location"]
        if "retriever" not in vector:
@@ -15,41 +37,53 @@ def migrate_to_v1_vectorstore_mongo():
            vector["remote_data"] = None
        vectors_collection.update_one({"_id": vector["_id"]}, {"$set": vector})

-    # move data from vectors_collection to sources_collection
-    for vector in vectors_collection.find():
+    # Move data from vectors_collection to sources_collection
+    for vector in tqdm(vectors, desc="Moving to sources"):
        sources_collection.insert_one(vector)

    vectors_collection.drop()
-
    client.close()
+    logger.info("Migration completed")

 def migrate_faiss_to_v1_vectorstore():
-    client = pymongo.MongoClient("mongodb://localhost:27017/")
-    db = client["docsgpt"]
+    client = pymongo.MongoClient(MONGO_URI)
+    db = client[DB_NAME]
    vectors_collection = db["vectors"]

-    for vector in vectors_collection.find():
+    vectors = list(vectors_collection.find())
+    for vector in tqdm(vectors, desc="Migrating FAISS vectors"):
        old_path = f"./application/indexes/{vector['user']}/{vector['name']}"
        new_path = f"./application/indexes/{vector['_id']}"
        try:
-            os.rename(old_path, new_path)
+            os.makedirs(os.path.dirname(new_path), exist_ok=True)
+            shutil.move(old_path, new_path)
        except OSError as e:
-            print(f"Error moving {old_path} to {new_path}: {e}")
+            logger.error(f"Error moving {old_path} to {new_path}: {e}")

    client.close()
+    logger.info("FAISS migration completed")

 def migrate_mongo_atlas_vector_to_v1_vectorstore():
-    client = pymongo.MongoClient("mongodb+srv://<username>:<password>@<cluster>/<dbname>?retryWrites=true&w=majority")
-    db = client["docsgpt"]
+    client = pymongo.MongoClient(MONGO_ATLAS_URI)
+    db = client[DB_NAME]
    vectors_collection = db["vectors"]
-
-    # mongodb atlas collection
    documents_collection = db["documents"]

-    for vector in vectors_collection.find():
-        documents_collection.update_many({"store": vector["user"] + "/" + vector["name"]}, {"$set": {"source_id": str(vector["_id"])}})
+    # Backup collections before migration
+    backup_collection(vectors_collection, "vectors_backup")
+    backup_collection(documents_collection, "documents_backup")
+
+    vectors = list(vectors_collection.find())
+    for vector in tqdm(vectors, desc="Updating Mongo Atlas vectors"):
+        documents_collection.update_many(
+            {"store": vector["user"] + "/" + vector["name"]},
+            {"$set": {"source_id": str(vector["_id"])}}
+        )

    client.close()
+    logger.info("Mongo Atlas migration completed")

-migrate_faiss_to_v1_vectorstore()
-migrate_to_v1_vectorstore_mongo()
+if __name__ == "__main__":
+    migrate_faiss_to_v1_vectorstore()
+    migrate_to_v1_vectorstore_mongo()
+    migrate_mongo_atlas_vector_to_v1_vectorstore()