Merge branch 'arc53:main' into patch-1

This commit is contained in:
Shatanik Mahanty
2024-10-08 12:52:28 +05:30
committed by GitHub
18 changed files with 288 additions and 107 deletions

View File

@@ -40,6 +40,8 @@ if settings.LLM_NAME == "openai":
gpt_model = "gpt-3.5-turbo"
elif settings.LLM_NAME == "anthropic":
gpt_model = "claude-2"
elif settings.LLM_NAME == "groq":
gpt_model = "llama3-8b-8192"
if settings.MODEL_NAME: # in case there is particular model name configured
gpt_model = settings.MODEL_NAME

45
application/llm/groq.py Normal file
View File

@@ -0,0 +1,45 @@
from application.llm.base import BaseLLM
class GroqLLM(BaseLLM):
def __init__(self, api_key=None, user_api_key=None, *args, **kwargs):
from openai import OpenAI
super().__init__(*args, **kwargs)
self.client = OpenAI(api_key=api_key, base_url="https://api.groq.com/openai/v1")
self.api_key = api_key
self.user_api_key = user_api_key
def _raw_gen(
self,
baseself,
model,
messages,
stream=False,
**kwargs
):
response = self.client.chat.completions.create(
model=model, messages=messages, stream=stream, **kwargs
)
return response.choices[0].message.content
def _raw_gen_stream(
self,
baseself,
model,
messages,
stream=True,
**kwargs
):
response = self.client.chat.completions.create(
model=model, messages=messages, stream=stream, **kwargs
)
for line in response:
# import sys
# print(line.choices[0].delta.content, file=sys.stderr)
if line.choices[0].delta.content is not None:
yield line.choices[0].delta.content

View File

@@ -1,3 +1,4 @@
from application.llm.groq import GroqLLM
from application.llm.openai import OpenAILLM, AzureOpenAILLM
from application.llm.sagemaker import SagemakerAPILLM
from application.llm.huggingface import HuggingFaceLLM
@@ -17,6 +18,7 @@ class LLMCreator:
"anthropic": AnthropicLLM,
"docsgpt": DocsGPTAPILLM,
"premai": PremAILLM,
"groq": GroqLLM
}
@classmethod

View File

@@ -10,13 +10,14 @@ from application.parser.file.epub_parser import EpubParser
from application.parser.file.html_parser import HTMLParser
from application.parser.file.markdown_parser import MarkdownParser
from application.parser.file.rst_parser import RstParser
from application.parser.file.tabular_parser import PandasCSVParser
from application.parser.file.tabular_parser import PandasCSVParser,ExcelParser
from application.parser.schema.base import Document
DEFAULT_FILE_EXTRACTOR: Dict[str, BaseParser] = {
".pdf": PDFParser(),
".docx": DocxParser(),
".csv": PandasCSVParser(),
".xlsx":ExcelParser(),
".epub": EpubParser(),
".md": MarkdownParser(),
".rst": RstParser(),

View File

@@ -113,3 +113,68 @@ class PandasCSVParser(BaseParser):
return (self._row_joiner).join(text_list)
else:
return text_list
class ExcelParser(BaseParser):
r"""Excel (.xlsx) parser.
Parses Excel files using Pandas `read_excel` function.
If special parameters are required, use the `pandas_config` dict.
Args:
concat_rows (bool): whether to concatenate all rows into one document.
If set to False, a Document will be created for each row.
True by default.
col_joiner (str): Separator to use for joining cols per row.
Set to ", " by default.
row_joiner (str): Separator to use for joining each row.
Only used when `concat_rows=True`.
Set to "\n" by default.
pandas_config (dict): Options for the `pandas.read_excel` function call.
Refer to https://pandas.pydata.org/docs/reference/api/pandas.read_excel.html
for more information.
Set to empty dict by default, this means pandas will try to figure
out the table structure on its own.
"""
def __init__(
self,
*args: Any,
concat_rows: bool = True,
col_joiner: str = ", ",
row_joiner: str = "\n",
pandas_config: dict = {},
**kwargs: Any
) -> None:
"""Init params."""
super().__init__(*args, **kwargs)
self._concat_rows = concat_rows
self._col_joiner = col_joiner
self._row_joiner = row_joiner
self._pandas_config = pandas_config
def _init_parser(self) -> Dict:
"""Init parser."""
return {}
def parse_file(self, file: Path, errors: str = "ignore") -> Union[str, List[str]]:
"""Parse file."""
try:
import pandas as pd
except ImportError:
raise ValueError("pandas module is required to read Excel files.")
df = pd.read_excel(file, **self._pandas_config)
text_list = df.apply(
lambda row: (self._col_joiner).join(row.astype(str).tolist()), axis=1
).tolist()
if self._concat_rows:
return (self._row_joiner).join(text_list)
else:
return text_list

View File

@@ -49,6 +49,7 @@ openapi3-parser==1.1.18
orjson==3.10.7
packaging==24.1
pandas==2.2.3
openpyxl==3.1.5
pathable==0.4.3
pillow==10.4.0
portalocker==2.10.1

View File

@@ -22,7 +22,7 @@ class FaissStore(BaseVectorStore):
else:
self.docsearch = FAISS.load_local(self.path, embeddings, allow_dangerous_deserialization=True)
except Exception:
raise # Just re-raise the exception without assigning to e
raise
self.assert_embedding_dimensions(embeddings)

View File

@@ -9,7 +9,6 @@
"version": "0.0.0",
"dependencies": {
"@reduxjs/toolkit": "^2.2.7",
"@vercel/analytics": "^1.3.1",
"chart.js": "^4.4.4",
"i18next": "^23.15.1",
"i18next-browser-languagedetector": "^8.0.0",
@@ -2089,26 +2088,6 @@
"resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.2.0.tgz",
"integrity": "sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ=="
},
"node_modules/@vercel/analytics": {
"version": "1.3.1",
"resolved": "https://registry.npmjs.org/@vercel/analytics/-/analytics-1.3.1.tgz",
"integrity": "sha512-xhSlYgAuJ6Q4WQGkzYTLmXwhYl39sWjoMA3nHxfkvG+WdBT25c563a7QhwwKivEOZtPJXifYHR1m2ihoisbWyA==",
"dependencies": {
"server-only": "^0.0.1"
},
"peerDependencies": {
"next": ">= 13",
"react": "^18 || ^19"
},
"peerDependenciesMeta": {
"next": {
"optional": true
},
"react": {
"optional": true
}
}
},
"node_modules/@vitejs/plugin-react": {
"version": "4.3.1",
"resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-4.3.1.tgz",
@@ -8451,11 +8430,6 @@
"semver": "bin/semver.js"
}
},
"node_modules/server-only": {
"version": "0.0.1",
"resolved": "https://registry.npmjs.org/server-only/-/server-only-0.0.1.tgz",
"integrity": "sha512-qepMx2JxAa5jjfzxG79yPPq+8BuFToHd1hm7kI+Z4zAq1ftQiP7HcxMhDDItrbtwVeLg/cY2JnKnrcFkmiswNA=="
},
"node_modules/set-function-length": {
"version": "1.2.2",
"resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz",

View File

@@ -20,7 +20,6 @@
},
"dependencies": {
"@reduxjs/toolkit": "^2.2.7",
"@vercel/analytics": "^1.3.1",
"chart.js": "^4.4.4",
"i18next": "^23.15.1",
"i18next-browser-languagedetector": "^8.0.0",

View File

@@ -3,7 +3,6 @@ import Navigation from './Navigation';
import Conversation from './conversation/Conversation';
import About from './About';
import PageNotFound from './PageNotFound';
import { inject } from '@vercel/analytics';
import { useMediaQuery } from './hooks';
import { useState } from 'react';
import Setting from './settings';
@@ -11,7 +10,6 @@ import './locale/i18n';
import { Outlet } from 'react-router-dom';
import { SharedConversation } from './conversation/SharedConversation';
import { useDarkTheme } from './hooks';
inject();
function MainLayout() {
const { isMobile } = useMediaQuery();

View File

@@ -209,24 +209,28 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
<img
src={Expand}
alt="menu toggle"
className={`${
!navOpen ? 'rotate-180' : 'rotate-0'
} m-auto transition-all duration-200`}
className={`${!navOpen ? 'rotate-180' : 'rotate-0'
} m-auto transition-all duration-200`}
/>
</button>
)}
<div
ref={navRef}
className={`${
!navOpen && '-ml-96 md:-ml-[18rem]'
} duration-20 fixed top-0 z-20 flex h-full w-72 flex-col border-r-[1px] border-b-0 bg-white transition-all dark:border-r-purple-taupe dark:bg-chinese-black dark:text-white`}
className={`${!navOpen && '-ml-96 md:-ml-[18rem]'
} duration-20 fixed top-0 z-20 flex h-full w-72 flex-col border-r-[1px] border-b-0 bg-white transition-all dark:border-r-purple-taupe dark:bg-chinese-black dark:text-white`}
>
<div
className={'visible mt-2 flex h-[6vh] w-full justify-between md:h-12'}
>
<div className="my-auto mx-4 flex cursor-pointer gap-1.5">
<img className="mb-2 h-10" src={DocsGPT3} alt="" />
<p className="my-auto text-2xl font-semibold">DocsGPT</p>
<div className="my-auto mx-4 flex cursor-pointer gap-1.5" onClick={() => {
if (isMobile) {
setNavOpen(!navOpen);
}
}}>
<a href="/" className="flex gap-1.5">
<img className="mb-2 h-10" src={DocsGPT3} alt="" />
<p className="my-auto text-2xl font-semibold">DocsGPT</p>
</a>
</div>
<button
className="float-right mr-5"
@@ -237,18 +241,21 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
<img
src={Expand}
alt="menu toggle"
className={`${
!navOpen ? 'rotate-180' : 'rotate-0'
} m-auto transition-all duration-200`}
className={`${!navOpen ? 'rotate-180' : 'rotate-0'
} m-auto transition-all duration-200`}
/>
</button>
</div>
<NavLink
to={'/'}
onClick={resetConversation}
onClick={() => {
if (isMobile) {
setNavOpen(!navOpen);
}
resetConversation();
}}
className={({ isActive }) =>
`${
isActive ? 'bg-gray-3000 dark:bg-transparent' : ''
`${isActive ? 'bg-gray-3000 dark:bg-transparent' : ''
} group sticky mx-4 mt-4 flex cursor-pointer gap-2.5 rounded-3xl border border-silver p-3 hover:border-rainy-gray hover:bg-gray-3000 dark:border-purple-taupe dark:text-white dark:hover:bg-transparent`
}
>
@@ -273,6 +280,7 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
key={conversation.id}
conversation={conversation}
selectConversation={(id) => handleConversationClick(id)}
onCoversationClick={() => {if (isMobile) { setNavOpen(false) }}}
onDeleteConversation={(id) => handleDeleteConversation(id)}
onSave={(conversation) =>
updateConversationName(conversation)
@@ -296,24 +304,38 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
isDocsListOpen={isDocsListOpen}
setIsDocsListOpen={setIsDocsListOpen}
handleDeleteClick={handleDeleteClick}
handlePostDocumentSelect={(option?: string) => {
if (isMobile) {
setNavOpen(!navOpen)
}
}}
/>
<img
className="mt-2 h-9 w-9 hover:cursor-pointer"
src={UploadIcon}
onClick={() => setUploadModalState('ACTIVE')}
></img>
onClick={() => {
setUploadModalState('ACTIVE')
if (isMobile) {
setNavOpen(!navOpen);
}
}
}></img>
</div>
<p className="ml-5 mt-3 text-sm font-semibold">{t('sourceDocs')}</p>
</div>
<div className="flex flex-col gap-2 border-b-[1px] py-2 dark:border-b-purple-taupe">
<NavLink
onClick={() => {
if (isMobile) {
setNavOpen(!navOpen);
}
resetConversation();
}}
to="/settings"
className={({ isActive }) =>
`my-auto mx-4 flex h-9 cursor-pointer gap-4 rounded-3xl hover:bg-gray-100 dark:hover:bg-[#28292E] ${
isActive ? 'bg-gray-3000 dark:bg-transparent' : ''
`my-auto mx-4 flex h-9 cursor-pointer gap-4 rounded-3xl hover:bg-gray-100 dark:hover:bg-[#28292E] ${isActive ? 'bg-gray-3000 dark:bg-transparent' : ''
}`
}
onClick={resetConversation}
>
<img
src={SettingGear}
@@ -327,13 +349,17 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
</div>
<div className="flex justify-between gap-2 border-b-[1.5px] py-2 dark:border-b-purple-taupe">
<NavLink
onClick={() => {
if (isMobile) {
setNavOpen(!navOpen);
}
resetConversation();
}}
to="/about"
className={({ isActive }) =>
`my-auto mx-4 flex h-9 cursor-pointer gap-4 rounded-3xl hover:bg-gray-100 dark:hover:bg-[#28292E] ${
isActive ? 'bg-gray-3000 dark:bg-[#28292E]' : ''
`my-auto mx-4 flex h-9 cursor-pointer gap-4 rounded-3xl hover:bg-gray-100 dark:hover:bg-[#28292E] ${isActive ? 'bg-gray-3000 dark:bg-[#28292E]' : ''
}`
}
onClick={resetConversation}
>
<img
src={Info}

View File

@@ -0,0 +1,96 @@
import React, { useCallback, useRef, useState } from 'react';
import ArrowLeft from '../assets/arrow-left.svg';
import ArrowRight from '../assets/arrow-right.svg';
import { useTranslation } from 'react-i18next';
type HiddenGradientType = 'left' | 'right' | undefined;
const useTabs = () => {
const { t } = useTranslation();
const tabs = [
t('settings.general.label'),
t('settings.documents.label'),
t('settings.apiKeys.label'),
t('settings.analytics.label'),
t('settings.logs.label'),
];
return tabs;
};
interface SettingsBarProps {
setActiveTab: React.Dispatch<React.SetStateAction<string>>;
activeTab: string;
}
const SettingsBar = ({ setActiveTab, activeTab }: SettingsBarProps) => {
const [hiddenGradient, setHiddenGradient] =
useState<HiddenGradientType>('left');
const containerRef = useRef<null | HTMLDivElement>(null);
const tabs = useTabs();
const scrollTabs = useCallback(
(direction: number) => {
if (containerRef.current) {
const container = containerRef.current;
container.scrollLeft += direction * 100; // Adjust the scroll amount as needed
if (container.scrollLeft === 0) {
setHiddenGradient('left');
} else if (
container.scrollLeft + container.offsetWidth ===
container.scrollWidth
) {
setHiddenGradient('right');
} else {
setHiddenGradient(undefined);
}
}
},
[containerRef.current],
);
return (
<div className="relative mt-6 flex flex-row items-center space-x-1 md:space-x-0 overflow-auto">
<div
className={`${hiddenGradient === 'left' ? 'hidden' : ''} md:hidden absolute inset-y-0 left-6 w-14 bg-gradient-to-r from-white dark:from-raisin-black pointer-events-none`}
></div>
<div
className={`${hiddenGradient === 'right' ? 'hidden' : ''} md:hidden absolute inset-y-0 right-6 w-14 bg-gradient-to-l from-white dark:from-raisin-black pointer-events-none`}
></div>
<div className="md:hidden z-10">
<button
onClick={() => scrollTabs(-1)}
className="flex h-6 w-6 items-center rounded-full justify-center transition-all hover:bg-gray-100"
>
<img src={ArrowLeft} alt="left-arrow" className="h-3" />
</button>
</div>
<div
ref={containerRef}
className="flex flex-nowrap overflow-x-auto no-scrollbar md:space-x-4 scroll-smooth snap-x"
>
{tabs.map((tab, index) => (
<button
key={index}
onClick={() => setActiveTab(tab)}
className={`snap-start h-9 rounded-3xl px-4 font-bold ${
activeTab === tab
? 'bg-purple-3000 text-purple-30 dark:bg-dark-charcoal'
: 'text-gray-6000'
}`}
>
{tab}
</button>
))}
</div>
<div className="md:hidden z-10">
<button
onClick={() => scrollTabs(1)}
className="flex h-6 w-6 rounded-full items-center justify-center hover:bg-gray-100"
>
<img src={ArrowRight} alt="right-arrow" className="h-3" />
</button>
</div>
</div>
);
};
export default SettingsBar;

View File

@@ -11,6 +11,7 @@ type Props = {
isDocsListOpen: boolean;
setIsDocsListOpen: React.Dispatch<React.SetStateAction<boolean>>;
handleDeleteClick: any;
handlePostDocumentSelect: any;
};
function SourceDropdown({
@@ -20,6 +21,7 @@ function SourceDropdown({
setIsDocsListOpen,
isDocsListOpen,
handleDeleteClick,
handlePostDocumentSelect, // Callback function fired after a document is selected
}: Props) {
const dispatch = useDispatch();
const { t } = useTranslation();
@@ -85,6 +87,7 @@ function SourceDropdown({
onClick={() => {
dispatch(setSelectedDocs(option));
setIsDocsListOpen(false);
handlePostDocumentSelect(option);
}}
>
<span
@@ -118,7 +121,9 @@ function SourceDropdown({
className="flex cursor-pointer items-center justify-between hover:bg-gray-100 dark:text-bright-gray dark:hover:bg-purple-taupe"
onClick={handleEmptyDocumentSelect}
>
<span className="ml-4 flex-1 overflow-hidden overflow-ellipsis whitespace-nowrap py-3">
<span className="ml-4 flex-1 overflow-hidden overflow-ellipsis whitespace-nowrap py-3" onClick = {() => {
handlePostDocumentSelect(null);
}}>
{t('none')}
</span>
</div>

View File

@@ -59,7 +59,12 @@ const ConversationBubble = forwardRef<
className={`flex flex-row-reverse self-end flex-wrap ${className}`}
>
<Avatar className="mt-2 text-2xl" avatar="🧑‍💻"></Avatar>
<div className="ml-10 mr-2 flex items-center rounded-[28px] bg-purple-30 py-[14px] px-[19px] text-white max-w-full whitespace-pre-wrap leading-normal break-normal">
<div
style={{
wordBreak: 'break-word',
}}
className="ml-10 mr-2 flex items-center rounded-[28px] bg-purple-30 py-[14px] px-[19px] text-white max-w-full whitespace-pre-wrap leading-normal"
>
{message}
</div>
</div>

View File

@@ -22,6 +22,7 @@ interface ConversationProps {
interface ConversationTileProps {
conversation: ConversationProps;
selectConversation: (arg1: string) => void;
onCoversationClick: () => void; //Callback to handle click on conversation tile regardless of selected or not
onDeleteConversation: (arg1: string) => void;
onSave: ({ name, id }: ConversationProps) => void;
}
@@ -29,6 +30,7 @@ interface ConversationTileProps {
export default function ConversationTile({
conversation,
selectConversation,
onCoversationClick,
onDeleteConversation,
onSave,
}: ConversationTileProps) {
@@ -90,6 +92,7 @@ export default function ConversationTile({
setIsHovered(false);
}}
onClick={() => {
onCoversationClick();
conversationId !== conversation.id &&
selectConversation(conversation.id);
}}
@@ -158,7 +161,12 @@ export default function ConversationTile({
</button>
)}
{isOpen && (
<div className="flex-start absolute z-30 flex w-32 translate-x-1 translate-y-5 flex-col rounded-xl bg-stone-100 text-sm text-black shadow-xl dark:bg-chinese-black dark:text-chinese-silver md:w-36">
<div
className="flex-start absolute z-30 flex w-32 translate-x-1 translate-y-5 flex-col rounded-xl bg-stone-100 text-sm text-black shadow-xl dark:bg-chinese-black dark:text-chinese-silver md:w-36"
style={{
top: `${(tileRef.current?.getBoundingClientRect().top ?? 0) + window.scrollY + 8}px`,
}}
>
<button
onClick={(event: SyntheticEvent) => {
event.stopPropagation();

View File

@@ -79,7 +79,7 @@
"remote": "Remote",
"name": "Name",
"choose": "Choose Files",
"info": "Please upload .pdf, .txt, .rst, .csv, .docx, .md, .zip limited to 25mb",
"info": "Please upload .pdf, .txt, .rst, .csv, .xlsx, .docx, .md, .zip limited to 25mb",
"uploadedFiles": "Uploaded Files",
"cancel": "Cancel",
"train": "Train",

View File

@@ -3,8 +3,7 @@ import { useTranslation } from 'react-i18next';
import { useDispatch, useSelector } from 'react-redux';
import userService from '../api/services/userService';
import ArrowLeft from '../assets/arrow-left.svg';
import ArrowRight from '../assets/arrow-right.svg';
import SettingsBar from '../components/SettingsBar';
import i18n from '../locale/i18n';
import { Doc } from '../models/misc';
import {
@@ -21,13 +20,6 @@ import Widgets from './Widgets';
export default function Settings() {
const dispatch = useDispatch();
const { t } = useTranslation();
const tabs = [
t('settings.general.label'),
t('settings.documents.label'),
t('settings.apiKeys.label'),
t('settings.analytics.label'),
t('settings.logs.label'),
];
const [activeTab, setActiveTab] = React.useState(t('settings.general.label'));
const [widgetScreenshot, setWidgetScreenshot] = React.useState<File | null>(
null,
@@ -61,39 +53,7 @@ export default function Settings() {
<p className="text-2xl font-bold text-eerie-black dark:text-bright-gray">
{t('settings.label')}
</p>
<div className="mt-6 flex flex-row items-center space-x-4 overflow-auto md:space-x-8 ">
<div className="md:hidden">
<button
onClick={() => scrollTabs(-1)}
className="flex h-8 w-8 items-center justify-center rounded-full border-2 border-purple-30 transition-all hover:bg-gray-100"
>
<img src={ArrowLeft} alt="left-arrow" className="h-6 w-6" />
</button>
</div>
<div className="flex flex-nowrap space-x-4 overflow-x-auto no-scrollbar md:space-x-8">
{tabs.map((tab, index) => (
<button
key={index}
onClick={() => setActiveTab(tab)}
className={`h-9 rounded-3xl px-4 font-bold ${
activeTab === tab
? 'bg-purple-3000 text-purple-30 dark:bg-dark-charcoal'
: 'text-gray-6000'
}`}
>
{tab}
</button>
))}
</div>
<div className="md:hidden">
<button
onClick={() => scrollTabs(1)}
className="flex h-8 w-8 items-center justify-center rounded-full border-2 border-purple-30 hover:bg-gray-100"
>
<img src={ArrowRight} alt="right-arrow" className="h-6 w-6" />
</button>
</div>
</div>
<SettingsBar activeTab={activeTab} setActiveTab={setActiveTab} />
{renderActiveTab()}
{/* {activeTab === 'Widgets' && (
@@ -105,13 +65,6 @@ export default function Settings() {
</div>
);
function scrollTabs(direction: number) {
const container = document.querySelector('.flex-nowrap');
if (container) {
container.scrollLeft += direction * 100; // Adjust the scroll amount as needed
}
}
function renderActiveTab() {
switch (activeTab) {
case t('settings.general.label'):

View File

@@ -275,6 +275,7 @@ function Upload({
'application/vnd.openxmlformats-officedocument.wordprocessingml.document':
['.docx'],
'text/csv': ['.csv'],
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': ['.xlsx'],
},
});