LLM/DocsGPT
1
0
Fork 0
mirror of https://github.com/arc53/DocsGPT.git synced 2025-11-29 08:33:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'arc53:main' into code/setting-ui

Browse Source
This commit is contained in:
Saksham Tyagi
2023-11-04 02:38:09 +05:30
committed by GitHub
parent 3d54a1abf3 b4d78376fb
commit d1c93754db
6 changed files with 351 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
application/api/user/routes.py
View File

@@ -153,7 +153,9 @@ def upload_file():
os.makedirs(save_dir)
file.save(os.path.join(save_dir, filename))
task = ingest.delay(settings.UPLOAD_FOLDER, [".rst", ".md", ".pdf", ".txt"], job_name, filename, user)
task = ingest.delay(settings.UPLOAD_FOLDER, [".rst", ".md", ".pdf", ".txt", ".docx",
".csv", ".epub", ".html", ".mdx"],
job_name, filename, user)
# task id
task_id = task.id
return {"status": "ok", "task_id": task_id}

1
application/requirements.txt
View File

@@ -20,6 +20,7 @@ click-repl==0.2.0
cryptography==41.0.4
dataclasses-json==0.5.7
decorator==5.1.1
docx2txt==0.8
dill==0.3.6
dnspython==2.3.0
ecdsa==0.18.0

314
frontend/package-lock.json generated
View File

@@ -17,7 +17,8 @@
"react-markdown": "^8.0.7",
"react-redux": "^8.0.5",
"react-router-dom": "^6.8.1",
"react-syntax-highlighter": "^15.5.0"
"react-syntax-highlighter": "^15.5.0",
"remark-gfm": "^3.0.0"
},
"devDependencies": {
"@types/react": "^18.0.27",
@@ -2077,6 +2078,15 @@
}
]
},
"node_modules/ccount": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/ccount/-/ccount-2.0.1.tgz",
"integrity": "sha512-eyrF0jiFpY+3drT6383f1qhkbGsLSifNAjA61IUjZjmLCWjItY6LB9ft9YhoDgwfmclB2zhu51Lc7+95b8NRAg==",
"funding": {
"type": "github",
"url": "https://github.com/sponsors/wooorm"
}
},
"node_modules/chalk": {
"version": "2.4.2",
"resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz",
@@ -4740,6 +4750,15 @@
"node": ">=8"
}
},
"node_modules/longest-streak": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/longest-streak/-/longest-streak-3.1.0.tgz",
"integrity": "sha512-9Ri+o0JYgehTaVBBDoMqIl8GXtbWg711O3srftcHhZ0dqnETqLaoIK0x17fUw9rFSlK/0NlsKe0Ahhyl5pXE2g==",
"funding": {
"type": "github",
"url": "https://github.com/sponsors/wooorm"
}
},
"node_modules/loose-envify": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz",
@@ -4785,6 +4804,15 @@
"node": ">=12"
}
},
"node_modules/markdown-table": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/markdown-table/-/markdown-table-3.0.3.tgz",
"integrity": "sha512-Z1NL3Tb1M9wH4XESsCDEksWoKTdlUafKc4pt0GRwjUyXaCFZ+dc3g2erqB6zm3szA2IUSi7VnPI+o/9jnxh9hw==",
"funding": {
"type": "github",
"url": "https://github.com/sponsors/wooorm"
}
},
"node_modules/mdast-util-definitions": {
"version": "5.1.2",
"resolved": "https://registry.npmjs.org/mdast-util-definitions/-/mdast-util-definitions-5.1.2.tgz",
@@ -4799,6 +4827,32 @@
"url": "https://opencollective.com/unified"
}
},
"node_modules/mdast-util-find-and-replace": {
"version": "2.2.2",
"resolved": "https://registry.npmjs.org/mdast-util-find-and-replace/-/mdast-util-find-and-replace-2.2.2.tgz",
"integrity": "sha512-MTtdFRz/eMDHXzeK6W3dO7mXUlF82Gom4y0oOgvHhh/HXZAGvIQDUvQ0SuUx+j2tv44b8xTHOm8K/9OoRFnXKw==",
"dependencies": {
"@types/mdast": "^3.0.0",
"escape-string-regexp": "^5.0.0",
"unist-util-is": "^5.0.0",
"unist-util-visit-parents": "^5.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/mdast-util-find-and-replace/node_modules/escape-string-regexp": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-5.0.0.tgz",
"integrity": "sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw==",
"engines": {
"node": ">=12"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/mdast-util-from-markdown": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/mdast-util-from-markdown/-/mdast-util-from-markdown-1.3.0.tgz",
@@ -4822,6 +4876,107 @@
"url": "https://opencollective.com/unified"
}
},
"node_modules/mdast-util-gfm": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/mdast-util-gfm/-/mdast-util-gfm-2.0.2.tgz",
"integrity": "sha512-qvZ608nBppZ4icQlhQQIAdc6S3Ffj9RGmzwUKUWuEICFnd1LVkN3EktF7ZHAgfcEdvZB5owU9tQgt99e2TlLjg==",
"dependencies": {
"mdast-util-from-markdown": "^1.0.0",
"mdast-util-gfm-autolink-literal": "^1.0.0",
"mdast-util-gfm-footnote": "^1.0.0",
"mdast-util-gfm-strikethrough": "^1.0.0",
"mdast-util-gfm-table": "^1.0.0",
"mdast-util-gfm-task-list-item": "^1.0.0",
"mdast-util-to-markdown": "^1.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/mdast-util-gfm-autolink-literal": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/mdast-util-gfm-autolink-literal/-/mdast-util-gfm-autolink-literal-1.0.3.tgz",
"integrity": "sha512-My8KJ57FYEy2W2LyNom4n3E7hKTuQk/0SES0u16tjA9Z3oFkF4RrC/hPAPgjlSpezsOvI8ObcXcElo92wn5IGA==",
"dependencies": {
"@types/mdast": "^3.0.0",
"ccount": "^2.0.0",
"mdast-util-find-and-replace": "^2.0.0",
"micromark-util-character": "^1.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/mdast-util-gfm-footnote": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/mdast-util-gfm-footnote/-/mdast-util-gfm-footnote-1.0.2.tgz",
"integrity": "sha512-56D19KOGbE00uKVj3sgIykpwKL179QsVFwx/DCW0u/0+URsryacI4MAdNJl0dh+u2PSsD9FtxPFbHCzJ78qJFQ==",
"dependencies": {
"@types/mdast": "^3.0.0",
"mdast-util-to-markdown": "^1.3.0",
"micromark-util-normalize-identifier": "^1.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/mdast-util-gfm-strikethrough": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/mdast-util-gfm-strikethrough/-/mdast-util-gfm-strikethrough-1.0.3.tgz",
"integrity": "sha512-DAPhYzTYrRcXdMjUtUjKvW9z/FNAMTdU0ORyMcbmkwYNbKocDpdk+PX1L1dQgOID/+vVs1uBQ7ElrBQfZ0cuiQ==",
"dependencies": {
"@types/mdast": "^3.0.0",
"mdast-util-to-markdown": "^1.3.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/mdast-util-gfm-table": {
"version": "1.0.7",
"resolved": "https://registry.npmjs.org/mdast-util-gfm-table/-/mdast-util-gfm-table-1.0.7.tgz",
"integrity": "sha512-jjcpmNnQvrmN5Vx7y7lEc2iIOEytYv7rTvu+MeyAsSHTASGCCRA79Igg2uKssgOs1i1po8s3plW0sTu1wkkLGg==",
"dependencies": {
"@types/mdast": "^3.0.0",
"markdown-table": "^3.0.0",
"mdast-util-from-markdown": "^1.0.0",
"mdast-util-to-markdown": "^1.3.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/mdast-util-gfm-task-list-item": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/mdast-util-gfm-task-list-item/-/mdast-util-gfm-task-list-item-1.0.2.tgz",
"integrity": "sha512-PFTA1gzfp1B1UaiJVyhJZA1rm0+Tzn690frc/L8vNX1Jop4STZgOE6bxUhnzdVSB+vm2GU1tIsuQcA9bxTQpMQ==",
"dependencies": {
"@types/mdast": "^3.0.0",
"mdast-util-to-markdown": "^1.3.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/mdast-util-phrasing": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/mdast-util-phrasing/-/mdast-util-phrasing-3.0.1.tgz",
"integrity": "sha512-WmI1gTXUBJo4/ZmSk79Wcb2HcjPJBzM1nlI/OUWA8yk2X9ik3ffNbBGsU+09BFmXaL1IBb9fiuvq6/KMiNycSg==",
"dependencies": {
"@types/mdast": "^3.0.0",
"unist-util-is": "^5.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/mdast-util-to-hast": {
"version": "12.3.0",
"resolved": "https://registry.npmjs.org/mdast-util-to-hast/-/mdast-util-to-hast-12.3.0.tgz",
@@ -4841,6 +4996,25 @@
"url": "https://opencollective.com/unified"
}
},
"node_modules/mdast-util-to-markdown": {
"version": "1.5.0",
"resolved": "https://registry.npmjs.org/mdast-util-to-markdown/-/mdast-util-to-markdown-1.5.0.tgz",
"integrity": "sha512-bbv7TPv/WC49thZPg3jXuqzuvI45IL2EVAr/KxF0BSdHsU0ceFHOmwQn6evxAh1GaoK/6GQ1wp4R4oW2+LFL/A==",
"dependencies": {
"@types/mdast": "^3.0.0",
"@types/unist": "^2.0.0",
"longest-streak": "^3.0.0",
"mdast-util-phrasing": "^3.0.0",
"mdast-util-to-string": "^3.0.0",
"micromark-util-decode-string": "^1.0.0",
"unist-util-visit": "^4.0.0",
"zwitch": "^2.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/mdast-util-to-string": {
"version": "3.2.0",
"resolved": "https://registry.npmjs.org/mdast-util-to-string/-/mdast-util-to-string-3.2.0.tgz",
@@ -4935,6 +5109,120 @@
"uvu": "^0.5.0"
}
},
"node_modules/micromark-extension-gfm": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/micromark-extension-gfm/-/micromark-extension-gfm-2.0.3.tgz",
"integrity": "sha512-vb9OoHqrhCmbRidQv/2+Bc6pkP0FrtlhurxZofvOEy5o8RtuuvTq+RQ1Vw5ZDNrVraQZu3HixESqbG+0iKk/MQ==",
"dependencies": {
"micromark-extension-gfm-autolink-literal": "^1.0.0",
"micromark-extension-gfm-footnote": "^1.0.0",
"micromark-extension-gfm-strikethrough": "^1.0.0",
"micromark-extension-gfm-table": "^1.0.0",
"micromark-extension-gfm-tagfilter": "^1.0.0",
"micromark-extension-gfm-task-list-item": "^1.0.0",
"micromark-util-combine-extensions": "^1.0.0",
"micromark-util-types": "^1.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/micromark-extension-gfm-autolink-literal": {
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/micromark-extension-gfm-autolink-literal/-/micromark-extension-gfm-autolink-literal-1.0.5.tgz",
"integrity": "sha512-z3wJSLrDf8kRDOh2qBtoTRD53vJ+CWIyo7uyZuxf/JAbNJjiHsOpG1y5wxk8drtv3ETAHutCu6N3thkOOgueWg==",
"dependencies": {
"micromark-util-character": "^1.0.0",
"micromark-util-sanitize-uri": "^1.0.0",
"micromark-util-symbol": "^1.0.0",
"micromark-util-types": "^1.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/micromark-extension-gfm-footnote": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/micromark-extension-gfm-footnote/-/micromark-extension-gfm-footnote-1.1.2.tgz",
"integrity": "sha512-Yxn7z7SxgyGWRNa4wzf8AhYYWNrwl5q1Z8ii+CSTTIqVkmGZF1CElX2JI8g5yGoM3GAman9/PVCUFUSJ0kB/8Q==",
"dependencies": {
"micromark-core-commonmark": "^1.0.0",
"micromark-factory-space": "^1.0.0",
"micromark-util-character": "^1.0.0",
"micromark-util-normalize-identifier": "^1.0.0",
"micromark-util-sanitize-uri": "^1.0.0",
"micromark-util-symbol": "^1.0.0",
"micromark-util-types": "^1.0.0",
"uvu": "^0.5.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/micromark-extension-gfm-strikethrough": {
"version": "1.0.7",
"resolved": "https://registry.npmjs.org/micromark-extension-gfm-strikethrough/-/micromark-extension-gfm-strikethrough-1.0.7.tgz",
"integrity": "sha512-sX0FawVE1o3abGk3vRjOH50L5TTLr3b5XMqnP9YDRb34M0v5OoZhG+OHFz1OffZ9dlwgpTBKaT4XW/AsUVnSDw==",
"dependencies": {
"micromark-util-chunked": "^1.0.0",
"micromark-util-classify-character": "^1.0.0",
"micromark-util-resolve-all": "^1.0.0",
"micromark-util-symbol": "^1.0.0",
"micromark-util-types": "^1.0.0",
"uvu": "^0.5.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/micromark-extension-gfm-table": {
"version": "1.0.7",
"resolved": "https://registry.npmjs.org/micromark-extension-gfm-table/-/micromark-extension-gfm-table-1.0.7.tgz",
"integrity": "sha512-3ZORTHtcSnMQEKtAOsBQ9/oHp9096pI/UvdPtN7ehKvrmZZ2+bbWhi0ln+I9drmwXMt5boocn6OlwQzNXeVeqw==",
"dependencies": {
"micromark-factory-space": "^1.0.0",
"micromark-util-character": "^1.0.0",
"micromark-util-symbol": "^1.0.0",
"micromark-util-types": "^1.0.0",
"uvu": "^0.5.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/micromark-extension-gfm-tagfilter": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/micromark-extension-gfm-tagfilter/-/micromark-extension-gfm-tagfilter-1.0.2.tgz",
"integrity": "sha512-5XWB9GbAUSHTn8VPU8/1DBXMuKYT5uOgEjJb8gN3mW0PNW5OPHpSdojoqf+iq1xo7vWzw/P8bAHY0n6ijpXF7g==",
"dependencies": {
"micromark-util-types": "^1.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/micromark-extension-gfm-task-list-item": {
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/micromark-extension-gfm-task-list-item/-/micromark-extension-gfm-task-list-item-1.0.5.tgz",
"integrity": "sha512-RMFXl2uQ0pNQy6Lun2YBYT9g9INXtWJULgbt01D/x8/6yJ2qpKyzdZD3pi6UIkzF++Da49xAelVKUeUMqd5eIQ==",
"dependencies": {
"micromark-factory-space": "^1.0.0",
"micromark-util-character": "^1.0.0",
"micromark-util-symbol": "^1.0.0",
"micromark-util-types": "^1.0.0",
"uvu": "^0.5.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/micromark-factory-destination": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/micromark-factory-destination/-/micromark-factory-destination-1.0.0.tgz",
@@ -6353,6 +6641,21 @@
"url": "https://github.com/sponsors/mysticatea"
}
},
"node_modules/remark-gfm": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/remark-gfm/-/remark-gfm-3.0.1.tgz",
"integrity": "sha512-lEFDoi2PICJyNrACFOfDD3JlLkuSbOa5Wd8EPt06HUdptv8Gn0bxYTdbU/XXQ3swAPkEaGxxPN9cbnMHvVu1Ig==",
"dependencies": {
"@types/mdast": "^3.0.0",
"mdast-util-gfm": "^2.0.0",
"micromark-extension-gfm": "^2.0.0",
"unified": "^10.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/remark-parse": {
"version": "10.0.1",
"resolved": "https://registry.npmjs.org/remark-parse/-/remark-parse-10.0.1.tgz",
@@ -7489,6 +7792,15 @@
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/zwitch": {
"version": "2.0.4",
"resolved": "https://registry.npmjs.org/zwitch/-/zwitch-2.0.4.tgz",
"integrity": "sha512-bXE4cR/kVZhKZX/RjPEflHaKVhUVl85noU3v6b8apfQEc1x4A+zBxjZ4lN8LqGd6WZ3dl98pY4o717VFmoPp+A==",
"funding": {
"type": "github",
"url": "https://github.com/sponsors/wooorm"
}
}
}
}

3
frontend/package.json
View File

@@ -28,7 +28,8 @@
"react-markdown": "^8.0.7",
"react-redux": "^8.0.5",
"react-router-dom": "^6.8.1",
"react-syntax-highlighter": "^15.5.0"
"react-syntax-highlighter": "^15.5.0",
"remark-gfm": "^3.0.0"
},
"devDependencies": {
"@types/react": "^18.0.27",

2
frontend/src/Navigation.tsx
View File

@@ -280,7 +280,7 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
onClick={() => setUploadModalState('ACTIVE')}
></img>
{isDocsListOpen && (
<div className="absolute top-12 left-0 right-6 ml-2 mr-4 max-h-52 overflow-y-scroll bg-white shadow-lg">
<div className="absolute top-12 left-0 right-6 z-10 ml-2 mr-4 max-h-52 overflow-y-scroll bg-white shadow-lg">
{docs ? (
docs.map((doc, index) => {
if (doc.model === embeddingsName) {

31
frontend/src/conversation/ConversationBubble.tsx
View File

@@ -1,5 +1,6 @@
import { forwardRef, useState } from 'react';
import Avatar from '../components/Avatar';
import remarkGfm from 'remark-gfm';
import { FEEDBACK, MESSAGE_TYPE } from './conversationModels';
import classes from './ConversationBubble.module.css';
import Alert from './../assets/alert.svg';
@@ -89,6 +90,7 @@ const ConversationBubble = forwardRef<
)}
<ReactMarkdown
className="max-w-screen-md whitespace-pre-wrap break-words"
remarkPlugins={[remarkGfm]}
components={{
code({ node, inline, className, children, ...props }) {
const match = /language-(\w+)/.exec(className || '');
@@ -126,6 +128,35 @@ const ConversationBubble = forwardRef<
</ol>
);
},
table({ children }) {
return (
<div className="relative overflow-x-auto rounded-lg border">
<table className="w-full text-left text-sm text-gray-700">
{children}
</table>
</div>
);
},
thead({ children }) {
return (
<thead className="text-xs uppercase text-gray-900 [&>.table-row]:bg-gray-50">
{children}
</thead>
);
},
tr({ children }) {
return (
<tr className="table-row border-b odd:bg-white even:bg-gray-50">
{children}
</tr>
);
},
td({ children }) {
return <td className="px-6 py-3">{children}</td>;
},
th({ children }) {
return <th className="px-6 py-3">{children}</th>;
},
}}
>
{message}