LLM/DocsGPT
1
0
Fork 0
mirror of https://github.com/arc53/DocsGPT.git synced 2025-11-29 08:33:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into feature/api-key-create

Browse Source
This commit is contained in:
Alex
2024-03-29 19:59:45 +00:00
committed by GitHub
parent 5daeb7f876 5e5e2b8aee
commit 794fc05ada
15 changed files with 340 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@@ -75,6 +75,7 @@ target/
# Jupyter Notebook
.ipynb_checkpoints
**/*.ipynb
# IPython
profile_default/

47
application/api/answer/routes.py
View File

@@ -29,12 +29,15 @@ prompts_collection = db["prompts"]
api_key_collection = db["api_keys"]
answer = Blueprint('answer', __name__)
if settings.LLM_NAME == "gpt4":
gpt_model = 'gpt-4'
gpt_model = ""
# to have some kind of default behaviour
if settings.LLM_NAME == "openai":
gpt_model = 'gpt-3.5-turbo'
elif settings.LLM_NAME == "anthropic":
gpt_model = 'claude-2'
else:
gpt_model = 'gpt-3.5-turbo'
if settings.MODEL_NAME: # in case there is particular model name configured
gpt_model = settings.MODEL_NAME
# load the prompts
current_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
@@ -102,9 +105,8 @@ def is_azure_configured():
return settings.OPENAI_API_BASE and settings.OPENAI_API_VERSION and settings.AZURE_DEPLOYMENT_NAME
def complete_stream(question, docsearch, chat_history, prompt_id, conversation_id):
def complete_stream(question, docsearch, chat_history, prompt_id, conversation_id, chunks=2):
llm = LLMCreator.create_llm(settings.LLM_NAME, api_key=settings.API_KEY)
if prompt_id == 'default':
prompt = chat_combine_template
elif prompt_id == 'creative':
@@ -113,8 +115,11 @@ def complete_stream(question, docsearch, chat_history, prompt_id, conversation_i
prompt = chat_combine_strict
else:
prompt = prompts_collection.find_one({"_id": ObjectId(prompt_id)})["content"]
docs = docsearch.search(question, k=2)
if chunks == 0:
docs = []
else:
docs = docsearch.search(question, k=chunks)
if settings.LLM_NAME == "llama.cpp":
docs = [docs[0]]
# join all page_content together with a newline
@@ -202,6 +207,10 @@ def stream():
prompt_id = data["prompt_id"]
else:
prompt_id = 'default'
if 'chunks' in data:
chunks = int(data["chunks"])
else:
chunks = 2
# check if active_docs is set
@@ -218,7 +227,8 @@ def stream():
complete_stream(question, docsearch,
chat_history=history,
prompt_id=prompt_id,
conversation_id=conversation_id), mimetype="text/event-stream"
conversation_id=conversation_id,
chunks=chunks), mimetype="text/event-stream"
)
@@ -239,6 +249,10 @@ def api_answer():
prompt_id = data["prompt_id"]
else:
prompt_id = 'default'
if 'chunks' in data:
chunks = int(data["chunks"])
else:
chunks = 2
if prompt_id == 'default':
prompt = chat_combine_template
@@ -266,7 +280,10 @@ def api_answer():
docs = docsearch.search(question, k=2)
if chunks == 0:
docs = []
else:
docs = docsearch.search(question, k=chunks)
# join all page_content together with a newline
docs_together = "\n".join([doc.page_content for doc in docs])
p_chat_combine = prompt.replace("{summaries}", docs_together)
@@ -361,9 +378,15 @@ def api_search():
vectorstore = get_vectorstore({"active_docs": data["active_docs"]})
else:
vectorstore = ""
if 'chunks' in data:
chunks = int(data["chunks"])
else:
chunks = 2
docsearch = VectorCreator.create_vectorstore(settings.VECTOR_STORE, vectorstore, settings.EMBEDDINGS_KEY)
docs = docsearch.search(question, k=2)
if chunks == 0:
docs = []
else:
docs = docsearch.search(question, k=chunks)
source_log_docs = []
for doc in docs:

1
application/core/settings.py
View File

@@ -9,6 +9,7 @@ current_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__
class Settings(BaseSettings):
LLM_NAME: str = "docsgpt"
MODEL_NAME: Optional[str] = None # when LLM_NAME is openai, MODEL_NAME can be e.g. gpt-4-turbo-preview or gpt-3.5-turbo
EMBEDDINGS_NAME: str = "huggingface_sentence-transformers/all-mpnet-base-v2"
CELERY_BROKER_URL: str = "redis://localhost:6379/0"
CELERY_RESULT_BACKEND: str = "redis://localhost:6379/1"

26
application/parser/remote/reddit_loader.py Normal file
View File

@@ -0,0 +1,26 @@
from application.parser.remote.base import BaseRemote
from langchain_community.document_loaders import RedditPostsLoader
class RedditPostsLoaderRemote(BaseRemote):
def load_data(self, inputs):
data = eval(inputs)
client_id = data.get("client_id")
client_secret = data.get("client_secret")
user_agent = data.get("user_agent")
categories = data.get("categories", ["new", "hot"])
mode = data.get("mode", "subreddit")
search_queries = data.get("search_queries")
number_posts = data.get("number_posts", 10)
self.loader = RedditPostsLoader(
client_id=client_id,
client_secret=client_secret,
user_agent=user_agent,
categories=categories,
mode=mode,
search_queries=search_queries,
number_posts=number_posts,
)
documents = self.loader.load()
print(f"Loaded {len(documents)} documents from Reddit")
return documents

10
application/parser/remote/remote_creator.py
View File

@@ -1,13 +1,15 @@
from application.parser.remote.sitemap_loader import SitemapLoader
from application.parser.remote.crawler_loader import CrawlerLoader
from application.parser.remote.web_loader import WebLoader
from application.parser.remote.reddit_loader import RedditPostsLoaderRemote
class RemoteCreator:
loaders = {
'url': WebLoader,
'sitemap': SitemapLoader,
'crawler': CrawlerLoader
"url": WebLoader,
"sitemap": SitemapLoader,
"crawler": CrawlerLoader,
"reddit": RedditPostsLoaderRemote,
}
@classmethod
@@ -15,4 +17,4 @@ class RemoteCreator:
loader_class = cls.loaders.get(type.lower())
if not loader_class:
raise ValueError(f"No LLM class found for type {type}")
return loader_class(*args, **kwargs)
return loader_class(*args, **kwargs)

132
application/worker.py
View File

@@ -15,23 +15,27 @@ from application.parser.schema.base import Document
from application.parser.token_func import group_split
try:
nltk.download('punkt', quiet=True)
nltk.download('averaged_perceptron_tagger', quiet=True)
nltk.download("punkt", quiet=True)
nltk.download("averaged_perceptron_tagger", quiet=True)
except FileExistsError:
pass
# Define a function to extract metadata from a given filename.
def metadata_from_filename(title):
store = '/'.join(title.split('/')[1:3])
return {'title': title, 'store': store}
store = "/".join(title.split("/")[1:3])
return {"title": title, "store": store}
# Define a function to generate a random string of a given length.
def generate_random_string(length):
return ''.join([string.ascii_letters[i % 52] for i in range(length)])
return "".join([string.ascii_letters[i % 52] for i in range(length)])
current_dir = os.path.dirname(
os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
)
current_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
# Define the main function for ingesting and processing documents.
def ingest_worker(self, directory, formats, name_job, filename, user):
@@ -62,38 +66,52 @@ def ingest_worker(self, directory, formats, name_job, filename, user):
token_check = True
min_tokens = 150
max_tokens = 1250
full_path = directory + '/' + user + '/' + name_job
full_path = directory + "/" + user + "/" + name_job
import sys
print(full_path, file=sys.stderr)
# check if API_URL env variable is set
file_data = {'name': name_job, 'file': filename, 'user': user}
response = requests.get(urljoin(settings.API_URL, "/api/download"), params=file_data)
file_data = {"name": name_job, "file": filename, "user": user}
response = requests.get(
urljoin(settings.API_URL, "/api/download"), params=file_data
)
# check if file is in the response
print(response, file=sys.stderr)
file = response.content
if not os.path.exists(full_path):
os.makedirs(full_path)
with open(full_path + '/' + filename, 'wb') as f:
with open(full_path + "/" + filename, "wb") as f:
f.write(file)
# check if file is .zip and extract it
if filename.endswith('.zip'):
with zipfile.ZipFile(full_path + '/' + filename, 'r') as zip_ref:
if filename.endswith(".zip"):
with zipfile.ZipFile(full_path + "/" + filename, "r") as zip_ref:
zip_ref.extractall(full_path)
os.remove(full_path + '/' + filename)
os.remove(full_path + "/" + filename)
self.update_state(state='PROGRESS', meta={'current': 1})
self.update_state(state="PROGRESS", meta={"current": 1})
raw_docs = SimpleDirectoryReader(input_dir=full_path, input_files=input_files, recursive=recursive,
required_exts=formats, num_files_limit=limit,
exclude_hidden=exclude, file_metadata=metadata_from_filename).load_data()
raw_docs = group_split(documents=raw_docs, min_tokens=min_tokens, max_tokens=max_tokens, token_check=token_check)
raw_docs = SimpleDirectoryReader(
input_dir=full_path,
input_files=input_files,
recursive=recursive,
required_exts=formats,
num_files_limit=limit,
exclude_hidden=exclude,
file_metadata=metadata_from_filename,
).load_data()
raw_docs = group_split(
documents=raw_docs,
min_tokens=min_tokens,
max_tokens=max_tokens,
token_check=token_check,
)
docs = [Document.to_langchain_format(raw_doc) for raw_doc in raw_docs]
call_openai_api(docs, full_path, self)
self.update_state(state='PROGRESS', meta={'current': 100})
self.update_state(state="PROGRESS", meta={"current": 100})
if sample:
for i in range(min(5, len(raw_docs))):
@@ -101,70 +119,80 @@ def ingest_worker(self, directory, formats, name_job, filename, user):
# get files from outputs/inputs/index.faiss and outputs/inputs/index.pkl
# and send them to the server (provide user and name in form)
file_data = {'name': name_job, 'user': user}
file_data = {"name": name_job, "user": user}
if settings.VECTOR_STORE == "faiss":
files = {'file_faiss': open(full_path + '/index.faiss', 'rb'),
'file_pkl': open(full_path + '/index.pkl', 'rb')}
response = requests.post(urljoin(settings.API_URL, "/api/upload_index"), files=files, data=file_data)
response = requests.get(urljoin(settings.API_URL, "/api/delete_old?path=" + full_path))
files = {
"file_faiss": open(full_path + "/index.faiss", "rb"),
"file_pkl": open(full_path + "/index.pkl", "rb"),
}
response = requests.post(
urljoin(settings.API_URL, "/api/upload_index"), files=files, data=file_data
)
response = requests.get(
urljoin(settings.API_URL, "/api/delete_old?path=" + full_path)
)
else:
response = requests.post(urljoin(settings.API_URL, "/api/upload_index"), data=file_data)
response = requests.post(
urljoin(settings.API_URL, "/api/upload_index"), data=file_data
)
# delete local
shutil.rmtree(full_path)
return {
'directory': directory,
'formats': formats,
'name_job': name_job,
'filename': filename,
'user': user,
'limited': False
"directory": directory,
"formats": formats,
"name_job": name_job,
"filename": filename,
"user": user,
"limited": False,
}
def remote_worker(self, source_data, name_job, user, directory = 'temp', loader = 'url'):
def remote_worker(self, source_data, name_job, user, loader, directory="temp"):
# sample = False
token_check = True
min_tokens = 150
max_tokens = 1250
full_path = directory + '/' + user + '/' + name_job
full_path = directory + "/" + user + "/" + name_job
if not os.path.exists(full_path):
os.makedirs(full_path)
self.update_state(state='PROGRESS', meta={'current': 1})
self.update_state(state="PROGRESS", meta={"current": 1})
# source_data {"data": [url]} for url type task just urls
# Use RemoteCreator to load data from URL
remote_loader = RemoteCreator.create_loader(loader)
raw_docs = remote_loader.load_data(source_data)
docs = group_split(documents=raw_docs, min_tokens=min_tokens, max_tokens=max_tokens, token_check=token_check)
docs = group_split(
documents=raw_docs,
min_tokens=min_tokens,
max_tokens=max_tokens,
token_check=token_check,
)
#docs = [Document.to_langchain_format(raw_doc) for raw_doc in raw_docs]
# docs = [Document.to_langchain_format(raw_doc) for raw_doc in raw_docs]
call_openai_api(docs, full_path, self)
self.update_state(state='PROGRESS', meta={'current': 100})
self.update_state(state="PROGRESS", meta={"current": 100})
# Proceed with uploading and cleaning as in the original function
file_data = {'name': name_job, 'user': user}
file_data = {"name": name_job, "user": user}
if settings.VECTOR_STORE == "faiss":
files = {'file_faiss': open(full_path + '/index.faiss', 'rb'),
'file_pkl': open(full_path + '/index.pkl', 'rb')}
requests.post(urljoin(settings.API_URL, "/api/upload_index"), files=files, data=file_data)
files = {
"file_faiss": open(full_path + "/index.faiss", "rb"),
"file_pkl": open(full_path + "/index.pkl", "rb"),
}
requests.post(
urljoin(settings.API_URL, "/api/upload_index"), files=files, data=file_data
)
requests.get(urljoin(settings.API_URL, "/api/delete_old?path=" + full_path))
else:
requests.post(urljoin(settings.API_URL, "/api/upload_index"), data=file_data)
shutil.rmtree(full_path)
return {
'urls': source_data,
'name_job': name_job,
'user': user,
'limited': False
}
return {"urls": source_data, "name_job": name_job, "user": user, "limited": False}

27
docs/package-lock.json generated
View File

@@ -7,7 +7,7 @@
"license": "MIT",
"dependencies": {
"@vercel/analytics": "^1.1.1",
"docsgpt": "^0.3.0",
"docsgpt": "^0.3.7",
"next": "^14.0.4",
"nextra": "^2.13.2",
"nextra-theme-docs": "^2.13.2",
@@ -422,6 +422,11 @@
"node": ">=6.9.0"
}
},
"node_modules/@bpmn-io/snarkdown": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/@bpmn-io/snarkdown/-/snarkdown-2.2.0.tgz",
"integrity": "sha512-bVD7FIoaBDZeCJkMRgnBPDeptPlto87wt2qaCjf5t8iLaevDmTPaREd6FpBEGsHlUdHFFZWRk4qAoEC5So2M0Q=="
},
"node_modules/@braintree/sanitize-url": {
"version": "6.0.4",
"resolved": "https://registry.npmjs.org/@braintree/sanitize-url/-/sanitize-url-6.0.4.tgz",
@@ -4958,11 +4963,12 @@
}
},
"node_modules/docsgpt": {
"version": "0.3.0",
"resolved": "https://registry.npmjs.org/docsgpt/-/docsgpt-0.3.0.tgz",
"integrity": "sha512-0yT2m+HAlJ+289p278c3Zi07bu2wr6zULOT/bYXtJ/nb59V2Vpfdj2xFB49+lYLSeVe8H+Ij5fFSNZ6RkVRfMQ==",
"version": "0.3.7",
"resolved": "https://registry.npmjs.org/docsgpt/-/docsgpt-0.3.7.tgz",
"integrity": "sha512-VHrXXOEFtjNTcpA8Blf3IzpLlJxOMhm/S5CM4FDjQEkdK9WWhI8yXd/0Rs/FS8oz7YbFrNxO758mlP7OtQtBBw==",
"dependencies": {
"@babel/plugin-transform-flow-strip-types": "^7.23.3",
"@bpmn-io/snarkdown": "^2.2.0",
"@parcel/resolver-glob": "^2.12.0",
"@parcel/transformer-svg-react": "^2.12.0",
"@parcel/transformer-typescript-tsc": "^2.12.0",
@@ -4972,6 +4978,7 @@
"@types/react-dom": "^18.2.19",
"class-variance-authority": "^0.7.0",
"clsx": "^2.1.0",
"dompurify": "^3.0.9",
"flow-bin": "^0.229.2",
"i": "^0.3.7",
"install": "^0.13.0",
@@ -5029,9 +5036,9 @@
}
},
"node_modules/dompurify": {
"version": "3.0.7",
"resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.0.7.tgz",
"integrity": "sha512-BViYTZoqP3ak/ULKOc101y+CtHDUvBsVgSxIF1ku0HmK6BRf+C03MC+tArMvOPtVtZp83DDh5puywKDu4sbVjQ=="
"version": "3.0.11",
"resolved": "https://registry.npmjs.org/dompurify/-/dompurify-3.0.11.tgz",
"integrity": "sha512-Fan4uMuyB26gFV3ovPoEoQbxRRPfTu3CvImyZnhGq5fsIEO+gEFLp45ISFt+kQBWsK5ulDdT0oV28jS1UrwQLg=="
},
"node_modules/domutils": {
"version": "2.8.0",
@@ -6206,9 +6213,9 @@
"integrity": "sha512-gfFQZrcTc8CnKXp6Y4/CBT3fTc0OVuDofpre4aEeEpSBPV5X5v4+Vmx+8snU7RLPrNHPKSgLxGo9YuQzz20o+w=="
},
"node_modules/katex": {
"version": "0.16.9",
"resolved": "https://registry.npmjs.org/katex/-/katex-0.16.9.tgz",
"integrity": "sha512-fsSYjWS0EEOwvy81j3vRA8TEAhQhKiqO+FQaKWp0m39qwOzHVBgAUBIXWj1pB+O2W3fIpNa6Y9KSKCVbfPhyAQ==",
"version": "0.16.10",
"resolved": "https://registry.npmjs.org/katex/-/katex-0.16.10.tgz",
"integrity": "sha512-ZiqaC04tp2O5utMsl2TEZTXxa6WSC4yo0fv5ML++D3QZv/vx2Mct0mTlRx3O+uUkjfuAgOkzsCmq5MiUEsDDdA==",
"funding": [
"https://opencollective.com/katex",
"https://github.com/sponsors/katex"

17
frontend/src/Setting.tsx
View File

@@ -9,6 +9,8 @@ import {
setPrompt,
selectSourceDocs,
setSourceDocs,
setChunks,
selectChunks,
} from './preferences/preferenceSlice';
import { Doc } from './preferences/preferenceApi';
import { useDarkTheme } from './hooks';
@@ -190,10 +192,13 @@ const Setting: React.FC = () => {
const General: React.FC = () => {
const themes = ['Light', 'Dark'];
const languages = ['English'];
const chunks = ['0', '2', '4', '6', '8', '10'];
const selectedChunks = useSelector(selectChunks);
const [isDarkTheme, toggleTheme] = useDarkTheme();
const [selectedTheme, setSelectedTheme] = useState(
isDarkTheme ? 'Dark' : 'Light',
);
const dispatch = useDispatch();
const [selectedLanguage, setSelectedLanguage] = useState(languages[0]);
return (
<div className="mt-[59px]">
@@ -208,7 +213,7 @@ const General: React.FC = () => {
}}
/>
</div>
<div>
<div className="mb-4">
<p className="font-bold text-jet dark:text-bright-gray">
Select Language
</p>
@@ -218,6 +223,16 @@ const General: React.FC = () => {
onSelect={setSelectedLanguage}
/>
</div>
<div>
<p className="font-bold text-jet dark:text-bright-gray">
Chunks processed per query
</p>
<Dropdown
options={chunks}
selectedValue={selectedChunks}
onSelect={(value: string) => dispatch(setChunks(value))}
/>
</div>
</div>
);
};

4
frontend/src/components/Dropdown.tsx
View File

@@ -39,10 +39,10 @@ function Dropdown({
isOpen
? typeof selectedValue === 'string'
? 'rounded-t-xl'
: 'rounded-t-2xl'
: 'rounded-t-3xl'
: typeof selectedValue === 'string'
? 'rounded-xl'
: 'rounded-full'
: 'rounded-3xl'
}`}
>
{typeof selectedValue === 'string' ? (

5
frontend/src/conversation/ConversationBubble.tsx
View File

@@ -160,7 +160,10 @@ const ConversationBubble = forwardRef<
>
{message}
</ReactMarkdown>
{DisableSourceFE || type === 'ERROR' ? null : (
{DisableSourceFE ||
type === 'ERROR' ||
!sources ||
sources.length === 0 ? null : (
<>
<span className="mt-3 h-px w-full bg-[#DEDEDE]"></span>
<div className="mt-3 flex w-full flex-row flex-wrap items-center justify-start gap-2">

6
frontend/src/conversation/conversationApi.ts
View File

@@ -10,6 +10,7 @@ export function fetchAnswerApi(
history: Array<any> = [],
conversationId: string | null,
promptId: string | null,
chunks: string,
): Promise<
| {
result: any;
@@ -62,6 +63,7 @@ export function fetchAnswerApi(
active_docs: docPath,
conversation_id: conversationId,
prompt_id: promptId,
chunks: chunks,
}),
signal,
})
@@ -91,6 +93,7 @@ export function fetchAnswerSteaming(
history: Array<any> = [],
conversationId: string | null,
promptId: string | null,
chunks: string,
onEvent: (event: MessageEvent) => void,
): Promise<Answer> {
let namePath = selectedDocs.name;
@@ -124,6 +127,7 @@ export function fetchAnswerSteaming(
history: JSON.stringify(history),
conversation_id: conversationId,
prompt_id: promptId,
chunks: chunks,
};
fetch(apiHost + '/stream', {
method: 'POST',
@@ -185,6 +189,7 @@ export function searchEndpoint(
selectedDocs: Doc,
conversation_id: string | null,
history: Array<any> = [],
chunks: string,
) {
/*
"active_docs": "default",
@@ -216,6 +221,7 @@ export function searchEndpoint(
active_docs: docPath,
conversation_id,
history,
chunks: chunks,
};
return fetch(`${apiHost}/api/search`, {
method: 'POST',

3
frontend/src/conversation/conversationSlice.ts
View File

@@ -27,6 +27,7 @@ export const fetchAnswer = createAsyncThunk<Answer, { question: string }>(
state.conversation.queries,
state.conversation.conversationId,
state.preference.prompt.id,
state.preference.chunks,
(event) => {
const data = JSON.parse(event.data);
@@ -49,6 +50,7 @@ export const fetchAnswer = createAsyncThunk<Answer, { question: string }>(
state.preference.selectedDocs!,
state.conversation.conversationId,
state.conversation.queries,
state.preference.chunks,
).then((sources) => {
//dispatch streaming sources
dispatch(
@@ -83,6 +85,7 @@ export const fetchAnswer = createAsyncThunk<Answer, { question: string }>(
state.conversation.queries,
state.conversation.conversationId,
state.preference.prompt.id,
state.preference.chunks,
);
if (answer) {
let sourcesPrepped = [];

17
frontend/src/preferences/preferenceSlice.ts
View File

@@ -10,6 +10,7 @@ interface Preference {
apiKey: string;
prompt: { name: string; id: string; type: string };
selectedDocs: Doc | null;
chunks: string;
sourceDocs: Doc[] | null;
conversations: { name: string; id: string }[] | null;
}
@@ -17,6 +18,7 @@ interface Preference {
const initialState: Preference = {
apiKey: 'xxx',
prompt: { name: 'default', id: 'default', type: 'public' },
chunks: '2',
selectedDocs: {
name: 'default',
language: 'default',
@@ -51,6 +53,9 @@ export const prefSlice = createSlice({
setPrompt: (state, action) => {
state.prompt = action.payload;
},
setChunks: (state, action) => {
state.chunks = action.payload;
},
},
});
@@ -60,6 +65,7 @@ export const {
setSourceDocs,
setConversations,
setPrompt,
setChunks,
} = prefSlice.actions;
export default prefSlice.reducer;
@@ -91,6 +97,16 @@ prefListenerMiddleware.startListening({
},
});
prefListenerMiddleware.startListening({
matcher: isAnyOf(setChunks),
effect: (action, listenerApi) => {
localStorage.setItem(
'DocsGPTChunks',
JSON.stringify((listenerApi.getState() as RootState).preference.chunks),
);
},
});
export const selectApiKey = (state: RootState) => state.preference.apiKey;
export const selectApiKeyStatus = (state: RootState) =>
!!state.preference.apiKey;
@@ -105,3 +121,4 @@ export const selectConversations = (state: RootState) =>
export const selectConversationId = (state: RootState) =>
state.conversation.conversationId;
export const selectPrompt = (state: RootState) => state.preference.prompt;
export const selectChunks = (state: RootState) => state.preference.chunks;

2
frontend/src/store.ts
View File

@@ -8,11 +8,13 @@ import {
const key = localStorage.getItem('DocsGPTApiKey');
const prompt = localStorage.getItem('DocsGPTPrompt');
const doc = localStorage.getItem('DocsGPTRecentDocs');
const chunks = localStorage.getItem('DocsGPTChunks');
const store = configureStore({
preloadedState: {
preference: {
apiKey: key ?? '',
chunks: JSON.parse(chunks ?? '2').toString(),
selectedDocs: doc !== null ? JSON.parse(doc) : null,
prompt:
prompt !== null

149
frontend/src/upload/Upload.tsx
View File

@@ -17,10 +17,18 @@ export default function Upload({
const [docName, setDocName] = useState('');
const [urlName, setUrlName] = useState('');
const [url, setUrl] = useState('');
const [redditData, setRedditData] = useState({
client_id: '',
client_secret: '',
user_agent: '',
search_queries: [''],
number_posts: 10,
});
const urlOptions: { label: string; value: string }[] = [
{ label: 'Crawler', value: 'crawler' },
// { label: 'Sitemap', value: 'sitemap' },
{ label: 'Link', value: 'url' },
{ label: 'Reddit', value: 'reddit' },
];
const [urlType, setUrlType] = useState<{ label: string; value: string }>({
label: 'Link',
@@ -163,7 +171,6 @@ export default function Upload({
};
const uploadRemote = () => {
console.log('here');
const formData = new FormData();
formData.append('name', urlName);
formData.append('user', 'local');
@@ -171,6 +178,13 @@ export default function Upload({
formData.append('source', urlType?.value);
}
formData.append('data', url);
if (
redditData.client_id.length > 0 &&
redditData.client_secret.length > 0
) {
formData.set('name', 'other');
formData.set('data', JSON.stringify(redditData));
}
const apiHost = import.meta.env.VITE_API_HOST;
const xhr = new XMLHttpRequest();
xhr.upload.addEventListener('progress', (event) => {
@@ -202,6 +216,19 @@ export default function Upload({
['.docx'],
},
});
const handleChange = (e: React.ChangeEvent<HTMLInputElement>) => {
const { name, value } = e.target;
if (name === 'search_queries' && value.length > 0) {
setRedditData({
...redditData,
[name]: value.split(',').map((item) => item.trim()),
});
} else
setRedditData({
...redditData,
[name]: value,
});
};
let view;
if (progress?.type === 'UPLOAD') {
view = <UploadProgress></UploadProgress>;
@@ -281,30 +308,102 @@ export default function Upload({
setUrlType(value)
}
/>
<input
placeholder="Enter name"
type="text"
className="h-10 w-full rounded-full border-2 border-silver px-3 outline-none dark:bg-transparent dark:text-silver"
value={urlName}
onChange={(e) => setUrlName(e.target.value)}
></input>
<div className="relative bottom-12 left-2 mt-[-18.39px]">
<span className="bg-white px-2 text-xs text-silver dark:bg-outer-space dark:text-silver">
Name
</span>
</div>
<input
placeholder="URL Link"
type="text"
className="h-10 w-full rounded-full border-2 border-silver px-3 outline-none dark:bg-transparent dark:text-silver"
value={url}
onChange={(e) => setUrl(e.target.value)}
></input>
<div className="relative bottom-12 left-2 mt-[-18.39px]">
<span className="bg-white px-2 text-xs text-silver dark:bg-outer-space dark:text-silver">
Link
</span>
</div>
{urlType.label !== 'Reddit' ? (
<>
<input
placeholder="Enter name"
type="text"
className="h-10 w-full rounded-full border-2 border-silver px-3 outline-none dark:bg-transparent dark:text-silver"
value={urlName}
onChange={(e) => setUrlName(e.target.value)}
></input>
<div className="relative bottom-12 left-2 mt-[-18.39px]">
<span className="bg-white px-2 text-xs text-silver dark:bg-outer-space dark:text-silver">
Name
</span>
</div>
<input
placeholder="URL Link"
type="text"
className="h-10 w-full rounded-full border-2 border-silver px-3 outline-none dark:bg-transparent dark:text-silver"
value={url}
onChange={(e) => setUrl(e.target.value)}
></input>
<div className="relative bottom-12 left-2 mt-[-18.39px]">
<span className="bg-white px-2 text-xs text-silver dark:bg-outer-space dark:text-silver">
Link
</span>
</div>
</>
) : (
<>
<input
placeholder="Enter client ID"
type="text"
className="h-10 w-full rounded-full border-2 border-silver px-3 outline-none dark:bg-transparent dark:text-silver"
name="client_id"
value={redditData.client_id}
onChange={handleChange}
></input>
<div className="relative bottom-12 left-2 mt-[-18.39px]">
<span className="bg-white px-2 text-xs text-silver dark:bg-outer-space dark:text-silver">
Client ID
</span>
</div>
<input
placeholder="Enter client secret"
type="text"
className="h-10 w-full rounded-full border-2 border-silver px-3 outline-none dark:bg-transparent dark:text-silver"
name="client_secret"
value={redditData.client_secret}
onChange={handleChange}
></input>
<div className="relative bottom-12 left-2 mt-[-18.39px]">
<span className="bg-white px-2 text-xs text-silver dark:bg-outer-space dark:text-silver">
Client secret
</span>
</div>
<input
placeholder="Enter user agent"
type="text"
className="h-10 w-full rounded-full border-2 border-silver px-3 outline-none dark:bg-transparent dark:text-silver"
name="user_agent"
value={redditData.user_agent}
onChange={handleChange}
></input>
<div className="relative bottom-12 left-2 mt-[-18.39px]">
<span className="bg-white px-2 text-xs text-silver dark:bg-outer-space dark:text-silver">
User agent
</span>
</div>
<input
placeholder="Enter search queries"
type="text"
className="h-10 w-full rounded-full border-2 border-silver px-3 outline-none dark:bg-transparent dark:text-silver"
name="search_queries"
value={redditData.search_queries}
onChange={handleChange}
></input>
<div className="relative bottom-12 left-2 mt-[-18.39px]">
<span className="bg-white px-2 text-xs text-silver dark:bg-outer-space dark:text-silver">
Search queries
</span>
</div>
<input
placeholder="Enter number of posts"
type="number"
className="h-10 w-full rounded-full border-2 border-silver px-3 outline-none dark:bg-transparent dark:text-silver"
name="number_posts"
value={redditData.number_posts}
onChange={handleChange}
></input>
<div className="relative bottom-12 left-2 mt-[-18.39px]">
<span className="bg-white px-2 text-xs text-silver dark:bg-outer-space dark:text-silver">
Number of posts
</span>
</div>
</>
)}
</>
)}
<div className="flex flex-row-reverse">