setIsApiModalOpen(true)}
+ onClick={() => dispatch(toggleApiKeyModal())}
>
Reset Key
@@ -155,32 +152,12 @@ function DesktopNavigation({
);
}
-export default function Navigation({
- isMobile,
- isMenuOpen,
- setIsMenuOpen,
- setIsApiModalOpen,
-}: {
- isMobile: boolean;
- isMenuOpen: boolean;
- setIsMenuOpen: React.Dispatch
>;
- setIsApiModalOpen: React.Dispatch>;
-}) {
+export default function Navigation() {
+ const isMobile = useMediaQuery('(max-width: 768px)');
+
if (isMobile) {
- return (
-
- );
+ return ;
} else {
- return (
-
- );
+ return ;
}
}
diff --git a/frontend/src/components/Navigation/PastChat.tsx b/frontend/src/components/Navigation/PastChat.tsx
deleted file mode 100644
index d677ec4a..00000000
--- a/frontend/src/components/Navigation/PastChat.tsx
+++ /dev/null
@@ -1 +0,0 @@
-export default function PastChat() {}
diff --git a/frontend/src/hooks.ts b/frontend/src/hooks.ts
new file mode 100644
index 00000000..45bfb83e
--- /dev/null
+++ b/frontend/src/hooks.ts
@@ -0,0 +1,22 @@
+import { useState, useEffect } from 'react';
+
+export function useMediaQuery(query: string): boolean {
+ const [matches, setMatches] = useState(false);
+
+ useEffect(() => {
+ const media = window.matchMedia(query);
+
+ if (media.matches !== matches) {
+ setMatches(media.matches);
+ }
+
+ const listener = () => {
+ setMatches(media.matches);
+ };
+
+ media.addEventListener('resize', listener);
+ return () => media.removeEventListener('resize', listener);
+ }, [matches, query]);
+
+ return matches;
+}
diff --git a/frontend/src/components/Navigation/imgs/arrow.svg b/frontend/src/imgs/arrow.svg
similarity index 100%
rename from frontend/src/components/Navigation/imgs/arrow.svg
rename to frontend/src/imgs/arrow.svg
diff --git a/frontend/src/components/Navigation/imgs/exit.svg b/frontend/src/imgs/exit.svg
similarity index 100%
rename from frontend/src/components/Navigation/imgs/exit.svg
rename to frontend/src/imgs/exit.svg
diff --git a/frontend/src/components/Navigation/imgs/hamburger.svg b/frontend/src/imgs/hamburger.svg
similarity index 100%
rename from frontend/src/components/Navigation/imgs/hamburger.svg
rename to frontend/src/imgs/hamburger.svg
diff --git a/frontend/src/components/Navigation/imgs/info.svg b/frontend/src/imgs/info.svg
similarity index 100%
rename from frontend/src/components/Navigation/imgs/info.svg
rename to frontend/src/imgs/info.svg
diff --git a/frontend/src/components/Navigation/imgs/key.svg b/frontend/src/imgs/key.svg
similarity index 100%
rename from frontend/src/components/Navigation/imgs/key.svg
rename to frontend/src/imgs/key.svg
diff --git a/frontend/src/components/Navigation/imgs/link.svg b/frontend/src/imgs/link.svg
similarity index 100%
rename from frontend/src/components/Navigation/imgs/link.svg
rename to frontend/src/imgs/link.svg
diff --git a/frontend/src/index.css b/frontend/src/index.css
index 8c23069d..1b6d459d 100644
--- a/frontend/src/index.css
+++ b/frontend/src/index.css
@@ -30,6 +30,7 @@ body {
margin: 0;
min-height: 100vh;
overflow-x: hidden;
+ font-family: 'Inter', sans-serif;
}
/**
diff --git a/frontend/src/main.tsx b/frontend/src/main.tsx
index 399758a4..6146911d 100644
--- a/frontend/src/main.tsx
+++ b/frontend/src/main.tsx
@@ -2,12 +2,16 @@ import React from 'react';
import ReactDOM from 'react-dom/client';
import App from './App';
import { BrowserRouter } from 'react-router-dom';
+import { Provider } from 'react-redux';
+import store from './store';
import './index.css';
ReactDOM.createRoot(document.getElementById('root') as HTMLElement).render(
-
+
+
+
,
);
diff --git a/frontend/src/store.ts b/frontend/src/store.ts
new file mode 100644
index 00000000..e5dcda3f
--- /dev/null
+++ b/frontend/src/store.ts
@@ -0,0 +1,48 @@
+import { configureStore, createSlice, PayloadAction } from '@reduxjs/toolkit';
+
+interface State {
+ isApiKeyModalOpen: boolean;
+ apiKey: string;
+ isMenuOpen: boolean;
+}
+
+const initialState: State = {
+ isApiKeyModalOpen: false,
+ apiKey: '',
+ isMenuOpen: false,
+};
+
+export const slice = createSlice({
+ name: 'app',
+ initialState,
+ reducers: {
+ toggleApiKeyModal: (state) => {
+ state.isApiKeyModalOpen = !state.isApiKeyModalOpen;
+ console.log('showApiKeyModal', state.isApiKeyModalOpen);
+ },
+ setApiKey: (state, action: PayloadAction) => {
+ state.apiKey = action.payload;
+ console.log('setApiKey', action.payload);
+ },
+ toggleIsMenuOpen: (state) => {
+ state.isMenuOpen = !state.isMenuOpen;
+ },
+ },
+});
+
+export const { toggleApiKeyModal, setApiKey, toggleIsMenuOpen } = slice.actions;
+
+const store = configureStore({
+ reducer: {
+ app: slice.reducer,
+ },
+});
+
+type RootState = ReturnType;
+
+export const selectIsApiKeyModalOpen = (state: RootState) =>
+ state.app.isApiKeyModalOpen;
+export const selectApiKey = (state: RootState) => state.app.apiKey;
+export const selectIsMenuOpen = (state: RootState) => state.app.isMenuOpen;
+
+export default store;
diff --git a/scripts/ingest.py b/scripts/ingest.py
index cebb6c33..3082cf4e 100644
--- a/scripts/ingest.py
+++ b/scripts/ingest.py
@@ -1,6 +1,9 @@
import sys
import nltk
import dotenv
+import typer
+
+from typing import List, Optional
from langchain.text_splitter import RecursiveCharacterTextSplitter
@@ -10,28 +13,52 @@ from parser.open_ai_func import call_openai_api, get_user_permission
dotenv.load_dotenv()
-#Specify your folder HERE
-directory_to_ingest = 'inputs'
+app = typer.Typer(add_completion=False)
-nltk.download('punkt')
-nltk.download('averaged_perceptron_tagger')
+nltk.download('punkt', quiet=True)
+nltk.download('averaged_perceptron_tagger', quiet=True)
#Splits all files in specified folder to documents
-raw_docs = SimpleDirectoryReader(input_dir=directory_to_ingest).load_data()
-raw_docs = [Document.to_langchain_format(raw_doc) for raw_doc in raw_docs]
-# Here we split the documents, as needed, into smaller chunks.
-# We do this due to the context limits of the LLMs.
-text_splitter = RecursiveCharacterTextSplitter()
-docs = text_splitter.split_documents(raw_docs)
+@app.command()
+def ingest(directory: Optional[str] = typer.Option("inputs",
+ help="Path to the directory for index creation."),
+ files: Optional[List[str]] = typer.Option(None,
+ help="""File paths to use (Optional; overrides directory).
+ E.g. --files inputs/1.md --files inputs/2.md"""),
+ recursive: Optional[bool] = typer.Option(True,
+ help="Whether to recursively search in subdirectories."),
+ limit: Optional[int] = typer.Option(None,
+ help="Maximum number of files to read."),
+ formats: Optional[List[str]] = typer.Option([".rst", ".md"],
+ help="""List of required extensions (list with .)
+ Currently supported: .rst, .md, .pdf, .docx, .csv, .epub"""),
+ exclude: Optional[bool] = typer.Option(True, help="Whether to exclude hidden files (dotfiles).")):
-# Here we check for command line arguments for bot calls.
-# If no argument exists or the permission_bypass_flag argument is not '-y',
-# user permission is requested to call the API.
-if len(sys.argv) > 1:
- permission_bypass_flag = sys.argv[1]
- if permission_bypass_flag == '-y':
- call_openai_api(docs)
+ """
+ Creates index from specified location or files.
+ By default /inputs folder is used, .rst and .md are parsed.
+ """
+ raw_docs = SimpleDirectoryReader(input_dir=directory, input_files=files, recursive=recursive,
+ required_exts=formats, num_files_limit=limit,
+ exclude_hidden=exclude).load_data()
+ raw_docs = [Document.to_langchain_format(raw_doc) for raw_doc in raw_docs]
+ print(raw_docs)
+ # Here we split the documents, as needed, into smaller chunks.
+ # We do this due to the context limits of the LLMs.
+ text_splitter = RecursiveCharacterTextSplitter()
+ docs = text_splitter.split_documents(raw_docs)
+
+ # Here we check for command line arguments for bot calls.
+ # If no argument exists or the permission_bypass_flag argument is not '-y',
+ # user permission is requested to call the API.
+ if len(sys.argv) > 1:
+ permission_bypass_flag = sys.argv[1]
+ if permission_bypass_flag == '-y':
+ call_openai_api(docs)
+ else:
+ get_user_permission(docs)
else:
get_user_permission(docs)
-else:
- get_user_permission(docs)
\ No newline at end of file
+
+if __name__ == "__main__":
+ app()
diff --git a/scripts/ingest_rst.py b/scripts/old/ingest_rst.py
similarity index 100%
rename from scripts/ingest_rst.py
rename to scripts/old/ingest_rst.py
diff --git a/scripts/ingest_rst_sphinx.py b/scripts/old/ingest_rst_sphinx.py
similarity index 88%
rename from scripts/ingest_rst_sphinx.py
rename to scripts/old/ingest_rst_sphinx.py
index 9d6c8ece..ecc71570 100644
--- a/scripts/ingest_rst_sphinx.py
+++ b/scripts/old/ingest_rst_sphinx.py
@@ -29,6 +29,18 @@ def convert_rst_to_txt(src_dir, dst_dir):
f"-D source_suffix=.rst " \
f"-C {dst_dir} "
sphinx_main(args.split())
+ elif file.endswith(".md"):
+ # Rename the .md file to .rst file
+ src_file = os.path.join(root, file)
+ dst_file = os.path.join(root, file.replace(".md", ".rst"))
+ os.rename(src_file, dst_file)
+ # Convert the .rst file to .txt file using sphinx-build
+ args = f". -b text -D extensions=sphinx.ext.autodoc " \
+ f"-D master_doc={dst_file} " \
+ f"-D source_suffix=.rst " \
+ f"-C {dst_dir} "
+ sphinx_main(args.split())
+
def num_tokens_from_string(string: str, encoding_name: str) -> int:
# Function to convert string to tokens and estimate user cost.
diff --git a/scripts/parser/file/rst_parser.py b/scripts/parser/file/rst_parser.py
index 0b887d4e..7c97b326 100644
--- a/scripts/parser/file/rst_parser.py
+++ b/scripts/parser/file/rst_parser.py
@@ -24,6 +24,8 @@ class RstParser(BaseParser):
remove_hyperlinks: bool = True,
remove_images: bool = True,
remove_table_excess: bool = True,
+ remove_interpreters: bool = True,
+ remove_directives: bool = True,
remove_whitespaces_excess: bool = True,
#Be carefull with remove_characters_excess, might cause data loss
remove_characters_excess: bool = True,
@@ -34,6 +36,8 @@ class RstParser(BaseParser):
self._remove_hyperlinks = remove_hyperlinks
self._remove_images = remove_images
self._remove_table_excess = remove_table_excess
+ self._remove_interpreters = remove_interpreters
+ self._remove_directives = remove_directives
self._remove_whitespaces_excess = remove_whitespaces_excess
self._remove_characters_excess = remove_characters_excess
@@ -95,6 +99,18 @@ class RstParser(BaseParser):
content = re.sub(pattern, r"\1", content)
return content
+ def remove_directives(self, content: str) -> str:
+ """Removes reStructuredText Directives"""
+ pattern = r"`\.\.([^:]+)::"
+ content = re.sub(pattern, "", content)
+ return content
+
+ def remove_interpreters(self, content: str) -> str:
+ """Removes reStructuredText Interpreted Text Roles"""
+ pattern = r":(\w+):"
+ content = re.sub(pattern, "", content)
+ return content
+
def remove_table_excess(self, content: str) -> str:
"""Pattern to remove grid table separators"""
pattern = r"^\+[-]+\+[-]+\+$"
@@ -129,6 +145,10 @@ class RstParser(BaseParser):
content = self.remove_images(content)
if self._remove_table_excess:
content = self.remove_table_excess(content)
+ if self._remove_directives:
+ content = self.remove_directives(content)
+ if self._remove_interpreters:
+ content = self.remove_interpreters(content)
rst_tups = self.rst_to_tups(content)
if self._remove_whitespaces_excess:
rst_tups = self.remove_whitespaces_excess(rst_tups)
diff --git a/scripts/parser/open_ai_func.py b/scripts/parser/open_ai_func.py
index 500e4888..70091324 100644
--- a/scripts/parser/open_ai_func.py
+++ b/scripts/parser/open_ai_func.py
@@ -14,10 +14,38 @@ def num_tokens_from_string(string: str, encoding_name: str) -> int:
def call_openai_api(docs):
# Function to create a vector store from the documents and save it to disk.
- store = FAISS.from_documents(docs, OpenAIEmbeddings())
+ from tqdm import tqdm
+ docs_test = [docs[0]]
+ # remove the first element from docs
+ docs.pop(0)
+ # cut first n docs if you want to restart
+ #docs = docs[:n]
+ c1 = 0
+ store = FAISS.from_documents(docs_test, OpenAIEmbeddings())
+ for i in tqdm(docs, desc="Embedding ๐ฆ", unit="docs", total=len(docs), bar_format='{l_bar}{bar}| Time Left: {remaining}'):
+ try:
+ import time
+ store.add_texts([i.page_content], metadatas=[i.metadata])
+ except Exception as e:
+ print(e)
+ print("Error on ", i)
+ print("Saving progress")
+ print(f"stopped at {c1} out of {len(docs)}")
+ faiss.write_index(store.index, "docs.index")
+ store_index_bak = store.index
+ store.index = None
+ with open("faiss_store.pkl", "wb") as f:
+ pickle.dump(store, f)
+ print("Sleeping for 60 seconds and trying again")
+ time.sleep(60)
+ faiss.write_index(store_index_bak, "docs.index")
+ store.index = store_index_bak
+ store.add_texts([i.page_content], metadatas=[i.metadata])
+ c1 += 1
+
+
faiss.write_index(store.index, "docs.index")
store.index = None
-
with open("faiss_store.pkl", "wb") as f:
pickle.dump(store, f)
@@ -41,4 +69,4 @@ def get_user_permission(docs):
elif user_input == "":
call_openai_api(docs)
else:
- print("The API was not called. No money was spent.")
\ No newline at end of file
+ print("The API was not called. No money was spent.")