LLM/DocsGPT
1
0
Fork 0
mirror of https://github.com/arc53/DocsGPT.git synced 2025-11-29 08:33:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1930 from Ankit-Matth/feature/multi-select-sources

Browse Source 
Added support for Multi select sources
This commit is contained in:
Alex
2025-09-10 17:48:02 +01:00
committed by GitHub
parent 4b9153069e 188d118fc0
commit e0eb7f456e
24 changed files with 1168 additions and 520 deletions
Download Patch File Download Diff File Expand all files Collapse all files

98
application/api/answer/services/stream_processor.py
View File

@@ -69,11 +69,8 @@ class StreamProcessor:
self.decoded_token.get("sub") if self.decoded_token is not None else None
)
self.conversation_id = self.data.get("conversation_id")
self.source = (
{"active_docs": self.data["active_docs"]}
if "active_docs" in self.data
else {}
)
self.source = {}
self.all_sources = []
self.attachments = []
self.history = []
self.agent_config = {}
@@ -85,6 +82,8 @@ class StreamProcessor:
def initialize(self):
"""Initialize all required components for processing"""
self._configure_agent()
self._configure_source()
self._configure_retriever()
self._configure_agent()
self._load_conversation_history()
@@ -171,13 +170,77 @@ class StreamProcessor:
source = data.get("source")
if isinstance(source, DBRef):
source_doc = self.db.dereference(source)
data["source"] = str(source_doc["_id"])
data["retriever"] = source_doc.get("retriever", data.get("retriever"))
data["chunks"] = source_doc.get("chunks", data.get("chunks"))
if source_doc:
data["source"] = str(source_doc["_id"])
data["retriever"] = source_doc.get("retriever", data.get("retriever"))
data["chunks"] = source_doc.get("chunks", data.get("chunks"))
else:
data["source"] = None
elif source == "default":
data["source"] = "default"
else:
data["source"] = None
# Handle multiple sources
sources = data.get("sources", [])
if sources and isinstance(sources, list):
sources_list = []
for i, source_ref in enumerate(sources):
if source_ref == "default":
processed_source = {
"id": "default",
"retriever": "classic",
"chunks": data.get("chunks", "2"),
}
sources_list.append(processed_source)
elif isinstance(source_ref, DBRef):
source_doc = self.db.dereference(source_ref)
if source_doc:
processed_source = {
"id": str(source_doc["_id"]),
"retriever": source_doc.get("retriever", "classic"),
"chunks": source_doc.get("chunks", data.get("chunks", "2")),
}
sources_list.append(processed_source)
data["sources"] = sources_list
else:
data["sources"] = []
return data
def _configure_source(self):
"""Configure the source based on agent data"""
api_key = self.data.get("api_key") or self.agent_key
if api_key:
agent_data = self._get_data_from_api_key(api_key)
if agent_data.get("sources") and len(agent_data["sources"]) > 0:
source_ids = [
source["id"] for source in agent_data["sources"] if source.get("id")
]
if source_ids:
self.source = {"active_docs": source_ids}
else:
self.source = {}
self.all_sources = agent_data["sources"]
elif agent_data.get("source"):
self.source = {"active_docs": agent_data["source"]}
self.all_sources = [
{
"id": agent_data["source"],
"retriever": agent_data.get("retriever", "classic"),
}
]
else:
self.source = {}
self.all_sources = []
return
if "active_docs" in self.data:
self.source = {"active_docs": self.data["active_docs"]}
return
self.source = {}
self.all_sources = []
def _configure_agent(self):
"""Configure the agent based on request data"""
agent_id = self.data.get("agent_id")
@@ -203,7 +266,13 @@ class StreamProcessor:
if data_key.get("retriever"):
self.retriever_config["retriever_name"] = data_key["retriever"]
if data_key.get("chunks") is not None:
self.retriever_config["chunks"] = data_key["chunks"]
try:
self.retriever_config["chunks"] = int(data_key["chunks"])
except (ValueError, TypeError):
logger.warning(
f"Invalid chunks value: {data_key['chunks']}, using default value 2"
)
self.retriever_config["chunks"] = 2
elif self.agent_key:
data_key = self._get_data_from_api_key(self.agent_key)
self.agent_config.update(
@@ -224,7 +293,13 @@ class StreamProcessor:
if data_key.get("retriever"):
self.retriever_config["retriever_name"] = data_key["retriever"]
if data_key.get("chunks") is not None:
self.retriever_config["chunks"] = data_key["chunks"]
try:
self.retriever_config["chunks"] = int(data_key["chunks"])
except (ValueError, TypeError):
logger.warning(
f"Invalid chunks value: {data_key['chunks']}, using default value 2"
)
self.retriever_config["chunks"] = 2
else:
self.agent_config.update(
{
@@ -243,7 +318,8 @@ class StreamProcessor:
"token_limit": self.data.get("token_limit", settings.DEFAULT_MAX_HISTORY),
}
if "isNoneDoc" in self.data and self.data["isNoneDoc"]:
api_key = self.data.get("api_key") or self.agent_key
if not api_key and "isNoneDoc" in self.data and self.data["isNoneDoc"]:
self.retriever_config["chunks"] = 0
def create_agent(self):

1
application/api/connector/routes.py
View File

@@ -1,6 +1,5 @@
import datetime
import json
import logging
from bson.objectid import ObjectId

578
application/api/user/routes.py
View File

File diff suppressed because it is too large Load Diff

4
application/retriever/base.py
View File

@@ -5,10 +5,6 @@ class BaseRetriever(ABC):
def __init__(self):
pass
@abstractmethod
def gen(self, *args, **kwargs):
pass
@abstractmethod
def search(self, *args, **kwargs):
pass

116
application/retriever/classic_rag.py
View File

@@ -1,4 +1,5 @@
import logging
from application.core.settings import settings
from application.llm.llm_creator import LLMCreator
from application.retriever.base import BaseRetriever
@@ -20,10 +21,20 @@ class ClassicRAG(BaseRetriever):
api_key=settings.API_KEY,
decoded_token=None,
):
self.original_question = ""
"""Initialize ClassicRAG retriever with vectorstore sources and LLM configuration"""
self.original_question = source.get("question", "")
self.chat_history = chat_history if chat_history is not None else []
self.prompt = prompt
self.chunks = chunks
if isinstance(chunks, str):
try:
self.chunks = int(chunks)
except ValueError:
logging.warning(
f"Invalid chunks value '{chunks}', using default value 2"
)
self.chunks = 2
else:
self.chunks = chunks
self.gpt_model = gpt_model
self.token_limit = (
token_limit
@@ -44,25 +55,52 @@ class ClassicRAG(BaseRetriever):
user_api_key=self.user_api_key,
decoded_token=decoded_token,
)
self.vectorstore = source["active_docs"] if "active_docs" in source else None
if "active_docs" in source and source["active_docs"] is not None:
if isinstance(source["active_docs"], list):
self.vectorstores = source["active_docs"]
else:
self.vectorstores = [source["active_docs"]]
else:
self.vectorstores = []
self.question = self._rephrase_query()
self.decoded_token = decoded_token
self._validate_vectorstore_config()
def _validate_vectorstore_config(self):
"""Validate vectorstore IDs and remove any empty/invalid entries"""
if not self.vectorstores:
logging.warning("No vectorstores configured for retrieval")
return
invalid_ids = [
vs_id for vs_id in self.vectorstores if not vs_id or not vs_id.strip()
]
if invalid_ids:
logging.warning(f"Found invalid vectorstore IDs: {invalid_ids}")
self.vectorstores = [
vs_id for vs_id in self.vectorstores if vs_id and vs_id.strip()
]
def _rephrase_query(self):
"""Rephrase user query with chat history context for better retrieval"""
if (
not self.original_question
or not self.chat_history
or self.chat_history == []
or self.chunks == 0
or self.vectorstore is None
or not self.vectorstores
):
return self.original_question
prompt = f"""Given the following conversation history:
{self.chat_history}
Rephrase the following user question to be a standalone search query
that captures all relevant context from the conversation:
"""
messages = [
@@ -79,44 +117,62 @@ class ClassicRAG(BaseRetriever):
return self.original_question
def _get_data(self):
if self.chunks == 0 or self.vectorstore is None:
docs = []
else:
docsearch = VectorCreator.create_vectorstore(
settings.VECTOR_STORE, self.vectorstore, settings.EMBEDDINGS_KEY
)
docs_temp = docsearch.search(self.question, k=self.chunks)
docs = [
{
"title": i.metadata.get(
"title", i.metadata.get("post_title", i.page_content)
).split("/")[-1],
"text": i.page_content,
"source": (
i.metadata.get("source")
if i.metadata.get("source")
else "local"
),
}
for i in docs_temp
]
"""Retrieve relevant documents from configured vectorstores"""
if self.chunks == 0 or not self.vectorstores:
return []
all_docs = []
chunks_per_source = max(1, self.chunks // len(self.vectorstores))
return docs
for vectorstore_id in self.vectorstores:
if vectorstore_id:
try:
docsearch = VectorCreator.create_vectorstore(
settings.VECTOR_STORE, vectorstore_id, settings.EMBEDDINGS_KEY
)
docs_temp = docsearch.search(self.question, k=chunks_per_source)
def gen():
pass
for doc in docs_temp:
if hasattr(doc, "page_content") and hasattr(doc, "metadata"):
page_content = doc.page_content
metadata = doc.metadata
else:
page_content = doc.get("text", doc.get("page_content", ""))
metadata = doc.get("metadata", {})
title = metadata.get(
"title", metadata.get("post_title", page_content)
)
if isinstance(title, str):
title = title.split("/")[-1]
else:
title = str(title).split("/")[-1]
all_docs.append(
{
"title": title,
"text": page_content,
"source": metadata.get("source") or vectorstore_id,
}
)
except Exception as e:
logging.error(
f"Error searching vectorstore {vectorstore_id}: {e}",
exc_info=True,
)
continue
return all_docs
def search(self, query: str = ""):
"""Search for documents using optional query override"""
if query:
self.original_question = query
self.question = self._rephrase_query()
return self._get_data()
def get_params(self):
"""Return current retriever configuration parameters"""
return {
"question": self.original_question,
"rephrased_question": self.question,
"source": self.vectorstore,
"sources": self.vectorstores,
"chunks": self.chunks,
"token_limit": self.token_limit,
"gpt_model": self.gpt_model,

77
application/vectorstore/base.py
View File

@@ -1,20 +1,28 @@
from abc import ABC, abstractmethod
import os
from sentence_transformers import SentenceTransformer
from abc import ABC, abstractmethod
from langchain_openai import OpenAIEmbeddings
from sentence_transformers import SentenceTransformer
from application.core.settings import settings
class EmbeddingsWrapper:
def __init__(self, model_name, *args, **kwargs):
self.model = SentenceTransformer(model_name, config_kwargs={'allow_dangerous_deserialization': True}, *args, **kwargs)
self.model = SentenceTransformer(
model_name,
config_kwargs={"allow_dangerous_deserialization": True},
*args,
**kwargs
)
self.dimension = self.model.get_sentence_embedding_dimension()
def embed_query(self, query: str):
return self.model.encode(query).tolist()
def embed_documents(self, documents: list):
return self.model.encode(documents).tolist()
def __call__(self, text):
if isinstance(text, str):
return self.embed_query(text)
@@ -24,15 +32,14 @@ class EmbeddingsWrapper:
raise ValueError("Input must be a string or a list of strings")
class EmbeddingsSingleton:
_instances = {}
@staticmethod
def get_instance(embeddings_name, *args, **kwargs):
if embeddings_name not in EmbeddingsSingleton._instances:
EmbeddingsSingleton._instances[embeddings_name] = EmbeddingsSingleton._create_instance(
embeddings_name, *args, **kwargs
EmbeddingsSingleton._instances[embeddings_name] = (
EmbeddingsSingleton._create_instance(embeddings_name, *args, **kwargs)
)
return EmbeddingsSingleton._instances[embeddings_name]
@@ -40,9 +47,15 @@ class EmbeddingsSingleton:
def _create_instance(embeddings_name, *args, **kwargs):
embeddings_factory = {
"openai_text-embedding-ada-002": OpenAIEmbeddings,
"huggingface_sentence-transformers/all-mpnet-base-v2": lambda: EmbeddingsWrapper("sentence-transformers/all-mpnet-base-v2"),
"huggingface_sentence-transformers-all-mpnet-base-v2": lambda: EmbeddingsWrapper("sentence-transformers/all-mpnet-base-v2"),
"huggingface_hkunlp/instructor-large": lambda: EmbeddingsWrapper("hkunlp/instructor-large"),
"huggingface_sentence-transformers/all-mpnet-base-v2": lambda: EmbeddingsWrapper(
"sentence-transformers/all-mpnet-base-v2"
),
"huggingface_sentence-transformers-all-mpnet-base-v2": lambda: EmbeddingsWrapper(
"sentence-transformers/all-mpnet-base-v2"
),
"huggingface_hkunlp/instructor-large": lambda: EmbeddingsWrapper(
"hkunlp/instructor-large"
),
}
if embeddings_name in embeddings_factory:
@@ -50,34 +63,63 @@ class EmbeddingsSingleton:
else:
return EmbeddingsWrapper(embeddings_name, *args, **kwargs)
class BaseVectorStore(ABC):
def __init__(self):
pass
@abstractmethod
def search(self, *args, **kwargs):
"""Search for similar documents/chunks in the vectorstore"""
pass
@abstractmethod
def add_texts(self, texts, metadatas=None, *args, **kwargs):
"""Add texts with their embeddings to the vectorstore"""
pass
def delete_index(self, *args, **kwargs):
"""Delete the entire index/collection"""
pass
def save_local(self, *args, **kwargs):
"""Save vectorstore to local storage"""
pass
def get_chunks(self, *args, **kwargs):
"""Get all chunks from the vectorstore"""
pass
def add_chunk(self, text, metadata=None, *args, **kwargs):
"""Add a single chunk to the vectorstore"""
pass
def delete_chunk(self, chunk_id, *args, **kwargs):
"""Delete a specific chunk from the vectorstore"""
pass
def is_azure_configured(self):
return settings.OPENAI_API_BASE and settings.OPENAI_API_VERSION and settings.AZURE_DEPLOYMENT_NAME
return (
settings.OPENAI_API_BASE
and settings.OPENAI_API_VERSION
and settings.AZURE_DEPLOYMENT_NAME
)
def _get_embeddings(self, embeddings_name, embeddings_key=None):
if embeddings_name == "openai_text-embedding-ada-002":
if self.is_azure_configured():
os.environ["OPENAI_API_TYPE"] = "azure"
embedding_instance = EmbeddingsSingleton.get_instance(
embeddings_name,
model=settings.AZURE_EMBEDDINGS_DEPLOYMENT_NAME
embeddings_name, model=settings.AZURE_EMBEDDINGS_DEPLOYMENT_NAME
)
else:
embedding_instance = EmbeddingsSingleton.get_instance(
embeddings_name,
openai_api_key=embeddings_key
embeddings_name, openai_api_key=embeddings_key
)
elif embeddings_name == "huggingface_sentence-transformers/all-mpnet-base-v2":
if os.path.exists("./models/all-mpnet-base-v2"):
embedding_instance = EmbeddingsSingleton.get_instance(
embeddings_name = "./models/all-mpnet-base-v2",
embeddings_name="./models/all-mpnet-base-v2",
)
else:
embedding_instance = EmbeddingsSingleton.get_instance(
@@ -87,4 +129,3 @@ class BaseVectorStore(ABC):
embedding_instance = EmbeddingsSingleton.get_instance(embeddings_name)
return embedding_instance

171
frontend/src/agents/NewAgent.tsx
View File

@@ -45,6 +45,7 @@ export default function NewAgent({ mode }: { mode: 'new' | 'edit' | 'draft' }) {
description: '',
image: '',
source: '',
sources: [],
chunks: '',
retriever: '',
prompt_id: 'default',
@@ -150,7 +151,41 @@ export default function NewAgent({ mode }: { mode: 'new' | 'edit' | 'draft' }) {
const formData = new FormData();
formData.append('name', agent.name);
formData.append('description', agent.description);
formData.append('source', agent.source);
if (selectedSourceIds.size > 1) {
const sourcesArray = Array.from(selectedSourceIds)
.map((id) => {
const sourceDoc = sourceDocs?.find(
(source) =>
source.id === id || source.retriever === id || source.name === id,
);
if (sourceDoc?.name === 'Default' && !sourceDoc?.id) {
return 'default';
}
return sourceDoc?.id || id;
})
.filter(Boolean);
formData.append('sources', JSON.stringify(sourcesArray));
formData.append('source', '');
} else if (selectedSourceIds.size === 1) {
const singleSourceId = Array.from(selectedSourceIds)[0];
const sourceDoc = sourceDocs?.find(
(source) =>
source.id === singleSourceId ||
source.retriever === singleSourceId ||
source.name === singleSourceId,
);
let finalSourceId;
if (sourceDoc?.name === 'Default' && !sourceDoc?.id)
finalSourceId = 'default';
else finalSourceId = sourceDoc?.id || singleSourceId;
formData.append('source', String(finalSourceId));
formData.append('sources', JSON.stringify([]));
} else {
formData.append('source', '');
formData.append('sources', JSON.stringify([]));
}
formData.append('chunks', agent.chunks);
formData.append('retriever', agent.retriever);
formData.append('prompt_id', agent.prompt_id);
@@ -196,7 +231,41 @@ export default function NewAgent({ mode }: { mode: 'new' | 'edit' | 'draft' }) {
const formData = new FormData();
formData.append('name', agent.name);
formData.append('description', agent.description);
formData.append('source', agent.source);
if (selectedSourceIds.size > 1) {
const sourcesArray = Array.from(selectedSourceIds)
.map((id) => {
const sourceDoc = sourceDocs?.find(
(source) =>
source.id === id || source.retriever === id || source.name === id,
);
if (sourceDoc?.name === 'Default' && !sourceDoc?.id) {
return 'default';
}
return sourceDoc?.id || id;
})
.filter(Boolean);
formData.append('sources', JSON.stringify(sourcesArray));
formData.append('source', '');
} else if (selectedSourceIds.size === 1) {
const singleSourceId = Array.from(selectedSourceIds)[0];
const sourceDoc = sourceDocs?.find(
(source) =>
source.id === singleSourceId ||
source.retriever === singleSourceId ||
source.name === singleSourceId,
);
let finalSourceId;
if (sourceDoc?.name === 'Default' && !sourceDoc?.id)
finalSourceId = 'default';
else finalSourceId = sourceDoc?.id || singleSourceId;
formData.append('source', String(finalSourceId));
formData.append('sources', JSON.stringify([]));
} else {
formData.append('source', '');
formData.append('sources', JSON.stringify([]));
}
formData.append('chunks', agent.chunks);
formData.append('retriever', agent.retriever);
formData.append('prompt_id', agent.prompt_id);
@@ -293,9 +362,33 @@ export default function NewAgent({ mode }: { mode: 'new' | 'edit' | 'draft' }) {
throw new Error('Failed to fetch agent');
}
const data = await response.json();
if (data.source) setSelectedSourceIds(new Set([data.source]));
else if (data.retriever)
if (data.sources && data.sources.length > 0) {
const mappedSources = data.sources.map((sourceId: string) => {
if (sourceId === 'default') {
const defaultSource = sourceDocs?.find(
(source) => source.name === 'Default',
);
return defaultSource?.retriever || 'classic';
}
return sourceId;
});
setSelectedSourceIds(new Set(mappedSources));
} else if (data.source) {
if (data.source === 'default') {
const defaultSource = sourceDocs?.find(
(source) => source.name === 'Default',
);
setSelectedSourceIds(
new Set([defaultSource?.retriever || 'classic']),
);
} else {
setSelectedSourceIds(new Set([data.source]));
}
} else if (data.retriever) {
setSelectedSourceIds(new Set([data.retriever]));
}
if (data.tools) setSelectedToolIds(new Set(data.tools));
if (data.status === 'draft') setEffectiveMode('draft');
if (data.json_schema) {
@@ -311,25 +404,57 @@ export default function NewAgent({ mode }: { mode: 'new' | 'edit' | 'draft' }) {
}, [agentId, mode, token]);
useEffect(() => {
const selectedSource = Array.from(selectedSourceIds).map((id) =>
sourceDocs?.find(
(source) =>
source.id === id || source.retriever === id || source.name === id,
),
);
if (selectedSource[0]?.model === embeddingsName) {
if (selectedSource[0] && 'id' in selectedSource[0]) {
const selectedSources = Array.from(selectedSourceIds)
.map((id) =>
sourceDocs?.find(
(source) =>
source.id === id || source.retriever === id || source.name === id,
),
)
.filter(Boolean);
if (selectedSources.length > 0) {
// Handle multiple sources
if (selectedSources.length > 1) {
// Multiple sources selected - store in sources array
const sourceIds = selectedSources
.map((source) => source?.id)
.filter((id): id is string => Boolean(id));
setAgent((prev) => ({
...prev,
source: selectedSource[0]?.id || 'default',
sources: sourceIds,
source: '', // Clear single source for multiple sources
retriever: '',
}));
} else
setAgent((prev) => ({
...prev,
source: '',
retriever: selectedSource[0]?.retriever || 'classic',
}));
} else {
// Single source selected - maintain backward compatibility
const selectedSource = selectedSources[0];
if (selectedSource?.model === embeddingsName) {
if (selectedSource && 'id' in selectedSource) {
setAgent((prev) => ({
...prev,
source: selectedSource?.id || 'default',
sources: [], // Clear sources array for single source
retriever: '',
}));
} else {
setAgent((prev) => ({
...prev,
source: '',
sources: [], // Clear sources array
retriever: selectedSource?.retriever || 'classic',
}));
}
}
}
} else {
// No sources selected
setAgent((prev) => ({
...prev,
source: '',
sources: [],
retriever: '',
}));
}
}, [selectedSourceIds]);
@@ -510,7 +635,7 @@ export default function NewAgent({ mode }: { mode: 'new' | 'edit' | 'draft' }) {
)
.filter(Boolean)
.join(', ')
: 'Select source'}
: 'Select sources'}
</button>
<MultiSelectPopup
isOpen={isSourcePopupOpen}
@@ -526,12 +651,10 @@ export default function NewAgent({ mode }: { mode: 'new' | 'edit' | 'draft' }) {
selectedIds={selectedSourceIds}
onSelectionChange={(newSelectedIds: Set<string | number>) => {
setSelectedSourceIds(newSelectedIds);
setIsSourcePopupOpen(false);
}}
title="Select Source"
title="Select Sources"
searchPlaceholder="Search sources..."
noOptionsMessage="No source available"
singleSelect={true}
noOptionsMessage="No sources available"
/>
</div>
<div className="mt-3">

1
frontend/src/agents/types/index.ts
View File

@@ -10,6 +10,7 @@ export type Agent = {
description: string;
image: string;
source: string;
sources?: string[];
chunks: string;
retriever: string;
prompt_id: string;

15
frontend/src/api/services/userService.ts
View File

@@ -90,7 +90,10 @@ const userService = {
path?: string,
search?: string,
): Promise<any> =>
apiClient.get(endpoints.USER.GET_CHUNKS(docId, page, perPage, path, search), token),
apiClient.get(
endpoints.USER.GET_CHUNKS(docId, page, perPage, path, search),
token,
),
addChunk: (data: any, token: string | null): Promise<any> =>
apiClient.post(endpoints.USER.ADD_CHUNK, data, token),
deleteChunk: (
@@ -105,16 +108,20 @@ const userService = {
apiClient.get(endpoints.USER.DIRECTORY_STRUCTURE(docId), token),
manageSourceFiles: (data: FormData, token: string | null): Promise<any> =>
apiClient.postFormData(endpoints.USER.MANAGE_SOURCE_FILES, data, token),
syncConnector: (docId: string, provider: string, token: string | null): Promise<any> => {
syncConnector: (
docId: string,
provider: string,
token: string | null,
): Promise<any> => {
const sessionToken = getSessionToken(provider);
return apiClient.post(
endpoints.USER.SYNC_CONNECTOR,
{
source_id: docId,
session_token: sessionToken,
provider: provider
provider: provider,
},
token
token,
);
},
};

42
frontend/src/components/ConnectorAuth.tsx
View File

@@ -16,7 +16,12 @@ const providerLabel = (provider: string) => {
return map[provider] || provider.replace(/_/g, ' ');
};
const ConnectorAuth: React.FC<ConnectorAuthProps> = ({ provider, onSuccess, onError, label }) => {
const ConnectorAuth: React.FC<ConnectorAuthProps> = ({
provider,
onSuccess,
onError,
label,
}) => {
const token = useSelector(selectToken);
const completedRef = useRef(false);
const intervalRef = useRef<number | null>(null);
@@ -31,8 +36,12 @@ const ConnectorAuth: React.FC<ConnectorAuthProps> = ({ provider, onSuccess, onEr
const handleAuthMessage = (event: MessageEvent) => {
const successGeneric = event.data?.type === 'connector_auth_success';
const successProvider = event.data?.type === `${provider}_auth_success` || event.data?.type === 'google_drive_auth_success';
const errorProvider = event.data?.type === `${provider}_auth_error` || event.data?.type === 'google_drive_auth_error';
const successProvider =
event.data?.type === `${provider}_auth_success` ||
event.data?.type === 'google_drive_auth_success';
const errorProvider =
event.data?.type === `${provider}_auth_error` ||
event.data?.type === 'google_drive_auth_error';
if (successGeneric || successProvider) {
completedRef.current = true;
@@ -54,12 +63,17 @@ const ConnectorAuth: React.FC<ConnectorAuthProps> = ({ provider, onSuccess, onEr
cleanup();
const apiHost = import.meta.env.VITE_API_HOST;
const authResponse = await fetch(`${apiHost}/api/connectors/auth?provider=${provider}`, {
headers: { Authorization: `Bearer ${token}` },
});
const authResponse = await fetch(
`${apiHost}/api/connectors/auth?provider=${provider}`,
{
headers: { Authorization: `Bearer ${token}` },
},
);
if (!authResponse.ok) {
throw new Error(`Failed to get authorization URL: ${authResponse.status}`);
throw new Error(
`Failed to get authorization URL: ${authResponse.status}`,
);
}
const authData = await authResponse.json();
@@ -70,10 +84,12 @@ const ConnectorAuth: React.FC<ConnectorAuthProps> = ({ provider, onSuccess, onEr
const authWindow = window.open(
authData.authorization_url,
`${provider}-auth`,
'width=500,height=600,scrollbars=yes,resizable=yes'
'width=500,height=600,scrollbars=yes,resizable=yes',
);
if (!authWindow) {
throw new Error('Failed to open authentication window. Please allow popups.');
throw new Error(
'Failed to open authentication window. Please allow popups.',
);
}
window.addEventListener('message', handleAuthMessage as any);
@@ -98,10 +114,13 @@ const ConnectorAuth: React.FC<ConnectorAuthProps> = ({ provider, onSuccess, onEr
return (
<button
onClick={handleAuth}
className="w-full flex items-center justify-center gap-2 rounded-lg bg-blue-500 px-4 py-3 text-white hover:bg-blue-600 transition-colors"
className="flex w-full items-center justify-center gap-2 rounded-lg bg-blue-500 px-4 py-3 text-white transition-colors hover:bg-blue-600"
>
<svg className="h-5 w-5" viewBox="0 0 24 24">
<path fill="currentColor" d="M6.28 3l5.72 10H24l-5.72-10H6.28zm11.44 0L12 13l5.72 10H24L18.28 3h-.56zM0 13l5.72 10h5.72L5.72 13H0z"/>
<path
fill="currentColor"
d="M6.28 3l5.72 10H24l-5.72-10H6.28zm11.44 0L12 13l5.72 10H24L18.28 3h-.56zM0 13l5.72 10h5.72L5.72 13H0z"
/>
</svg>
{buttonLabel}
</button>
@@ -109,4 +128,3 @@ const ConnectorAuth: React.FC<ConnectorAuthProps> = ({ provider, onSuccess, onEr
};
export default ConnectorAuth;

2
frontend/src/components/ConnectorTreeComponent.tsx
View File

@@ -227,8 +227,6 @@ const ConnectorTreeComponent: React.FC<ConnectorTreeComponentProps> = ({
return current;
};
const getMenuRef = (id: string) => {
if (!menuRefs.current[id]) {
menuRefs.current[id] = React.createRef();

92
frontend/src/components/FileTreeComponent.tsx
View File

@@ -11,9 +11,7 @@ import FolderIcon from '../assets/folder.svg';
import ArrowLeft from '../assets/arrow-left.svg';
import ThreeDots from '../assets/three-dots.svg';
import EyeView from '../assets/eye-view.svg';
import OutlineSource from '../assets/outline-source.svg';
import Trash from '../assets/red-trash.svg';
import SearchIcon from '../assets/search.svg';
import { useOutsideAlerter } from '../hooks';
import ConfirmationModal from '../modals/ConfirmationModal';
@@ -129,8 +127,6 @@ const FileTreeComponent: React.FC<FileTreeComponentProps> = ({
}
}, [docId, token]);
const navigateToDirectory = (dirName: string) => {
setCurrentPath((prev) => [...prev, dirName]);
};
@@ -438,18 +434,18 @@ const FileTreeComponent: React.FC<FileTreeComponentProps> = ({
const renderPathNavigation = () => {
return (
<div className="mb-0 min-h-[38px] flex flex-col gap-2 text-base sm:flex-row sm:items-center sm:justify-between">
<div className="mb-0 flex min-h-[38px] flex-col gap-2 text-base sm:flex-row sm:items-center sm:justify-between">
{/* Left side with path navigation */}
<div className="flex w-full items-center sm:w-auto">
<button
className="mr-3 flex h-[29px] w-[29px] items-center justify-center rounded-full border p-2 text-sm text-gray-400 dark:border-0 dark:bg-[#28292D] dark:text-gray-500 dark:hover:bg-[#2E2F34] font-medium"
className="mr-3 flex h-[29px] w-[29px] items-center justify-center rounded-full border p-2 text-sm font-medium text-gray-400 dark:border-0 dark:bg-[#28292D] dark:text-gray-500 dark:hover:bg-[#2E2F34]"
onClick={handleBackNavigation}
>
<img src={ArrowLeft} alt="left-arrow" className="h-3 w-3" />
</button>
<div className="flex flex-wrap items-center">
<span className="text-[#7D54D1] font-semibold break-words">
<span className="font-semibold break-words text-[#7D54D1]">
{sourceName}
</span>
{currentPath.length > 0 && (
@@ -480,8 +476,7 @@ const FileTreeComponent: React.FC<FileTreeComponentProps> = ({
</div>
</div>
<div className="flex relative flex-row flex-nowrap items-center gap-2 w-full sm:w-auto justify-end mt-2 sm:mt-0">
<div className="relative mt-2 flex w-full flex-row flex-nowrap items-center justify-end gap-2 sm:mt-0 sm:w-auto">
{processingRef.current && (
<div className="text-sm text-gray-500">
{currentOpRef.current === 'add'
@@ -490,13 +485,13 @@ const FileTreeComponent: React.FC<FileTreeComponentProps> = ({
</div>
)}
{renderFileSearch()}
{renderFileSearch()}
{/* Add file button */}
{!processingRef.current && (
<button
onClick={handleAddFile}
className="bg-purple-30 hover:bg-violets-are-blue flex h-[38px] min-w-[108px] items-center justify-center rounded-full px-4 text-[14px] whitespace-nowrap text-white font-medium"
className="bg-purple-30 hover:bg-violets-are-blue flex h-[38px] min-w-[108px] items-center justify-center rounded-full px-4 text-[14px] font-medium whitespace-nowrap text-white"
title={t('settings.sources.addFile')}
>
{t('settings.sources.addFile')}
@@ -538,32 +533,32 @@ const FileTreeComponent: React.FC<FileTreeComponentProps> = ({
const parentRow =
currentPath.length > 0
? [
<tr
key="parent-dir"
className="cursor-pointer border-b border-[#D1D9E0] hover:bg-[#ECEEEF] dark:border-[#6A6A6A] dark:hover:bg-[#27282D]"
onClick={navigateUp}
>
<td className="px-2 py-2 lg:px-4">
<div className="flex items-center">
<img
src={FolderIcon}
alt={t('settings.sources.parentFolderAlt')}
className="mr-2 h-4 w-4 flex-shrink-0"
/>
<span className="truncate text-sm dark:text-[#E0E0E0]">
..
</span>
</div>
</td>
<td className="px-2 py-2 text-sm lg:px-4 dark:text-[#E0E0E0]">
-
</td>
<td className="px-2 py-2 text-sm lg:px-4 dark:text-[#E0E0E0]">
-
</td>
<td className="w-10 px-2 py-2 text-sm lg:px-4"></td>
</tr>,
]
<tr
key="parent-dir"
className="cursor-pointer border-b border-[#D1D9E0] hover:bg-[#ECEEEF] dark:border-[#6A6A6A] dark:hover:bg-[#27282D]"
onClick={navigateUp}
>
<td className="px-2 py-2 lg:px-4">
<div className="flex items-center">
<img
src={FolderIcon}
alt={t('settings.sources.parentFolderAlt')}
className="mr-2 h-4 w-4 flex-shrink-0"
/>
<span className="truncate text-sm dark:text-[#E0E0E0]">
..
</span>
</div>
</td>
<td className="px-2 py-2 text-sm lg:px-4 dark:text-[#E0E0E0]">
-
</td>
<td className="px-2 py-2 text-sm lg:px-4 dark:text-[#E0E0E0]">
-
</td>
<td className="w-10 px-2 py-2 text-sm lg:px-4"></td>
</tr>,
]
: [];
// Render directories first, then files
@@ -604,7 +599,7 @@ const FileTreeComponent: React.FC<FileTreeComponentProps> = ({
<div ref={menuRef} className="relative">
<button
onClick={(e) => handleMenuClick(e, itemId)}
className="inline-flex h-[35px] w-[24px] shrink-0 items-center justify-center rounded-md transition-colors hover:bg-[#EBEBEB] dark:hover:bg-[#26272E] font-medium"
className="inline-flex h-[35px] w-[24px] shrink-0 items-center justify-center rounded-md font-medium transition-colors hover:bg-[#EBEBEB] dark:hover:bg-[#26272E]"
aria-label={t('settings.sources.menuAlt')}
>
<img
@@ -660,7 +655,7 @@ const FileTreeComponent: React.FC<FileTreeComponentProps> = ({
<div ref={menuRef} className="relative">
<button
onClick={(e) => handleMenuClick(e, itemId)}
className="inline-flex h-[35px] w-[24px] shrink-0 items-center justify-center rounded-md transition-colors hover:bg-[#EBEBEB] dark:hover:bg-[#26272E] font-medium"
className="inline-flex h-[35px] w-[24px] shrink-0 items-center justify-center rounded-md font-medium transition-colors hover:bg-[#EBEBEB] dark:hover:bg-[#26272E]"
aria-label={t('settings.sources.menuAlt')}
>
<img
@@ -752,14 +747,12 @@ const FileTreeComponent: React.FC<FileTreeComponentProps> = ({
}
}}
placeholder={t('settings.sources.searchFiles')}
className={`w-full h-[38px] border border-[#D1D9E0] px-4 py-2 dark:border-[#6A6A6A]
${searchQuery ? 'rounded-t-[24px]' : 'rounded-[24px]'}
bg-transparent focus:outline-none dark:text-[#E0E0E0]`}
className={`h-[38px] w-full border border-[#D1D9E0] px-4 py-2 dark:border-[#6A6A6A] ${searchQuery ? 'rounded-t-[24px]' : 'rounded-[24px]'} bg-transparent focus:outline-none dark:text-[#E0E0E0]`}
/>
{searchQuery && (
<div className="absolute top-full left-0 right-0 z-10 max-h-[calc(100vh-200px)] w-full overflow-hidden rounded-b-[12px] border border-t-0 border-[#D1D9E0] bg-white shadow-lg dark:border-[#6A6A6A] dark:bg-[#1F2023] transition-all duration-200">
<div className="max-h-[calc(100vh-200px)] overflow-y-auto overflow-x-hidden overscroll-contain">
<div className="absolute top-full right-0 left-0 z-10 max-h-[calc(100vh-200px)] w-full overflow-hidden rounded-b-[12px] border border-t-0 border-[#D1D9E0] bg-white shadow-lg transition-all duration-200 dark:border-[#6A6A6A] dark:bg-[#1F2023]">
<div className="max-h-[calc(100vh-200px)] overflow-x-hidden overflow-y-auto overscroll-contain">
{searchResults.length === 0 ? (
<div className="py-2 text-center text-sm text-gray-500 dark:text-gray-400">
{t('settings.sources.noResults')}
@@ -770,10 +763,11 @@ const FileTreeComponent: React.FC<FileTreeComponentProps> = ({
key={index}
onClick={() => handleSearchSelect(result)}
title={result.path}
className={`flex min-w-0 cursor-pointer items-center px-3 py-2 hover:bg-[#ECEEEF] dark:hover:bg-[#27282D] ${index !== searchResults.length - 1
className={`flex min-w-0 cursor-pointer items-center px-3 py-2 hover:bg-[#ECEEEF] dark:hover:bg-[#27282D] ${
index !== searchResults.length - 1
? 'border-b border-[#D1D9E0] dark:border-[#6A6A6A]'
: ''
}`}
}`}
>
<img
src={result.isFile ? FileIcon : FolderIcon}
@@ -784,7 +778,7 @@ const FileTreeComponent: React.FC<FileTreeComponentProps> = ({
}
className="mr-2 h-4 w-4 flex-shrink-0"
/>
<span className="text-sm dark:text-[#E0E0E0] truncate flex-1">
<span className="flex-1 truncate text-sm dark:text-[#E0E0E0]">
{result.path.split('/').pop() || result.path}
</span>
</div>
@@ -866,7 +860,9 @@ const FileTreeComponent: React.FC<FileTreeComponentProps> = ({
message={
itemToDelete?.isFile
? t('settings.sources.confirmDelete')
: t('settings.sources.deleteDirectoryWarning', { name: itemToDelete?.name })
: t('settings.sources.deleteDirectoryWarning', {
name: itemToDelete?.name,
})
}
modalState={deleteModalState}
setModalState={setDeleteModalState}

10
frontend/src/components/MessageInput.tsx
View File

@@ -368,8 +368,8 @@ export default function MessageInput({
className="xs:px-3 xs:py-1.5 dark:border-purple-taupe flex max-w-[130px] items-center rounded-[32px] border border-[#AAAAAA] px-2 py-1 transition-colors hover:bg-gray-100 sm:max-w-[150px] dark:hover:bg-[#2C2E3C]"
onClick={() => setIsSourcesPopupOpen(!isSourcesPopupOpen)}
title={
selectedDocs
? selectedDocs.name
selectedDocs && selectedDocs.length > 0
? selectedDocs.map((doc) => doc.name).join(', ')
: t('conversation.sources.title')
}
>
@@ -379,8 +379,10 @@ export default function MessageInput({
className="mr-1 h-3.5 w-3.5 shrink-0 sm:mr-1.5 sm:h-4"
/>
<span className="xs:text-[12px] dark:text-bright-gray truncate overflow-hidden text-[10px] font-medium text-[#5D5D5D] sm:text-[14px]">
{selectedDocs
? selectedDocs.name
{selectedDocs && selectedDocs.length > 0
? selectedDocs.length === 1
? selectedDocs[0].name
: `${selectedDocs.length} sources selected`
: t('conversation.sources.title')}
</span>
{!isTouch && (

38
frontend/src/components/SourcesPopup.tsx
View File

@@ -17,7 +17,7 @@ type SourcesPopupProps = {
isOpen: boolean;
onClose: () => void;
anchorRef: React.RefObject<HTMLButtonElement | null>;
handlePostDocumentSelect: (doc: Doc | null) => void;
handlePostDocumentSelect: (doc: Doc[] | null) => void;
setUploadModalState: React.Dispatch<React.SetStateAction<ActiveState>>;
};
@@ -149,9 +149,13 @@ export default function SourcesPopup({
if (option.model === embeddingsName) {
const isSelected =
selectedDocs &&
(option.id
? selectedDocs.id === option.id
: selectedDocs.date === option.date);
Array.isArray(selectedDocs) &&
selectedDocs.length > 0 &&
selectedDocs.some((doc) =>
option.id
? doc.id === option.id
: doc.date === option.date,
);
return (
<div
@@ -159,11 +163,29 @@ export default function SourcesPopup({
className="border-opacity-80 dark:border-dim-gray flex cursor-pointer items-center border-b border-[#D9D9D9] p-3 transition-colors hover:bg-gray-100 dark:text-[14px] dark:hover:bg-[#2C2E3C]"
onClick={() => {
if (isSelected) {
dispatch(setSelectedDocs(null));
handlePostDocumentSelect(null);
const updatedDocs =
selectedDocs && Array.isArray(selectedDocs)
? selectedDocs.filter((doc) =>
option.id
? doc.id !== option.id
: doc.date !== option.date,
)
: [];
dispatch(
setSelectedDocs(
updatedDocs.length > 0 ? updatedDocs : null,
),
);
handlePostDocumentSelect(
updatedDocs.length > 0 ? updatedDocs : null,
);
} else {
dispatch(setSelectedDocs(option));
handlePostDocumentSelect(option);
const updatedDocs =
selectedDocs && Array.isArray(selectedDocs)
? [...selectedDocs, option]
: [option];
dispatch(setSelectedDocs(updatedDocs));
handlePostDocumentSelect(updatedDocs);
}
}}
>

46
frontend/src/conversation/conversationHandlers.ts
View File

@@ -7,7 +7,7 @@ export function handleFetchAnswer(
question: string,
signal: AbortSignal,
token: string | null,
selectedDocs: Doc | null,
selectedDocs: Doc[] | null,
conversationId: string | null,
promptId: string | null,
chunks: string,
@@ -52,10 +52,17 @@ export function handleFetchAnswer(
payload.attachments = attachments;
}
if (selectedDocs && 'id' in selectedDocs) {
payload.active_docs = selectedDocs.id as string;
if (selectedDocs && Array.isArray(selectedDocs)) {
if (selectedDocs.length > 1) {
// Handle multiple documents
payload.active_docs = selectedDocs.map((doc) => doc.id!);
payload.retriever = selectedDocs[0]?.retriever as string;
} else if (selectedDocs.length === 1 && 'id' in selectedDocs[0]) {
// Handle single document (backward compatibility)
payload.active_docs = selectedDocs[0].id as string;
payload.retriever = selectedDocs[0].retriever as string;
}
}
payload.retriever = selectedDocs?.retriever as string;
return conversationService
.answer(payload, token, signal)
.then((response) => {
@@ -84,7 +91,7 @@ export function handleFetchAnswerSteaming(
question: string,
signal: AbortSignal,
token: string | null,
selectedDocs: Doc | null,
selectedDocs: Doc[] | null,
conversationId: string | null,
promptId: string | null,
chunks: string,
@@ -112,10 +119,17 @@ export function handleFetchAnswerSteaming(
payload.attachments = attachments;
}
if (selectedDocs && 'id' in selectedDocs) {
payload.active_docs = selectedDocs.id as string;
if (selectedDocs && Array.isArray(selectedDocs)) {
if (selectedDocs.length > 1) {
// Handle multiple documents
payload.active_docs = selectedDocs.map((doc) => doc.id!);
payload.retriever = selectedDocs[0]?.retriever as string;
} else if (selectedDocs.length === 1 && 'id' in selectedDocs[0]) {
// Handle single document (backward compatibility)
payload.active_docs = selectedDocs[0].id as string;
payload.retriever = selectedDocs[0].retriever as string;
}
}
payload.retriever = selectedDocs?.retriever as string;
return new Promise<Answer>((resolve, reject) => {
conversationService
@@ -171,7 +185,7 @@ export function handleFetchAnswerSteaming(
export function handleSearch(
question: string,
token: string | null,
selectedDocs: Doc | null,
selectedDocs: Doc[] | null,
conversation_id: string | null,
chunks: string,
token_limit: number,
@@ -183,9 +197,17 @@ export function handleSearch(
token_limit: token_limit,
isNoneDoc: selectedDocs === null,
};
if (selectedDocs && 'id' in selectedDocs)
payload.active_docs = selectedDocs.id as string;
payload.retriever = selectedDocs?.retriever as string;
if (selectedDocs && Array.isArray(selectedDocs)) {
if (selectedDocs.length > 1) {
// Handle multiple documents
payload.active_docs = selectedDocs.map((doc) => doc.id!);
payload.retriever = selectedDocs[0]?.retriever as string;
} else if (selectedDocs.length === 1 && 'id' in selectedDocs[0]) {
// Handle single document (backward compatibility)
payload.active_docs = selectedDocs[0].id as string;
payload.retriever = selectedDocs[0].retriever as string;
}
}
return conversationService
.search(payload, token)
.then((response) => response.json())

2
frontend/src/conversation/conversationModels.ts
View File

@@ -54,7 +54,7 @@ export interface Query {
export interface RetrievalPayload {
question: string;
active_docs?: string;
active_docs?: string | string[];
retriever?: string;
conversation_id: string | null;
prompt_id?: string | null;

4
frontend/src/hooks/useDefaultDocument.ts
View File

@@ -18,11 +18,11 @@ export default function useDefaultDocument() {
const fetchDocs = () => {
getDocs(token).then((data) => {
dispatch(setSourceDocs(data));
if (!selectedDoc)
if (!selectedDoc || (Array.isArray(selectedDoc) && selectedDoc.length === 0))
Array.isArray(data) &&
data?.forEach((doc: Doc) => {
if (doc.model && doc.name === 'default') {
dispatch(setSelectedDocs(doc));
dispatch(setSelectedDocs([doc]));
}
});
});

14
frontend/src/modals/ShareConversationModal.tsx
View File

@@ -60,7 +60,7 @@ export const ShareConversationModal = ({
const [sourcePath, setSourcePath] = useState<{
label: string;
value: string;
} | null>(preSelectedDoc ? extractDocPaths([preSelectedDoc])[0] : null);
} | null>(preSelectedDoc ? extractDocPaths(preSelectedDoc)[0] : null);
const handleCopyKey = (url: string) => {
navigator.clipboard.writeText(url);
@@ -105,14 +105,14 @@ export const ShareConversationModal = ({
return (
<WrapperModal close={close}>
<div className="flex flex-col gap-2">
<h2 className="text-xl font-medium text-eerie-black dark:text-chinese-white">
<h2 className="text-eerie-black dark:text-chinese-white text-xl font-medium">
{t('modals.shareConv.label')}
</h2>
<p className="text-sm text-eerie-black dark:text-silver/60">
<p className="text-eerie-black dark:text-silver/60 text-sm">
{t('modals.shareConv.note')}
</p>
<div className="flex items-center justify-between">
<span className="text-lg text-eerie-black dark:text-white">
<span className="text-eerie-black text-lg dark:text-white">
{t('modals.shareConv.option')}
</span>
<ToggleSwitch
@@ -136,19 +136,19 @@ export const ShareConversationModal = ({
</div>
)}
<div className="flex items-baseline justify-between gap-2">
<span className="no-scrollbar w-full overflow-x-auto whitespace-nowrap rounded-full border-2 border-silver px-4 py-3 text-eerie-black dark:border-silver/40 dark:text-white">
<span className="no-scrollbar border-silver text-eerie-black dark:border-silver/40 w-full overflow-x-auto rounded-full border-2 px-4 py-3 whitespace-nowrap dark:text-white">
{`${domain}/share/${identifier ?? '....'}`}
</span>
{status === 'fetched' ? (
<button
className="my-1 h-10 w-28 rounded-full bg-purple-30 p-2 text-sm text-white hover:bg-violets-are-blue"
className="bg-purple-30 hover:bg-violets-are-blue my-1 h-10 w-28 rounded-full p-2 text-sm text-white"
onClick={() => handleCopyKey(`${domain}/share/${identifier}`)}
>
{isCopied ? t('modals.saveKey.copied') : t('modals.saveKey.copy')}
</button>
) : (
<button
className="my-1 flex h-10 w-28 items-center justify-evenly rounded-full bg-purple-30 p-2 text-center text-sm font-normal text-white hover:bg-violets-are-blue"
className="bg-purple-30 hover:bg-violets-are-blue my-1 flex h-10 w-28 items-center justify-evenly rounded-full p-2 text-center text-sm font-normal text-white"
onClick={() => {
shareCoversationPublicly(allowPrompt);
}}

33
frontend/src/preferences/preferenceApi.ts
View File

@@ -90,9 +90,9 @@ export function getLocalApiKey(): string | null {
return key;
}
export function getLocalRecentDocs(): string | null {
const doc = localStorage.getItem('DocsGPTRecentDocs');
return doc;
export function getLocalRecentDocs(): Doc[] | null {
const docs = localStorage.getItem('DocsGPTRecentDocs');
return docs ? JSON.parse(docs) as Doc[] : null;
}
export function getLocalPrompt(): string | null {
@@ -108,19 +108,20 @@ export function setLocalPrompt(prompt: string): void {
localStorage.setItem('DocsGPTPrompt', prompt);
}
export function setLocalRecentDocs(doc: Doc | null): void {
localStorage.setItem('DocsGPTRecentDocs', JSON.stringify(doc));
export function setLocalRecentDocs(docs: Doc[] | null): void {
if (docs && docs.length > 0) {
localStorage.setItem('DocsGPTRecentDocs', JSON.stringify(docs));
let docPath = 'default';
if (doc?.type === 'local') {
docPath = 'local' + '/' + doc.name + '/';
docs.forEach((doc) => {
let docPath = 'default';
if (doc.type === 'local') {
docPath = 'local' + '/' + doc.name + '/';
}
userService
.checkDocs({ docs: docPath }, null)
.then((response) => response.json());
});
} else {
localStorage.removeItem('DocsGPTRecentDocs');
}
userService
.checkDocs(
{
docs: docPath,
},
null,
)
.then((response) => response.json());
}

21
frontend/src/preferences/preferenceSlice.ts
View File

@@ -15,7 +15,7 @@ export interface Preference {
prompt: { name: string; id: string; type: string };
chunks: string;
token_limit: number;
selectedDocs: Doc | null;
selectedDocs: Doc[] | null;
sourceDocs: Doc[] | null;
conversations: {
data: { name: string; id: string }[] | null;
@@ -34,15 +34,16 @@ const initialState: Preference = {
prompt: { name: 'default', id: 'default', type: 'public' },
chunks: '2',
token_limit: 2000,
selectedDocs: {
id: 'default',
name: 'default',
type: 'remote',
date: 'default',
docLink: 'default',
model: 'openai_text-embedding-ada-002',
retriever: 'classic',
} as Doc,
selectedDocs: [
{
id: 'default',
name: 'default',
type: 'remote',
date: 'default',
model: 'openai_text-embedding-ada-002',
retriever: 'classic',
},
] as Doc[],
sourceDocs: null,
conversations: {
data: null,

71
frontend/src/settings/Sources.tsx
View File

@@ -1,4 +1,3 @@
import React, { useCallback, useEffect, useRef, useState } from 'react';
import { useTranslation } from 'react-i18next';
import { useDispatch, useSelector } from 'react-redux';
@@ -272,27 +271,27 @@ export default function Sources({
return documentToView ? (
<div className="mt-8 flex flex-col">
{documentToView.isNested ? (
documentToView.type === 'connector' ? (
<ConnectorTreeComponent
docId={documentToView.id || ''}
sourceName={documentToView.name}
onBackToDocuments={() => setDocumentToView(undefined)}
/>
{documentToView.isNested ? (
documentToView.type === 'connector' ? (
<ConnectorTreeComponent
docId={documentToView.id || ''}
sourceName={documentToView.name}
onBackToDocuments={() => setDocumentToView(undefined)}
/>
) : (
<FileTreeComponent
docId={documentToView.id || ''}
sourceName={documentToView.name}
onBackToDocuments={() => setDocumentToView(undefined)}
/>
)
) : (
<FileTreeComponent
docId={documentToView.id || ''}
sourceName={documentToView.name}
onBackToDocuments={() => setDocumentToView(undefined)}
<Chunks
documentId={documentToView.id || ''}
documentName={documentToView.name}
handleGoBack={() => setDocumentToView(undefined)}
/>
)
) : (
<Chunks
documentId={documentToView.id || ''}
documentName={documentToView.name}
handleGoBack={() => setDocumentToView(undefined)}
/>
)}
)}
</div>
) : (
<div className="mt-8 flex w-full max-w-full flex-col overflow-hidden">
@@ -319,7 +318,7 @@ export default function Sources({
setSearchTerm(e.target.value);
setCurrentPage(1);
}}
className="w-full h-[32px] rounded-full border border-silver dark:border-silver/40 bg-transparent px-3 text-sm text-jet dark:text-bright-gray placeholder:text-gray-400 dark:placeholder:text-gray-500 outline-none focus:border-silver dark:focus:border-silver/60"
className="border-silver dark:border-silver/40 text-jet dark:text-bright-gray focus:border-silver dark:focus:border-silver/60 h-[32px] w-full rounded-full border bg-transparent px-3 text-sm outline-none placeholder:text-gray-400 dark:placeholder:text-gray-500"
/>
</div>
</div>
@@ -336,7 +335,7 @@ export default function Sources({
</div>
<div className="relative w-full">
{loading ? (
<div className="w-full grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-3 xl:grid-cols-4 gap-6 px-2 py-4">
<div className="grid w-full grid-cols-1 gap-6 px-2 py-4 sm:grid-cols-2 lg:grid-cols-3 xl:grid-cols-4">
<SkeletonLoader component="sourceCards" count={rowsPerPage} />
</div>
) : !currentDocuments?.length ? (
@@ -351,19 +350,19 @@ export default function Sources({
</p>
</div>
) : (
<div className="w-full grid grid-cols-1 sm:grid-cols-2 lg:grid-cols-3 xl:grid-cols-4 gap-6 px-2 py-4">
{currentDocuments.map((document, index) => {
const docId = document.id ? document.id.toString() : '';
<div className="grid w-full grid-cols-1 gap-6 px-2 py-4 sm:grid-cols-2 lg:grid-cols-3 xl:grid-cols-4">
{currentDocuments.map((document, index) => {
const docId = document.id ? document.id.toString() : '';
return (
<div key={docId} className="relative">
<div
className={`flex h-[130px] w-full flex-col rounded-2xl bg-[#F9F9F9] p-3 transition-all duration-200 dark:bg-[#383838] ${
activeMenuId === docId || syncMenuState.docId === docId
? 'scale-[1.05]'
: 'hover:scale-[1.05]'
}`}
>
return (
<div key={docId} className="relative">
<div
className={`flex h-[130px] w-full flex-col rounded-2xl bg-[#F9F9F9] p-3 transition-all duration-200 dark:bg-[#383838] ${
activeMenuId === docId || syncMenuState.docId === docId
? 'scale-[1.05]'
: 'hover:scale-[1.05]'
}`}
>
<div className="w-full flex-1">
<div className="flex w-full items-center justify-between gap-2">
<h3
@@ -427,7 +426,7 @@ export default function Sources({
<img
src={CalendarIcon}
alt=""
className="w-[14px] h-[14px]"
className="h-[14px] w-[14px]"
/>
<span className="font-inter text-[12px] leading-[18px] font-[500] text-[#848484] dark:text-[#848484]">
{document.date ? formatDate(document.date) : ''}
@@ -437,7 +436,7 @@ export default function Sources({
<img
src={DiscIcon}
alt=""
className="w-[14px] h-[14px]"
className="h-[14px] w-[14px]"
/>
<span className="font-inter text-[12px] leading-[18px] font-[500] text-[#848484] dark:text-[#848484]">
{document.tokens

242
frontend/src/upload/Upload.tsx
View File

@@ -4,7 +4,11 @@ import { useTranslation } from 'react-i18next';
import { useDispatch, useSelector } from 'react-redux';
import userService from '../api/services/userService';
import { getSessionToken, setSessionToken, removeSessionToken } from '../utils/providerUtils';
import {
getSessionToken,
setSessionToken,
removeSessionToken,
} from '../utils/providerUtils';
import { formatDate } from '../utils/dateTimeUtils';
import { formatBytes } from '../utils/stringUtils';
import FileUpload from '../assets/file_upload.svg';
@@ -63,7 +67,9 @@ function Upload({
const [userEmail, setUserEmail] = useState<string>('');
const [authError, setAuthError] = useState<string>('');
const [currentFolderId, setCurrentFolderId] = useState<string | null>(null);
const [folderPath, setFolderPath] = useState<Array<{id: string | null, name: string}>>([{id: null, name: 'My Drive'}]);
const [folderPath, setFolderPath] = useState<
Array<{ id: string | null; name: string }>
>([{ id: null, name: 'My Drive' }]);
const [nextPageToken, setNextPageToken] = useState<string | null>(null);
const [hasMoreFiles, setHasMoreFiles] = useState<boolean>(false);
@@ -337,7 +343,8 @@ function Upload({
data?.find(
(d: Doc) => d.type?.toLowerCase() === 'local',
),
));
),
);
});
setProgress(
(progress) =>
@@ -454,23 +461,31 @@ function Upload({
if (ingestor.type === 'google_drive') {
const sessionToken = getSessionToken(ingestor.type);
const selectedItems = googleDriveFiles.filter(file => selectedFiles.includes(file.id));
const selectedItems = googleDriveFiles.filter((file) =>
selectedFiles.includes(file.id),
);
const selectedFolderIds = selectedItems
.filter(item => item.type === 'application/vnd.google-apps.folder' || item.isFolder)
.map(folder => folder.id);
.filter(
(item) =>
item.type === 'application/vnd.google-apps.folder' || item.isFolder,
)
.map((folder) => folder.id);
const selectedFileIds = selectedItems
.filter(item => item.type !== 'application/vnd.google-apps.folder' && !item.isFolder)
.map(file => file.id);
.filter(
(item) =>
item.type !== 'application/vnd.google-apps.folder' &&
!item.isFolder,
)
.map((file) => file.id);
configData = {
file_ids: selectedFileIds,
folder_ids: selectedFolderIds,
recursive: ingestor.config.recursive,
session_token: sessionToken || null
session_token: sessionToken || null,
};
} else {
configData = { ...ingestor.config };
}
@@ -522,14 +537,20 @@ function Upload({
try {
const apiHost = import.meta.env.VITE_API_HOST;
const validateResponse = await fetch(`${apiHost}/api/connectors/validate-session`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${token}`
const validateResponse = await fetch(
`${apiHost}/api/connectors/validate-session`,
{
method: 'POST',
headers: {
'Content-Type': 'application/json',
Authorization: `Bearer ${token}`,
},
body: JSON.stringify({
provider: 'google_drive',
session_token: sessionToken,
}),
},
body: JSON.stringify({ provider: 'google_drive', session_token: sessionToken })
});
);
if (!validateResponse.ok) {
removeSessionToken(ingestor.type);
@@ -545,15 +566,16 @@ function Upload({
// reset pagination state and files
setGoogleDriveFiles([]);
setNextPageToken(null);
setHasMoreFiles(false);
loadGoogleDriveFiles(sessionToken, null, null, false);
} else {
removeSessionToken(ingestor.type);
setIsGoogleDriveConnected(false);
setAuthError(validateData.error || 'Session expired. Please reconnect your Google Drive account and make sure to grant offline access.');
setAuthError(
validateData.error ||
'Session expired. Please reconnect your Google Drive account and make sure to grant offline access.',
);
}
} catch (error) {
console.error('Error validating Google Drive session:', error);
@@ -566,7 +588,7 @@ function Upload({
sessionToken: string,
folderId?: string | null,
pageToken?: string | null,
append: boolean = false,
append = false,
) => {
setIsLoadingFiles(true);
@@ -587,9 +609,9 @@ function Upload({
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${token}`
Authorization: `Bearer ${token}`,
},
body: JSON.stringify({ ...requestBody, provider: 'google_drive' })
body: JSON.stringify({ ...requestBody, provider: 'google_drive' }),
});
if (!filesResponse.ok) {
@@ -599,28 +621,31 @@ function Upload({
const filesData = await filesResponse.json();
if (filesData.success && Array.isArray(filesData.files)) {
setGoogleDriveFiles(prev => append ? [...prev, ...filesData.files] : filesData.files);
setGoogleDriveFiles((prev) =>
append ? [...prev, ...filesData.files] : filesData.files,
);
setNextPageToken(filesData.next_page_token || null);
setHasMoreFiles(Boolean(filesData.has_more));
} else {
throw new Error(filesData.error || 'Failed to load files');
}
} catch (error) {
console.error('Error loading Google Drive files:', error);
setAuthError(error instanceof Error ? error.message : 'Failed to load files. Please make sure your Google Drive account is properly connected and you granted offline access during authorization.');
setAuthError(
error instanceof Error
? error.message
: 'Failed to load files. Please make sure your Google Drive account is properly connected and you granted offline access during authorization.',
);
} finally {
setIsLoadingFiles(false);
}
};
// Handle file selection
const handleFileSelect = (fileId: string) => {
setSelectedFiles(prev => {
setSelectedFiles((prev) => {
if (prev.includes(fileId)) {
return prev.filter(id => id !== fileId);
return prev.filter((id) => id !== fileId);
} else {
return [...prev, fileId];
}
@@ -631,7 +656,7 @@ function Upload({
const sessionToken = getSessionToken(ingestor.type);
if (sessionToken) {
setCurrentFolderId(folderId);
setFolderPath(prev => [...prev, {id: folderId, name: folderName}]);
setFolderPath((prev) => [...prev, { id: folderId, name: folderName }]);
setGoogleDriveFiles([]);
setNextPageToken(null);
@@ -662,7 +687,7 @@ function Upload({
if (selectedFiles.length === googleDriveFiles.length) {
setSelectedFiles([]);
} else {
setSelectedFiles(googleDriveFiles.map(file => file.id));
setSelectedFiles(googleDriveFiles.map((file) => file.id));
}
};
@@ -829,7 +854,7 @@ function Upload({
{files.map((file) => (
<p
key={file.name}
className="text-gray-6000 dark:text-[#ececf1] truncate overflow-hidden text-ellipsis"
className="text-gray-6000 truncate overflow-hidden text-ellipsis dark:text-[#ececf1]"
title={file.name}
>
{file.name}
@@ -905,10 +930,13 @@ function Upload({
) : (
<div className="space-y-4">
{/* Connection Status */}
<div className="w-full flex items-center justify-between rounded-lg bg-green-500 px-4 py-2 text-white text-sm">
<div className="flex w-full items-center justify-between rounded-lg bg-green-500 px-4 py-2 text-sm text-white">
<div className="flex items-center gap-2">
<svg className="h-4 w-4" viewBox="0 0 24 24">
<path fill="currentColor" d="M9 16.17L4.83 12l-1.42 1.41L9 19 21 7l-1.41-1.41z"/>
<path
fill="currentColor"
d="M9 16.17L4.83 12l-1.42 1.41L9 19 21 7l-1.41-1.41z"
/>
</svg>
<span>Connected as {userEmail}</span>
</div>
@@ -927,28 +955,41 @@ function Upload({
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Authorization': `Bearer ${token}`
Authorization: `Bearer ${token}`,
},
body: JSON.stringify({ provider: ingestor.type, session_token: getSessionToken(ingestor.type) })
}).catch(err => console.error('Error disconnecting from Google Drive:', err));
body: JSON.stringify({
provider: ingestor.type,
session_token: getSessionToken(ingestor.type),
}),
}).catch((err) =>
console.error(
'Error disconnecting from Google Drive:',
err,
),
);
}}
className="text-white hover:text-gray-200 text-xs underline"
className="text-xs text-white underline hover:text-gray-200"
>
Disconnect
</button>
</div>
{/* File Browser */}
<div className="border border-gray-200 rounded-lg dark:border-gray-600">
<div className="p-3 border-b border-gray-200 dark:border-gray-600 bg-gray-50 dark:bg-gray-800 rounded-t-lg">
<div className="rounded-lg border border-gray-200 dark:border-gray-600">
<div className="rounded-t-lg border-b border-gray-200 bg-gray-50 p-3 dark:border-gray-600 dark:bg-gray-800">
{/* Breadcrumb navigation */}
<div className="flex items-center gap-1 mb-2">
<div className="mb-2 flex items-center gap-1">
{folderPath.map((path, index) => (
<div key={path.id || 'root'} className="flex items-center gap-1">
{index > 0 && <span className="text-gray-400">/</span>}
<div
key={path.id || 'root'}
className="flex items-center gap-1"
>
{index > 0 && (
<span className="text-gray-400">/</span>
)}
<button
onClick={() => navigateBack(index)}
className="text-sm text-blue-600 hover:text-blue-800 dark:text-blue-400 hover:underline"
className="text-sm text-blue-600 hover:text-blue-800 hover:underline dark:text-blue-400"
disabled={index === folderPath.length - 1}
>
{path.name}
@@ -966,18 +1007,24 @@ function Upload({
onClick={handleSelectAll}
className="text-xs text-blue-600 hover:text-blue-800 dark:text-blue-400"
>
{selectedFiles.length === googleDriveFiles.length ? 'Deselect All' : 'Select All'}
{selectedFiles.length === googleDriveFiles.length
? 'Deselect All'
: 'Select All'}
</button>
)}
</div>
{selectedFiles.length > 0 && (
<p className="text-xs text-gray-500 mt-1">
{selectedFiles.length} file{selectedFiles.length !== 1 ? 's' : ''} selected
<p className="mt-1 text-xs text-gray-500">
{selectedFiles.length} file
{selectedFiles.length !== 1 ? 's' : ''} selected
</p>
)}
</div>
<div className="max-h-72 overflow-y-auto" ref={scrollContainerRef}>
<div
className="max-h-72 overflow-y-auto"
ref={scrollContainerRef}
>
{isLoadingFiles && googleDriveFiles.length === 0 ? (
<div className="p-4 text-center">
<div className="inline-flex items-center gap-2 text-sm text-gray-600 dark:text-gray-400">
@@ -996,47 +1043,76 @@ function Upload({
<div
key={file.id}
className={`p-3 transition-colors ${
selectedFiles.includes(file.id) ? 'bg-blue-50 dark:bg-blue-900/20' : ''
selectedFiles.includes(file.id)
? 'bg-blue-50 dark:bg-blue-900/20'
: ''
}`}
>
<div className="flex items-center gap-3">
<div className="flex-shrink-0">
<input
type="checkbox"
checked={selectedFiles.includes(file.id)}
onChange={() => handleFileSelect(file.id)}
className="h-4 w-4 text-blue-600 rounded border-gray-300 focus:ring-blue-500"
checked={selectedFiles.includes(
file.id,
)}
onChange={() =>
handleFileSelect(file.id)
}
className="h-4 w-4 rounded border-gray-300 text-blue-600 focus:ring-blue-500"
/>
</div>
{file.type === 'application/vnd.google-apps.folder' || file.isFolder ? (
{file.type ===
'application/vnd.google-apps.folder' ||
file.isFolder ? (
<div
className="text-lg cursor-pointer hover:text-blue-600"
onClick={() => handleFolderClick(file.id, file.name)}
className="cursor-pointer text-lg hover:text-blue-600"
onClick={() =>
handleFolderClick(file.id, file.name)
}
>
<img src={FolderIcon} alt="Folder" className="h-6 w-6" />
<img
src={FolderIcon}
alt="Folder"
className="h-6 w-6"
/>
</div>
) : (
<div className="text-lg">
<img src={FileIcon} alt="File" className="h-6 w-6" />
<img
src={FileIcon}
alt="File"
className="h-6 w-6"
/>
</div>
)}
<div className="flex-1 min-w-0">
<div className="min-w-0 flex-1">
<p
className={`text-sm font-medium truncate dark:text-[#ececf1] ${
file.type === 'application/vnd.google-apps.folder' || file.isFolder
className={`truncate text-sm font-medium dark:text-[#ececf1] ${
file.type ===
'application/vnd.google-apps.folder' ||
file.isFolder
? 'cursor-pointer hover:text-blue-600'
: ''
}`}
onClick={() => {
if (file.type === 'application/vnd.google-apps.folder' || file.isFolder) {
handleFolderClick(file.id, file.name);
if (
file.type ===
'application/vnd.google-apps.folder' ||
file.isFolder
) {
handleFolderClick(
file.id,
file.name,
);
}
}}
>
{file.name}
</p>
<p className="text-xs text-gray-500 dark:text-gray-400">
{file.size && `${formatBytes(file.size)} • `}Modified {formatDate(file.modifiedTime)}
{file.size &&
`${formatBytes(file.size)} • `}
Modified {formatDate(file.modifiedTime)}
</p>
</div>
</div>
@@ -1044,7 +1120,7 @@ function Upload({
))}
</div>
<div className="p-4 flex items-center justify-center border-t border-gray-100 dark:border-gray-800">
<div className="flex items-center justify-center border-t border-gray-100 p-4 dark:border-gray-800">
{isLoadingFiles && (
<div className="inline-flex items-center gap-2 text-sm text-gray-600 dark:text-gray-400">
<div className="h-4 w-4 animate-spin rounded-full border-2 border-blue-500 border-t-transparent"></div>
@@ -1052,16 +1128,16 @@ function Upload({
</div>
)}
{!hasMoreFiles && !isLoadingFiles && (
<span className="text-sm text-gray-500 dark:text-gray-400">All files loaded</span>
<span className="text-sm text-gray-500 dark:text-gray-400">
All files loaded
</span>
)}
</div>
</>
)}
</div>
<div className="hidden" aria-hidden="true">
</div>
<div className="hidden" aria-hidden="true"></div>
</div>
</div>
)}
@@ -1110,8 +1186,7 @@ function Upload({
>
{ingestor.type === 'google_drive' && selectedFiles.length > 0
? `Train with ${selectedFiles.length} file${selectedFiles.length !== 1 ? 's' : ''}`
: t('modals.uploadDoc.train')
}
: t('modals.uploadDoc.train')}
</button>
)}
</div>
@@ -1121,27 +1196,38 @@ function Upload({
useEffect(() => {
const scrollContainer = scrollContainerRef.current;
const handleScroll = () => {
if (!scrollContainer) return;
const { scrollTop, scrollHeight, clientHeight } = scrollContainer;
const isNearBottom = scrollHeight - scrollTop - clientHeight < 50;
if (isNearBottom && hasMoreFiles && !isLoadingFiles && nextPageToken) {
const sessionToken = getSessionToken(ingestor.type);
if (sessionToken) {
loadGoogleDriveFiles(sessionToken, currentFolderId, nextPageToken, true);
loadGoogleDriveFiles(
sessionToken,
currentFolderId,
nextPageToken,
true,
);
}
}
};
scrollContainer?.addEventListener('scroll', handleScroll);
return () => {
scrollContainer?.removeEventListener('scroll', handleScroll);
};
}, [hasMoreFiles, isLoadingFiles, nextPageToken, currentFolderId, ingestor.type]);
}, [
hasMoreFiles,
isLoadingFiles,
nextPageToken,
currentFolderId,
ingestor.type,
]);
return (
<WrapperModal

7
frontend/src/upload/types/ingestor.ts
View File

@@ -29,7 +29,12 @@ export interface GoogleDriveIngestorConfig extends BaseIngestorConfig {
token_info?: any;
}
export type IngestorType = 'crawler' | 'github' | 'reddit' | 'url' | 'google_drive';
export type IngestorType =
| 'crawler'
| 'github'
| 'reddit'
| 'url'
| 'google_drive';
export interface IngestorConfig {
type: IngestorType;

3
frontend/src/utils/providerUtils.ts
View File

@@ -3,7 +3,6 @@
* Follows the convention: {provider}_session_token
*/
export const getSessionToken = (provider: string): string | null => {
return localStorage.getItem(`${provider}_session_token`);
};
@@ -14,4 +13,4 @@ export const setSessionToken = (provider: string, token: string): void => {
export const removeSessionToken = (provider: string): void => {
localStorage.removeItem(`${provider}_session_token`);
};
};