LLM/DocsGPT
1
0
Fork 0
mirror of https://github.com/arc53/DocsGPT.git synced 2026-03-01 15:51:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(oneDrive): shared user files

Browse Source
This commit is contained in:
ManishMadan2882
2026-02-25 02:13:33 +05:30
parent 11e59540fb
commit e87da7e5a9
10 changed files with 309 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
application/api/connector/routes.py
View File

@@ -205,12 +205,13 @@ class ConnectorsCallback(Resource):
@connectors_ns.route("/api/connectors/files")
class ConnectorFiles(Resource):
@api.expect(api.model("ConnectorFilesModel", {
"provider": fields.String(required=True),
"session_token": fields.String(required=True),
"folder_id": fields.String(required=False),
"limit": fields.Integer(required=False),
"provider": fields.String(required=True),
"session_token": fields.String(required=True),
"folder_id": fields.String(required=False),
"limit": fields.Integer(required=False),
"page_token": fields.String(required=False),
"search_query": fields.String(required=False)
"search_query": fields.String(required=False),
"shared": fields.Boolean(required=False)
}))
@api.doc(description="List files from a connector provider (supports pagination and search)")
def post(self):
@@ -222,6 +223,7 @@ class ConnectorFiles(Resource):
limit = data.get('limit', 10)
page_token = data.get('page_token')
search_query = data.get('search_query')
shared = data.get('shared', False)
if not provider or not session_token:
return make_response(jsonify({"success": False, "error": "provider and session_token are required"}), 400)
@@ -240,7 +242,8 @@ class ConnectorFiles(Resource):
'list_only': True,
'session_token': session_token,
'folder_id': folder_id,
'page_token': page_token
'page_token': page_token,
'shared': shared
}
if search_query:
input_config['search_query'] = search_query

119
application/parser/connectors/share_point/loader.py
View File

@@ -73,6 +73,12 @@ class SharePointLoader(BaseConnectorLoader):
logging.error(f"Failed to refresh token: {e}")
raise ValueError("Failed to refresh access token")
def _get_item_url(self, item_ref: str) -> str:
if ':' in item_ref:
drive_id, item_id = item_ref.split(':', 1)
return f"{self.GRAPH_API_BASE}/drives/{drive_id}/items/{item_id}"
return f"{self.GRAPH_API_BASE}/me/drive/items/{item_ref}"
def _process_file(self, file_metadata: Dict[str, Any], load_content: bool = True) -> Optional[Document]:
try:
drive_item_id = file_metadata.get('id')
@@ -126,9 +132,17 @@ class SharePointLoader(BaseConnectorLoader):
load_content = not list_only
page_token = inputs.get('page_token')
search_query = inputs.get('search_query')
shared = inputs.get('shared', False)
self.next_page_token = None
if file_ids:
if shared and not file_ids and not folder_id:
documents = self._list_shared_items(
limit=limit,
load_content=load_content,
page_token=page_token,
search_query=search_query
)
elif file_ids:
for file_id in file_ids:
try:
doc = self._load_file_by_id(file_id, load_content=load_content)
@@ -161,7 +175,7 @@ class SharePointLoader(BaseConnectorLoader):
self._ensure_valid_token()
try:
url = f"{self.GRAPH_API_BASE}/me/drive/items/{file_id}"
url = self._get_item_url(file_id)
params = {'$select': 'id,name,file,createdDateTime,lastModifiedDateTime,size'}
response = requests.get(url, headers=self._get_headers(), params=params)
response.raise_for_status()
@@ -193,13 +207,17 @@ class SharePointLoader(BaseConnectorLoader):
documents: List[Document] = []
try:
url = f"{self.GRAPH_API_BASE}/me/drive/items/{parent_id}/children"
url = f"{self._get_item_url(parent_id)}/children"
params = {'$top': min(100, limit) if limit else 100, '$select': 'id,name,file,folder,createdDateTime,lastModifiedDateTime,size'}
if page_token:
params['$skipToken'] = page_token
if search_query:
search_url = f"{self.GRAPH_API_BASE}/me/drive/search(q='{search_query}')"
if ':' in parent_id:
drive_id = parent_id.split(':', 1)[0]
search_url = f"{self.GRAPH_API_BASE}/drives/{drive_id}/root/search(q='{search_query}')"
else:
search_url = f"{self.GRAPH_API_BASE}/me/drive/search(q='{search_query}')"
response = requests.get(search_url, headers=self._get_headers(), params=params)
else:
response = requests.get(url, headers=self._get_headers(), params=params)
@@ -248,11 +266,94 @@ class SharePointLoader(BaseConnectorLoader):
logging.error(f"Error listing items under parent {parent_id}: {e}")
return documents
def _list_shared_items(self, limit: int = 100, load_content: bool = False,
page_token: Optional[str] = None,
search_query: Optional[str] = None) -> List[Document]:
self._ensure_valid_token()
documents: List[Document] = []
try:
url = f"{self.GRAPH_API_BASE}/me/drive/sharedWithMe"
params = {'$top': min(100, limit) if limit else 100}
if page_token:
params['$skipToken'] = page_token
response = requests.get(url, headers=self._get_headers(), params=params)
response.raise_for_status()
results = response.json()
for item in results.get('value', []):
remote = item.get('remoteItem', {})
drive_id = remote.get('parentReference', {}).get('driveId')
item_id = remote.get('id')
name = remote.get('name', item.get('name', 'Unknown'))
if not drive_id or not item_id:
continue
if search_query and search_query.lower() not in name.lower():
continue
composite_id = f"{drive_id}:{item_id}"
if 'folder' in remote:
doc_metadata = {
'file_name': name,
'mime_type': 'folder',
'size': remote.get('size'),
'created_time': remote.get('createdDateTime'),
'modified_time': remote.get('lastModifiedDateTime'),
'source': 'share_point',
'is_folder': True
}
documents.append(Document(text="", doc_id=composite_id, extra_info=doc_metadata))
elif 'file' in remote:
mime_type = remote.get('file', {}).get('mimeType', 'application/octet-stream')
if mime_type not in self.SUPPORTED_MIME_TYPES:
continue
doc_metadata = {
'file_name': name,
'mime_type': mime_type,
'size': remote.get('size'),
'created_time': remote.get('createdDateTime'),
'modified_time': remote.get('lastModifiedDateTime'),
'source': 'share_point'
}
if load_content:
content = self._download_file_content(composite_id)
if content is None:
continue
documents.append(Document(text=content, doc_id=composite_id, extra_info=doc_metadata))
else:
documents.append(Document(text="", doc_id=composite_id, extra_info=doc_metadata))
if limit and len(documents) >= limit:
break
next_link = results.get('@odata.nextLink')
if next_link:
from urllib.parse import urlparse, parse_qs
parsed = urlparse(next_link)
query_params = parse_qs(parsed.query)
skiptoken_list = query_params.get('$skiptoken')
self.next_page_token = skiptoken_list[0] if skiptoken_list else None
else:
self.next_page_token = None
return documents
except Exception as e:
logging.error(f"Error listing shared items: {e}")
return documents
def _download_file_content(self, file_id: str) -> Optional[str]:
self._ensure_valid_token()
try:
url = f"{self.GRAPH_API_BASE}/me/drive/items/{file_id}/content"
url = f"{self._get_item_url(file_id)}/content"
response = requests.get(url, headers=self._get_headers())
response.raise_for_status()
@@ -297,7 +398,7 @@ class SharePointLoader(BaseConnectorLoader):
def _download_single_file(self, file_id: str, local_dir: str) -> bool:
try:
url = f"{self.GRAPH_API_BASE}/me/drive/items/{file_id}"
url = self._get_item_url(file_id)
params = {'$select': 'id,name,file'}
response = requests.get(url, headers=self._get_headers(), params=params)
response.raise_for_status()
@@ -314,7 +415,7 @@ class SharePointLoader(BaseConnectorLoader):
os.makedirs(local_dir, exist_ok=True)
full_path = os.path.join(local_dir, file_name)
download_url = f"{self.GRAPH_API_BASE}/me/drive/items/{file_id}/content"
download_url = f"{self._get_item_url(file_id)}/content"
download_response = requests.get(download_url, headers=self._get_headers())
download_response.raise_for_status()
@@ -331,7 +432,7 @@ class SharePointLoader(BaseConnectorLoader):
try:
os.makedirs(local_dir, exist_ok=True)
url = f"{self.GRAPH_API_BASE}/me/drive/items/{folder_id}/children"
url = f"{self._get_item_url(folder_id)}/children"
params = {'$top': 1000}
while url:
@@ -415,7 +516,7 @@ class SharePointLoader(BaseConnectorLoader):
for folder_id in folder_ids:
try:
url = f"{self.GRAPH_API_BASE}/me/drive/items/{folder_id}"
url = self._get_item_url(folder_id)
params = {'$select': 'id,name'}
response = requests.get(url, headers=self._get_headers(), params=params)
response.raise_for_status()

237
frontend/src/components/FilePicker.tsx
View File

@@ -92,9 +92,11 @@ export const FilePicker: React.FC<CloudFilePickerProps> = ({
const [authError, setAuthError] = useState<string>('');
const [isConnected, setIsConnected] = useState(false);
const [userEmail, setUserEmail] = useState<string>('');
const [shared, setShared] = useState(false);
const scrollContainerRef = useRef<HTMLDivElement>(null);
const searchTimeoutRef = useRef<ReturnType<typeof setTimeout> | null>(null);
const abortControllerRef = useRef<AbortController | null>(null);
const isFolder = (file: CloudFile) => {
return (
@@ -110,7 +112,13 @@ export const FilePicker: React.FC<CloudFilePickerProps> = ({
folderId: string | null,
pageToken?: string,
searchQuery = '',
isShared = false,
) => {
// Cancel any in-flight request so stale responses never overwrite new state
abortControllerRef.current?.abort();
const controller = new AbortController();
abortControllerRef.current = controller;
setIsLoading(true);
const apiHost = import.meta.env.VITE_API_HOST;
@@ -119,20 +127,25 @@ export const FilePicker: React.FC<CloudFilePickerProps> = ({
}
try {
const body: Record<string, unknown> = {
provider: provider,
session_token: sessionToken,
folder_id: folderId,
limit: 10,
page_token: pageToken,
search_query: searchQuery,
};
if (isShared) {
body.shared = true;
}
const response = await fetch(`${apiHost}/api/connectors/files`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
Authorization: `Bearer ${token}`,
},
body: JSON.stringify({
provider: provider,
session_token: sessionToken,
folder_id: folderId,
limit: 10,
page_token: pageToken,
search_query: searchQuery,
}),
body: JSON.stringify(body),
signal: controller.signal,
});
const data = await response.json();
@@ -149,12 +162,15 @@ export const FilePicker: React.FC<CloudFilePickerProps> = ({
}
}
} catch (err) {
if ((err as Error).name === 'AbortError') return;
console.error('Error loading files:', err);
if (!pageToken) {
setFiles([]);
}
} finally {
setIsLoading(false);
if (!controller.signal.aborted) {
setIsLoading(false);
}
}
},
[token, provider],
@@ -207,7 +223,7 @@ export const FilePicker: React.FC<CloudFilePickerProps> = ({
name: getProviderConfig(provider).rootName,
},
]);
loadCloudFiles(sessionToken, null, undefined, '');
loadCloudFiles(sessionToken, null, undefined, '', false);
} else {
removeSessionToken(provider);
setIsConnected(false);
@@ -242,6 +258,7 @@ export const FilePicker: React.FC<CloudFilePickerProps> = ({
currentFolderId,
nextPageToken,
searchQuery,
shared,
);
}
}
@@ -253,6 +270,7 @@ export const FilePicker: React.FC<CloudFilePickerProps> = ({
searchQuery,
provider,
loadCloudFiles,
shared,
]);
useEffect(() => {
@@ -268,6 +286,7 @@ export const FilePicker: React.FC<CloudFilePickerProps> = ({
if (searchTimeoutRef.current) {
clearTimeout(searchTimeoutRef.current);
}
abortControllerRef.current?.abort();
};
}, []);
@@ -281,7 +300,7 @@ export const FilePicker: React.FC<CloudFilePickerProps> = ({
searchTimeoutRef.current = setTimeout(() => {
const sessionToken = getSessionToken(provider);
if (sessionToken) {
loadCloudFiles(sessionToken, currentFolderId, undefined, query);
loadCloudFiles(sessionToken, currentFolderId, undefined, query, shared);
}
}, 300);
};
@@ -299,7 +318,7 @@ export const FilePicker: React.FC<CloudFilePickerProps> = ({
const sessionToken = getSessionToken(provider);
if (sessionToken) {
loadCloudFiles(sessionToken, folderId, undefined, '');
loadCloudFiles(sessionToken, folderId, undefined, '', shared);
}
};
@@ -315,7 +334,25 @@ export const FilePicker: React.FC<CloudFilePickerProps> = ({
const sessionToken = getSessionToken(provider);
if (sessionToken) {
loadCloudFiles(sessionToken, newFolderId, undefined, '');
loadCloudFiles(sessionToken, newFolderId, undefined, '', shared);
}
};
const handleTabSwitch = (isShared: boolean) => {
if (isShared === shared) return;
setShared(isShared);
setFiles([]);
setCurrentFolderId(null);
setSearchQuery('');
setNextPageToken(null);
setHasMoreFiles(false);
const rootName = isShared
? t('filePicker.sharedWithMe')
: getProviderConfig(provider).rootName;
setFolderPath([{ id: null, name: rootName }]);
const sessionToken = getSessionToken(provider);
if (sessionToken) {
loadCloudFiles(sessionToken, null, undefined, '', isShared);
}
};
@@ -350,7 +387,7 @@ export const FilePicker: React.FC<CloudFilePickerProps> = ({
if (data.session_token) {
setSessionToken(provider, data.session_token);
loadCloudFiles(data.session_token, null);
loadCloudFiles(data.session_token, null, undefined, '', shared);
}
}}
onError={(error) => {
@@ -394,9 +431,32 @@ export const FilePicker: React.FC<CloudFilePickerProps> = ({
/>
{isConnected && (
<div className="mt-3 rounded-lg border border-[#D7D7D7] dark:border-[#6A6A6A]">
<div className="mt-3 overflow-hidden rounded-lg border border-[#D7D7D7] dark:border-[#6A6A6A]">
<div className="rounded-t-lg border-[#EEE6FF78] dark:border-[#6A6A6A]">
{/* Breadcrumb navigation */}
{provider === 'share_point' && (
<div className="flex border-b border-[#D7D7D7] dark:border-[#6A6A6A]">
<button
onClick={() => handleTabSwitch(false)}
className={`px-4 py-2 text-sm font-medium ${
!shared
? 'border-b-2 border-[#A076F6] text-[#A076F6]'
: 'text-gray-500 hover:text-gray-700 dark:text-gray-400 dark:hover:text-gray-300'
}`}
>
{t('filePicker.myFiles')}
</button>
<button
onClick={() => handleTabSwitch(true)}
className={`px-4 py-2 text-sm font-medium ${
shared
? 'border-b-2 border-[#A076F6] text-[#A076F6]'
: 'text-gray-500 hover:text-gray-700 dark:text-gray-400 dark:hover:text-gray-300'
}`}
>
{t('filePicker.sharedWithMe')}
</button>
</div>
)}
<div className="rounded-t-lg bg-[#EEE6FF78] px-4 pt-4 dark:bg-[#2A262E]">
<div className="mb-2 flex items-center gap-1">
{folderPath.map((path, index) => (
@@ -443,7 +503,7 @@ export const FilePicker: React.FC<CloudFilePickerProps> = ({
</div>
</div>
<div className="h-72">
<div className="h-72 border-t border-[#D7D7D7] dark:border-[#6A6A6A]">
<TableContainer
ref={scrollContainerRef}
height="288px"
@@ -468,68 +528,95 @@ export const FilePicker: React.FC<CloudFilePickerProps> = ({
</TableRow>
</TableHead>
<TableBody>
{files.map((file, index) => (
<TableRow
key={`${file.id}-${index}`}
onClick={() => {
if (isFolder(file)) {
handleFolderClick(file.id, file.name);
} else {
handleFileSelect(file.id, false);
}
}}
>
<TableCell width="40px" align="center">
<div
className="mx-auto flex h-5 w-5 shrink-0 cursor-pointer items-center justify-center border border-[#EEE6FF78] p-[0.5px] text-sm dark:border-[#6A6A6A]"
onClick={(e) => {
e.stopPropagation();
handleFileSelect(file.id, isFolder(file));
{isLoading && files.length === 0
? Array.from({ length: 5 }).map((_, i) => (
<TableRow key={`skeleton-${i}`}>
<TableCell width="40px" align="center">
<div className="mx-auto h-5 w-5 animate-pulse rounded bg-gray-200 dark:bg-gray-700" />
</TableCell>
<TableCell>
<div className="h-4 w-48 animate-pulse rounded bg-gray-200 dark:bg-gray-700" />
</TableCell>
<TableCell>
<div className="h-4 w-24 animate-pulse rounded bg-gray-200 dark:bg-gray-700" />
</TableCell>
<TableCell>
<div className="h-4 w-16 animate-pulse rounded bg-gray-200 dark:bg-gray-700" />
</TableCell>
</TableRow>
))
: files.map((file, index) => (
<TableRow
key={`${file.id}-${index}`}
onClick={() => {
if (isFolder(file)) {
handleFolderClick(file.id, file.name);
} else {
handleFileSelect(file.id, false);
}
}}
>
{(isFolder(file)
? selectedFolders
: selectedFiles
).includes(file.id) && (
<img
src={CheckIcon}
alt="Selected"
className="h-4 w-4"
/>
)}
</div>
</TableCell>
<TableCell>
<div className="flex min-w-0 items-center gap-3">
<div className="flex-shrink-0">
<img
src={isFolder(file) ? FolderIcon : FileIcon}
alt={isFolder(file) ? 'Folder' : 'File'}
className="h-6 w-6"
/>
</div>
<span className="truncate">{file.name}</span>
</div>
</TableCell>
<TableCell className="text-xs">
{formatDate(file.modifiedTime)}
</TableCell>
<TableCell className="text-xs">
{file.size ? formatBytes(file.size) : '-'}
</TableCell>
</TableRow>
))}
<TableCell width="40px" align="center">
<div
className="mx-auto flex h-5 w-5 shrink-0 cursor-pointer items-center justify-center border border-[#EEE6FF78] p-[0.5px] text-sm dark:border-[#6A6A6A]"
onClick={(e) => {
e.stopPropagation();
handleFileSelect(file.id, isFolder(file));
}}
>
{(isFolder(file)
? selectedFolders
: selectedFiles
).includes(file.id) && (
<img
src={CheckIcon}
alt="Selected"
className="h-4 w-4"
/>
)}
</div>
</TableCell>
<TableCell>
<div className="flex min-w-0 items-center gap-3">
<div className="shrink-0">
<img
src={
isFolder(file) ? FolderIcon : FileIcon
}
alt={isFolder(file) ? 'Folder' : 'File'}
className="h-6 w-6"
/>
</div>
<span className="truncate">{file.name}</span>
</div>
</TableCell>
<TableCell className="text-xs">
{formatDate(file.modifiedTime)}
</TableCell>
<TableCell className="text-xs">
{file.size ? formatBytes(file.size) : '-'}
</TableCell>
</TableRow>
))}
{isLoading && files.length > 0 &&
Array.from({ length: 3 }).map((_, i) => (
<TableRow key={`load-more-skeleton-${i}`}>
<TableCell width="40px" align="center">
<div className="mx-auto h-5 w-5 animate-pulse rounded bg-gray-200 dark:bg-gray-700" />
</TableCell>
<TableCell>
<div className="h-4 w-48 animate-pulse rounded bg-gray-200 dark:bg-gray-700" />
</TableCell>
<TableCell>
<div className="h-4 w-24 animate-pulse rounded bg-gray-200 dark:bg-gray-700" />
</TableCell>
<TableCell>
<div className="h-4 w-16 animate-pulse rounded bg-gray-200 dark:bg-gray-700" />
</TableCell>
</TableRow>
))}
</TableBody>
</Table>
{isLoading && (
<div className="flex items-center justify-center border-t border-[#EEE6FF78] p-4 dark:border-[#6A6A6A]">
<div className="inline-flex items-center gap-2 text-sm text-gray-600 dark:text-gray-400">
<div className="h-4 w-4 animate-spin rounded-full border-2 border-blue-500 border-t-transparent"></div>
Loading more files...
</div>
</div>
)}
</>
}
</TableContainer>

5
frontend/src/locale/de.json
View File

@@ -671,7 +671,10 @@
"itemsSelected": "{{count}} ausgewählt",
"name": "Name",
"lastModified": "Zuletzt geändert",
"size": "Größe"
"size": "Größe",
"myFiles": "Meine Dateien",
"sharedWithMe": "Mit mir geteilt",
"loadingMore": "Weitere Dateien laden..."
},
"actionButtons": {
"openNewChat": "Neuen Chat öffnen",

5
frontend/src/locale/en.json
View File

@@ -694,7 +694,10 @@
"itemsSelected": "{{count}} selected",
"name": "Name",
"lastModified": "Last Modified",
"size": "Size"
"size": "Size",
"myFiles": "My Files",
"sharedWithMe": "Shared with Me",
"loadingMore": "Loading more files..."
},
"actionButtons": {
"openNewChat": "Open New Chat",

5
frontend/src/locale/es.json
View File

@@ -693,7 +693,10 @@
"itemsSelected": "{{count}} seleccionados",
"name": "Nombre",
"lastModified": "Última modificación",
"size": "Tamaño"
"size": "Tamaño",
"myFiles": "Mis archivos",
"sharedWithMe": "Compartido conmigo",
"loadingMore": "Cargando más archivos..."
},
"actionButtons": {
"openNewChat": "Abrir nuevo chat",

5
frontend/src/locale/jp.json
View File

@@ -693,7 +693,10 @@
"itemsSelected": "{{count}} 件選択済み",
"name": "名前",
"lastModified": "最終更新日",
"size": "サイズ"
"size": "サイズ",
"myFiles": "マイファイル",
"sharedWithMe": "共有アイテム",
"loadingMore": "さらに読み込み中..."
},
"actionButtons": {
"openNewChat": "新しいチャットを開く",

5
frontend/src/locale/ru.json
View File

@@ -693,7 +693,10 @@
"itemsSelected": "{{count}} выбрано",
"name": "Имя",
"lastModified": "Последнее изменение",
"size": "Размер"
"size": "Размер",
"myFiles": "Мои файлы",
"sharedWithMe": "Доступные мне",
"loadingMore": "Загрузка файлов..."
},
"actionButtons": {
"openNewChat": "Открыть новый чат",

5
frontend/src/locale/zh-TW.json
View File

@@ -693,7 +693,10 @@
"itemsSelected": "已選擇 {{count}} 項",
"name": "名稱",
"lastModified": "最後修改",
"size": "大小"
"size": "大小",
"myFiles": "我的檔案",
"sharedWithMe": "與我共用",
"loadingMore": "載入更多檔案..."
},
"actionButtons": {
"openNewChat": "開啟新聊天",

5
frontend/src/locale/zh.json
View File

@@ -693,7 +693,10 @@
"itemsSelected": "已选择 {{count}} 项",
"name": "名称",
"lastModified": "最后修改",
"size": "大小"
"size": "大小",
"myFiles": "我的文件",
"sharedWithMe": "与我共享",
"loadingMore": "加载更多文件..."
},
"actionButtons": {
"openNewChat": "打开新聊天",