LLM/DocsGPT
1
0
Fork 0
mirror of https://github.com/arc53/DocsGPT.git synced 2026-04-27 20:40:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add Amazon S3 support and synchronization features (#2244)

Browse Source 
* Add Amazon S3 support and synchronization features

* refactor: remove unused variable in load_data test
This commit is contained in:
Alex
2025-12-30 18:26:51 +00:00
committed by GitHub
parent f910a82683
commit 9e7f1ad1c0
22 changed files with 1841 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
frontend/src/api/endpoints.ts
View File

@@ -34,6 +34,7 @@ const endpoints = {
FEEDBACK_ANALYTICS: '/api/get_feedback_analytics',
LOGS: `/api/get_user_logs`,
MANAGE_SYNC: '/api/manage_sync',
SYNC_SOURCE: '/api/sync_source',
GET_AVAILABLE_TOOLS: '/api/available_tools',
GET_USER_TOOLS: '/api/get_tools',
CREATE_TOOL: '/api/create_tool',

2
frontend/src/api/services/userService.ts
View File

@@ -72,6 +72,8 @@ const userService = {
apiClient.post(endpoints.USER.LOGS, data, token),
manageSync: (data: any, token: string | null): Promise<any> =>
apiClient.post(endpoints.USER.MANAGE_SYNC, data, token),
syncSource: (data: any, token: string | null): Promise<any> =>
apiClient.post(endpoints.USER.SYNC_SOURCE, data, token),
getAvailableTools: (token: string | null): Promise<any> =>
apiClient.get(endpoints.USER.GET_AVAILABLE_TOOLS, token),
getUserTools: (token: string | null): Promise<any> =>

7
frontend/src/assets/s3.svg Normal file
View File

@@ -0,0 +1,7 @@
<svg width="20" height="20" viewBox="0 0 20 20" fill="none" xmlns="http://www.w3.org/2000/svg">
<path d="M4 6C4 4.5 16 4.5 16 6L14.5 18C14.4 18.6 13.9 19 13.3 19H6.7C6.1 19 5.6 18.6 5.5 18L4 6Z" fill="none" stroke="black" stroke-width="1.5" stroke-linejoin="round"/>
<path d="M4 6C4 7.5 16 7.5 16 6" fill="none" stroke="black" stroke-width="1.5" stroke-linecap="round"/>
<circle cx="11.8" cy="9.2" r="1" fill="black"/>
<path d="M12.8 9.6L15.4 10.8" fill="none" stroke="black" stroke-width="1.2" stroke-linecap="round"/>
<ellipse cx="16.6" cy="11.2" rx="2" ry="1.1" fill="none" stroke="black" stroke-width="1.2"/>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 630 B

5
frontend/src/locale/de.json
View File

@@ -67,6 +67,7 @@
"preLoaded": "Vorgeladen",
"private": "Privat",
"sync": "Synchronisieren",
"syncNow": "Jetzt synchronisieren",
"syncing": "Synchronisiere...",
"syncConfirmation": "Bist du sicher, dass du \"{{sourceName}}\" synchronisieren möchtest? Dies aktualisiert den Inhalt mit deinem Cloud-Speicher und kann Änderungen an einzelnen Chunks überschreiben.",
"syncFrequency": {
@@ -316,6 +317,10 @@
"google_drive": {
"label": "Google Drive",
"heading": "Von Google Drive hochladen"
},
"s3": {
"label": "Amazon S3",
"heading": "Inhalt von Amazon S3 hinzufügen"
}
},
"connectors": {

5
frontend/src/locale/en.json
View File

@@ -67,6 +67,7 @@
"preLoaded": "Pre-loaded",
"private": "Private",
"sync": "Sync",
"syncNow": "Sync now",
"syncing": "Syncing...",
"syncConfirmation": "Are you sure you want to sync \"{{sourceName}}\"? This will update the content with your cloud storage and may override any edits you made to individual chunks.",
"syncFrequency": {
@@ -316,6 +317,10 @@
"google_drive": {
"label": "Google Drive",
"heading": "Upload from Google Drive"
},
"s3": {
"label": "Amazon S3",
"heading": "Add content from Amazon S3"
}
},
"connectors": {

5
frontend/src/locale/es.json
View File

@@ -67,6 +67,7 @@
"preLoaded": "Precargado",
"private": "Privado",
"sync": "Sincronizar",
"syncNow": "Sincronizar ahora",
"syncing": "Sincronizando...",
"syncConfirmation": "¿Estás seguro de que deseas sincronizar \"{{sourceName}}\"? Esto actualizará el contenido con tu almacenamiento en la nube y puede anular cualquier edición que hayas realizado en fragmentos individuales.",
"syncFrequency": {
@@ -316,6 +317,10 @@
"google_drive": {
"label": "Google Drive",
"heading": "Subir desde Google Drive"
},
"s3": {
"label": "Amazon S3",
"heading": "Agregar contenido desde Amazon S3"
}
},
"connectors": {

5
frontend/src/locale/jp.json
View File

@@ -67,6 +67,7 @@
"preLoaded": "プリロード済み",
"private": "プライベート",
"sync": "同期",
"syncNow": "今すぐ同期",
"syncing": "同期中...",
"syncConfirmation": "\"{{sourceName}}\"を同期してもよろしいですか？これにより、コンテンツがクラウドストレージで更新され、個々のチャンクに加えた編集が上書きされる可能性があります。",
"syncFrequency": {
@@ -316,6 +317,10 @@
"google_drive": {
"label": "Google Drive",
"heading": "Google Driveからアップロード"
},
"s3": {
"label": "Amazon S3",
"heading": "Amazon S3からコンテンツを追加"
}
},
"connectors": {

5
frontend/src/locale/ru.json
View File

@@ -67,6 +67,7 @@
"preLoaded": "Предзагруженный",
"private": "Частный",
"sync": "Синхронизация",
"syncNow": "Синхронизировать сейчас",
"syncing": "Синхронизация...",
"syncConfirmation": "Вы уверены, что хотите синхронизировать \"{{sourceName}}\"? Это обновит содержимое с вашим облачным хранилищем и может перезаписать любые изменения, внесенные вами в отдельные фрагменты.",
"syncFrequency": {
@@ -316,6 +317,10 @@
"google_drive": {
"label": "Google Drive",
"heading": "Загрузить из Google Drive"
},
"s3": {
"label": "Amazon S3",
"heading": "Добавить контент из Amazon S3"
}
},
"connectors": {

5
frontend/src/locale/zh-TW.json
View File

@@ -67,6 +67,7 @@
"preLoaded": "預載入",
"private": "私人",
"sync": "同步",
"syncNow": "立即同步",
"syncing": "同步中...",
"syncConfirmation": "您確定要同步 \"{{sourceName}}\" 嗎？這將使用您的雲端儲存更新內容，並可能覆蓋您對個別文本塊所做的任何編輯。",
"syncFrequency": {
@@ -316,6 +317,10 @@
"google_drive": {
"label": "Google Drive",
"heading": "從Google Drive上傳"
},
"s3": {
"label": "Amazon S3",
"heading": "從Amazon S3新增內容"
}
},
"connectors": {

5
frontend/src/locale/zh.json
View File

@@ -67,6 +67,7 @@
"preLoaded": "预加载",
"private": "私有",
"sync": "同步",
"syncNow": "立即同步",
"syncing": "同步中...",
"syncConfirmation": "您确定要同步 \"{{sourceName}}\" 吗？这将使用您的云存储更新内容，并可能覆盖您对单个文本块所做的任何编辑。",
"syncFrequency": {
@@ -316,6 +317,10 @@
"google_drive": {
"label": "Google Drive",
"heading": "从Google Drive上传"
},
"s3": {
"label": "Amazon S3",
"heading": "从Amazon S3添加内容"
}
},
"connectors": {

1
frontend/src/models/misc.ts
View File

@@ -13,6 +13,7 @@ export type Doc = {
retriever?: string;
syncFrequency?: string;
isNested?: boolean;
provider?: string;
};
export type GetDocsResponse = {

65
frontend/src/settings/Sources.tsx
View File

@@ -201,6 +201,61 @@ export default function Sources({
});
};
const getConnectorProvider = async (doc: Doc): Promise<string | null> => {
if (doc.provider) {
return doc.provider;
}
if (!doc.id) {
return null;
}
try {
const directoryResponse = await userService.getDirectoryStructure(
doc.id,
token,
);
const directoryData = await directoryResponse.json();
return directoryData?.provider ?? null;
} catch (error) {
console.error('Error fetching connector provider:', error);
return null;
}
};
const handleSyncNow = async (doc: Doc) => {
if (!doc.id) {
return;
}
try {
if (doc.type?.startsWith('connector')) {
const provider = await getConnectorProvider(doc);
if (!provider) {
console.error('Sync now failed: provider not found');
return;
}
const response = await userService.syncConnector(
doc.id,
provider,
token,
);
const data = await response.json();
if (!data.success) {
console.error('Sync now failed:', data.error || data.message);
}
return;
}
const response = await userService.syncSource(
{ source_id: doc.id },
token,
);
const data = await response.json();
if (!data.success) {
console.error('Sync now failed:', data.error || data.message);
}
} catch (error) {
console.error('Error syncing source:', error);
}
};
const [documentToDelete, setDocumentToDelete] = useState<{
index: number;
document: Doc;
@@ -250,6 +305,16 @@ export default function Sources({
iconHeight: 14,
variant: 'primary',
});
actions.push({
icon: SyncIcon,
label: t('settings.sources.syncNow'),
onClick: () => {
handleSyncNow(document);
},
iconWidth: 14,
iconHeight: 14,
variant: 'primary',
});
}
actions.push({

59
frontend/src/upload/types/ingestor.ts
View File

@@ -4,6 +4,7 @@ import UrlIcon from '../../assets/url.svg';
import GithubIcon from '../../assets/github.svg';
import RedditIcon from '../../assets/reddit.svg';
import DriveIcon from '../../assets/drive.svg';
import S3Icon from '../../assets/s3.svg';
export type IngestorType =
| 'crawler'
@@ -11,7 +12,8 @@ export type IngestorType =
| 'reddit'
| 'url'
| 'google_drive'
| 'local_file';
| 'local_file'
| 's3';
export interface IngestorConfig {
type: IngestorType | null;
@@ -147,6 +149,50 @@ export const IngestorFormSchemas: IngestorSchema[] = [
},
],
},
{
key: 's3',
label: 'Amazon S3',
icon: S3Icon,
heading: 'Add content from Amazon S3',
fields: [
{
name: 'aws_access_key_id',
label: 'AWS Access Key ID',
type: 'string',
required: true,
},
{
name: 'aws_secret_access_key',
label: 'AWS Secret Access Key',
type: 'string',
required: true,
},
{
name: 'bucket',
label: 'Bucket Name',
type: 'string',
required: true,
},
{
name: 'prefix',
label: 'Path Prefix (optional)',
type: 'string',
required: false,
},
{
name: 'region',
label: 'AWS Region',
type: 'string',
required: false,
},
{
name: 'endpoint_url',
label: 'Custom Endpoint URL (optional)',
type: 'string',
required: false,
},
],
},
];
export const IngestorDefaultConfigs: Record<
@@ -175,6 +221,17 @@ export const IngestorDefaultConfigs: Record<
},
},
local_file: { name: '', config: { files: [] } },
s3: {
name: '',
config: {
aws_access_key_id: '',
aws_secret_access_key: '',
bucket: '',
prefix: '',
region: 'us-east-1',
endpoint_url: '',
},
},
};
export interface IngestorOption {