From 33a28a64ec5a2aae91a173c648c99273ed274898 Mon Sep 17 00:00:00 2001 From: ManishMadan2882 Date: Thu, 2 Jan 2025 01:33:49 +0530 Subject: [PATCH 01/22] (fix:ui/mobile) button squashed --- frontend/src/conversation/SharedConversation.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/src/conversation/SharedConversation.tsx b/frontend/src/conversation/SharedConversation.tsx index 5ab1d639..c6f73351 100644 --- a/frontend/src/conversation/SharedConversation.tsx +++ b/frontend/src/conversation/SharedConversation.tsx @@ -236,7 +236,7 @@ export const SharedConversation = () => { -
+
{apiKey ? (
{ ) : ( From 0751debff7ad82c83d0f64bd9c00e72a15c83fc8 Mon Sep 17 00:00:00 2001 From: ManishMadan2882 Date: Thu, 2 Jan 2025 02:31:29 +0530 Subject: [PATCH 02/22] (feat:nav) add enter/esc to rename in tile --- frontend/src/conversation/ConversationTile.tsx | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/frontend/src/conversation/ConversationTile.tsx b/frontend/src/conversation/ConversationTile.tsx index dd3e6e1d..c29b9d71 100644 --- a/frontend/src/conversation/ConversationTile.tsx +++ b/frontend/src/conversation/ConversationTile.tsx @@ -115,6 +115,19 @@ export default function ConversationTile({ setConversationsName(conversation.name); setIsEdit(false); } + + const handleRenameKeyDown = (e: React.KeyboardEvent) => { + e.stopPropagation(); + if (e.key === 'Enter') { + handleSaveConversation({ + id: conversation.id, + name: conversationName, + }); + } else if (e.key === 'Escape') { + onClear(); + } + }; + return ( <>
setConversationsName(e.target.value)} + onKeyDown={handleRenameKeyDown} /> ) : (

@@ -239,7 +253,7 @@ export default function ConversationTile({ > Edit Date: Thu, 2 Jan 2025 13:32:25 +0530 Subject: [PATCH 03/22] (fix: sources) inconcistent capitalisation --- application/api/user/routes.py | 2 +- frontend/src/locale/ru.json | 274 ++++++++++++++++----------------- 2 files changed, 138 insertions(+), 138 deletions(-) diff --git a/application/api/user/routes.py b/application/api/user/routes.py index fa0ff382..6b000816 100644 --- a/application/api/user/routes.py +++ b/application/api/user/routes.py @@ -538,7 +538,7 @@ class CombinedJson(Resource): user = "local" data = [ { - "name": "default", + "name": "Default", "date": "default", "model": settings.EMBEDDINGS_NAME, "location": "remote", diff --git a/frontend/src/locale/ru.json b/frontend/src/locale/ru.json index d23092a1..defbf94a 100644 --- a/frontend/src/locale/ru.json +++ b/frontend/src/locale/ru.json @@ -1,142 +1,142 @@ { - "language": "Русский", - "chat": "Чат", - "chats": "Чаты", - "newChat": "Новый чат", - "myPlan": "Мой план", - "about": "О", - "inputPlaceholder": "Введите свое сообщение здесь...", - "tagline": "DocsGPT использует GenAI, пожалуйста, проверьте важную информацию, используя источники.", - "sourceDocs": "Источник", - "none": "Нет", - "cancel": "Отмена", - "demo": [ - { - "header": "Узнайте о DocsGPT", - "query": "Что такое DocsGPT?" - }, - { - "header": "Обобщить документацию", - "query": "Обобщить текущий контекст" - }, - { - "header": "Написать код", - "query": "Написать код для запроса API к /api/answer" - }, - { - "header": "Помощь в обучении", - "query": "Написать потенциальные вопросы для контекста" - } - ], - "settings": { - "label": "Настройки", - "general": { - "label": "Общие", - "selectTheme": "Выбрать тему", - "light": "Светлая", - "dark": "Темная", - "selectLanguage": "Выбрать язык", - "chunks": "Обработанные фрагменты на запрос", - "prompt": "Активная подсказка", - "deleteAllLabel": "Удалить все беседы", - "deleteAllBtn": "Удалить все", - "addNew": "Добавить новый", - "convHistory": "История разговоров", - "none": "Нет", - "low": "Низкий", - "medium": "Средний", - "high": "Высокий", - "unlimited": "Без ограничений", - "default": "по умолчанию" - }, - "documents": { - "label": "Документы", - "name": "Название документа", - "date": "Дата вектора", - "type": "Тип", - "tokenUsage": "Использование токена", - "noData": "Нет существующих документов" - }, - "apiKeys": { - "label": "Чат-боты", - "name": "Название", - "key": "Ключ API", - "sourceDoc": "Исходный документ", - "createNew": "Создать новый", - "noData": "Нет существующих чат-ботов" - }, - "analytics": { - "label": "Analytics" - }, - "logs": { - "label": "Журналы" - } + "language": "Русский", + "chat": "Чат", + "chats": "Чаты", + "newChat": "Новый чат", + "myPlan": "Мой план", + "about": "О", + "inputPlaceholder": "Введите свое сообщение здесь...", + "tagline": "DocsGPT использует GenAI, пожалуйста, проверьте важную информацию, используя источники.", + "sourceDocs": "Источник", + "none": "Нет", + "cancel": "Отмена", + "demo": [ + { + "header": "Узнайте о DocsGPT", + "query": "Что такое DocsGPT?" }, - "modals": { - "uploadDoc": { - "label": "Загрузить новую документацию", - "select": "Выберите способ загрузки документа в DocsGPT", - "file": "Загрузить с устройства", - "back": "Назад", - "wait": "Пожалуйста, подождите ...", - "remote": "Собрать с веб-сайта", - "start": "Начать чат", - "name": "Имя", - "choose": "Выбрать файлы", - "info": "Загрузите .pdf, .txt, .rst, .csv, .xlsx, .docx, .md, .zip с ограничением до 25 МБ", - "uploadedFiles": "Загруженные файлы", - "cancel": "Отмена", - "train": "Обучение", - "link": "Ссылка", - "urlLink": "URL-ссылка", - "repoUrl": "URL-адрес репозитория", - "reddit": { - "id": "ID клиента", - "secret": "Секрет клиента", - "agent": "Агент пользователя", - "searchQueries": "Поисковые запросы", - "numberOfPosts": "Количество сообщений" - }, - "drag": { - "title": "Загрузите исходный файл", - "description": "Перетащите сюда свой файл, чтобы добавить его в качестве источника." - } - }, - "createAPIKey": { - "label": "Создать новый ключ API", - "apiKeyName": "Имя ключа API", - "chunks": "Обработано фрагментов на запрос", - "prompt": "Выбрать активный запрос", - "sourceDoc": "Исходный документ", - "create": "Создать" - }, - "saveKey": { - "note": "Пожалуйста, сохраните свой ключ", - "disclaimer": "Это единственный раз, когда будет показан ваш ключ.", - "copy": "Копировать", - "copied": "Скопировано", - "confirm": "Я сохранил ключ" - }, - "deleteConv": { - "confirm": "Вы уверены, что хотите удалить все разговоры?", - "delete": "Удалить" - }, - "shareConv": { - "label": "Создать публичную страницу для общего доступа", - "note": "Исходный документ, личная информация и дальнейший разговор останутся конфиденциальными", - "create": "Создать", - "option": "Разрешить пользователям запрашивать дальнейшие действия" - } + { + "header": "Обобщить документацию", + "query": "Обобщить текущий контекст" }, - "sharedConv": { - "subtitle": "Создано с помощью", - "button": "Начать работу с DocsGPT", - "meta": "DocsGPT использует GenAI, пожалуйста, проверьте важную информацию с помощью источников." + { + "header": "Написать код", + "query": "Написать код для запроса API к /api/answer" }, - "convTile": { - "share": "Поделиться", - "delete": "Удалить", - "rename": "Переименовать", - "deleteWarning": "Вы уверены, что хотите удалить этот разговор?" + { + "header": "Помощь в обучении", + "query": "Написать потенциальные вопросы для контекста" } -} \ No newline at end of file + ], + "settings": { + "label": "Настройки", + "general": { + "label": "Общие", + "selectTheme": "Выбрать тему", + "light": "Светлая", + "dark": "Темная", + "selectLanguage": "Выбрать язык", + "chunks": "Обработанные фрагменты на запрос", + "prompt": "Активная подсказка", + "deleteAllLabel": "Удалить все беседы", + "deleteAllBtn": "Удалить все", + "addNew": "Добавить новый", + "convHistory": "История разговоров", + "none": "Нет", + "low": "Низкий", + "medium": "Средний", + "high": "Высокий", + "unlimited": "Без ограничений", + "default": "по умолчанию" + }, + "documents": { + "label": "Документы", + "name": "Название документа", + "date": "Дата вектора", + "type": "Тип", + "tokenUsage": "Использование токена", + "noData": "Нет существующих документов" + }, + "apiKeys": { + "label": "Чат-боты", + "name": "Название", + "key": "Ключ API", + "sourceDoc": "Исходный документ", + "createNew": "Создать новый", + "noData": "Нет существующих чат-ботов" + }, + "analytics": { + "label": "Analytics" + }, + "logs": { + "label": "Журналы" + } + }, + "modals": { + "uploadDoc": { + "label": "Загрузить новую документацию", + "select": "Выберите способ загрузки документа в DocsGPT", + "file": "Загрузить с устройства", + "back": "Назад", + "wait": "Пожалуйста, подождите ...", + "remote": "Собрать с веб-сайта", + "start": "Начать чат", + "name": "Имя", + "choose": "Выбрать файлы", + "info": "Загрузите .pdf, .txt, .rst, .csv, .xlsx, .docx, .md, .zip с ограничением до 25 МБ", + "uploadedFiles": "Загруженные файлы", + "cancel": "Отмена", + "train": "Обучение", + "link": "Ссылка", + "urlLink": "URL-ссылка", + "repoUrl": "URL-адрес репозитория", + "reddit": { + "id": "ID клиента", + "secret": "Секрет клиента", + "agent": "Агент пользователя", + "searchQueries": "Поисковые запросы", + "numberOfPosts": "Количество сообщений" + }, + "drag": { + "title": "Загрузите исходный файл", + "description": "Перетащите сюда свой файл, чтобы добавить его в качестве источника." + } + }, + "createAPIKey": { + "label": "Создать новый ключ API", + "apiKeyName": "Имя ключа API", + "chunks": "Обработано фрагментов на запрос", + "prompt": "Выбрать активный запрос", + "sourceDoc": "Исходный документ", + "create": "Создать" + }, + "saveKey": { + "note": "Пожалуйста, сохраните свой ключ", + "disclaimer": "Это единственный раз, когда будет показан ваш ключ.", + "copy": "Копировать", + "copied": "Скопировано", + "confirm": "Я сохранил ключ" + }, + "deleteConv": { + "confirm": "Вы уверены, что хотите удалить все разговоры?", + "delete": "Удалить" + }, + "shareConv": { + "label": "Создать публичную страницу для общего доступа", + "note": "Исходный документ, личная информация и дальнейший разговор останутся конфиденциальными", + "create": "Создать", + "option": "Разрешить пользователям запрашивать дальнейшие действия" + } + }, + "sharedConv": { + "subtitle": "Создано с помощью", + "button": "Начать работу с DocsGPT", + "meta": "DocsGPT использует GenAI, пожалуйста, проверьте важную информацию с помощью источников." + }, + "convTile": { + "share": "Поделиться", + "delete": "Удалить", + "rename": "Переименовать", + "deleteWarning": "Вы уверены, что хотите удалить этот разговор?" + } +} From 2480dc83b23eabbd4027c22292c61384a32476d3 Mon Sep 17 00:00:00 2001 From: ManishMadan2882 Date: Fri, 3 Jan 2025 19:29:57 +0530 Subject: [PATCH 04/22] (fix/locale) missing keys in locale --- frontend/src/locale/ru.json | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/frontend/src/locale/ru.json b/frontend/src/locale/ru.json index defbf94a..98bfaf07 100644 --- a/frontend/src/locale/ru.json +++ b/frontend/src/locale/ru.json @@ -10,6 +10,9 @@ "sourceDocs": "Источник", "none": "Нет", "cancel": "Отмена", + "help": "Помощь", + "emailUs": "Напишите нам", + "documentation": "Документация", "demo": [ { "header": "Узнайте о DocsGPT", @@ -54,7 +57,7 @@ "name": "Название документа", "date": "Дата вектора", "type": "Тип", - "tokenUsage": "Использование токена", + "tokenUsage": "Использование токенов", "noData": "Нет существующих документов" }, "apiKeys": { @@ -66,10 +69,13 @@ "noData": "Нет существующих чат-ботов" }, "analytics": { - "label": "Analytics" + "label": "Аналитика" }, "logs": { "label": "Журналы" + }, + "tools": { + "label": "Инструменты" } }, "modals": { @@ -83,7 +89,7 @@ "start": "Начать чат", "name": "Имя", "choose": "Выбрать файлы", - "info": "Загрузите .pdf, .txt, .rst, .csv, .xlsx, .docx, .md, .zip с ограничением до 25 МБ", + "info": "Пожалуйста, загрузите .pdf, .txt, .rst, .csv, .xlsx, .docx, .md, .html, .epub, .json, .pptx, .zip с ограничением до 25 МБ", "uploadedFiles": "Загруженные файлы", "cancel": "Отмена", "train": "Обучение", @@ -106,7 +112,7 @@ "label": "Создать новый ключ API", "apiKeyName": "Имя ключа API", "chunks": "Обработано фрагментов на запрос", - "prompt": "Выбрать активный запрос", + "prompt": "Выбрать активную подсказку", "sourceDoc": "Исходный документ", "create": "Создать" }, @@ -125,7 +131,7 @@ "label": "Создать публичную страницу для общего доступа", "note": "Исходный документ, личная информация и дальнейший разговор останутся конфиденциальными", "create": "Создать", - "option": "Разрешить пользователям запрашивать дальнейшие действия" + "option": "Разрешить пользователям продолжить диалог" } }, "sharedConv": { From 69c5c6d6b8839e95b8539cb148fa98399c55ad33 Mon Sep 17 00:00:00 2001 From: ManishMadan2882 Date: Fri, 3 Jan 2025 19:30:43 +0530 Subject: [PATCH 05/22] (fix/locale) missing keys in chinese --- frontend/src/locale/zh-TW.json | 25 ++++++++++++++++++------- frontend/src/locale/zh.json | 9 ++++++--- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/frontend/src/locale/zh-TW.json b/frontend/src/locale/zh-TW.json index d5f633d2..46c645de 100644 --- a/frontend/src/locale/zh-TW.json +++ b/frontend/src/locale/zh-TW.json @@ -57,39 +57,49 @@ "name": "文件名稱", "date": "向量日期", "type": "類型", - "tokenUsage": "Token 使用量" + "tokenUsage": "Token 使用量", + "noData": "沒有現有的文件" }, "apiKeys": { - "label": "API 金鑰", + "label": "聊天機器人", "name": "名稱", "key": "API 金鑰", "sourceDoc": "來源文件", - "createNew": "新增" + "createNew": "新增", + "noData": "沒有現有的聊天機器人" }, "analytics": { "label": "分析" }, "logs": { "label": "日誌" + }, + "tools": { + "label": "工具" } }, "modals": { "uploadDoc": { "label": "上傳新文件", + "select": "選擇如何將文件上傳到 DocsGPT", "file": "從檔案", - "remote": "遠端", + "back": "返回", + "wait": "請稍候...", + "remote": "從網站收集", + "start": "開始對話", "name": "名稱", "choose": "選擇檔案", - "info": "請上傳 .pdf, .txt, .rst, .docx, .md, .json, .pptx, .zip 檔案，大小限制為 25MB", + "info": "請上傳 .pdf, .txt, .rst, .csv, .xlsx, .docx, .md, .html, .epub, .json, .pptx, .zip 檔案，大小限制為 25MB", "uploadedFiles": "已上傳的檔案", "cancel": "取消", "train": "訓練", "link": "連結", "urlLink": "URL 連結", + "repoUrl": "儲存庫 URL", "reddit": { "id": "用戶端 ID", "secret": "用戶端金鑰", - "agent": "使用者代理(User-Agent)", + "agent": "使用者代理", "searchQueries": "搜尋查詢", "numberOfPosts": "貼文數量" }, @@ -120,7 +130,8 @@ "shareConv": { "label": "建立公開頁面以分享", "note": "來源文件、個人資訊和後續對話將保持私密", - "create": "建立" + "create": "建立", + "option": "允許用戶進行更多查詢" } }, "sharedConv": { diff --git a/frontend/src/locale/zh.json b/frontend/src/locale/zh.json index 1a0d1f94..12a2ec65 100644 --- a/frontend/src/locale/zh.json +++ b/frontend/src/locale/zh.json @@ -34,7 +34,7 @@ "settings": { "label": "设置", "general": { - "label": "般", + "label": "一般", "selectTheme": "选择主题", "light": "浅色", "dark": "暗色", @@ -73,6 +73,9 @@ }, "logs": { "label": "日志" + }, + "tools": { + "label": "工具" } }, "modals": { @@ -86,7 +89,7 @@ "start": "开始聊天", "name": "名称", "choose": "选择文件", - "info": "请上传 .pdf, .txt, .rst, .csv, .xlsx, .docx, .md, .html, .epub, .json, .pptx, .zip 文件，限 25MB", + "info": "请上传 .pdf, .txt, .rst, .csv, .xlsx, .docx, .md, .html, .epub, .json, .pptx, .zip 文件，限制为 25MB", "uploadedFiles": "已上传文件", "cancel": "取消", "train": "训练", @@ -128,7 +131,7 @@ "label": "创建用于分享的公共页面", "note": "源文档、个人信息和后续对话将保持私密", "create": "创建", - "option": "允许用户进行更多查询。" + "option": "允许用户进行更多查询" } }, "sharedConv": { From 1992acaf6191feaf392fbda8067c0267c599ae7c Mon Sep 17 00:00:00 2001 From: ManishMadan2882 Date: Sun, 5 Jan 2025 01:44:29 +0530 Subject: [PATCH 06/22] (fix:locale) add missing keys in jp --- frontend/src/locale/jp.json | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/frontend/src/locale/jp.json b/frontend/src/locale/jp.json index 5f572ea4..2b4d8ba1 100644 --- a/frontend/src/locale/jp.json +++ b/frontend/src/locale/jp.json @@ -73,6 +73,9 @@ }, "logs": { "label": "ログ" + }, + "tools": { + "label": "ツール" } }, "modals": { @@ -86,7 +89,7 @@ "start": "チャットを開始する", "name": "名前", "choose": "ファイルを選択", - "info": ".pdf, .txt, .rst, .docx, .md, .json, .pptx, .zipファイルを25MBまでアップロードしてください", + "info": ".pdf, .txt, .rst, .csv, .xlsx, .docx, .md, .html, .epub, .json, .pptx, .zipファイルを25MBまでアップロードしてください", "uploadedFiles": "アップロードされたファイル", "cancel": "キャンセル", "train": "トレーニング", From b9540ba2bcdb148ea9c9bb897264b176a33ca84d Mon Sep 17 00:00:00 2001 From: ManishMadan2882 Date: Sun, 5 Jan 2025 02:04:18 +0530 Subject: [PATCH 07/22] (fix:locale) add missing keys/semantics in es --- frontend/src/locale/es.json | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/frontend/src/locale/es.json b/frontend/src/locale/es.json index 79aa1b3b..b7cc056e 100644 --- a/frontend/src/locale/es.json +++ b/frontend/src/locale/es.json @@ -8,7 +8,7 @@ "inputPlaceholder": "Escribe tu mensaje aquí...", "tagline": "DocsGPT utiliza GenAI, por favor revisa información crítica utilizando fuentes.", "sourceDocs": "Fuente", - "none": "Nada", + "none": "Ninguno", "cancel": "Cancelar", "help": "Asistencia", "emailUs": "Envíanos un correo", @@ -36,21 +36,21 @@ "general": { "label": "General", "selectTheme": "Seleccionar Tema", - "light": "de luz", - "dark": "oscura", + "light": "Claro", + "dark": "Oscuro", "selectLanguage": "Seleccionar Idioma", "chunks": "Trozos procesados por consulta", "prompt": "Prompt Activo", "deleteAllLabel": "Eliminar toda la Conversación", "deleteAllBtn": "Eliminar todo", "addNew": "Agregar Nuevo", - "convHistory": "Historia conversacional", - "none": "ninguno", + "convHistory": "Historial de conversaciones", + "none": "Ninguno", "low": "Bajo", "medium": "Medio", "high": "Alto", "unlimited": "Ilimitado", - "default": "predeterminada" + "default": "predeterminado" }, "documents": { "label": "Documentos", @@ -73,6 +73,9 @@ }, "logs": { "label": "Registros" + }, + "tools": { + "label": "Herramientas" } }, "modals": { @@ -81,12 +84,12 @@ "select": "Elija cómo cargar su documento en DocsGPT", "file": "Subir desde el dispositivo", "back": "Atrás", - "wait": "Espere por favor ...", + "wait": "Por favor espera ...", "remote": "Recoger desde un sitio web", "start": "Empezar a chatear", "name": "Nombre", "choose": "Seleccionar Archivos", - "info": "Por favor, suba archivos .pdf, .txt, .rst, .csv, .xlsx, .docx, .md, .html, .epub, .json, .pptx, .zip limitados a 25 MB", + "info": "Por favor, suba archivos .pdf, .txt, .rst, .csv, .xlsx, .docx, .md, .html, .epub, .json, .pptx, .zip limitados a 25MB", "uploadedFiles": "Archivos Subidos", "cancel": "Cancelar", "train": "Entrenar", @@ -108,7 +111,7 @@ "createAPIKey": { "label": "Crear Nueva Clave de API", "apiKeyName": "Nombre de la Clave de API", - "chunks": "Fragmentos procesados por consulta", + "chunks": "Trozos procesados por consulta", "prompt": "Seleccione el prompt activo", "sourceDoc": "Documento Fuente", "create": "Crear" @@ -126,7 +129,7 @@ }, "shareConv": { "label": "Crear una página pública para compartir", - "note": "El documento original, la información personal y las conversaciones posteriores permanecerán privadas", + "note": "El documento fuente, información personal y conversaciones posteriores permanecerán privadas", "create": "Crear", "option": "Permitir a los usuarios realizar más consultas." } From fa108126bbc12e3bdba321dfd40968d34bb96ac7 Mon Sep 17 00:00:00 2001 From: ManishMadan2882 Date: Sun, 5 Jan 2025 03:39:33 +0530 Subject: [PATCH 08/22] (feat:settings) sync with locales --- frontend/src/locale/en.json | 28 +++++++++++++++++++------ frontend/src/locale/es.json | 26 ++++++++++++++++++----- frontend/src/locale/jp.json | 6 +++++- frontend/src/locale/ru.json | 6 +++++- frontend/src/locale/zh-TW.json | 6 +++++- frontend/src/locale/zh.json | 6 +++++- frontend/src/settings/Documents.tsx | 20 +++++++++--------- frontend/src/settings/General.tsx | 32 +++++++---------------------- frontend/src/settings/Logs.tsx | 12 +++++++---- 9 files changed, 88 insertions(+), 54 deletions(-) diff --git a/frontend/src/locale/en.json b/frontend/src/locale/en.json index c6ae8023..730b6d3d 100644 --- a/frontend/src/locale/en.json +++ b/frontend/src/locale/en.json @@ -41,16 +41,16 @@ "selectLanguage": "Select Language", "chunks": "Chunks processed per query", "prompt": "Active Prompt", - "deleteAllLabel": "Delete all Conversation", - "deleteAllBtn": "Delete all", + "deleteAllLabel": "Delete All Conversations", + "deleteAllBtn": "Delete All", "addNew": "Add New", - "convHistory": "Conversational history", + "convHistory": "Conversation History", "none": "None", "low": "Low", "medium": "Medium", "high": "High", "unlimited": "Unlimited", - "default": "default" + "default": "Default" }, "documents": { "label": "Documents", @@ -58,7 +58,19 @@ "date": "Vector Date", "type": "Type", "tokenUsage": "Token Usage", - "noData": "No existing Documents" + "noData": "No existing Documents", + "searchPlaceholder": "Search...", + "addNew": "Add New", + "addNewTitle": "Add New Document", + "preLoaded": "Pre-loaded", + "private": "Private", + "sync": "Sync", + "syncFrequency": { + "never": "Never", + "daily": "Daily", + "weekly": "Weekly", + "monthly": "Monthly" + } }, "apiKeys": { "label": "Chatbots", @@ -72,7 +84,11 @@ "label": "Analytics" }, "logs": { - "label": "Logs" + "label": "Logs", + "filterByChatbot": "Filter by chatbot", + "none": "None", + "selectChatbotPlaceholder": "Select chatbot", + "apiGeneratedConversations": "API generated / chatbot conversations" }, "tools": { "label": "Tools" diff --git a/frontend/src/locale/es.json b/frontend/src/locale/es.json index b7cc056e..a91f179a 100644 --- a/frontend/src/locale/es.json +++ b/frontend/src/locale/es.json @@ -1,5 +1,5 @@ { - "language": "Spanish", + "language": "Español", "chat": "Chat", "chats": "Chats", "newChat": "Nuevo Chat", @@ -41,7 +41,7 @@ "selectLanguage": "Seleccionar Idioma", "chunks": "Trozos procesados por consulta", "prompt": "Prompt Activo", - "deleteAllLabel": "Eliminar toda la Conversación", + "deleteAllLabel": "Eliminar todas las conversaciones", "deleteAllBtn": "Eliminar todo", "addNew": "Agregar Nuevo", "convHistory": "Historial de conversaciones", @@ -50,7 +50,7 @@ "medium": "Medio", "high": "Alto", "unlimited": "Ilimitado", - "default": "predeterminado" + "default": "Predeterminado" }, "documents": { "label": "Documentos", @@ -58,7 +58,19 @@ "date": "Fecha Vector", "type": "Tipo", "tokenUsage": "Uso de Tokens", - "noData": "No hay documentos existentes" + "noData": "No hay documentos existentes", + "searchPlaceholder": "Buscar...", + "addNew": "Agregar Nuevo", + "addNewTitle": "Agregar Nuevo Documento", + "preLoaded": "Precargado", + "private": "Privado", + "sync": "Sincronizar", + "syncFrequency": { + "never": "Nunca", + "daily": "Diario", + "weekly": "Semanal", + "monthly": "Mensual" + } }, "apiKeys": { "label": "Chatbots", @@ -72,7 +84,11 @@ "label": "Analítica" }, "logs": { - "label": "Registros" + "label": "Registros", + "filterByChatbot": "Filtrar por chatbot", + "none": "Ninguno", + "selectChatbotPlaceholder": "Seleccionar chatbot", + "apiGeneratedConversations": "Conversaciones generadas por API / chatbot" }, "tools": { "label": "Herramientas" diff --git a/frontend/src/locale/jp.json b/frontend/src/locale/jp.json index 2b4d8ba1..29343a55 100644 --- a/frontend/src/locale/jp.json +++ b/frontend/src/locale/jp.json @@ -72,7 +72,11 @@ "label": "分析" }, "logs": { - "label": "ログ" + "label": "ログ", + "filterByChatbot": "チャットボットでフィルタ", + "none": "なし", + "selectChatbotPlaceholder": "チャットボットを選択", + "apiGeneratedConversations": "API生成/チャットボット会話" }, "tools": { "label": "ツール" diff --git a/frontend/src/locale/ru.json b/frontend/src/locale/ru.json index 98bfaf07..b16c2140 100644 --- a/frontend/src/locale/ru.json +++ b/frontend/src/locale/ru.json @@ -72,7 +72,11 @@ "label": "Аналитика" }, "logs": { - "label": "Журналы" + "label": "Регистры", + "filterByChatbot": "Фильтр по чат-боту", + "none": "Нет", + "selectChatbotPlaceholder": "Выберите чат-бота", + "apiGeneratedConversations": "Разговоры, сгенерированные API / чат-ботом" }, "tools": { "label": "Инструменты" diff --git a/frontend/src/locale/zh-TW.json b/frontend/src/locale/zh-TW.json index 46c645de..952a4df1 100644 --- a/frontend/src/locale/zh-TW.json +++ b/frontend/src/locale/zh-TW.json @@ -72,7 +72,11 @@ "label": "分析" }, "logs": { - "label": "日誌" + "label": "日誌", + "filterByChatbot": "按聊天機器人篩選", + "none": "無", + "selectChatbotPlaceholder": "選擇聊天機器人", + "apiGeneratedConversations": "API生成/聊天機器人對話" }, "tools": { "label": "工具" diff --git a/frontend/src/locale/zh.json b/frontend/src/locale/zh.json index 12a2ec65..4edccfb8 100644 --- a/frontend/src/locale/zh.json +++ b/frontend/src/locale/zh.json @@ -72,7 +72,11 @@ "label": "分析" }, "logs": { - "label": "日志" + "label": "日志", + "filterByChatbot": "按聊天机器人筛选", + "none": "无", + "selectChatbotPlaceholder": "选择聊天机器人", + "apiGeneratedConversations": "API生成/聊天机器人对话" }, "tools": { "label": "工具" diff --git a/frontend/src/settings/Documents.tsx b/frontend/src/settings/Documents.tsx index 8e68f226..71ba63f6 100644 --- a/frontend/src/settings/Documents.tsx +++ b/frontend/src/settings/Documents.tsx @@ -54,10 +54,10 @@ const Documents: React.FC = ({ const [totalPages, setTotalPages] = useState(1); const currentDocuments = paginatedDocuments ?? []; const syncOptions = [ - { label: 'Never', value: 'never' }, - { label: 'Daily', value: 'daily' }, - { label: 'Weekly', value: 'weekly' }, - { label: 'Monthly', value: 'monthly' }, + { label: t('settings.documents.syncFrequency.never'), value: 'never' }, + { label: t('settings.documents.syncFrequency.daily'), value: 'daily' }, + { label: t('settings.documents.syncFrequency.weekly'), value: 'weekly' }, + { label: t('settings.documents.syncFrequency.monthly'), value: 'monthly' }, ]; const refreshDocs = useCallback( @@ -153,7 +153,7 @@ const Documents: React.FC = ({

= ({
{loading ? ( ) : ( -
+
@@ -270,7 +270,7 @@ const Documents: React.FC = ({ {document.type !== 'remote' && ( Delete { @@ -282,7 +282,7 @@ const Documents: React.FC = ({ {document.syncFrequency && (
{ handleManageSync(document, value); diff --git a/frontend/src/settings/General.tsx b/frontend/src/settings/General.tsx index 4868d252..227b0a6e 100644 --- a/frontend/src/settings/General.tsx +++ b/frontend/src/settings/General.tsx @@ -21,33 +21,15 @@ export default function General() { t, i18n: { changeLanguage, language }, } = useTranslation(); - const themes = ['Light', 'Dark']; + const themes = [t('settings.general.light'), t('settings.general.dark')]; const languageOptions = [ - { - label: 'English', - value: 'en', - }, - { - label: 'Spanish', - value: 'es', - }, - { - label: 'Japanese', - value: 'jp', - }, - { - label: 'Mandarin', - value: 'zh', - }, - { - label: 'Traditional Chinese', - value: 'zhTW', - }, - { - label: 'Russian', - value: 'ru', - }, + { label: 'English', value: 'en' }, + { label: 'Español', value: 'es' }, + { label: '日本語', value: 'jp' }, + { label: '普通话', value: 'zh' }, + { label: '繁體中文（臺灣）', value: 'zhTW' }, + { label: 'Русский', value: 'ru' }, ]; const chunks = ['0', '2', '4', '6', '8', '10']; const token_limits = new Map([ diff --git a/frontend/src/settings/Logs.tsx b/frontend/src/settings/Logs.tsx index 1e248d46..0f452e97 100644 --- a/frontend/src/settings/Logs.tsx +++ b/frontend/src/settings/Logs.tsx @@ -1,4 +1,5 @@ import React, { useState, useEffect, useRef, useCallback } from 'react'; +import { useTranslation } from 'react-i18next'; import userService from '../api/services/userService'; import ChevronRight from '../assets/chevron-right.svg'; @@ -8,6 +9,7 @@ import { APIKeyData, LogData } from './types'; import CoppyButton from '../components/CopyButton'; export default function Logs() { + const { t } = useTranslation(); const [chatbots, setChatbots] = useState([]); const [selectedChatbot, setSelectedChatbot] = useState(); const [logs, setLogs] = useState([]); @@ -66,7 +68,7 @@ export default function Logs() {

- Filter by chatbot + {t('settings.logs.filterByChatbot')}

{loadingChatbots ? ( @@ -78,9 +80,9 @@ export default function Logs() { label: chatbot.name, value: chatbot.id, })), - { label: 'None', value: '' }, + { label: t('settings.logs.none'), value: '' }, ]} - placeholder="Select chatbot" + placeholder={t('settings.logs.selectChatbotPlaceholder')} onSelect={(chatbot: { label: string; value: string }) => { setSelectedChatbot( chatbots.find((item) => item.id === chatbot.value), @@ -120,6 +122,7 @@ type LogsTableProps = { }; function LogsTable({ logs, setPage }: LogsTableProps) { + const { t } = useTranslation(); const observerRef = useRef(); const firstObserver = useCallback((node: HTMLDivElement) => { if (observerRef.current) { @@ -134,7 +137,7 @@ function LogsTable({ logs, setPage }: LogsTableProps) {

- API generated / chatbot conversations + {t('settings.logs.apiGeneratedConversations')}

Date: Tue, 7 Jan 2025 20:28:59 +0530 Subject: [PATCH 09/22] (fix:locales) sync conversation --- .../src/conversation/ConversationBubble.tsx | 36 ++++++++++++------- frontend/src/locale/en.json | 16 +++++++++ frontend/src/locale/jp.json | 34 +++++++++++++++++- frontend/src/locale/zh-TW.json | 2 +- frontend/src/settings/index.tsx | 4 +-- 5 files changed, 75 insertions(+), 17 deletions(-) diff --git a/frontend/src/conversation/ConversationBubble.tsx b/frontend/src/conversation/ConversationBubble.tsx index d88b249e..e410b7c5 100644 --- a/frontend/src/conversation/ConversationBubble.tsx +++ b/frontend/src/conversation/ConversationBubble.tsx @@ -8,6 +8,7 @@ import { vscDarkPlus } from 'react-syntax-highlighter/dist/cjs/styles/prism'; import rehypeKatex from 'rehype-katex'; import remarkGfm from 'remark-gfm'; import remarkMath from 'remark-math'; +import { useTranslation } from 'react-i18next'; import DocsGPT3 from '../assets/cute_docsgpt3.svg'; import Dislike from '../assets/dislike.svg?react'; @@ -62,6 +63,7 @@ const ConversationBubble = forwardRef< }, ref, ) { + const { t } = useTranslation(); // const bubbleRef = useRef(null); const chunks = useSelector(selectChunks); const selectedDocs = useSelector(selectSelectedDocs); @@ -113,13 +115,13 @@ const ConversationBubble = forwardRef< {isEditClicked && (
{status === 'loading' ? ( + alt={t('loading')} + /> ) : (
- handleQuestionSubmission()} - src={isDarkTheme ? SendDark : Send} - > + aria-label={t('send')} + className="flex items-center justify-center" + > + {t('send')} +
)}
diff --git a/frontend/src/locale/en.json b/frontend/src/locale/en.json index 070deba6..9f9cbb21 100644 --- a/frontend/src/locale/en.json +++ b/frontend/src/locale/en.json @@ -166,6 +166,8 @@ "copied": "Copied", "speak": "Speak", "answer": "Answer", + "send": "Send message", + "loading": "Loading response", "edit": { "placeholder": "Type the updated query..." }, diff --git a/frontend/src/preferences/PromptsModal.tsx b/frontend/src/preferences/PromptsModal.tsx index 3aa8c54c..3a1f9bfa 100644 --- a/frontend/src/preferences/PromptsModal.tsx +++ b/frontend/src/preferences/PromptsModal.tsx @@ -29,8 +29,9 @@ function AddPrompt({ setNewPromptName(''); setNewPromptContent(''); }} + aria-label="Close add prompt modal" > - + Close modal

@@ -40,7 +41,11 @@ function AddPrompt({ Add your custom prompt and save it to DocsGPT

+
+
@@ -104,8 +114,9 @@ function EditPrompt({ onClick={() => { setModalState('INACTIVE'); }} + aria-label="Close edit prompt modal" > - + Close modal

@@ -115,13 +126,17 @@ function EditPrompt({ Edit your custom prompt and save it to DocsGPT

+ setEditPromptName(e.target.value)} - > + />
Prompt Name @@ -132,10 +147,15 @@ function EditPrompt({ Prompt Text
+
diff --git a/frontend/src/settings/APIKeys.tsx b/frontend/src/settings/APIKeys.tsx index 6775ba87..038e4bbb 100644 --- a/frontend/src/settings/APIKeys.tsx +++ b/frontend/src/settings/APIKeys.tsx @@ -115,12 +115,20 @@ export default function APIKeys() {
- - + - - + + @@ -146,7 +154,7 @@ export default function APIKeys() { diff --git a/frontend/src/settings/Logs.tsx b/frontend/src/settings/Logs.tsx index 0f452e97..f1ebfe72 100644 --- a/frontend/src/settings/Logs.tsx +++ b/frontend/src/settings/Logs.tsx @@ -67,9 +67,12 @@ export default function Logs() {
-

+

+ {loadingChatbots ? ( ) : ( diff --git a/frontend/src/settings/Tools.tsx b/frontend/src/settings/Tools.tsx index f58487a3..3dc31e35 100644 --- a/frontend/src/settings/Tools.tsx +++ b/frontend/src/settings/Tools.tsx @@ -1,4 +1,5 @@ import React from 'react'; +import { useTranslation } from 'react-i18next'; import userService from '../api/services/userService'; import CogwheelIcon from '../assets/cogwheel.svg'; @@ -18,6 +19,7 @@ export default function Tools() { React.useState('INACTIVE'); const [userTools, setUserTools] = React.useState([]); const [selectedTool, setSelectedTool] = React.useState(null); + const { t } = useTranslation(); const getUserTools = () => { userService @@ -70,12 +72,15 @@ export default function Tools() {
+ setSearchTerm(e.target.value)} /> @@ -86,7 +91,7 @@ export default function Tools() { setAddToolModalState('ACTIVE'); }} > - Add Tool + {t('settings.tools.addTool')}
@@ -98,10 +103,10 @@ export default function Tools() {
No tools found - No tools found + {t('settings.tools.noToolsFound')}
) : ( userTools @@ -119,15 +124,19 @@ export default function Tools() {
{`${tool.displayName} @@ -146,6 +155,11 @@ export default function Tools() { htmlFor={`toolToggle-${index}`} className="relative inline-block h-6 w-10 cursor-pointer rounded-full bg-gray-300 dark:bg-[#D2D5DA33]/20 transition [-webkit-tap-highlight-color:_transparent] has-[:checked]:bg-[#0C9D35CC] has-[:checked]:dark:bg-[#0C9D35CC]" > + + {t('settings.tools.toggleToolAria', { + toolName: tool.displayName, + })} +
Date: Mon, 13 Jan 2025 18:20:08 +0530 Subject: [PATCH 14/22] (fix/widget) centered the toolkit msg --- extensions/react-widget/publish.sh | 102 ++++++++++++------ .../react-widget/src/components/SearchBar.tsx | 10 +- 2 files changed, 80 insertions(+), 32 deletions(-) diff --git a/extensions/react-widget/publish.sh b/extensions/react-widget/publish.sh index c4545d85..129c4bcf 100755 --- a/extensions/react-widget/publish.sh +++ b/extensions/react-widget/publish.sh @@ -1,43 +1,85 @@ #!/bin/bash -## chmod +x publish.sh - to upgrade ownership set -e -cat package.json >> package_copy.json -cat package-lock.json >> package-lock_copy.json + +# Create backup of original files +cp package.json package_original.json +cp package-lock.json package-lock_original.json + +# Store the latest version after publishing +LATEST_VERSION="" + publish_package() { - PACKAGE_NAME=$1 - BUILD_COMMAND=$2 - # Update package name in package.json - jq --arg name "$PACKAGE_NAME" '.name=$name' package.json > temp.json && mv temp.json package.json + PACKAGE_NAME=$1 + BUILD_COMMAND=$2 + IS_REACT=$3 - # Remove 'target' key if the package name is 'docsgpt-react' - if [ "$PACKAGE_NAME" = "docsgpt-react" ]; then - jq 'del(.targets)' package.json > temp.json && mv temp.json package.json - fi + echo "Preparing to publish ${PACKAGE_NAME}..." + + # Restore original package.json state before each publish + cp package_original.json package.json + cp package-lock_original.json package-lock.json - if [ -d "dist" ]; then - echo "Deleting existing dist directory..." - rm -rf dist - fi + # Update package name in package.json + jq --arg name "$PACKAGE_NAME" '.name=$name' package.json > temp.json && mv temp.json package.json - npm version patch + # Handle targets based on package type + if [ "$IS_REACT" = "true" ]; then + echo "Removing targets for React library build..." + jq 'del(.targets)' package.json > temp.json && mv temp.json package.json + fi - npm run "$BUILD_COMMAND" + # Clean dist directory + if [ -d "dist" ]; then + echo "Cleaning dist directory..." + rm -rf dist + fi - # Publish to npm - npm publish - # Clean up - mv package_copy.json package.json - mv package-lock_copy.json package-lock.json - echo "Published ${PACKAGE_NAME}" + # update version and store it + LATEST_VERSION=$(npm version patch) + echo "New version: ${LATEST_VERSION}" + + # Build package + npm run "$BUILD_COMMAND" + + # Replace npm publish with npm pack for testing + npm publish + + echo "Successfully packaged ${PACKAGE_NAME}" + + # Log the bundle size + TARBALL="${PACKAGE_NAME}-${LATEST_VERSION#v}.tgz" + if [ -f "$TARBALL" ]; then + BUNDLE_SIZE=$(du -h "$TARBALL" | cut -f1) + echo "Bundle size for ${PACKAGE_NAME}: ${BUNDLE_SIZE}" + else + echo "Error: ${TARBALL} not found." + exit 1 + fi } -# Publish docsgpt package -publish_package "docsgpt" "build" +# First publish docsgpt (HTML bundle) +publish_package "docsgpt" "build" "false" -# Publish docsgpt-react package -publish_package "docsgpt-react" "build:react" +# Then publish docsgpt-react (React library) +publish_package "docsgpt-react" "build:react" "true" +# Restore original state but keep the updated version +cp package_original.json package.json +cp package-lock_original.json package-lock.json -rm -rf package_copy.json -rm -rf package-lock_copy.json -echo "---Process completed---" \ No newline at end of file +# Update the version in the final package.json +jq --arg version "${LATEST_VERSION#v}" '.version=$version' package.json > temp.json && mv temp.json package.json + +# Run npm install to update package-lock.json with the new version +npm install --package-lock-only + +# Cleanup backup files +rm -f package_original.json +rm -f package-lock_original.json +rm -f temp.json + +echo "---Process completed---" +echo "Final version in package.json: $(jq -r '.version' package.json)" +echo "Final version in package-lock.json: $(jq -r '.version' package-lock.json)" +echo "Generated test packages:" +ls *.tgz diff --git a/extensions/react-widget/src/components/SearchBar.tsx b/extensions/react-widget/src/components/SearchBar.tsx index c7344e43..c647991f 100644 --- a/extensions/react-widget/src/components/SearchBar.tsx +++ b/extensions/react-widget/src/components/SearchBar.tsx @@ -242,14 +242,20 @@ white-space: pre-wrap; const Toolkit = styled.kbd` position: absolute; right: 4px; - top: 4px; + top: 50%; + transform: translateY(-50%); background-color: ${(props) => props.theme.primary.bg}; color: ${(props) => props.theme.secondary.text}; font-weight: 600; font-size: 10px; - padding: 3px; + padding: 3px 6px; border: 1px solid ${(props) => props.theme.secondary.text}; border-radius: 4px; + display: flex; + align-items: center; + justify-content: center; + z-index: 1; + pointer-events: none; ` const Loader = styled.div` margin: 2rem auto; From 838525b452b3d766eee69c35595c309ce02241c0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Jan 2025 20:06:18 +0000 Subject: [PATCH 15/22] build(deps): bump tqdm from 4.66.5 to 4.67.1 in /application Bumps [tqdm](https://github.com/tqdm/tqdm) from 4.66.5 to 4.67.1. - [Release notes](https://github.com/tqdm/tqdm/releases) - [Commits](https://github.com/tqdm/tqdm/compare/v4.66.5...v4.67.1) --- updated-dependencies: - dependency-name: tqdm dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- application/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/requirements.txt b/application/requirements.txt index a54e0ddd..41d742eb 100644 --- a/application/requirements.txt +++ b/application/requirements.txt @@ -77,7 +77,7 @@ sentence-transformers==3.3.1 tiktoken==0.8.0 tokenizers==0.21.0 torch==2.5.1 -tqdm==4.66.5 +tqdm==4.67.1 transformers==4.47.1 typing-extensions==4.12.2 typing-inspect==0.9.0 From e0912f0cf0e4c49a88a264633e061c487758a943 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Jan 2025 20:06:20 +0000 Subject: [PATCH 16/22] build(deps): bump openapi3-parser from 1.1.18 to 1.1.19 in /application Bumps [openapi3-parser](https://github.com/manchenkoff/openapi3-parser) from 1.1.18 to 1.1.19. - [Release notes](https://github.com/manchenkoff/openapi3-parser/releases) - [Commits](https://github.com/manchenkoff/openapi3-parser/compare/v1.1.18...v1.1.19) --- updated-dependencies: - dependency-name: openapi3-parser dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- application/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/requirements.txt b/application/requirements.txt index a54e0ddd..5d6451cd 100644 --- a/application/requirements.txt +++ b/application/requirements.txt @@ -46,7 +46,7 @@ numpy==2.2.1 openai==1.59.5 openapi-schema-validator==0.6.2 openapi-spec-validator==0.6.0 -openapi3-parser==1.1.18 +openapi3-parser==1.1.19 orjson==3.10.14 packaging==24.1 pandas==2.2.3 From 1438fea76bd74f17ae94ddb47d8f75f8ec2e3231 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Jan 2025 20:06:27 +0000 Subject: [PATCH 17/22] build(deps): bump elastic-transport in /application Bumps [elastic-transport](https://github.com/elastic/elastic-transport-python) from 8.15.1 to 8.17.0. - [Release notes](https://github.com/elastic/elastic-transport-python/releases) - [Changelog](https://github.com/elastic/elastic-transport-python/blob/v8.17.0/CHANGELOG.md) - [Commits](https://github.com/elastic/elastic-transport-python/compare/v8.15.1...v8.17.0) --- updated-dependencies: - dependency-name: elastic-transport dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- application/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/requirements.txt b/application/requirements.txt index a54e0ddd..d750b9aa 100644 --- a/application/requirements.txt +++ b/application/requirements.txt @@ -6,7 +6,7 @@ dataclasses-json==0.6.7 docx2txt==0.8 duckduckgo-search==6.3.0 ebooklib==0.18 -elastic-transport==8.15.1 +elastic-transport==8.17.0 elasticsearch==8.17.0 escodegen==1.0.11 esprima==4.0.1 From 7369b02bf40bd4661ff41e19c53a1e113b012d72 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 Jan 2025 20:06:32 +0000 Subject: [PATCH 18/22] build(deps): bump langsmith from 0.2.6 to 0.2.10 in /application Bumps [langsmith](https://github.com/langchain-ai/langsmith-sdk) from 0.2.6 to 0.2.10. - [Release notes](https://github.com/langchain-ai/langsmith-sdk/releases) - [Commits](https://github.com/langchain-ai/langsmith-sdk/compare/v0.2.6...v0.2.10) --- updated-dependencies: - dependency-name: langsmith dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- application/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/requirements.txt b/application/requirements.txt index a54e0ddd..499d6251 100644 --- a/application/requirements.txt +++ b/application/requirements.txt @@ -33,7 +33,7 @@ langchain-community==0.3.14 langchain-core==0.3.29 langchain-openai==0.3.0 langchain-text-splitters==0.3.5 -langsmith==0.2.6 +langsmith==0.2.10 lazy-object-proxy==1.10.0 lxml==5.3.0 markupsafe==3.0.2 From 9e6f970bc400bcec3d8ffb191c6d90927f982920 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 14 Jan 2025 20:35:08 +0000 Subject: [PATCH 19/22] build(deps): bump flask from 3.0.3 to 3.1.0 in /application Bumps [flask](https://github.com/pallets/flask) from 3.0.3 to 3.1.0. - [Release notes](https://github.com/pallets/flask/releases) - [Changelog](https://github.com/pallets/flask/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/flask/compare/3.0.3...3.1.0) --- updated-dependencies: - dependency-name: flask dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- application/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/requirements.txt b/application/requirements.txt index df50d9d6..e2b3c129 100644 --- a/application/requirements.txt +++ b/application/requirements.txt @@ -11,7 +11,7 @@ elasticsearch==8.17.0 escodegen==1.0.11 esprima==4.0.1 esutils==1.0.1 -Flask==3.0.3 +Flask==3.1.0 faiss-cpu==1.9.0.post1 flask-restx==1.3.0 gTTS==2.5.4 From 850b79f459e8c722562bbce8474619f9ca224360 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 14 Jan 2025 20:35:19 +0000 Subject: [PATCH 20/22] build(deps): bump transformers from 4.47.1 to 4.48.0 in /application Bumps [transformers](https://github.com/huggingface/transformers) from 4.47.1 to 4.48.0. - [Release notes](https://github.com/huggingface/transformers/releases) - [Commits](https://github.com/huggingface/transformers/compare/v4.47.1...v4.48.0) --- updated-dependencies: - dependency-name: transformers dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- application/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/requirements.txt b/application/requirements.txt index df50d9d6..5b1a1404 100644 --- a/application/requirements.txt +++ b/application/requirements.txt @@ -78,7 +78,7 @@ tiktoken==0.8.0 tokenizers==0.21.0 torch==2.5.1 tqdm==4.67.1 -transformers==4.47.1 +transformers==4.48.0 typing-extensions==4.12.2 typing-inspect==0.9.0 tzdata==2024.2 From 13fcbe3e74dad4b6f2b37bc093f022d941be9d0b Mon Sep 17 00:00:00 2001 From: Pavel Date: Wed, 15 Jan 2025 01:08:09 +0300 Subject: [PATCH 21/22] scraper with markdownify --- application/api/user/routes.py | 2 +- application/parser/remote/crawler_loader.py | 41 +++--- application/parser/remote/crawler_markdown.py | 139 ++++++++++++++++++ application/requirements.txt | 4 +- application/worker.py | 82 ++++++----- 5 files changed, 211 insertions(+), 57 deletions(-) create mode 100644 application/parser/remote/crawler_markdown.py diff --git a/application/api/user/routes.py b/application/api/user/routes.py index 4bcbd719..e516bb71 100644 --- a/application/api/user/routes.py +++ b/application/api/user/routes.py @@ -2105,4 +2105,4 @@ class DeleteTool(Resource): except Exception as err: return {"success": False, "error": str(err)}, 400 - return {"success": True}, 200 + return {"success": True}, 200 \ No newline at end of file diff --git a/application/parser/remote/crawler_loader.py b/application/parser/remote/crawler_loader.py index 76325ae6..c2da230b 100644 --- a/application/parser/remote/crawler_loader.py +++ b/application/parser/remote/crawler_loader.py @@ -2,16 +2,16 @@ import requests from urllib.parse import urlparse, urljoin from bs4 import BeautifulSoup from application.parser.remote.base import BaseRemote +from application.parser.schema.base import Document +from langchain_community.document_loaders import WebBaseLoader class CrawlerLoader(BaseRemote): def __init__(self, limit=10): - from langchain_community.document_loaders import WebBaseLoader self.loader = WebBaseLoader # Initialize the document loader self.limit = limit # Set the limit for the number of pages to scrape def load_data(self, inputs): url = inputs - # Check if the input is a list and if it is, use the first element if isinstance(url, list) and url: url = url[0] @@ -19,24 +19,29 @@ class CrawlerLoader(BaseRemote): if not urlparse(url).scheme: url = "http://" + url - visited_urls = set() # Keep track of URLs that have been visited - base_url = urlparse(url).scheme + "://" + urlparse(url).hostname # Extract the base URL - urls_to_visit = [url] # List of URLs to be visited, starting with the initial URL - loaded_content = [] # Store the loaded content from each URL + visited_urls = set() + base_url = urlparse(url).scheme + "://" + urlparse(url).hostname + urls_to_visit = [url] + loaded_content = [] - # Continue crawling until there are no more URLs to visit while urls_to_visit: - current_url = urls_to_visit.pop(0) # Get the next URL to visit - visited_urls.add(current_url) # Mark the URL as visited + current_url = urls_to_visit.pop(0) + visited_urls.add(current_url) - # Try to load and process the content from the current URL try: - response = requests.get(current_url) # Fetch the content of the current URL - response.raise_for_status() # Raise an exception for HTTP errors - loader = self.loader([current_url]) # Initialize the document loader for the current URL - loaded_content.extend(loader.load()) # Load the content and add it to the loaded_content list + response = requests.get(current_url) + response.raise_for_status() + loader = self.loader([current_url]) + docs = loader.load() + # Convert the loaded documents to your Document schema + for doc in docs: + loaded_content.append( + Document( + doc.page_content, + extra_info=doc.metadata + ) + ) except Exception as e: - # Print an error message if loading or processing fails and continue with the next URL print(f"Error processing URL {current_url}: {e}") continue @@ -45,15 +50,15 @@ class CrawlerLoader(BaseRemote): all_links = [ urljoin(current_url, a['href']) for a in soup.find_all('a', href=True) - if base_url in urljoin(current_url, a['href']) # Ensure links are from the same domain + if base_url in urljoin(current_url, a['href']) ] # Add new links to the list of URLs to visit if they haven't been visited yet urls_to_visit.extend([link for link in all_links if link not in visited_urls]) - urls_to_visit = list(set(urls_to_visit)) # Remove duplicate URLs + urls_to_visit = list(set(urls_to_visit)) # Stop crawling if the limit of pages to scrape is reached if self.limit is not None and len(visited_urls) >= self.limit: break - return loaded_content # Return the loaded content from all visited URLs + return loaded_content \ No newline at end of file diff --git a/application/parser/remote/crawler_markdown.py b/application/parser/remote/crawler_markdown.py new file mode 100644 index 00000000..3d199332 --- /dev/null +++ b/application/parser/remote/crawler_markdown.py @@ -0,0 +1,139 @@ +import requests +from urllib.parse import urlparse, urljoin +from bs4 import BeautifulSoup +from application.parser.remote.base import BaseRemote +import re +from markdownify import markdownify +from application.parser.schema.base import Document +import tldextract + +class CrawlerLoader(BaseRemote): + def __init__(self, limit=10, allow_subdomains=False): + """ + Given a URL crawl web pages up to `self.limit`, + convert HTML content to Markdown, and returning a list of Document objects. + + :param limit: The maximum number of pages to crawl. + :param allow_subdomains: If True, crawl pages on subdomains of the base domain. + """ + self.limit = limit + self.allow_subdomains = allow_subdomains + self.session = requests.Session() + + def load_data(self, inputs): + url = inputs + if isinstance(url, list) and url: + url = url[0] + + # Ensure the URL has a scheme (if not, default to http) + if not urlparse(url).scheme: + url = "http://" + url + + # Keep track of visited URLs to avoid revisiting the same page + visited_urls = set() + + # Determine the base domain for link filtering using tldextract + base_domain = self._get_base_domain(url) + urls_to_visit = {url} + documents = [] + + while urls_to_visit: + current_url = urls_to_visit.pop() + + # Skip if already visited + if current_url in visited_urls: + continue + visited_urls.add(current_url) + + # Fetch the page content + html_content = self._fetch_page(current_url) + if html_content is None: + continue + + # Convert the HTML to Markdown for cleaner text formatting + title, language, processed_markdown = self._process_html_to_markdown(html_content, current_url) + if processed_markdown: + # Create a Document for each visited page + documents.append( + Document( + processed_markdown, # content + None, # doc_id + None, # embedding + {"source": current_url, "title": title, "language": language} # extra_info + ) + ) + + # Extract links and filter them according to domain rules + new_links = self._extract_links(html_content, current_url) + filtered_links = self._filter_links(new_links, base_domain) + + # Add any new, not-yet-visited links to the queue + urls_to_visit.update(link for link in filtered_links if link not in visited_urls) + + # If we've reached the limit, stop crawling + if self.limit is not None and len(visited_urls) >= self.limit: + break + + return documents + + def _fetch_page(self, url): + try: + response = self.session.get(url, timeout=10) + response.raise_for_status() + return response.text + except requests.exceptions.RequestException as e: + print(f"Error fetching URL {url}: {e}") + return None + + def _process_html_to_markdown(self, html_content, current_url): + soup = BeautifulSoup(html_content, 'html.parser') + title_tag = soup.find('title') + title = title_tag.text.strip() if title_tag else "No Title" + + # Extract language + language_tag = soup.find('html') + language = language_tag.get('lang', 'en') if language_tag else "en" + + markdownified = markdownify(html_content, heading_style="ATX", newline_style="BACKSLASH") + # Reduce sequences of more than two newlines to exactly three + markdownified = re.sub(r'\n{3,}', '\n\n\n', markdownified) + return title, language, markdownified + + def _extract_links(self, html_content, current_url): + soup = BeautifulSoup(html_content, 'html.parser') + links = [] + for a in soup.find_all('a', href=True): + full_url = urljoin(current_url, a['href']) + links.append((full_url, a.text.strip())) + return links + + def _get_base_domain(self, url): + extracted = tldextract.extract(url) + # Reconstruct the domain as domain.suffix + base_domain = f"{extracted.domain}.{extracted.suffix}" + return base_domain + + def _filter_links(self, links, base_domain): + """ + Filter the extracted links to only include those that match the crawling criteria: + - If allow_subdomains is True, allow any link whose domain ends with the base_domain. + - If allow_subdomains is False, only allow exact matches of the base_domain. + """ + filtered = [] + for link, _ in links: + parsed_link = urlparse(link) + if not parsed_link.netloc: + continue + + extracted = tldextract.extract(parsed_link.netloc) + link_base = f"{extracted.domain}.{extracted.suffix}" + + if self.allow_subdomains: + # For subdomains: sub.example.com ends with example.com + if link_base == base_domain or link_base.endswith("." + base_domain): + filtered.append(link) + else: + # Exact domain match + if link_base == base_domain: + filtered.append(link) + return filtered \ No newline at end of file diff --git a/application/requirements.txt b/application/requirements.txt index ca8ba68f..7b756ed3 100644 --- a/application/requirements.txt +++ b/application/requirements.txt @@ -86,4 +86,6 @@ urllib3==2.3.0 vine==5.1.0 wcwidth==0.2.13 werkzeug==3.1.3 -yarl==1.18.3 \ No newline at end of file +yarl==1.18.3 +markdownify==0.14.1 +tldextract==5.1.3 \ No newline at end of file diff --git a/application/worker.py b/application/worker.py index f4f181e5..df0bbe7d 100755 --- a/application/worker.py +++ b/application/worker.py @@ -203,53 +203,61 @@ def remote_worker( sync_frequency="never", operation_mode="upload", doc_id=None, -): +): full_path = os.path.join(directory, user, name_job) - if not os.path.exists(full_path): os.makedirs(full_path) + self.update_state(state="PROGRESS", meta={"current": 1}) - logging.info( - f"Remote job: {full_path}", - extra={"user": user, "job": name_job, "source_data": source_data}, - ) + try: + logging.info("Initializing remote loader with type: %s", loader) + remote_loader = RemoteCreator.create_loader(loader) + raw_docs = remote_loader.load_data(source_data) - remote_loader = RemoteCreator.create_loader(loader) - raw_docs = remote_loader.load_data(source_data) + chunker = Chunker( + chunking_strategy="classic_chunk", + max_tokens=MAX_TOKENS, + min_tokens=MIN_TOKENS, + duplicate_headers=False + ) + docs = chunker.chunk(documents=raw_docs) + docs = [Document.to_langchain_format(raw_doc) for raw_doc in raw_docs] + tokens = count_tokens_docs(docs) + logging.info("Total tokens calculated: %d", tokens) - chunker = Chunker( - chunking_strategy="classic_chunk", - max_tokens=MAX_TOKENS, - min_tokens=MIN_TOKENS, - duplicate_headers=False - ) - docs = chunker.chunk(documents=raw_docs) + if operation_mode == "upload": + id = ObjectId() + embed_and_store_documents(docs, full_path, id, self) + elif operation_mode == "sync": + if not doc_id or not ObjectId.is_valid(doc_id): + logging.error("Invalid doc_id provided for sync operation: %s", doc_id) + raise ValueError("doc_id must be provided for sync operation.") + id = ObjectId(doc_id) + embed_and_store_documents(docs, full_path, id, self) - tokens = count_tokens_docs(docs) - if operation_mode == "upload": - id = ObjectId() - embed_and_store_documents(docs, full_path, id, self) - elif operation_mode == "sync": - if not doc_id or not ObjectId.is_valid(doc_id): - raise ValueError("doc_id must be provided for sync operation.") - id = ObjectId(doc_id) - embed_and_store_documents(docs, full_path, id, self) - self.update_state(state="PROGRESS", meta={"current": 100}) + self.update_state(state="PROGRESS", meta={"current": 100}) - file_data = { - "name": name_job, - "user": user, - "tokens": tokens, - "retriever": retriever, - "id": str(id), - "type": loader, - "remote_data": source_data, - "sync_frequency": sync_frequency, - } - upload_index(full_path, file_data) + file_data = { + "name": name_job, + "user": user, + "tokens": tokens, + "retriever": retriever, + "id": str(id), + "type": loader, + "remote_data": source_data, + "sync_frequency": sync_frequency, + } + upload_index(full_path, file_data) - shutil.rmtree(full_path) + except Exception as e: + logging.error("Error in remote_worker task: %s", str(e), exc_info=True) + raise + finally: + if os.path.exists(full_path): + shutil.rmtree(full_path) + + logging.info("remote_worker task completed successfully") return {"urls": source_data, "name_job": name_job, "user": user, "limited": False} def sync( From cab6305462db0aca429fa16fb4d66975b96c4d13 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 15 Jan 2025 20:59:45 +0000 Subject: [PATCH 22/22] build(deps): bump primp from 0.9.3 to 0.10.0 in /application Bumps [primp](https://github.com/deedy5/primp) from 0.9.3 to 0.10.0. - [Release notes](https://github.com/deedy5/primp/releases) - [Commits](https://github.com/deedy5/primp/compare/v0.9.3...v0.10.0) --- updated-dependencies: - dependency-name: primp dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- application/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/requirements.txt b/application/requirements.txt index 0ad92e27..3fc6d02d 100644 --- a/application/requirements.txt +++ b/application/requirements.txt @@ -55,7 +55,7 @@ pathable==0.4.4 pillow==11.1.0 portalocker==2.10.1 prance==23.6.21.0 -primp==0.9.3 +primp==0.10.0 prompt-toolkit==3.0.48 protobuf==5.29.3 py==1.11.0
{t('settings.apiKeys.name')} + + {t('settings.apiKeys.name')} + {t('settings.apiKeys.sourceDoc')} {t('settings.apiKeys.key')} + {t('settings.apiKeys.key')} +
Delete handleDeleteKey(element.id)} diff --git a/frontend/src/settings/Analytics.tsx b/frontend/src/settings/Analytics.tsx index 8baad361..ccb5ffcb 100644 --- a/frontend/src/settings/Analytics.tsx +++ b/frontend/src/settings/Analytics.tsx @@ -164,7 +164,7 @@ export default function Analytics() {
-

+

Filter by chatbot

diff --git a/frontend/src/settings/General.tsx b/frontend/src/settings/General.tsx index 227b0a6e..216ec1e0 100644 --- a/frontend/src/settings/General.tsx +++ b/frontend/src/settings/General.tsx @@ -81,9 +81,9 @@ export default function General() { return (
-

+

+
-

+

+ @@ -115,9 +115,9 @@ export default function General() { />
-

+

+
-

+

+ ({ value: value, @@ -163,16 +163,14 @@ export default function General() { />
-

+

+
diff --git a/frontend/src/settings/Prompts.tsx b/frontend/src/settings/Prompts.tsx index 6e1810e5..611b0b90 100644 --- a/frontend/src/settings/Prompts.tsx +++ b/frontend/src/settings/Prompts.tsx @@ -168,7 +168,7 @@ export default function Prompts({ />
- {' '} + {t('settings.documents.actions')}