Merge branch 'arc53:main' into Fixes-#1260

2025-11-30 17:13:15 +00:00 · 2024-11-17 16:18:11 +05:30
parent a44319d815 5971ff884e
commit 626689cbe0
15 changed files with 194 additions and 30 deletions
--- a/application/api/answer/routes.py
+++ b/application/api/answer/routes.py
@@ -241,6 +241,7 @@ def complete_stream(
        yield f"data: {data}\n\n"
    except Exception as e:
        print("\033[91merr", str(e), file=sys.stderr)
+        traceback.print_exc()
        data = json.dumps(
            {
                "type": "error",
--- a/application/api/user/routes.py
+++ b/application/api/user/routes.py
@@ -358,7 +358,7 @@ class UploadFile(Resource):
                for file in files:
                    filename = secure_filename(file.filename)
                    file.save(os.path.join(temp_dir, filename))
-
+                    print(f"Saved file: {filename}")
                zip_path = shutil.make_archive(
                    base_name=os.path.join(save_dir, job_name),
                    format="zip",
@@ -366,6 +366,26 @@ class UploadFile(Resource):
                )
                final_filename = os.path.basename(zip_path)
                shutil.rmtree(temp_dir)
+                task = ingest.delay(
+                    settings.UPLOAD_FOLDER,
+                    [
+                        ".rst",
+                        ".md",
+                        ".pdf",
+                        ".txt",
+                        ".docx",
+                        ".csv",
+                        ".epub",
+                        ".html",
+                        ".mdx",
+                        ".json",
+                        ".xlsx",
+                        ".pptx",
+                    ],
+                    job_name,
+                    final_filename,
+                    user,
+                )
            else:
                file = files[0]
                final_filename = secure_filename(file.filename)
@@ -392,9 +412,10 @@ class UploadFile(Resource):
                    final_filename,
                    user,
                )
-        except Exception as err:
-            return make_response(jsonify({"success": False, "error": str(err)}), 400)

+        except Exception as err:
+            print(f"Error: {err}")
+            return make_response(jsonify({"success": False, "error": str(err)}), 400)
        return make_response(jsonify({"success": True, "task_id": task.id}), 200)


@@ -465,6 +486,11 @@ class TaskStatus(Resource):

            task = celery.AsyncResult(task_id)
            task_meta = task.info
+            print(f"Task status: {task.status}")
+            if not isinstance(
+                task_meta, (dict, list, str, int, float, bool, type(None))
+            ):
+                task_meta = str(task_meta)  # Convert to a string representation
        except Exception as err:
            return make_response(jsonify({"success": False, "error": str(err)}), 400)

--- a/application/llm/google_ai.py
+++ b/application/llm/google_ai.py
@@ -0,0 +1,48 @@
+from application.llm.base import BaseLLM
+
+class GoogleLLM(BaseLLM):
+
+    def __init__(self, api_key=None, user_api_key=None, *args, **kwargs):
+
+        super().__init__(*args, **kwargs)
+        self.api_key = api_key
+        self.user_api_key = user_api_key
+
+    def _clean_messages_google(self, messages):
+        return [
+            {
+                "role": "model" if message["role"] == "system" else message["role"],
+                "parts": [message["content"]],
+            }
+            for message in messages[1:]
+        ]
+
+    def _raw_gen(
+        self,
+        baseself,
+        model,
+        messages,
+        stream=False,
+        **kwargs
+    ):  
+        import google.generativeai as genai
+        genai.configure(api_key=self.api_key)
+        model = genai.GenerativeModel(model, system_instruction=messages[0]["content"])
+        response = model.generate_content(self._clean_messages_google(messages))
+        return response.text
+
+    def _raw_gen_stream(
+        self,
+        baseself,
+        model,
+        messages,
+        stream=True,
+        **kwargs
+    ):  
+        import google.generativeai as genai
+        genai.configure(api_key=self.api_key)
+        model = genai.GenerativeModel(model, system_instruction=messages[0]["content"])
+        response = model.generate_content(self._clean_messages_google(messages), stream=True)
+        for line in response:
+            if line.text is not None:
+                yield line.text
--- a/application/llm/llm_creator.py
+++ b/application/llm/llm_creator.py
@@ -6,6 +6,7 @@ from application.llm.llama_cpp import LlamaCpp
 from application.llm.anthropic import AnthropicLLM
 from application.llm.docsgpt_provider import DocsGPTAPILLM
 from application.llm.premai import PremAILLM
+from application.llm.google_ai import GoogleLLM


 class LLMCreator:
@@ -18,7 +19,8 @@ class LLMCreator:
        "anthropic": AnthropicLLM,
        "docsgpt": DocsGPTAPILLM,
        "premai": PremAILLM,
-        "groq": GroqLLM
+        "groq": GroqLLM,
+        "google": GoogleLLM
    }

    @classmethod
--- a/application/parser/remote/reddit_loader.py
+++ b/application/parser/remote/reddit_loader.py
@@ -1,10 +1,19 @@
 from application.parser.remote.base import BaseRemote
 from langchain_community.document_loaders import RedditPostsLoader
+import json


 class RedditPostsLoaderRemote(BaseRemote):
    def load_data(self, inputs):
-        data = eval(inputs)
+        try:
+            data = json.loads(inputs)
+        except json.JSONDecodeError as e:
+            raise ValueError(f"Invalid JSON input: {e}")
+
+        required_fields = ["client_id", "client_secret", "user_agent", "search_queries"]
+        missing_fields = [field for field in required_fields if field not in data]
+        if missing_fields:
+            raise ValueError(f"Missing required fields: {', '.join(missing_fields)}")
        client_id = data.get("client_id")
        client_secret = data.get("client_secret")
        user_agent = data.get("user_agent")
--- a/application/retriever/classic_rag.py
+++ b/application/retriever/classic_rag.py
@@ -45,7 +45,6 @@ class ClassicRAG(BaseRetriever):
                settings.VECTOR_STORE, self.vectorstore, settings.EMBEDDINGS_KEY
            )
            docs_temp = docsearch.search(self.question, k=self.chunks)
-            print(docs_temp)
            docs = [
                {
                    "title": i.metadata.get(
@@ -60,8 +59,6 @@ class ClassicRAG(BaseRetriever):
                }
                for i in docs_temp
            ]
-        if settings.LLM_NAME == "llama.cpp":
-            docs = [docs[0]]

        return docs

--- a/frontend/src/Navigation.tsx
+++ b/frontend/src/Navigation.tsx
@@ -18,6 +18,7 @@ import SourceDropdown from './components/SourceDropdown';
 import {
  setConversation,
  updateConversationId,
+  handleAbort,
 } from './conversation/conversationSlice';
 import ConversationTile from './conversation/ConversationTile';
 import { useDarkTheme, useMediaQuery, useOutsideAlerter } from './hooks';
@@ -180,6 +181,7 @@ export default function Navigation({ navOpen, setNavOpen }: NavigationProps) {
  };

  const resetConversation = () => {
+    handleAbort();
    dispatch(setConversation([]));
    dispatch(
      updateConversationId({
--- a/frontend/src/components/Dropdown.tsx
+++ b/frontend/src/components/Dropdown.tsx
@@ -140,7 +140,7 @@ function Dropdown({
                            : option.description
                        }`}
              </span>
-              {showEdit && onEdit && (
+              {showEdit && onEdit && option.type !== 'public' && (
                <img
                  src={Edit}
                  alt="Edit"
--- a/frontend/src/conversation/Conversation.tsx
+++ b/frontend/src/conversation/Conversation.tsx
@@ -1,30 +1,31 @@
 import { Fragment, useEffect, useRef, useState } from 'react';
 import { useTranslation } from 'react-i18next';
 import { useDispatch, useSelector } from 'react-redux';
-import newChatIcon from '../assets/openNewChat.svg';
+import { useNavigate } from 'react-router-dom';
+import Hero from '../Hero';
 import ArrowDown from '../assets/arrow-down.svg';
+import newChatIcon from '../assets/openNewChat.svg';
 import Send from '../assets/send.svg';
 import SendDark from '../assets/send_dark.svg';
+import ShareIcon from '../assets/share.svg';
 import SpinnerDark from '../assets/spinner-dark.svg';
 import Spinner from '../assets/spinner.svg';
 import RetryIcon from '../components/RetryIcon';
-import { useNavigate } from 'react-router-dom';
-import Hero from '../Hero';
 import { useDarkTheme, useMediaQuery } from '../hooks';
 import { ShareConversationModal } from '../modals/ShareConversationModal';
-import { setConversation, updateConversationId } from './conversationSlice';
 import { selectConversationId } from '../preferences/preferenceSlice';
 import { AppDispatch } from '../store';
 import conversationService from '../api/services/conversationService';
 import ConversationBubble from './ConversationBubble';
 import { handleSendFeedback } from './conversationHandlers';
 import { FEEDBACK, Query } from './conversationModels';
-import ShareIcon from '../assets/share.svg';
 import {
  addQuery,
  fetchAnswer,
  selectQueries,
  selectStatus,
+  setConversation,
+  updateConversationId,
  updateQuery,
 } from './conversationSlice';

@@ -329,14 +330,14 @@ export default function Conversation() {
        )}
      </div>

-      <div className="flex w-11/12 flex-col items-end self-center rounded-2xl bg-opacity-0 pb-1 sm:w-[62%] h-auto">
+      <div className="flex w-11/12 flex-col items-end self-center rounded-2xl bg-opacity-0 z-3 sm:w-[62%] h-auto">
        <div className="flex w-full items-center rounded-[40px] border border-silver bg-white py-1 dark:bg-raisin-black">
          <textarea
            id="inputbox"
            ref={inputRef}
            tabIndex={1}
            placeholder={t('inputPlaceholder')}
-            className={`inputbox-style h-16 w-full overflow-y-auto overflow-x-hidden whitespace-pre-wrap rounded-full bg-white pt-5 pb-[22px] text-base leading-tight opacity-100 focus:outline-none dark:bg-raisin-black dark:text-bright-gray`}
+            className={`inputbox-style w-full overflow-y-auto overflow-x-hidden whitespace-pre-wrap rounded-full bg-transparent py-5 text-base leading-tight opacity-100 focus:outline-none dark:bg-transparent dark:text-bright-gray`}
            onInput={handleInput}
            onKeyDown={(e) => {
              if (e.key === 'Enter' && !e.shiftKey) {
--- a/frontend/src/conversation/ConversationBubble.tsx
+++ b/frontend/src/conversation/ConversationBubble.tsx
@@ -1,12 +1,14 @@
+import 'katex/dist/katex.min.css';
+
 import { forwardRef, useState } from 'react';
 import ReactMarkdown from 'react-markdown';
 import { useSelector } from 'react-redux';
 import { Prism as SyntaxHighlighter } from 'react-syntax-highlighter';
 import { vscDarkPlus } from 'react-syntax-highlighter/dist/cjs/styles/prism';
+import rehypeKatex from 'rehype-katex';
 import remarkGfm from 'remark-gfm';
 import remarkMath from 'remark-math';
-import rehypeKatex from 'rehype-katex';
-import 'katex/dist/katex.min.css';
+
 import DocsGPT3 from '../assets/cute_docsgpt3.svg';
 import Dislike from '../assets/dislike.svg?react';
 import Document from '../assets/document.svg';
@@ -17,13 +19,13 @@ import Edit from '../assets/edit.svg';
 import Avatar from '../components/Avatar';
 import CopyButton from '../components/CopyButton';
 import Sidebar from '../components/Sidebar';
+import SpeakButton from '../components/TextToSpeechButton';
 import {
  selectChunks,
  selectSelectedDocs,
 } from '../preferences/preferenceSlice';
 import classes from './ConversationBubble.module.css';
 import { FEEDBACK, MESSAGE_TYPE } from './conversationModels';
-import SpeakButton from '../components/TextToSpeechButton';

 const DisableSourceFE = import.meta.env.VITE_DISABLE_SOURCE_FE || false;

@@ -58,6 +60,7 @@ const ConversationBubble = forwardRef<
  },
  ref,
 ) {
+  // const bubbleRef = useRef<HTMLDivElement | null>(null);
  const chunks = useSelector(selectChunks);
  const selectedDocs = useSelector(selectSelectedDocs);
  const [isLikeHovered, setIsLikeHovered] = useState(false);
@@ -209,7 +212,7 @@ const ConversationBubble = forwardRef<
                />
                <p className="text-base font-semibold">Sources</p>
              </div>
-              <div className="ml-3 mr-5 max-w-[90vw] md:max-w-[70vw] lg:max-w-[50vw]">
+              <div className="fade-in ml-3 mr-5 max-w-[90vw] md:max-w-[70vw] lg:max-w-[50vw]">
                <div className="grid grid-cols-2 gap-2 lg:grid-cols-4">
                  {sources?.slice(0, 3)?.map((source, index) => (
                    <div key={index} className="relative">
@@ -258,7 +261,7 @@ const ConversationBubble = forwardRef<
                      </div>
                      {activeTooltip === index && (
                        <div
-                          className={`absolute left-1/2 z-30 max-h-48 w-40 translate-x-[-50%] translate-y-[3px] rounded-xl bg-[#FBFBFB] p-4 text-black shadow-xl dark:bg-chinese-black dark:text-chinese-silver sm:w-56`}
+                          className={`absolute left-1/2 z-50 max-h-48 w-40 translate-x-[-50%] translate-y-[3px] rounded-xl bg-[#FBFBFB] p-4 text-black shadow-xl dark:bg-chinese-black dark:text-chinese-silver sm:w-56`}
                          onMouseOver={() => setActiveTooltip(index)}
                          onMouseOut={() => setActiveTooltip(null)}
                        >
@@ -299,14 +302,14 @@ const ConversationBubble = forwardRef<
            <p className="text-base font-semibold">Answer</p>
          </div>
          <div
-            className={`ml-2 mr-5 flex max-w-[90vw] rounded-[28px] bg-gray-1000 py-[14px] px-7 dark:bg-gun-metal md:max-w-[70vw] lg:max-w-[50vw] ${
+            className={`fade-in-bubble ml-2 mr-5 flex max-w-[90vw] rounded-[28px] bg-gray-1000 py-[14px] px-7 dark:bg-gun-metal md:max-w-[70vw] lg:max-w-[50vw] ${
              type === 'ERROR'
                ? 'relative flex-row items-center rounded-full border border-transparent bg-[#FFE7E7] p-2 py-5 text-sm font-normal text-red-3000  dark:border-red-2000 dark:text-white'
                : 'flex-col rounded-3xl'
            }`}
          >
            <ReactMarkdown
-              className="whitespace-pre-wrap break-normal leading-normal"
+              className="fade-in whitespace-pre-wrap break-normal leading-normal"
              remarkPlugins={[remarkGfm, remarkMath]}
              rehypePlugins={[rehypeKatex]}
              components={{
--- a/frontend/src/conversation/conversationSlice.ts
+++ b/frontend/src/conversation/conversationSlice.ts
@@ -17,9 +17,23 @@ const initialState: ConversationState = {

 const API_STREAMING = import.meta.env.VITE_API_STREAMING === 'true';

+let abortController: AbortController | null = null;
+export function handleAbort() {
+  if (abortController) {
+    abortController.abort();
+    abortController = null;
+  }
+}
+
 export const fetchAnswer = createAsyncThunk<Answer, { question: string }>(
  'fetchAnswer',
-  async ({ question }, { dispatch, getState, signal }) => {
+  async ({ question }, { dispatch, getState }) => {
+    if (abortController) {
+      abortController.abort();
+    }
+    abortController = new AbortController();
+    const { signal } = abortController;
+
    let isSourceUpdated = false;
    const state = getState() as RootState;
    if (state.preference) {
--- a/frontend/src/index.css
+++ b/frontend/src/index.css
@@ -514,3 +514,29 @@ input:-webkit-autofill:focus {
 .logs-table {
  font-family: 'IBMPlexMono-Medium', system-ui;
 }
+
+.fade-in {
+  animation: fadeIn 0.5s ease-in-out;
+}
+
+@keyframes fadeIn {
+  0% {
+    opacity: 0;
+  }
+  100% {
+    opacity: 1;
+  }
+}
+
+.fade-in-bubble {
+  opacity: 0;
+  transform: translateY(10px);
+  animation: fadeInUp 0.5s forwards;
+}
+
+@keyframes fadeInUp {
+  to {
+    opacity: 1;
+    transform: translateY(0);
+  }
+}
--- a/frontend/src/preferences/PromptsModal.tsx
+++ b/frontend/src/preferences/PromptsModal.tsx
@@ -1,6 +1,7 @@
 import { ActiveState } from '../models/misc';
 import Exit from '../assets/exit.svg';
 import Input from '../components/Input';
+import React from 'react';

 function AddPrompt({
  setModalState,
@@ -9,6 +10,7 @@ function AddPrompt({
  setNewPromptName,
  newPromptContent,
  setNewPromptContent,
+  disableSave,
 }: {
  setModalState: (state: ActiveState) => void;
  handleAddPrompt?: () => void;
@@ -16,6 +18,7 @@ function AddPrompt({
  setNewPromptName: (name: string) => void;
  newPromptContent: string;
  setNewPromptContent: (content: string) => void;
+  disableSave: boolean;
 }) {
  return (
    <div className="relative">
@@ -23,6 +26,8 @@ function AddPrompt({
        className="absolute top-3 right-4 m-2 w-3"
        onClick={() => {
          setModalState('INACTIVE');
+          setNewPromptName('');
+          setNewPromptContent('');
        }}
      >
        <img className="filter dark:invert" src={Exit} />
@@ -41,7 +46,7 @@ function AddPrompt({
            className="h-10 rounded-lg"
            value={newPromptName}
            onChange={(e) => setNewPromptName(e.target.value)}
-          ></Input>
+          />
          <div className="relative bottom-12 left-3 mt-[-3.00px]">
            <span className="bg-white px-1 text-xs text-silver dark:bg-outer-space dark:text-silver">
              Prompt Name
@@ -62,6 +67,8 @@ function AddPrompt({
          <button
            onClick={handleAddPrompt}
            className="rounded-3xl bg-purple-30 px-5 py-2 text-sm text-white transition-all hover:opacity-90"
+            disabled={disableSave}
+            title={disableSave && newPromptName ? 'Name already exists' : ''}
          >
            Save
          </button>
@@ -79,6 +86,7 @@ function EditPrompt({
  editPromptContent,
  setEditPromptContent,
  currentPromptEdit,
+  disableSave,
 }: {
  setModalState: (state: ActiveState) => void;
  handleEditPrompt?: (id: string, type: string) => void;
@@ -87,6 +95,7 @@ function EditPrompt({
  editPromptContent: string;
  setEditPromptContent: (content: string) => void;
  currentPromptEdit: { name: string; id: string; type: string };
+  disableSave: boolean;
 }) {
  return (
    <div className="relative">
@@ -140,7 +149,8 @@ function EditPrompt({
              handleEditPrompt &&
                handleEditPrompt(currentPromptEdit.id, currentPromptEdit.type);
            }}
-            disabled={currentPromptEdit.type === 'public'}
+            disabled={currentPromptEdit.type === 'public' || disableSave}
+            title={disableSave && editPromptName ? 'Name already exists' : ''}
          >
            Save
          </button>
@@ -151,6 +161,7 @@ function EditPrompt({
 }

 export default function PromptsModal({
+  existingPrompts,
  modalState,
  setModalState,
  type,
@@ -166,6 +177,7 @@ export default function PromptsModal({
  handleAddPrompt,
  handleEditPrompt,
 }: {
+  existingPrompts: { name: string; id: string; type: string }[];
  modalState: ActiveState;
  setModalState: (state: ActiveState) => void;
  type: 'ADD' | 'EDIT';
@@ -181,6 +193,25 @@ export default function PromptsModal({
  handleAddPrompt?: () => void;
  handleEditPrompt?: (id: string, type: string) => void;
 }) {
+  const [disableSave, setDisableSave] = React.useState(true);
+  const handlePrompNameChange = (edit: boolean, newName: string) => {
+    const nameExists = existingPrompts.find(
+      (prompt) => newName === prompt.name,
+    );
+
+    if (newName && !nameExists) {
+      setDisableSave(false);
+    } else {
+      setDisableSave(true);
+    }
+
+    if (edit) {
+      setEditPromptName(newName);
+    } else {
+      setNewPromptName(newName);
+    }
+  };
+
  let view;

  if (type === 'ADD') {
@@ -189,9 +220,10 @@ export default function PromptsModal({
        setModalState={setModalState}
        handleAddPrompt={handleAddPrompt}
        newPromptName={newPromptName}
-        setNewPromptName={setNewPromptName}
+        setNewPromptName={handlePrompNameChange.bind(null, false)}
        newPromptContent={newPromptContent}
        setNewPromptContent={setNewPromptContent}
+        disableSave={disableSave}
      />
    );
  } else if (type === 'EDIT') {
@@ -200,10 +232,11 @@ export default function PromptsModal({
        setModalState={setModalState}
        handleEditPrompt={handleEditPrompt}
        editPromptName={editPromptName}
-        setEditPromptName={setEditPromptName}
+        setEditPromptName={handlePrompNameChange.bind(null, true)}
        editPromptContent={editPromptContent}
        setEditPromptContent={setEditPromptContent}
        currentPromptEdit={currentPromptEdit}
+        disableSave={disableSave}
      />
    );
  } else {
--- a/frontend/src/settings/Prompts.tsx
+++ b/frontend/src/settings/Prompts.tsx
@@ -58,7 +58,8 @@ export default function Prompts({
      }
      setModalState('INACTIVE');
      onSelectPrompt(newPromptName, newPrompt.id, newPromptContent);
-      setNewPromptName(newPromptName);
+      setNewPromptName('');
+      setNewPromptContent('');
    } catch (error) {
      console.error(error);
    }
@@ -178,6 +179,7 @@ export default function Prompts({
        </div>
      </div>
      <PromptsModal
+        existingPrompts={prompts}
        type={modalType}
        modalState={modalState}
        setModalState={setModalState}
--- a/frontend/src/upload/Upload.tsx
+++ b/frontend/src/upload/Upload.tsx
@@ -76,7 +76,7 @@ function Upload({
        <div className="relative w-32 h-32 rounded-full">
          <div className="absolute inset-0 rounded-full shadow-[0_0_10px_2px_rgba(0,0,0,0.3)_inset] dark:shadow-[0_0_10px_2px_rgba(0,0,0,0.3)_inset]"></div>
          <div
-            className={`absolute inset-0 rounded-full ${progressPercent === 100 ? 'shadow-xl shadow-lime-300/50 dark:shadow-lime-300/50 bg-gradient-to-r from-white to-gray-400 dark:bg-gradient-to-br dark:from-gray-500 dark:to-gray-300' : 'shadow-[0_2px_0_#FF3D00_inset] dark:shadow-[0_2px_0_#FF3D00_inset]'}`}
+            className={`absolute inset-0 rounded-full ${progressPercent === 100 ? 'shadow-xl shadow-lime-300/50 dark:shadow-lime-300/50 bg-gradient-to-r from-white to-gray-400 dark:bg-gradient-to-br dark:from-gray-500 dark:to-gray-300' : 'shadow-[0_4px_0_#7D54D1] dark:shadow-[0_4px_0_#7D54D1]'}`}
            style={{
              animation: `${progressPercent === 100 ? 'none' : 'rotate 2s linear infinite'}`,
            }}