Merge branch 'main' into feature/api-key-create

frontend/src/Setting.tsx

@@ -9,6 +9,8 @@ import {
   setPrompt,
   selectSourceDocs,
   setSourceDocs,
+  setChunks,
+  selectChunks,
 } from './preferences/preferenceSlice';
 import { Doc } from './preferences/preferenceApi';
 import { useDarkTheme } from './hooks';
@@ -190,10 +192,13 @@ const Setting: React.FC = () => {
 const General: React.FC = () => {
   const themes = ['Light', 'Dark'];
   const languages = ['English'];
+  const chunks = ['0', '2', '4', '6', '8', '10'];
+  const selectedChunks = useSelector(selectChunks);
   const [isDarkTheme, toggleTheme] = useDarkTheme();
   const [selectedTheme, setSelectedTheme] = useState(
     isDarkTheme ? 'Dark' : 'Light',
   );
+  const dispatch = useDispatch();
   const [selectedLanguage, setSelectedLanguage] = useState(languages[0]);
   return (
     <div className="mt-[59px]">
@@ -208,7 +213,7 @@ const General: React.FC = () => {
           }}
         />
       </div>
-      <div>
+      <div className="mb-4">
         <p className="font-bold text-jet dark:text-bright-gray">
           Select Language
         </p>
@@ -218,6 +223,16 @@ const General: React.FC = () => {
           onSelect={setSelectedLanguage}
         />
       </div>
+      <div>
+        <p className="font-bold text-jet dark:text-bright-gray">
+          Chunks processed per query
+        </p>
+        <Dropdown
+          options={chunks}
+          selectedValue={selectedChunks}
+          onSelect={(value: string) => dispatch(setChunks(value))}
+        />
+      </div>
     </div>
   );
 };

frontend/src/components/Dropdown.tsx

@@ -39,10 +39,10 @@ function Dropdown({
           isOpen
             ? typeof selectedValue === 'string'
               ? 'rounded-t-xl'
-              : 'rounded-t-2xl'
+              : 'rounded-t-3xl'
             : typeof selectedValue === 'string'
             ? 'rounded-xl'
-            : 'rounded-full'
+            : 'rounded-3xl'
         }`}
       >
         {typeof selectedValue === 'string' ? (

frontend/src/conversation/ConversationBubble.tsx

@@ -160,7 +160,10 @@ const ConversationBubble = forwardRef<
             >
               {message}
             </ReactMarkdown>
-            {DisableSourceFE || type === 'ERROR' ? null : (
+            {DisableSourceFE ||
+            type === 'ERROR' ||
+            !sources ||
+            sources.length === 0 ? null : (
               <>
                 <span className="mt-3 h-px w-full bg-[#DEDEDE]"></span>
                 <div className="mt-3 flex w-full flex-row flex-wrap items-center justify-start gap-2">

frontend/src/conversation/conversationApi.ts

@@ -10,6 +10,7 @@ export function fetchAnswerApi(
   history: Array<any> = [],
   conversationId: string | null,
   promptId: string | null,
+  chunks: string,
 ): Promise<
   | {
       result: any;
@@ -62,6 +63,7 @@ export function fetchAnswerApi(
       active_docs: docPath,
       conversation_id: conversationId,
       prompt_id: promptId,
+      chunks: chunks,
     }),
     signal,
   })
@@ -91,6 +93,7 @@ export function fetchAnswerSteaming(
   history: Array<any> = [],
   conversationId: string | null,
   promptId: string | null,
+  chunks: string,
   onEvent: (event: MessageEvent) => void,
 ): Promise<Answer> {
   let namePath = selectedDocs.name;
@@ -124,6 +127,7 @@ export function fetchAnswerSteaming(
       history: JSON.stringify(history),
       conversation_id: conversationId,
       prompt_id: promptId,
+      chunks: chunks,
     };
     fetch(apiHost + '/stream', {
       method: 'POST',
@@ -185,6 +189,7 @@ export function searchEndpoint(
   selectedDocs: Doc,
   conversation_id: string | null,
   history: Array<any> = [],
+  chunks: string,
 ) {
   /*
   "active_docs": "default",
@@ -216,6 +221,7 @@ export function searchEndpoint(
     active_docs: docPath,
     conversation_id,
     history,
+    chunks: chunks,
   };
   return fetch(`${apiHost}/api/search`, {
     method: 'POST',

frontend/src/conversation/conversationSlice.ts

@@ -27,6 +27,7 @@ export const fetchAnswer = createAsyncThunk<Answer, { question: string }>(
           state.conversation.queries,
           state.conversation.conversationId,
           state.preference.prompt.id,
+          state.preference.chunks,
 
           (event) => {
             const data = JSON.parse(event.data);
@@ -49,6 +50,7 @@ export const fetchAnswer = createAsyncThunk<Answer, { question: string }>(
                 state.preference.selectedDocs!,
                 state.conversation.conversationId,
                 state.conversation.queries,
+                state.preference.chunks,
               ).then((sources) => {
                 //dispatch streaming sources
                 dispatch(
@@ -83,6 +85,7 @@ export const fetchAnswer = createAsyncThunk<Answer, { question: string }>(
           state.conversation.queries,
           state.conversation.conversationId,
           state.preference.prompt.id,
+          state.preference.chunks,
         );
         if (answer) {
           let sourcesPrepped = [];

frontend/src/preferences/preferenceSlice.ts

@@ -10,6 +10,7 @@ interface Preference {
   apiKey: string;
   prompt: { name: string; id: string; type: string };
   selectedDocs: Doc | null;
+  chunks: string;
   sourceDocs: Doc[] | null;
   conversations: { name: string; id: string }[] | null;
 }
@@ -17,6 +18,7 @@ interface Preference {
 const initialState: Preference = {
   apiKey: 'xxx',
   prompt: { name: 'default', id: 'default', type: 'public' },
+  chunks: '2',
   selectedDocs: {
     name: 'default',
     language: 'default',
@@ -51,6 +53,9 @@ export const prefSlice = createSlice({
     setPrompt: (state, action) => {
       state.prompt = action.payload;
     },
+    setChunks: (state, action) => {
+      state.chunks = action.payload;
+    },
   },
 });
 
@@ -60,6 +65,7 @@ export const {
   setSourceDocs,
   setConversations,
   setPrompt,
+  setChunks,
 } = prefSlice.actions;
 export default prefSlice.reducer;
 
@@ -91,6 +97,16 @@ prefListenerMiddleware.startListening({
   },
 });
 
+prefListenerMiddleware.startListening({
+  matcher: isAnyOf(setChunks),
+  effect: (action, listenerApi) => {
+    localStorage.setItem(
+      'DocsGPTChunks',
+      JSON.stringify((listenerApi.getState() as RootState).preference.chunks),
+    );
+  },
+});
+
 export const selectApiKey = (state: RootState) => state.preference.apiKey;
 export const selectApiKeyStatus = (state: RootState) =>
   !!state.preference.apiKey;
@@ -105,3 +121,4 @@ export const selectConversations = (state: RootState) =>
 export const selectConversationId = (state: RootState) =>
   state.conversation.conversationId;
 export const selectPrompt = (state: RootState) => state.preference.prompt;
+export const selectChunks = (state: RootState) => state.preference.chunks;

frontend/src/store.ts

@@ -8,11 +8,13 @@ import {
 const key = localStorage.getItem('DocsGPTApiKey');
 const prompt = localStorage.getItem('DocsGPTPrompt');
 const doc = localStorage.getItem('DocsGPTRecentDocs');
+const chunks = localStorage.getItem('DocsGPTChunks');
 
 const store = configureStore({
   preloadedState: {
     preference: {
       apiKey: key ?? '',
+      chunks: JSON.parse(chunks ?? '2').toString(),
       selectedDocs: doc !== null ? JSON.parse(doc) : null,
       prompt:
         prompt !== null

frontend/src/upload/Upload.tsx

@@ -17,10 +17,18 @@ export default function Upload({
   const [docName, setDocName] = useState('');
   const [urlName, setUrlName] = useState('');
   const [url, setUrl] = useState('');
+  const [redditData, setRedditData] = useState({
+    client_id: '',
+    client_secret: '',
+    user_agent: '',
+    search_queries: [''],
+    number_posts: 10,
+  });
   const urlOptions: { label: string; value: string }[] = [
     { label: 'Crawler', value: 'crawler' },
     // { label: 'Sitemap', value: 'sitemap' },
     { label: 'Link', value: 'url' },
+    { label: 'Reddit', value: 'reddit' },
   ];
   const [urlType, setUrlType] = useState<{ label: string; value: string }>({
     label: 'Link',
@@ -163,7 +171,6 @@ export default function Upload({
   };
 
   const uploadRemote = () => {
-    console.log('here');
     const formData = new FormData();
     formData.append('name', urlName);
     formData.append('user', 'local');
@@ -171,6 +178,13 @@ export default function Upload({
       formData.append('source', urlType?.value);
     }
     formData.append('data', url);
+    if (
+      redditData.client_id.length > 0 &&
+      redditData.client_secret.length > 0
+    ) {
+      formData.set('name', 'other');
+      formData.set('data', JSON.stringify(redditData));
+    }
     const apiHost = import.meta.env.VITE_API_HOST;
     const xhr = new XMLHttpRequest();
     xhr.upload.addEventListener('progress', (event) => {
@@ -202,6 +216,19 @@ export default function Upload({
         ['.docx'],
     },
   });
+  const handleChange = (e: React.ChangeEvent<HTMLInputElement>) => {
+    const { name, value } = e.target;
+    if (name === 'search_queries' && value.length > 0) {
+      setRedditData({
+        ...redditData,
+        [name]: value.split(',').map((item) => item.trim()),
+      });
+    } else
+      setRedditData({
+        ...redditData,
+        [name]: value,
+      });
+  };
   let view;
   if (progress?.type === 'UPLOAD') {
     view = <UploadProgress></UploadProgress>;
@@ -281,30 +308,102 @@ export default function Upload({
                 setUrlType(value)
               }
             />
-            <input
-              placeholder="Enter name"
-              type="text"
-              className="h-10 w-full rounded-full border-2 border-silver px-3 outline-none dark:bg-transparent dark:text-silver"
-              value={urlName}
-              onChange={(e) => setUrlName(e.target.value)}
-            ></input>
-            <div className="relative bottom-12 left-2 mt-[-18.39px]">
-              <span className="bg-white px-2 text-xs text-silver dark:bg-outer-space dark:text-silver">
-                Name
-              </span>
-            </div>
-            <input
-              placeholder="URL Link"
-              type="text"
-              className="h-10 w-full rounded-full border-2 border-silver px-3 outline-none dark:bg-transparent dark:text-silver"
-              value={url}
-              onChange={(e) => setUrl(e.target.value)}
-            ></input>
-            <div className="relative bottom-12 left-2 mt-[-18.39px]">
-              <span className="bg-white px-2 text-xs text-silver dark:bg-outer-space dark:text-silver">
-                Link
-              </span>
-            </div>
+            {urlType.label !== 'Reddit' ? (
+              <>
+                <input
+                  placeholder="Enter name"
+                  type="text"
+                  className="h-10 w-full rounded-full border-2 border-silver px-3 outline-none dark:bg-transparent dark:text-silver"
+                  value={urlName}
+                  onChange={(e) => setUrlName(e.target.value)}
+                ></input>
+                <div className="relative bottom-12 left-2 mt-[-18.39px]">
+                  <span className="bg-white px-2 text-xs text-silver dark:bg-outer-space dark:text-silver">
+                    Name
+                  </span>
+                </div>
+                <input
+                  placeholder="URL Link"
+                  type="text"
+                  className="h-10 w-full rounded-full border-2 border-silver px-3 outline-none dark:bg-transparent dark:text-silver"
+                  value={url}
+                  onChange={(e) => setUrl(e.target.value)}
+                ></input>
+                <div className="relative bottom-12 left-2 mt-[-18.39px]">
+                  <span className="bg-white px-2 text-xs text-silver dark:bg-outer-space dark:text-silver">
+                    Link
+                  </span>
+                </div>
+              </>
+            ) : (
+              <>
+                <input
+                  placeholder="Enter client ID"
+                  type="text"
+                  className="h-10 w-full rounded-full border-2 border-silver px-3 outline-none dark:bg-transparent dark:text-silver"
+                  name="client_id"
+                  value={redditData.client_id}
+                  onChange={handleChange}
+                ></input>
+                <div className="relative bottom-12 left-2 mt-[-18.39px]">
+                  <span className="bg-white px-2 text-xs text-silver dark:bg-outer-space dark:text-silver">
+                    Client ID
+                  </span>
+                </div>
+                <input
+                  placeholder="Enter client secret"
+                  type="text"
+                  className="h-10 w-full rounded-full border-2 border-silver px-3 outline-none dark:bg-transparent dark:text-silver"
+                  name="client_secret"
+                  value={redditData.client_secret}
+                  onChange={handleChange}
+                ></input>
+                <div className="relative bottom-12 left-2 mt-[-18.39px]">
+                  <span className="bg-white px-2 text-xs text-silver dark:bg-outer-space dark:text-silver">
+                    Client secret
+                  </span>
+                </div>
+                <input
+                  placeholder="Enter user agent"
+                  type="text"
+                  className="h-10 w-full rounded-full border-2 border-silver px-3 outline-none dark:bg-transparent dark:text-silver"
+                  name="user_agent"
+                  value={redditData.user_agent}
+                  onChange={handleChange}
+                ></input>
+                <div className="relative bottom-12 left-2 mt-[-18.39px]">
+                  <span className="bg-white px-2 text-xs text-silver dark:bg-outer-space dark:text-silver">
+                    User agent
+                  </span>
+                </div>
+                <input
+                  placeholder="Enter search queries"
+                  type="text"
+                  className="h-10 w-full rounded-full border-2 border-silver px-3 outline-none dark:bg-transparent dark:text-silver"
+                  name="search_queries"
+                  value={redditData.search_queries}
+                  onChange={handleChange}
+                ></input>
+                <div className="relative bottom-12 left-2 mt-[-18.39px]">
+                  <span className="bg-white px-2 text-xs text-silver dark:bg-outer-space dark:text-silver">
+                    Search queries
+                  </span>
+                </div>
+                <input
+                  placeholder="Enter number of posts"
+                  type="number"
+                  className="h-10 w-full rounded-full border-2 border-silver px-3 outline-none dark:bg-transparent dark:text-silver"
+                  name="number_posts"
+                  value={redditData.number_posts}
+                  onChange={handleChange}
+                ></input>
+                <div className="relative bottom-12 left-2 mt-[-18.39px]">
+                  <span className="bg-white px-2 text-xs text-silver dark:bg-outer-space dark:text-silver">
+                    Number of posts
+                  </span>
+                </div>
+              </>
+            )}
           </>
         )}
         <div className="flex flex-row-reverse">