Merge branch 'main' into feat/agent-schema-response

application/api/answer/routes/base.py

@@ -179,6 +179,12 @@ class BaseAnswerResource:
                 log_data["structured_output"] = True
                 if schema_info:
                     log_data["schema"] = schema_info
+  
+            # clean up text fields to be no longer than 10000 characters
+            for key, value in log_data.items():
+                if isinstance(value, str) and len(value) > 10000:
+                    log_data[key] = value[:10000]
+            
             self.user_logs_collection.insert_one(log_data)
 
             # End of stream

application/api/answer/services/conversation_service.py

@@ -66,6 +66,11 @@ class ConversationService:
         if not user_id:
             raise ValueError("User ID not found in token")
         current_time = datetime.now(timezone.utc)
+        
+        # clean up in sources array such that we save max 1k characters for text part
+        for source in sources:
+            if "text" in source and isinstance(source["text"], str):
+                source["text"] = source["text"][:1000]
 
         if conversation_id is not None and index is not None:
             # Update existing conversation with new query

application/api/user/routes.py

@@ -64,7 +64,7 @@ attachments_collection = db["attachments"]
 
 try:
     agents_collection.create_index(
-        [("shared_publicly", 1)],
+        [("shared", 1)],
         name="shared_index",
         background=True,
     )

application/logging.py

@@ -136,6 +136,8 @@ def _log_to_mongodb(
         mongo = MongoDB.get_client()
         db = mongo[settings.MONGO_DB_NAME]
         user_logs_collection = db["stack_logs"]
+        
+
 
         log_entry = {
             "endpoint": endpoint,
@@ -147,6 +149,11 @@ def _log_to_mongodb(
             "stacks": stacks,
             "timestamp": datetime.datetime.now(datetime.timezone.utc),
         }
+        # clean up text fields to be no longer than 10000 characters
+        for key, value in log_entry.items():
+            if isinstance(value, str) and len(value) > 10000:
+                log_entry[key] = value[:10000]
+    
         user_logs_collection.insert_one(log_entry)
         logging.debug(f"Logged activity to MongoDB: {activity_id}")
 

application/worker.py

@@ -471,9 +471,13 @@ def attachment_worker(self, file_info, user):
             .load_data()[0]
             .text,
         )
-
+        
+        
         token_count = num_tokens_from_string(content)
-
+        if token_count > 100000:
+            content = content[:250000]
+            token_count = num_tokens_from_string(content)
+        
         self.update_state(
             state="PROGRESS", meta={"current": 80, "status": "Storing in database"}
         )