Merge branch 'main' of https://github.com/arc53/DocsGPT

2025-11-29 08:33:20 +00:00 · 2025-08-12 01:43:57 +00:00
parent 65c33ad915 f94a093e8c
commit d72558eb36
5 changed files with 38 additions and 17 deletions
--- a/application/worker.py
+++ b/application/worker.py
@@ -746,9 +746,13 @@ def attachment_worker(self, file_info, user):
            .load_data()[0]
            .text,
        )
-
+        
+        
        token_count = num_tokens_from_string(content)
-
+        if token_count > 100000:
+            content = content[:250000]
+            token_count = num_tokens_from_string(content)
+        
        self.update_state(
            state="PROGRESS", meta={"current": 80, "status": "Storing in database"}
        )