Merge branch 'main' into googleai-compatability-tools

application/api/user/routes.py

@@ -185,8 +185,13 @@ class SubmitFeedback(Resource):
                 ),
                 "answer": fields.String(required=False, description="The AI answer"),
                 "feedback": fields.String(required=True, description="User feedback"),
-                "question_index":fields.Integer(required=True, description="The question number in that particular conversation"),
-                "conversation_id":fields.String(required=True, description="id of the particular conversation"),
+                "question_index": fields.Integer(
+                    required=True,
+                    description="The question number in that particular conversation",
+                ),
+                "conversation_id": fields.String(
+                    required=True, description="id of the particular conversation"
+                ),
                 "api_key": fields.String(description="Optional API key"),
             },
         )
@@ -196,21 +201,24 @@ class SubmitFeedback(Resource):
     )
     def post(self):
         data = request.get_json()
-        required_fields = [ "feedback","conversation_id","question_index"]
+        required_fields = ["feedback", "conversation_id", "question_index"]
         missing_fields = check_required_fields(data, required_fields)
         if missing_fields:
             return missing_fields
 
         try:
             conversations_collection.update_one(
-            {"_id": ObjectId(data["conversation_id"]), f"queries.{data['question_index']}": {"$exists": True}},
-            {
-                "$set": {
-                    f"queries.{data['question_index']}.feedback": data["feedback"]
-                }
-            }
-        )
-           
+                {
+                    "_id": ObjectId(data["conversation_id"]),
+                    f"queries.{data['question_index']}": {"$exists": True},
+                },
+                {
+                    "$set": {
+                        f"queries.{data['question_index']}.feedback": data["feedback"]
+                    }
+                },
+            )
+
         except Exception as err:
             return make_response(jsonify({"success": False, "error": str(err)}), 400)
 
@@ -253,11 +261,9 @@ class DeleteOldIndexes(Resource):
                 jsonify({"success": False, "message": "Missing required fields"}), 400
             )
 
-        doc = sources_collection.find_one(
-                {"_id": ObjectId(source_id), "user": "local"}
-        )
+        doc = sources_collection.find_one({"_id": ObjectId(source_id), "user": "local"})
         if not doc:
-                return make_response(jsonify({"status": "not found"}), 404)
+            return make_response(jsonify({"status": "not found"}), 404)
         try:
             if settings.VECTOR_STORE == "faiss":
                 shutil.rmtree(os.path.join(current_dir, "indexes", str(doc["_id"])))
@@ -271,7 +277,7 @@ class DeleteOldIndexes(Resource):
             pass
         except Exception as err:
             return make_response(jsonify({"success": False, "error": str(err)}), 400)
-        
+
         sources_collection.delete_one({"_id": ObjectId(source_id)})
         return make_response(jsonify({"success": True}), 200)
 
@@ -498,7 +504,9 @@ class PaginatedSources(Resource):
 
         total_documents = sources_collection.count_documents(query)
         total_pages = max(1, math.ceil(total_documents / rows_per_page))
-        page = min(max(1, page), total_pages) # add this to make sure page inbound is within the range
+        page = min(
+            max(1, page), total_pages
+        )  # add this to make sure page inbound is within the range
         sort_order = 1 if sort_order == "asc" else -1
         skip = (page - 1) * rows_per_page
 
@@ -543,7 +551,7 @@ class CombinedJson(Resource):
         user = "local"
         data = [
             {
-                "name": "default",
+                "name": "Default",
                 "date": "default",
                 "model": settings.EMBEDDINGS_NAME,
                 "location": "remote",
@@ -2097,5 +2105,4 @@ class DeleteTool(Resource):
         except Exception as err:
             return {"success": False, "error": str(err)}, 400
 
-        return {"success": True}, 200
-      
+        return {"success": True}, 200

application/parser/remote/crawler_loader.py

@@ -2,16 +2,16 @@ import requests
 from urllib.parse import urlparse, urljoin
 from bs4 import BeautifulSoup
 from application.parser.remote.base import BaseRemote
+from application.parser.schema.base import Document
+from langchain_community.document_loaders import WebBaseLoader
 
 class CrawlerLoader(BaseRemote):
     def __init__(self, limit=10):
-        from langchain_community.document_loaders import WebBaseLoader
         self.loader = WebBaseLoader  # Initialize the document loader
         self.limit = limit  # Set the limit for the number of pages to scrape
 
     def load_data(self, inputs):
         url = inputs
-        # Check if the input is a list and if it is, use the first element
         if isinstance(url, list) and url:
             url = url[0]
 
@@ -19,24 +19,29 @@ class CrawlerLoader(BaseRemote):
         if not urlparse(url).scheme:
             url = "http://" + url
 
-        visited_urls = set()  # Keep track of URLs that have been visited
-        base_url = urlparse(url).scheme + "://" + urlparse(url).hostname  # Extract the base URL
-        urls_to_visit = [url]  # List of URLs to be visited, starting with the initial URL
-        loaded_content = []  # Store the loaded content from each URL
+        visited_urls = set()
+        base_url = urlparse(url).scheme + "://" + urlparse(url).hostname
+        urls_to_visit = [url]
+        loaded_content = []
 
-        # Continue crawling until there are no more URLs to visit
         while urls_to_visit:
-            current_url = urls_to_visit.pop(0)  # Get the next URL to visit
-            visited_urls.add(current_url)  # Mark the URL as visited
+            current_url = urls_to_visit.pop(0)
+            visited_urls.add(current_url)
 
-            # Try to load and process the content from the current URL
             try:
-                response = requests.get(current_url)  # Fetch the content of the current URL
-                response.raise_for_status()  # Raise an exception for HTTP errors
-                loader = self.loader([current_url])  # Initialize the document loader for the current URL
-                loaded_content.extend(loader.load())  # Load the content and add it to the loaded_content list
+                response = requests.get(current_url)
+                response.raise_for_status()
+                loader = self.loader([current_url])
+                docs = loader.load()
+                # Convert the loaded documents to your Document schema
+                for doc in docs:
+                    loaded_content.append(
+                        Document(
+                            doc.page_content,
+                            extra_info=doc.metadata
+                        )
+                    )
             except Exception as e:
-                # Print an error message if loading or processing fails and continue with the next URL
                 print(f"Error processing URL {current_url}: {e}")
                 continue
 
@@ -45,15 +50,15 @@ class CrawlerLoader(BaseRemote):
             all_links = [
                 urljoin(current_url, a['href'])
                 for a in soup.find_all('a', href=True)
-                if base_url in urljoin(current_url, a['href'])  # Ensure links are from the same domain
+                if base_url in urljoin(current_url, a['href'])
             ]
 
             # Add new links to the list of URLs to visit if they haven't been visited yet
             urls_to_visit.extend([link for link in all_links if link not in visited_urls])
-            urls_to_visit = list(set(urls_to_visit))  # Remove duplicate URLs
+            urls_to_visit = list(set(urls_to_visit))
 
             # Stop crawling if the limit of pages to scrape is reached
             if self.limit is not None and len(visited_urls) >= self.limit:
                 break
 
-        return loaded_content  # Return the loaded content from all visited URLs
+        return loaded_content

application/parser/remote/crawler_markdown.py

@@ -0,0 +1,139 @@
+import requests
+from urllib.parse import urlparse, urljoin
+from bs4 import BeautifulSoup
+from application.parser.remote.base import BaseRemote
+import re
+from markdownify import markdownify
+from application.parser.schema.base import Document
+import tldextract
+
+class CrawlerLoader(BaseRemote):
+    def __init__(self, limit=10, allow_subdomains=False):
+        """
+        Given a URL crawl web pages up to `self.limit`,
+        convert HTML content to Markdown, and returning a list of Document objects.
+
+        :param limit: The maximum number of pages to crawl.
+        :param allow_subdomains: If True, crawl pages on subdomains of the base domain.
+        """
+        self.limit = limit
+        self.allow_subdomains = allow_subdomains
+        self.session = requests.Session()
+
+    def load_data(self, inputs):
+        url = inputs
+        if isinstance(url, list) and url:
+            url = url[0]
+
+        # Ensure the URL has a scheme (if not, default to http)
+        if not urlparse(url).scheme:
+            url = "http://" + url
+
+        # Keep track of visited URLs to avoid revisiting the same page
+        visited_urls = set()
+
+        # Determine the base domain for link filtering using tldextract
+        base_domain = self._get_base_domain(url)
+        urls_to_visit = {url}
+        documents = []
+
+        while urls_to_visit:
+            current_url = urls_to_visit.pop()
+
+            # Skip if already visited
+            if current_url in visited_urls:
+                continue
+            visited_urls.add(current_url)
+
+            # Fetch the page content
+            html_content = self._fetch_page(current_url)
+            if html_content is None:
+                continue
+
+            # Convert the HTML to Markdown for cleaner text formatting
+            title, language, processed_markdown = self._process_html_to_markdown(html_content, current_url)
+            if processed_markdown:
+                # Create a Document for each visited page
+                documents.append(
+                    Document(
+                        processed_markdown,  # content
+                        None,  # doc_id
+                        None,  # embedding
+                        {"source": current_url, "title": title, "language": language} # extra_info
+                    )
+                )
+
+            # Extract links and filter them according to domain rules
+            new_links = self._extract_links(html_content, current_url)
+            filtered_links = self._filter_links(new_links, base_domain)
+
+            # Add any new, not-yet-visited links to the queue
+            urls_to_visit.update(link for link in filtered_links if link not in visited_urls)
+
+            # If we've reached the limit, stop crawling
+            if self.limit is not None and len(visited_urls) >= self.limit:
+                break
+
+        return documents
+
+    def _fetch_page(self, url):
+        try:
+            response = self.session.get(url, timeout=10)
+            response.raise_for_status()
+            return response.text
+        except requests.exceptions.RequestException as e:
+            print(f"Error fetching URL {url}: {e}")
+            return None
+
+    def _process_html_to_markdown(self, html_content, current_url):
+        soup = BeautifulSoup(html_content, 'html.parser')
+        title_tag = soup.find('title')
+        title = title_tag.text.strip() if title_tag else "No Title"
+
+        # Extract language
+        language_tag = soup.find('html')
+        language = language_tag.get('lang', 'en') if language_tag else "en"
+
+        markdownified = markdownify(html_content, heading_style="ATX", newline_style="BACKSLASH")
+        # Reduce sequences of more than two newlines to exactly three
+        markdownified = re.sub(r'\n{3,}', '\n\n\n', markdownified)
+        return title, language, markdownified
+
+    def _extract_links(self, html_content, current_url):
+        soup = BeautifulSoup(html_content, 'html.parser')
+        links = []
+        for a in soup.find_all('a', href=True):
+            full_url = urljoin(current_url, a['href'])
+            links.append((full_url, a.text.strip()))
+        return links
+
+    def _get_base_domain(self, url):
+        extracted = tldextract.extract(url)
+        # Reconstruct the domain as domain.suffix
+        base_domain = f"{extracted.domain}.{extracted.suffix}"
+        return base_domain
+
+    def _filter_links(self, links, base_domain):
+        """
+        Filter the extracted links to only include those that match the crawling criteria:
+        - If allow_subdomains is True, allow any link whose domain ends with the base_domain.
+        - If allow_subdomains is False, only allow exact matches of the base_domain.
+        """
+        filtered = []
+        for link, _ in links:
+            parsed_link = urlparse(link)
+            if not parsed_link.netloc:
+                continue
+
+            extracted = tldextract.extract(parsed_link.netloc)
+            link_base = f"{extracted.domain}.{extracted.suffix}"
+
+            if self.allow_subdomains:
+                # For subdomains: sub.example.com ends with example.com
+                if link_base == base_domain or link_base.endswith("." + base_domain):
+                    filtered.append(link)
+            else:
+                # Exact domain match
+                if link_base == base_domain:
+                    filtered.append(link)
+        return filtered

application/parser/remote/web_loader.py

@@ -1,5 +1,7 @@
 from application.parser.remote.base import BaseRemote
+from application.parser.schema.base import Document
 from langchain_community.document_loaders import WebBaseLoader
+from urllib.parse import urlparse
 
 headers = {
     "User-Agent": "Mozilla/5.0",
@@ -23,10 +25,20 @@ class WebLoader(BaseRemote):
             urls = [urls]
         documents = []
         for url in urls:
+            # Check if the URL scheme is provided, if not, assume http
+            if not urlparse(url).scheme:
+                url = "http://" + url
             try:
                 loader = self.loader([url], header_template=headers)
-                documents.extend(loader.load())
+                loaded_docs = loader.load()
+                for doc in loaded_docs:
+                    documents.append(
+                        Document(
+                            doc.page_content,
+                            extra_info=doc.metadata,
+                        )
+                    )
             except Exception as e:
                 print(f"Error processing URL {url}: {e}")
                 continue
-        return documents
+        return documents

application/requirements.txt

@@ -1,27 +1,27 @@
 anthropic==0.40.0
-boto3==1.34.153
+boto3==1.35.97
 beautifulsoup4==4.12.3
-celery==5.3.6
+celery==5.4.0
 dataclasses-json==0.6.7
 docx2txt==0.8
 duckduckgo-search==6.3.0
 ebooklib==0.18
-elastic-transport==8.15.1
+elastic-transport==8.17.0
 elasticsearch==8.17.0
 escodegen==1.0.11
 esprima==4.0.1
 esutils==1.0.1
-Flask==3.0.3
+Flask==3.1.0
 faiss-cpu==1.9.0.post1
 flask-restx==1.3.0
 google-genai==0.5.0
 google-generativeai==0.8.3
-gTTS==2.3.2
+gTTS==2.5.4
 gunicorn==23.0.0
 html2text==2024.2.26
 javalang==0.13.0
 jinja2==3.1.5
-jiter==0.5.0
+jiter==0.8.2
 jmespath==1.0.1
 joblib==1.4.2
 jsonpatch==1.33
@@ -30,62 +30,64 @@ jsonschema==4.23.0
 jsonschema-spec==0.2.4
 jsonschema-specifications==2023.7.1
 kombu==5.4.2
-langchain==0.3.13
-langchain-community==0.3.13
-langchain-core==0.3.28
-langchain-openai==0.2.14
-langchain-text-splitters==0.3.4
-langsmith==0.2.3
+langchain==0.3.14
+langchain-community==0.3.14
+langchain-core==0.3.29
+langchain-openai==0.3.0
+langchain-text-splitters==0.3.5
+langsmith==0.2.10
 lazy-object-proxy==1.10.0
 lxml==5.3.0
-markupsafe==2.1.5
-marshmallow==3.22.0
+markupsafe==3.0.2
+marshmallow==3.24.1
 mpmath==1.3.0
 multidict==6.1.0
 mypy-extensions==1.0.0
-networkx==3.3
+networkx==3.4.2
 numpy==2.2.1
-openai==1.58.1
+openai==1.59.5
 openapi-schema-validator==0.6.2
 openapi-spec-validator==0.6.0
-openapi3-parser==1.1.18
-orjson==3.10.7
+openapi3-parser==1.1.19
+orjson==3.10.14
 packaging==24.1
 pandas==2.2.3
 openpyxl==3.1.5
-pathable==0.4.3
-pillow==10.4.0
+pathable==0.4.4
+pillow==11.1.0
 portalocker==2.10.1
 prance==23.6.21.0
-primp==0.6.3
-prompt-toolkit==3.0.47
-protobuf==5.28.2
+primp==0.10.0
+prompt-toolkit==3.0.48
+protobuf==5.29.3
 py==1.11.0
-pydantic==2.9.2
-pydantic-core==2.23.4
-pydantic-settings==2.4.0
-pymongo==4.8.0
+pydantic==2.10.4
+pydantic-core==2.27.2
+pydantic-settings==2.7.1
+pymongo==4.10.1
 pypdf2==3.0.1
 python-dateutil==2.9.0.post0
 python-dotenv==1.0.1
 python-pptx==1.0.2
-qdrant-client==1.11.0
-redis==5.0.1
+qdrant-client==1.12.2
+redis==5.2.1
 referencing==0.30.2
-regex==2024.9.11
+regex==2024.11.6
 requests==2.32.3
 retry==0.9.2
 sentence-transformers==3.3.1
-tiktoken==0.7.0
+tiktoken==0.8.0
 tokenizers==0.21.0
-torch==2.4.1
-tqdm==4.66.5
-transformers==4.47.0
+torch==2.5.1
+tqdm==4.67.1
+transformers==4.48.0
 typing-extensions==4.12.2
 typing-inspect==0.9.0
 tzdata==2024.2
-urllib3==2.2.3
+urllib3==2.3.0
 vine==5.1.0
 wcwidth==0.2.13
 werkzeug==3.1.3
-yarl==1.18.3
+yarl==1.18.3
+markdownify==0.14.1
+tldextract==5.1.3

application/worker.py

@@ -203,53 +203,61 @@ def remote_worker(
     sync_frequency="never",
     operation_mode="upload",
     doc_id=None,
-):
+):  
     full_path = os.path.join(directory, user, name_job)
-
     if not os.path.exists(full_path):
         os.makedirs(full_path)
+
     self.update_state(state="PROGRESS", meta={"current": 1})
-    logging.info(
-        f"Remote job: {full_path}",
-        extra={"user": user, "job": name_job, "source_data": source_data},
-    )
+    try:
+        logging.info("Initializing remote loader with type: %s", loader)
+        remote_loader = RemoteCreator.create_loader(loader)
+        raw_docs = remote_loader.load_data(source_data)
 
-    remote_loader = RemoteCreator.create_loader(loader)
-    raw_docs = remote_loader.load_data(source_data)
+        chunker = Chunker(
+            chunking_strategy="classic_chunk",
+            max_tokens=MAX_TOKENS,
+            min_tokens=MIN_TOKENS,
+            duplicate_headers=False
+        )
+        docs = chunker.chunk(documents=raw_docs)
+        docs = [Document.to_langchain_format(raw_doc) for raw_doc in raw_docs]
+        tokens = count_tokens_docs(docs)
+        logging.info("Total tokens calculated: %d", tokens)
 
-    chunker = Chunker(
-        chunking_strategy="classic_chunk",
-        max_tokens=MAX_TOKENS,
-        min_tokens=MIN_TOKENS,
-        duplicate_headers=False
-    )
-    docs = chunker.chunk(documents=raw_docs)
+        if operation_mode == "upload":
+            id = ObjectId()
+            embed_and_store_documents(docs, full_path, id, self)
+        elif operation_mode == "sync":
+            if not doc_id or not ObjectId.is_valid(doc_id):
+                logging.error("Invalid doc_id provided for sync operation: %s", doc_id)
+                raise ValueError("doc_id must be provided for sync operation.")
+            id = ObjectId(doc_id)
+            embed_and_store_documents(docs, full_path, id, self)
 
-    tokens = count_tokens_docs(docs)
-    if operation_mode == "upload":
-        id = ObjectId()
-        embed_and_store_documents(docs, full_path, id, self)
-    elif operation_mode == "sync":
-        if not doc_id or not ObjectId.is_valid(doc_id):
-            raise ValueError("doc_id must be provided for sync operation.")
-        id = ObjectId(doc_id)
-        embed_and_store_documents(docs, full_path, id, self)
-    self.update_state(state="PROGRESS", meta={"current": 100})
+        self.update_state(state="PROGRESS", meta={"current": 100})
 
-    file_data = {
-        "name": name_job,
-        "user": user,
-        "tokens": tokens,
-        "retriever": retriever,
-        "id": str(id),
-        "type": loader,
-        "remote_data": source_data,
-        "sync_frequency": sync_frequency,
-    }
-    upload_index(full_path, file_data)
+        file_data = {
+            "name": name_job,
+            "user": user,
+            "tokens": tokens,
+            "retriever": retriever,
+            "id": str(id),
+            "type": loader,
+            "remote_data": source_data,
+            "sync_frequency": sync_frequency,
+        }
+        upload_index(full_path, file_data)
 
-    shutil.rmtree(full_path)
+    except Exception as e:
+        logging.error("Error in remote_worker task: %s", str(e), exc_info=True)
+        raise
 
+    finally:
+        if os.path.exists(full_path):
+            shutil.rmtree(full_path)
+
+    logging.info("remote_worker task completed successfully")
     return {"urls": source_data, "name_job": name_job, "user": user, "limited": False}
 
 def sync(