LLM/DocsGPT
1
0
Fork 0
mirror of https://github.com/arc53/DocsGPT.git synced 2025-11-29 16:43:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' of https://github.com/arc53/DocsGPT

Browse Source
This commit is contained in:
GH Action - Upstream Sync
2025-08-27 01:36:38 +00:00
parent 578c68205a 545caacfa3
commit f08067a161
3 changed files with 83 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
application/parser/embedding_pipeline.py
View File

@@ -6,6 +6,21 @@ from application.core.settings import settings
from application.vectorstore.vector_creator import VectorCreator
def sanitize_content(content: str) -> str:
"""
Remove NUL characters that can cause vector store ingestion to fail.
Args:
content (str): Raw content that may contain NUL characters
Returns:
str: Sanitized content with NUL characters removed
"""
if not content:
return content
return content.replace('\x00', '')
@retry(tries=10, delay=60)
def add_text_to_store_with_retry(store, doc, source_id):
"""
@@ -16,6 +31,9 @@ def add_text_to_store_with_retry(store, doc, source_id):
source_id: Unique identifier for the source.
"""
try:
# Sanitize content to remove NUL characters that cause ingestion failures
doc.page_content = sanitize_content(doc.page_content)
doc.metadata["source_id"] = str(source_id)
store.add_texts([doc.page_content], metadatas=[doc.metadata])
except Exception as e: