fix

2025-11-29 16:43:16 +00:00 · 2023-07-24 16:23:49 +01:00
parent dac76a867f
commit a64a30c088
2 changed files with 0 additions and 4 deletions
--- a/application/parser/token_func.py
+++ b/application/parser/token_func.py
@@ -47,7 +47,6 @@ def split_documents(documents: List[Document], max_tokens: int) -> List[Document
        else:
            header, body = separate_header_and_body(doc.text)
            if len(tiktoken.get_encoding("cl100k_base").encode(header)) > max_tokens:
-                print("header too long, skipping", file=sys.stderr)
                body = doc.text
                header = ""
            num_body_parts = ceil(token_length / max_tokens)