LLM/DocsGPT
1
0
Fork 0
mirror of https://github.com/arc53/DocsGPT.git synced 2025-11-29 08:33:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix tokens for header

Browse Source
This commit is contained in:
Alex
2023-07-24 16:14:08 +01:00
parent b2e86e105d
commit dac76a867f
2 changed files with 10 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
application/parser/token_func.py
View File

@@ -46,6 +46,10 @@ def split_documents(documents: List[Document], max_tokens: int) -> List[Document
docs.append(doc)
else:
header, body = separate_header_and_body(doc.text)
if len(tiktoken.get_encoding("cl100k_base").encode(header)) > max_tokens:
print("header too long, skipping", file=sys.stderr)
body = doc.text
header = ""
num_body_parts = ceil(token_length / max_tokens)
part_length = ceil(len(body) / num_body_parts)
body_parts = [body[i:i + part_length] for i in range(0, len(body), part_length)]