LLM/DocsGPT
1
0
Fork 0
mirror of https://github.com/arc53/DocsGPT.git synced 2025-11-29 16:43:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix

Browse Source
This commit is contained in:
Alex
2023-07-24 16:23:49 +01:00
parent dac76a867f
commit a64a30c088
2 changed files with 0 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
scripts/parser/token_func.py
View File

@@ -4,7 +4,6 @@ from typing import List
import tiktoken
from parser.schema.base import Document
import sys
def separate_header_and_body(text):
header_pattern = r"^(.*?\n){3}"
@@ -17,7 +16,6 @@ def separate_header_and_body(text):
def group_documents(documents: List[Document], min_tokens: int, max_tokens: int) -> List[Document]:
docs = []
current_group = None
print("Grouping", len(documents), "documents", file=sys.stderr)
for doc in documents:
doc_len = len(tiktoken.get_encoding("cl100k_base").encode(doc.text))
@@ -59,7 +57,6 @@ def split_documents(documents: List[Document], max_tokens: int) -> List[Document
embedding=doc.embedding,
extra_info=doc.extra_info)
docs.append(new_doc)
print("split into", len(docs), "documents", file=sys.stderr)
return docs