LLM/DocsGPT
1
0
Fork 0
mirror of https://github.com/arc53/DocsGPT.git synced 2026-01-20 05:50:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: tiktoken import in markdown parser

Browse Source
This commit is contained in:
Alex
2026-01-12 23:04:20 +00:00
parent f1d714b5c1
commit 2c55c6cd9a
2 changed files with 4 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
application/parser/file/markdown_parser.py
View File

@@ -7,8 +7,8 @@ import re
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple, Union, cast
import tiktoken
from application.parser.file.base_parser import BaseParser
from application.utils import num_tokens_from_string
class MarkdownParser(BaseParser):
@@ -38,7 +38,7 @@ class MarkdownParser(BaseParser):
def tups_chunk_append(self, tups: List[Tuple[Optional[str], str]], current_header: Optional[str],
current_text: str):
"""Append to tups chunk."""
num_tokens = len(tiktoken.get_encoding("cl100k_base").encode(current_text))
num_tokens = num_tokens_from_string(current_text)
if num_tokens > self._max_tokens:
chunks = [current_text[i:i + self._max_tokens] for i in range(0, len(current_text), self._max_tokens)]
for chunk in chunks: