LLM/DocsGPT
1
0
Fork 0
mirror of https://github.com/arc53/DocsGPT.git synced 2026-01-20 05:50:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: tiktoken import in markdown parser

Browse Source
This commit is contained in:
Alex
2026-01-12 23:04:20 +00:00
parent f1d714b5c1
commit 2c55c6cd9a
2 changed files with 4 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
tests/parser/file/test_markdown_parser.py
View File

@@ -2,9 +2,9 @@ from pathlib import Path
from unittest.mock import mock_open, patch
import pytest
import tiktoken
from application.parser.file.markdown_parser import MarkdownParser
from application import utils
class _Enc:
@@ -14,7 +14,7 @@ class _Enc:
@pytest.fixture(autouse=True)
def _patch_tokenizer(monkeypatch):
monkeypatch.setattr(tiktoken, "get_encoding", lambda _: _Enc())
monkeypatch.setattr(utils, "get_encoding", lambda: _Enc())
def test_markdown_init_parser():
parser = MarkdownParser()
@@ -57,4 +57,3 @@ def test_markdown_token_chunking_via_max_tokens():
assert len(tups) > 1
for _hdr, chunk in tups:
assert len(chunk) <= 4