LLM/DocsGPT
1
0
Fork 0
mirror of https://github.com/arc53/DocsGPT.git synced 2025-11-29 16:43:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

(fix:test_markdown) patch tokenizer

Browse Source
This commit is contained in:
ManishMadan2882
2025-09-30 13:35:03 +05:30
parent e24a0ac686
commit 402d5e054b
1 changed files with 8 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
tests/parser/file/test_markdown_parser.py
View File

@@ -1,21 +1,16 @@
from pathlib import Path
from unittest.mock import mock_open, patch
import sys, types
if "tiktoken" not in sys.modules:
fake_tt = types.ModuleType("tiktoken")
import pytest
class _Enc:
def encode(self, s: str):
return list(s)
class _Enc:
def encode(self, s: str):
return list(s)
def get_encoding(_: str):
return _Enc()
fake_tt.get_encoding = get_encoding
sys.modules["tiktoken"] = fake_tt
import tiktoken
@pytest.fixture(autouse=True)
def _patch_tokenizer(monkeypatch):
import application.parser.file.markdown_parser as mdp
monkeypatch.setattr(mdp.tiktoken, "get_encoding", lambda _: _Enc())
from application.parser.file.markdown_parser import MarkdownParser