mirror of
https://github.com/arc53/DocsGPT.git
synced 2025-11-29 16:43:16 +00:00
fix: Update count_tokens function in utils.py
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
from transformers import GPT2TokenizerFast
|
||||
|
||||
|
||||
tokenizer = GPT2TokenizerFast.from_pretrained('gpt2')
|
||||
def count_tokens(string):
|
||||
tokenizer = GPT2TokenizerFast.from_pretrained('gpt2')
|
||||
|
||||
return len(tokenizer(string)['input_ids'])
|
||||
Reference in New Issue
Block a user