mirror of
https://github.com/arc53/DocsGPT.git
synced 2025-11-29 08:33:20 +00:00
6 lines
175 B
Python
6 lines
175 B
Python
from transformers import GPT2TokenizerFast
|
|
|
|
tokenizer = GPT2TokenizerFast.from_pretrained('gpt2')
|
|
def count_tokens(string):
|
|
|
|
return len(tokenizer(string)['input_ids']) |