From 0a0fe20fa0821d799b3a691fde171c2e4444451a Mon Sep 17 00:00:00 2001
From: ManishMadan2882 <manishmadan321@gmail.com>
Date: Sat, 14 Dec 2024 04:53:06 +0530
Subject: [PATCH] (update) tokenizers

---
 application/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/application/requirements.txt b/application/requirements.txt
index 92ab06da..fc6d63f7 100644
--- a/application/requirements.txt
+++ b/application/requirements.txt
@@ -75,7 +75,7 @@ requests==2.32.3
 retry==0.9.2
 sentence-transformers==3.0.1
 tiktoken==0.7.0
-tokenizers==0.19.1
+tokenizers==0.21.0
 torch==2.4.1
 tqdm==4.66.5
 transformers==4.47.0