mirror of
https://github.com/arc53/DocsGPT.git
synced 2025-11-30 09:03:15 +00:00
20 lines
574 B
Python
20 lines
574 B
Python
import os
|
|
|
|
import dotenv
|
|
import tiktoken
|
|
from langchain import FAISS
|
|
from langchain.embeddings import OpenAIEmbeddings
|
|
|
|
dotenv.load_dotenv()
|
|
embeddings_key = os.getenv("API_KEY")
|
|
docsearch = FAISS.load_local('outputs', OpenAIEmbeddings(openai_api_key=embeddings_key))
|
|
|
|
d1 = docsearch.similarity_search("Whats new in 1.5.3?")
|
|
print(d1)
|
|
print("=====================================")
|
|
print("=====================================")
|
|
for i in d1:
|
|
print("docs length (tokens)")
|
|
doc_len = len(tiktoken.get_encoding("cl100k_base").encode(i.page_content))
|
|
print(doc_len)
|