Merge pull request #168 from arc53/feature/backend-uploads

Feature/backend uploads

scripts/parser/file/bulk.py

@@ -76,6 +76,8 @@ class SimpleDirectoryReader(BaseReader):
         self.exclude_hidden = exclude_hidden
         self.required_exts = required_exts
         self.num_files_limit = num_files_limit
+        print("input_files")
+        print(input_files)
 
         if input_files:
             self.input_files = []

scripts/parser/open_ai_func.py

@@ -23,6 +23,7 @@ def num_tokens_from_string(string: str, encoding_name: str) -> int:
 @retry(tries=10, delay=60)
 def store_add_texts_with_retry(store, i):
     store.add_texts([i.page_content], metadatas=[i.metadata])
+    #store_pine.add_texts([i.page_content], metadatas=[i.metadata])
 
 def call_openai_api(docs, folder_name):
 # Function to create a vector store from the documents and save it to disk.
@@ -38,7 +39,13 @@ def call_openai_api(docs, folder_name):
     # cut first n docs if you want to restart
     #docs = docs[:n]
     c1 = 0
+    # pinecone.init(
+    #     api_key="",  # find at app.pinecone.io
+    #     environment="us-east1-gcp"  # next to api key in console
+    # )
+    #index_name = "pandas"
     store = FAISS.from_documents(docs_test, OpenAIEmbeddings())
+    #store_pine = Pinecone.from_documents(docs_test, OpenAIEmbeddings(), index_name=index_name)
 
     # Uncomment for MPNet embeddings
     # model_name = "sentence-transformers/all-mpnet-base-v2"