feat: migrate store to source_id

application/api/answer/routes.py

@@ -21,7 +21,7 @@ logger = logging.getLogger(__name__)
 mongo = MongoClient(settings.MONGO_URI)
 db = mongo["docsgpt"]
 conversations_collection = db["conversations"]
-vectors_collection = db["vectors"]
+sources_collection = db["sources"]
 prompts_collection = db["prompts"]
 api_key_collection = db["api_keys"]
 answer = Blueprint("answer", __name__)
@@ -91,7 +91,7 @@ def get_data_from_api_key(api_key):
 
 
 def get_retriever(source_id: str):
-    doc = vectors_collection.find_one({"_id": ObjectId(source_id)})
+    doc = sources_collection.find_one({"_id": ObjectId(source_id)})
     if doc is None:
         raise Exception("Source document does not exist", 404)
     retriever_name = None if "retriever" not in doc else doc["retriever"]

application/api/internal/routes.py

@@ -9,7 +9,7 @@ from application.core.settings import settings
 mongo = MongoClient(settings.MONGO_URI)
 db = mongo["docsgpt"]
 conversations_collection = db["conversations"]
-vectors_collection = db["vectors"]
+sources_collection = db["sources"]
 
 current_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
@@ -60,8 +60,8 @@ def upload_index_files():
             os.makedirs(save_dir)
         file_faiss.save(os.path.join(save_dir, "index.faiss"))
         file_pkl.save(os.path.join(save_dir, "index.pkl"))
-    # create entry in vectors_collection
-    vectors_collection.insert_one(
+    # create entry in sources_collection
+    sources_collection.insert_one(
         {
             "_id": ObjectId(id),
             "user": user,

application/api/user/routes.py

@@ -17,7 +17,7 @@ from application.vectorstore.vector_creator import VectorCreator
 mongo = MongoClient(settings.MONGO_URI)
 db = mongo["docsgpt"]
 conversations_collection = db["conversations"]
-vectors_collection = db["vectors"]
+sources_collection = db["sources"]
 prompts_collection = db["prompts"]
 feedback_collection = db["feedback"]
 api_key_collection = db["api_keys"]
@@ -106,7 +106,7 @@ def delete_by_ids():
         return {"status": "error"}
 
     if settings.VECTOR_STORE == "faiss":
-        result = vectors_collection.delete_index(ids=ids)
+        result = sources_collection.delete_index(ids=ids)
         if result:
             return {"status": "ok"}
     return {"status": "error"}
@@ -116,9 +116,9 @@ def delete_by_ids():
 def delete_old():
     """Delete old indexes."""
     import shutil
-    path = request.args.get("path")
-    doc = vectors_collection.find_one({
-        "_id": ObjectId(path),
+    source_id = request.args.get("source_id")
+    doc = sources_collection.find_one({
+        "_id": ObjectId(source_id),
         "user": "local",
     })
     if(doc is None):
@@ -129,10 +129,10 @@ def delete_old():
         except FileNotFoundError:
             pass
     else:
-        vetorstore = VectorCreator.create_vectorstore(settings.VECTOR_STORE, path=str(doc["_id"]))
+        vetorstore = VectorCreator.create_vectorstore(settings.VECTOR_STORE, source_id=str(doc["_id"]))
         vetorstore.delete_index()
-    vectors_collection.delete_one({
-        "_id": ObjectId(path),
+    sources_collection.delete_one({
+        "_id": ObjectId(source_id),
     })
 
     return {"status": "ok"}
@@ -244,8 +244,8 @@ def combined_json():
         }
     ]
     # structure: name, language, version, description, fullName, date, docLink
-    # append data from vectors_collection in sorted order in descending order of date
-    for index in vectors_collection.find({"user": user}).sort("date", -1):
+    # append data from sources_collection in sorted order in descending order of date
+    for index in sources_collection.find({"user": user}).sort("date", -1):
         data.append(
             {
                 "id": str(index["_id"]),