From 458f2a3ff3a9edaa78290f2e1ed0f5237f787fb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=86=AF=E4=B8=8D=E6=B8=B8?= <71683364+mefengl@users.noreply.github.com> Date: Tue, 14 Feb 2023 22:05:16 +0800 Subject: [PATCH 1/2] fix: restore index back when continue process --- scripts/parser/open_ai_func.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/parser/open_ai_func.py b/scripts/parser/open_ai_func.py index cbd947ee..433e5bcd 100644 --- a/scripts/parser/open_ai_func.py +++ b/scripts/parser/open_ai_func.py @@ -32,11 +32,14 @@ def call_openai_api(docs): print("Saving progress") print(f"stopped at {c1} out of {len(docs)}") faiss.write_index(store.index, "docs.index") + store_index_bak = store.index store.index = None with open("faiss_store.pkl", "wb") as f: pickle.dump(store, f) print("Sleeping for 10 seconds and trying again") time.sleep(10) + faiss.write_index(store_index_bak, "docs.index") + store.index = store_index_bak store.add_texts([i.page_content], metadatas=[i.metadata]) c1 += 1 @@ -66,4 +69,4 @@ def get_user_permission(docs): elif user_input == "": call_openai_api(docs) else: - print("The API was not called. No money was spent.") \ No newline at end of file + print("The API was not called. No money was spent.") From 636783ca8a5843b3dfb3e399e14ff3634efb4962 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=86=AF=E4=B8=8D=E6=B8=B8?= <71683364+mefengl@users.noreply.github.com> Date: Tue, 14 Feb 2023 22:29:17 +0800 Subject: [PATCH 2/2] fix: avoid second error issue --- scripts/parser/open_ai_func.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/parser/open_ai_func.py b/scripts/parser/open_ai_func.py index 433e5bcd..70091324 100644 --- a/scripts/parser/open_ai_func.py +++ b/scripts/parser/open_ai_func.py @@ -36,8 +36,8 @@ def call_openai_api(docs): store.index = None with open("faiss_store.pkl", "wb") as f: pickle.dump(store, f) - print("Sleeping for 10 seconds and trying again") - time.sleep(10) + print("Sleeping for 60 seconds and trying again") + time.sleep(60) faiss.write_index(store_index_bak, "docs.index") store.index = store_index_bak store.add_texts([i.page_content], metadatas=[i.metadata])