LLM/DocsGPT
1
0
Fork 0
mirror of https://github.com/arc53/DocsGPT.git synced 2025-11-29 16:43:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: add support for directory list

Browse Source 
example: `python ingest.py --dir inputs1 --dir another --dir ../inputs`,
the outputs will be in `outputs/input_folder_name/`
This commit is contained in:
冯不游
2023-02-15 02:30:39 +08:00
parent 5883ce2685
commit b83589a308
2 changed files with 56 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

66
scripts/ingest.py
View File

@@ -1,3 +1,5 @@
from collections import defaultdict
import os
import sys
import nltk
import dotenv
@@ -18,13 +20,16 @@ app = typer.Typer(add_completion=False)
nltk.download('punkt', quiet=True)
nltk.download('averaged_perceptron_tagger', quiet=True)
#Splits all files in specified folder to documents
@app.command()
def ingest(directory: Optional[str] = typer.Option("inputs",
help="Path to the directory for index creation."),
files: Optional[List[str]] = typer.Option(None,
help="""File paths to use (Optional; overrides directory).
E.g. --files inputs/1.md --files inputs/2.md"""),
def ingest(yes: bool = typer.Option(False, "-y", "--yes", prompt=False),
dir: Optional[List[str]] = typer.Option(["inputs"],
help="""List of paths to directory for index creation.
E.g. --dir inputs --dir inputs2"""),
file: Optional[List[str]] = typer.Option(None,
help="""File paths to use (Optional; overrides dir).
E.g. --file inputs/1.md --file inputs/2.md"""),
recursive: Optional[bool] = typer.Option(True,
help="Whether to recursively search in subdirectories."),
limit: Optional[int] = typer.Option(None,
@@ -38,27 +43,40 @@ def ingest(directory: Optional[str] = typer.Option("inputs",
Creates index from specified location or files.
By default /inputs folder is used, .rst and .md are parsed.
"""
raw_docs = SimpleDirectoryReader(input_dir=directory, input_files=files, recursive=recursive,
required_exts=formats, num_files_limit=limit,
exclude_hidden=exclude).load_data()
raw_docs = [Document.to_langchain_format(raw_doc) for raw_doc in raw_docs]
print(raw_docs)
# Here we split the documents, as needed, into smaller chunks.
# We do this due to the context limits of the LLMs.
text_splitter = RecursiveCharacterTextSplitter()
docs = text_splitter.split_documents(raw_docs)
# Here we check for command line arguments for bot calls.
# If no argument exists or the permission_bypass_flag argument is not '-y',
# user permission is requested to call the API.
if len(sys.argv) > 1:
permission_bypass_flag = sys.argv[1]
if permission_bypass_flag == '-y':
call_openai_api(docs)
def process_one_docs(directory, folder_name):
raw_docs = SimpleDirectoryReader(input_dir=directory, input_files=file, recursive=recursive,
required_exts=formats, num_files_limit=limit,
exclude_hidden=exclude).load_data()
raw_docs = [Document.to_langchain_format(raw_doc) for raw_doc in raw_docs]
print(raw_docs)
# Here we split the documents, as needed, into smaller chunks.
# We do this due to the context limits of the LLMs.
text_splitter = RecursiveCharacterTextSplitter()
docs = text_splitter.split_documents(raw_docs)
# Here we check for command line arguments for bot calls.
# If no argument exists or the yes is not True, then the
# user permission is requested to call the API.
if len(sys.argv) > 1:
if yes:
call_openai_api(docs, folder_name)
else:
get_user_permission(docs, folder_name)
else:
get_user_permission(docs)
else:
get_user_permission(docs)
get_user_permission(docs, folder_name)
folder_counts = defaultdict(int)
folder_names = []
for dir_path in dir:
folder_name = os.path.basename(os.path.normpath(dir_path))
folder_counts[folder_name] += 1
if folder_counts[folder_name] > 1:
folder_name = f"{folder_name}_{folder_counts[folder_name]}"
folder_names.append(folder_name)
for directory, folder_name in zip(dir, folder_names):
process_one_docs(directory, folder_name)
if __name__ == "__main__":
app()