Calc + structure

scripts/parser/py2doc.py

@@ -0,0 +1,155 @@
+from pathlib import Path
+from langchain.llms import OpenAI
+from langchain.prompts import PromptTemplate
+import dotenv
+import ast
+import typer
+import tiktoken
+
+dotenv.load_dotenv()
+
+def get_functions(source_code):
+    tree = ast.parse(source_code)
+    functions = {}
+    for node in tree.body:
+        if isinstance(node, ast.FunctionDef):
+            functions[node.name] = ast.unparse(node)
+
+    return functions
+
+def get_functions_names(node):
+    functions = []
+    for child in node.body:
+        if isinstance(child, ast.FunctionDef):
+            functions.append(child.name)
+    return functions
+
+
+
+def get_classes(source_code):
+    tree = ast.parse(source_code)
+    classes = {}
+    for node in tree.body:
+        if isinstance(node, ast.ClassDef):
+            classes[node.name] = get_functions_names(node)
+    return classes
+
+def get_functions_in_class(source_code, class_name):
+    tree = ast.parse(source_code)
+    functions = []
+    for node in tree.body:
+        if isinstance(node, ast.ClassDef):
+            if node.name == class_name:
+                for function in node.body:
+                    if isinstance(function, ast.FunctionDef):
+                        functions.append(function.name)
+    return functions
+
+
+def parse_functions(functions_dict):
+    c1 = len(functions_dict)
+    c2 = 0
+    for source, functions in functions_dict.items():
+        c2 += 1
+        print(f"Processing file {c2}/{c1}")
+        f1 = len(functions)
+        f2 = 0
+        source_w = source.replace("inputs/", "")
+        source_w = source_w.replace(".py", ".md")
+        # this is how we check subfolders
+        if "/" in source_w:
+            subfolders = source_w.split("/")
+            subfolders = subfolders[:-1]
+            subfolders = "/".join(subfolders)
+            if not Path(f"outputs/{subfolders}").exists():
+                Path(f"outputs/{subfolders}").mkdir(parents=True)
+
+        for name, function in functions.items():
+            f2 += 1
+            print(f"Processing function {f2}/{f1}")
+            prompt = PromptTemplate(
+                input_variables=["code"],
+                template="Code: \n{code}, \nDocumentation: ",
+            )
+            llm = OpenAI(temperature=0)
+            response = llm(prompt.format(code=function))
+
+            if not Path(f"outputs/{source_w}").exists():
+                with open(f"outputs/{source_w}", "w") as f:
+                    f.write(f"# Function name: {name} \n\nFunction: \n```\n{function}\n```, \nDocumentation: \n{response}")
+            else:
+                with open(f"outputs/{source_w}", "a") as f:
+                    f.write(f"\n\n# Function name: {name} \n\nFunction: \n```\n{function}\n```, \nDocumentation: \n{response}")
+
+
+def parse_classes(classes_dict):
+    c1 = len(classes_dict)
+    c2 = 0
+    for source, classes in classes_dict.items():
+        c2 += 1
+        print(f"Processing file {c2}/{c1}")
+        f1 = len(classes)
+        f2 = 0
+        source_w = source.replace("inputs/", "")
+        source_w = source_w.replace(".py", ".md")
+
+        if "/" in source_w:
+            subfolders = source_w.split("/")
+            subfolders = subfolders[:-1]
+            subfolders = "/".join(subfolders)
+            if not Path(f"outputs/{subfolders}").exists():
+                Path(f"outputs/{subfolders}").mkdir(parents=True)
+
+        for name, function_names in classes.items():
+            print(f"Processing Class {f2}/{f1}")
+            f2 += 1
+            prompt = PromptTemplate(
+                input_variables=["class_name", "functions_names"],
+                template="Class name: {class_name} \nFunctions: {functions_names}, \nDocumentation: ",
+            )
+            llm = OpenAI(temperature=0)
+            response = llm(prompt.format(class_name=name, functions_names=function_names))
+
+            if not Path(f"outputs/{source_w}").exists():
+                with open(f"outputs/{source_w}", "w") as f:
+                    f.write(f"# Class name: {name} \n\nFunctions: \n{function_names}, \nDocumentation: \n{response}")
+            else:
+                with open(f"outputs/{source_w}", "a") as f:
+                    f.write(f"\n\n# Class name: {name} \n\nFunctions: \n{function_names}, \nDocumentation: \n{response}")
+
+
+#User permission
+def transform_to_docs(functions_dict, classes_dict):
+# Function to ask user permission to call the OpenAI api and spend their OpenAI funds.
+    # Here we convert dicts to a string and calculate the number of OpenAI tokens the string represents.
+    docs_content = ""
+    for key, value in functions_dict.items():
+        docs_content += str(key) + str(value)
+    for key, value in classes_dict.items():
+        docs_content += str(key) + str(value)
+
+    encoding = tiktoken.get_encoding("cl100k_base")
+    num_tokens = len(encoding.encode(docs_content))
+    total_price = ((num_tokens / 1000) * 0.02)
+
+    # Here we print the number of tokens and the approx user cost with some visually appealing formatting.
+    print(f"Number of Tokens = {format(num_tokens, ',d')}")
+    print(f"Approx Cost = ${format(total_price, ',.2f')}")
+    #Here we check for user permission before calling the API.
+    user_input = input("Price Okay? (Y/N) \n").lower()
+    if user_input == "y":
+        if not Path("outputs").exists():
+            Path("outputs").mkdir()
+        parse_functions(functions_dict)
+        print("Functions done!")
+        parse_classes(classes_dict)
+        print("All done!")
+    elif user_input == "":
+        if not Path("outputs").exists():
+            Path("outputs").mkdir()
+        parse_functions(functions_dict)
+        print("Functions done!")
+        parse_classes(classes_dict)
+        print("All done!")
+    else:
+        print("The API was not called. No money was spent.")