LLM/DocsGPT
1
0
Fork 0
mirror of https://github.com/arc53/DocsGPT.git synced 2025-11-30 00:53:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: fields to handle reddit loader + minor changes

Browse Source
This commit is contained in:
Siddhant Rai
2024-03-26 16:07:44 +05:30
parent 0c3970a266
commit eed1bfbe50
3 changed files with 120 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
application/parser/remote/reddit_loader.py
View File

@@ -4,12 +4,13 @@ from langchain_community.document_loaders import RedditPostsLoader
class RedditPostsLoaderRemote(BaseRemote):
def load_data(self, inputs):
client_id = inputs.get("client_id")
client_secret = inputs.get("client_secret")
user_agent = inputs.get("user_agent")
categories = inputs.get("categories", ["new", "hot"])
mode = inputs.get("mode", "subreddit")
search_queries = inputs.get("search_queries")
data = eval(inputs)
client_id = data.get("client_id")
client_secret = data.get("client_secret")
user_agent = data.get("user_agent")
categories = data.get("categories", ["new", "hot"])
mode = data.get("mode", "subreddit")
search_queries = data.get("search_queries")
self.loader = RedditPostsLoader(
client_id=client_id,
client_secret=client_secret,
@@ -17,11 +18,8 @@ class RedditPostsLoaderRemote(BaseRemote):
categories=categories,
mode=mode,
search_queries=search_queries,
number_posts=10,
)
documents = []
try:
documents.extend(self.loader.load())
except Exception as e:
print(f"Error processing Data: {e}")
documents = self.loader.load()
print(f"Loaded {len(documents)} documents from Reddit")
return documents[:5]
return documents