mirror of
https://github.com/arc53/DocsGPT.git
synced 2026-01-20 22:10:54 +00:00
9e7f1ad1c0
* Add Amazon S3 support and synchronization features * refactor: remove unused variable in load_data test
35 lines
1.2 KiB
Python
35 lines
1.2 KiB
Python
from application.parser.remote.sitemap_loader import SitemapLoader
|
|
from application.parser.remote.crawler_loader import CrawlerLoader
|
|
from application.parser.remote.web_loader import WebLoader
|
|
from application.parser.remote.reddit_loader import RedditPostsLoaderRemote
|
|
from application.parser.remote.github_loader import GitHubLoader
|
|
from application.parser.remote.s3_loader import S3Loader
|
|
|
|
|
|
class RemoteCreator:
|
|
"""
|
|
Factory class for creating remote content loaders.
|
|
|
|
These loaders fetch content from remote web sources like URLs,
|
|
sitemaps, web crawlers, social media platforms, etc.
|
|
|
|
For external knowledge base connectors (like Google Drive),
|
|
use ConnectorCreator instead.
|
|
"""
|
|
|
|
loaders = {
|
|
"url": WebLoader,
|
|
"sitemap": SitemapLoader,
|
|
"crawler": CrawlerLoader,
|
|
"reddit": RedditPostsLoaderRemote,
|
|
"github": GitHubLoader,
|
|
"s3": S3Loader,
|
|
}
|
|
|
|
@classmethod
|
|
def create_loader(cls, type, *args, **kwargs):
|
|
loader_class = cls.loaders.get(type.lower())
|
|
if not loader_class:
|
|
raise ValueError(f"No loader class found for type {type}")
|
|
return loader_class(*args, **kwargs)
|