fix: translation + auth

application/parser/remote/github_loader.py

@@ -1,34 +1,37 @@
-import os
 import base64
 import requests
 from typing import List
 from application.parser.remote.base import BaseRemote
-from application.parser.schema.base import Document
+from langchain_core.documents import Document
 
 class GitHubLoader(BaseRemote):
-    def __init__(self, access_token: str):
-        self.access_token = access_token
+    def __init__(self):
+        self.access_token = None
+        self.headers = {
+            "Authorization": f"token {self.access_token}"
+        } if self.access_token else {}
+        return
 
     def fetch_file_content(self, repo_url: str, file_path: str) -> str:
         url = f"https://api.github.com/repos/{repo_url}/contents/{file_path}"
-        headers = {
-            "Authorization": f"token {self.access_token}",
-            "Accept": "application/vnd.github.v3.raw"
-        }
-        response = requests.get(url, headers=headers)
-        response.raise_for_status()
-        content = response.json()
-        if content.get("encoding") == "base64":
-            return base64.b64decode(content["content"]).decode("utf-8")
-        return content["content"]
+        response = requests.get(url, headers=self.headers)
+
+        if response.status_code == 200:
+            content = response.json()
+            if content.get("encoding") == "base64":
+                try:
+                    decoded_content = base64.b64decode(content["content"]).decode("utf-8")
+                    return decoded_content
+                except Exception as e:
+                    raise
+            else:
+                return content["content"]
+        else:
+            response.raise_for_status()
 
     def fetch_repo_files(self, repo_url: str, path: str = "") -> List[str]:
         url = f"https://api.github.com/repos/{repo_url}/contents/{path}"
-        headers = {
-            "Authorization": f"token {self.access_token}",
-            "Accept": "application/vnd.github.v3.raw"
-        }
-        response = requests.get(url, headers=headers)
+        response = requests.get(url, headers=self.headers)
         response.raise_for_status()
         contents = response.json()
         files = []
@@ -45,5 +48,5 @@ class GitHubLoader(BaseRemote):
         documents = []
         for file_path in files:
             content = self.fetch_file_content(repo_name, file_path)
-            documents.append(Document(content=content, metadata={"file_path": file_path}))
+            documents.append(Document(page_content=content, metadata={"file_path": file_path}))
         return documents