cors + dependencies

application/Dockerfile

@@ -12,11 +12,13 @@ RUN pip install -r requirements.txt
 FROM python:3.10-slim-bullseye
 # Copy pre-built packages from builder stage
 COPY --from=builder /usr/local/lib/python3.10/site-packages/ /usr/local/lib/python3.10/site-packages/
+RUN pip install gunicorn==20.1.0
+RUN pip install celery==5.2.7
 WORKDIR /app
 COPY . /app
 ENV FLASK_APP=app.py
 ENV FLASK_DEBUG=true
-RUN pip install gunicorn==20.1.0
+
 
 EXPOSE 5001
 

application/app.py

@@ -400,10 +400,16 @@ def delete_old():
     """Delete old indexes."""
     import shutil
     path = request.args.get('path')
+    dirs = path.split('/')
     first_dir = path.split('/')[0]
+    for i in range(1, len(dirs)):
+        dirs[i] = secure_filename(dirs[i])
+
+
     # check that path strats with indexes or vectors
-    if first_dir not in ['indexes', 'vectors']:
+    if dirs[0] not in ['indexes', 'vectors']:
         return {"status": 'error'}
+    path = '/'.join(dirs)
     shutil.rmtree(path)
     vectors_collection.delete_one({'location': path})
     return {"status": 'ok'}
@@ -414,6 +420,7 @@ def after_request(response):
     response.headers.add('Access-Control-Allow-Origin', '*')
     response.headers.add('Access-Control-Allow-Headers', 'Content-Type,Authorization')
     response.headers.add('Access-Control-Allow-Methods', 'GET,PUT,POST,DELETE,OPTIONS')
+    response.headers.add('Access-Control-Allow-Credentials', 'true')
     return response
 
 

application/requirements.txt

@@ -3,18 +3,25 @@ aiohttp==3.8.4
 aiohttp-retry==2.8.3
 aiosignal==1.3.1
 aleph-alpha-client==2.16.1
+amqp==5.1.1
 async-timeout==4.0.2
 attrs==22.2.0
+billiard==3.6.4.0
 blobfile==2.0.1
 boto3==1.26.84
 botocore==1.29.84
 cffi==1.15.1
 charset-normalizer==3.1.0
 click==8.1.3
+click-didyoumean==0.3.0
+click-plugins==1.1.1
+click-repl==0.2.0
 cryptography==39.0.2
 dataclasses-json==0.5.7
+decorator==5.1.1
 deeplake==3.2.13
 dill==0.3.6
+dnspython==2.3.0
 ecdsa==0.18.0
 entrypoints==0.4
 faiss-cpu==1.7.3
@@ -29,6 +36,8 @@ idna==3.4
 itsdangerous==2.1.2
 Jinja2==3.1.2
 jmespath==1.0.1
+joblib==1.2.0
+kombu==5.2.4
 langchain==0.0.103
 lxml==4.9.2
 MarkupSafe==2.1.2
@@ -37,6 +46,7 @@ marshmallow-enum==1.5.1
 multidict==6.0.4
 multiprocess==0.70.14
 mypy-extensions==1.0.0
+nltk==3.8.1
 numcodecs==0.11.0
 numpy==1.24.2
 openai==0.27.0
@@ -45,18 +55,24 @@ pathos==0.3.0
 Pillow==9.4.0
 pox==0.3.2
 ppft==1.7.6.6
+prompt-toolkit==3.0.38
+py==1.11.0
 pyasn1==0.4.8
 pycares==4.3.0
 pycparser==2.21
 pycryptodomex==3.17
 pydantic==1.10.5
 PyJWT==2.6.0
+pymongo==4.3.3
 python-dateutil==2.8.2
 python-dotenv==1.0.0
 python-jose==3.3.0
+pytz==2022.7.1
 PyYAML==6.0
+redis==4.5.1
 regex==2022.10.31
 requests==2.28.2
+retry==0.9.2
 rsa==4.9
 s3transfer==0.6.0
 six==1.16.0
@@ -69,5 +85,7 @@ transformers==4.26.1
 typing-inspect==0.8.0
 typing_extensions==4.5.0
 urllib3==1.26.14
+vine==5.0.0
+wcwidth==0.2.6
 Werkzeug==2.2.3
 yarl==1.8.2

application/worker.py

@@ -8,11 +8,16 @@ from parser.schema.base import Document
 from parser.open_ai_func import call_openai_api
 from celery import current_task
 
-nltk.download('punkt', quiet=True)
-nltk.download('averaged_perceptron_tagger', quiet=True)
+
 import string
 import zipfile
 import shutil
+
+try:
+    nltk.download('punkt', quiet=True)
+    nltk.download('averaged_perceptron_tagger', quiet=True)
+except FileExistsError:
+    pass
 def generate_random_string(length):
     return ''.join([string.ascii_letters[i % 52] for i in range(length)])
 
@@ -29,7 +34,11 @@ def ingest_worker(self, directory, formats, name_job, filename, user):
     # filename = 'install.rst'
     # user = 'local'
     full_path = directory + '/' + user + '/' + name_job
-    url = 'http://localhost:5001/api/download'
+    # check if API_URL env variable is set
+    if not os.environ.get('API_URL'):
+        url = 'http://localhost:5001/api/download'
+    else:
+        url = os.environ.get('API_URL') + '/api/download'
     file_data = {'name': name_job, 'file': filename, 'user': user}
     response = requests.get(url, params=file_data)
     file = response.content
@@ -62,14 +71,20 @@ def ingest_worker(self, directory, formats, name_job, filename, user):
 
     # get files from outputs/inputs/index.faiss and outputs/inputs/index.pkl
     # and send them to the server (provide user and name in form)
-    url = 'http://localhost:5001/api/upload_index'
+    if not os.environ.get('API_URL'):
+        url = 'http://localhost:5001/api/upload_index'
+    else:
+        url = os.environ.get('API_URL') + '/api/upload_index'
     file_data = {'name': name_job, 'user': user}
     files = {'file_faiss': open(full_path + '/index.faiss', 'rb'),
              'file_pkl': open(full_path + '/index.pkl', 'rb')}
     response = requests.post(url, files=files, data=file_data)
 
     #deletes remote
-    url = 'http://localhost:5001/api/delete_old?path=' + 'inputs/' + user + '/' + name_job
+    if not os.environ.get('API_URL'):
+        url = 'http://localhost:5001/api/delete_old?path=' + 'inputs/' + user + '/' + name_job
+    else:
+        url = os.environ.get('API_URL') + '/api/delete_old?path=' + 'inputs/' + user + '/' + name_job
     response = requests.get(url)
     # delete local
     shutil.rmtree(full_path)

docker-compose.yaml

@@ -4,7 +4,7 @@ services:
   frontend:
     build: ./frontend
     environment:
-      - API_HOST=http://backend:5001
+      - VITE_API_HOST=http://localhost:5001
     ports:
       - "5173:5173"
     depends_on:
@@ -12,6 +12,12 @@ services:
 
   backend:
     build: ./application
+    environment:
+      - API_KEY=<your_api_key>
+      - EMBEDDINGS_KEY=<your_api_key>
+      - CELERY_BROKER_URL=redis://redis:6379/0
+      - CELERY_RESULT_BACKEND=redis://redis:6379/1
+      - MONGO_URI=mongodb://mongo:27017/docsgpt
     ports:
       - "5001:5001"
     volumes:
@@ -19,9 +25,17 @@ services:
     depends_on:
         - redis
         - mongo
+
   worker:
     build: ./application
-    command: celery -A app.celery worker -l info
+    command: celery -A app.celery worker -l INFO
+    environment:
+      - API_KEY=<your_api_key>
+      - EMBEDDINGS_KEY=<your_api_key>
+      - CELERY_BROKER_URL=redis://redis:6379/0
+      - CELERY_RESULT_BACKEND=redis://redis:6379/1
+      - MONGO_URI=mongodb://mongo:27017/docsgpt
+      - API_URL=http://backend:5001
     depends_on:
         - redis
         - mongo