From c4464455a16f923efdbc7f568e35d2c5a3cc9c80 Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 14 Mar 2023 14:29:36 +0000 Subject: [PATCH] cors + dependencies --- application/Dockerfile | 4 +++- application/app.py | 9 ++++++++- application/requirements.txt | 18 ++++++++++++++++++ application/worker.py | 25 ++++++++++++++++++++----- docker-compose.yaml | 18 ++++++++++++++++-- 5 files changed, 65 insertions(+), 9 deletions(-) diff --git a/application/Dockerfile b/application/Dockerfile index d48fc1b4..c494bacc 100644 --- a/application/Dockerfile +++ b/application/Dockerfile @@ -12,11 +12,13 @@ RUN pip install -r requirements.txt FROM python:3.10-slim-bullseye # Copy pre-built packages from builder stage COPY --from=builder /usr/local/lib/python3.10/site-packages/ /usr/local/lib/python3.10/site-packages/ +RUN pip install gunicorn==20.1.0 +RUN pip install celery==5.2.7 WORKDIR /app COPY . /app ENV FLASK_APP=app.py ENV FLASK_DEBUG=true -RUN pip install gunicorn==20.1.0 + EXPOSE 5001 diff --git a/application/app.py b/application/app.py index d31fadae..195ef4e7 100644 --- a/application/app.py +++ b/application/app.py @@ -400,10 +400,16 @@ def delete_old(): """Delete old indexes.""" import shutil path = request.args.get('path') + dirs = path.split('/') first_dir = path.split('/')[0] + for i in range(1, len(dirs)): + dirs[i] = secure_filename(dirs[i]) + + # check that path strats with indexes or vectors - if first_dir not in ['indexes', 'vectors']: + if dirs[0] not in ['indexes', 'vectors']: return {"status": 'error'} + path = '/'.join(dirs) shutil.rmtree(path) vectors_collection.delete_one({'location': path}) return {"status": 'ok'} @@ -414,6 +420,7 @@ def after_request(response): response.headers.add('Access-Control-Allow-Origin', '*') response.headers.add('Access-Control-Allow-Headers', 'Content-Type,Authorization') response.headers.add('Access-Control-Allow-Methods', 'GET,PUT,POST,DELETE,OPTIONS') + response.headers.add('Access-Control-Allow-Credentials', 'true') return response diff --git a/application/requirements.txt b/application/requirements.txt index 80e28929..f4f3539e 100644 --- a/application/requirements.txt +++ b/application/requirements.txt @@ -3,18 +3,25 @@ aiohttp==3.8.4 aiohttp-retry==2.8.3 aiosignal==1.3.1 aleph-alpha-client==2.16.1 +amqp==5.1.1 async-timeout==4.0.2 attrs==22.2.0 +billiard==3.6.4.0 blobfile==2.0.1 boto3==1.26.84 botocore==1.29.84 cffi==1.15.1 charset-normalizer==3.1.0 click==8.1.3 +click-didyoumean==0.3.0 +click-plugins==1.1.1 +click-repl==0.2.0 cryptography==39.0.2 dataclasses-json==0.5.7 +decorator==5.1.1 deeplake==3.2.13 dill==0.3.6 +dnspython==2.3.0 ecdsa==0.18.0 entrypoints==0.4 faiss-cpu==1.7.3 @@ -29,6 +36,8 @@ idna==3.4 itsdangerous==2.1.2 Jinja2==3.1.2 jmespath==1.0.1 +joblib==1.2.0 +kombu==5.2.4 langchain==0.0.103 lxml==4.9.2 MarkupSafe==2.1.2 @@ -37,6 +46,7 @@ marshmallow-enum==1.5.1 multidict==6.0.4 multiprocess==0.70.14 mypy-extensions==1.0.0 +nltk==3.8.1 numcodecs==0.11.0 numpy==1.24.2 openai==0.27.0 @@ -45,18 +55,24 @@ pathos==0.3.0 Pillow==9.4.0 pox==0.3.2 ppft==1.7.6.6 +prompt-toolkit==3.0.38 +py==1.11.0 pyasn1==0.4.8 pycares==4.3.0 pycparser==2.21 pycryptodomex==3.17 pydantic==1.10.5 PyJWT==2.6.0 +pymongo==4.3.3 python-dateutil==2.8.2 python-dotenv==1.0.0 python-jose==3.3.0 +pytz==2022.7.1 PyYAML==6.0 +redis==4.5.1 regex==2022.10.31 requests==2.28.2 +retry==0.9.2 rsa==4.9 s3transfer==0.6.0 six==1.16.0 @@ -69,5 +85,7 @@ transformers==4.26.1 typing-inspect==0.8.0 typing_extensions==4.5.0 urllib3==1.26.14 +vine==5.0.0 +wcwidth==0.2.6 Werkzeug==2.2.3 yarl==1.8.2 diff --git a/application/worker.py b/application/worker.py index 5523131d..1a538f7f 100644 --- a/application/worker.py +++ b/application/worker.py @@ -8,11 +8,16 @@ from parser.schema.base import Document from parser.open_ai_func import call_openai_api from celery import current_task -nltk.download('punkt', quiet=True) -nltk.download('averaged_perceptron_tagger', quiet=True) + import string import zipfile import shutil + +try: + nltk.download('punkt', quiet=True) + nltk.download('averaged_perceptron_tagger', quiet=True) +except FileExistsError: + pass def generate_random_string(length): return ''.join([string.ascii_letters[i % 52] for i in range(length)]) @@ -29,7 +34,11 @@ def ingest_worker(self, directory, formats, name_job, filename, user): # filename = 'install.rst' # user = 'local' full_path = directory + '/' + user + '/' + name_job - url = 'http://localhost:5001/api/download' + # check if API_URL env variable is set + if not os.environ.get('API_URL'): + url = 'http://localhost:5001/api/download' + else: + url = os.environ.get('API_URL') + '/api/download' file_data = {'name': name_job, 'file': filename, 'user': user} response = requests.get(url, params=file_data) file = response.content @@ -62,14 +71,20 @@ def ingest_worker(self, directory, formats, name_job, filename, user): # get files from outputs/inputs/index.faiss and outputs/inputs/index.pkl # and send them to the server (provide user and name in form) - url = 'http://localhost:5001/api/upload_index' + if not os.environ.get('API_URL'): + url = 'http://localhost:5001/api/upload_index' + else: + url = os.environ.get('API_URL') + '/api/upload_index' file_data = {'name': name_job, 'user': user} files = {'file_faiss': open(full_path + '/index.faiss', 'rb'), 'file_pkl': open(full_path + '/index.pkl', 'rb')} response = requests.post(url, files=files, data=file_data) #deletes remote - url = 'http://localhost:5001/api/delete_old?path=' + 'inputs/' + user + '/' + name_job + if not os.environ.get('API_URL'): + url = 'http://localhost:5001/api/delete_old?path=' + 'inputs/' + user + '/' + name_job + else: + url = os.environ.get('API_URL') + '/api/delete_old?path=' + 'inputs/' + user + '/' + name_job response = requests.get(url) # delete local shutil.rmtree(full_path) diff --git a/docker-compose.yaml b/docker-compose.yaml index a30ec7a2..172b6bae 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -4,7 +4,7 @@ services: frontend: build: ./frontend environment: - - API_HOST=http://backend:5001 + - VITE_API_HOST=http://localhost:5001 ports: - "5173:5173" depends_on: @@ -12,6 +12,12 @@ services: backend: build: ./application + environment: + - API_KEY= + - EMBEDDINGS_KEY= + - CELERY_BROKER_URL=redis://redis:6379/0 + - CELERY_RESULT_BACKEND=redis://redis:6379/1 + - MONGO_URI=mongodb://mongo:27017/docsgpt ports: - "5001:5001" volumes: @@ -19,9 +25,17 @@ services: depends_on: - redis - mongo + worker: build: ./application - command: celery -A app.celery worker -l info + command: celery -A app.celery worker -l INFO + environment: + - API_KEY= + - EMBEDDINGS_KEY= + - CELERY_BROKER_URL=redis://redis:6379/0 + - CELERY_RESULT_BACKEND=redis://redis:6379/1 + - MONGO_URI=mongodb://mongo:27017/docsgpt + - API_URL=http://backend:5001 depends_on: - redis - mongo