From 237975bfef940e806fef36daf9b406c24b29e039 Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 22 Dec 2023 13:25:57 +0000 Subject: [PATCH 1/3] fix: cve upgrades --- application/Dockerfile | 6 +- application/requirements.txt | 190 +++++++++++++++++++++-------------- 2 files changed, 115 insertions(+), 81 deletions(-) diff --git a/application/Dockerfile b/application/Dockerfile index 8c083926..7ea99661 100644 --- a/application/Dockerfile +++ b/application/Dockerfile @@ -1,14 +1,14 @@ -FROM python:3.10-slim-bullseye as builder +FROM python:3.11-slim-bullseye as builder # Tiktoken requires Rust toolchain, so build it in a separate stage RUN apt-get update && apt-get install -y gcc curl RUN curl https://sh.rustup.rs -sSf | sh -s -- -y && apt-get install --reinstall libc6-dev -y ENV PATH="/root/.cargo/bin:${PATH}" -RUN pip install --upgrade pip && pip install tiktoken==0.3.3 +RUN pip install --upgrade pip && pip install tiktoken==0.5.2 COPY requirements.txt . RUN pip install -r requirements.txt -FROM python:3.10-slim-bullseye +FROM python:3.11-slim-bullseye # Copy pre-built packages and binaries from builder stage COPY --from=builder /usr/local/ /usr/local/ diff --git a/application/requirements.txt b/application/requirements.txt index cec8655b..c0a16bfa 100644 --- a/application/requirements.txt +++ b/application/requirements.txt @@ -1,111 +1,145 @@ -aiodns==3.0.0 -aiohttp==3.8.6 +aiodns==3.1.1 +aiohttp==3.9.1 aiohttp-retry==2.8.3 aiosignal==1.3.1 -aleph-alpha-client==2.16.1 -amqp==5.1.1 -anthropic==0.5.0 -async-timeout==4.0.2 -attrs==22.2.0 -billiard==3.6.4.0 -blobfile==2.0.1 -boto3==1.28.20 -celery==5.2.7 -cffi==1.15.1 -charset-normalizer==3.1.0 -click==8.1.3 +amqp==5.2.0 +annotated-types==0.6.0 +anthropic==0.8.0 +anyio==4.2.0 +async-timeout==4.0.3 +attrs==23.1.0 +billiard==4.2.0 +blinker==1.7.0 +blobfile==2.1.1 +boto3==1.34.6 +botocore==1.34.6 +celery==5.3.6 +certifi==2023.11.17 +cffi==1.16.0 +chardet==5.2.0 +charset-normalizer==3.3.2 +click==8.1.7 click-didyoumean==0.3.0 click-plugins==1.1.1 -click-repl==0.2.0 -cryptography==41.0.4 -dataclasses-json==0.5.7 +click-repl==0.3.0 +cryptography==41.0.7 +dataclasses-json==0.6.3 decorator==5.1.1 +dill==0.3.7 +distro==1.8.0 +dnspython==2.4.2 docx2txt==0.8 -dill==0.3.6 -dnspython==2.3.0 ecdsa==0.18.0 -elasticsearch==8.9.0 +elastic-transport==8.11.0 +elasticsearch==8.11.1 entrypoints==0.4 -faiss-cpu==1.7.3 -filelock==3.9.0 -Flask==2.2.5 -Flask-Cors==3.0.10 -frozenlist==1.3.3 +faiss-cpu==1.7.4 +filelock==3.13.1 +Flask==3.0.0 +Flask-Cors==4.0.0 +frozenlist==1.4.1 +fsspec==2023.12.2 geojson==2.5.0 -gunicorn==20.1.0 -greenlet==2.0.2 -gpt4all==0.1.7 -huggingface-hub==0.19.4 +greenlet==3.0.3 +gunicorn==21.2.0 +h11==0.14.0 +httpcore==1.0.2 +httpx==0.26.0 +huggingface-hub==0.20.1 humbug==0.3.2 -idna==3.4 +idna==3.6 +iniconfig==2.0.0 itsdangerous==2.1.2 Jinja2==3.1.2 jmespath==1.0.1 -joblib==1.2.0 -kombu==5.2.4 -langchain==0.0.312 -loguru==0.6.0 -lxml==4.9.2 -MarkupSafe==2.1.2 -marshmallow==3.19.0 +joblib==1.3.2 +jsonpatch==1.33 +jsonpointer==2.4 +kombu==5.3.4 +langchain==0.0.352 +langchain-community==0.0.6 +langchain-core==0.1.3 +langsmith==0.0.74 +lazy-object-proxy==1.10.0 +loguru==0.7.2 +lxml==4.9.4 +MarkupSafe==2.1.3 +marshmallow==3.20.1 marshmallow-enum==1.5.1 mpmath==1.3.0 multidict==6.0.4 -multiprocess==0.70.14 +multiprocess==0.70.15 mypy-extensions==1.0.0 -networkx==3.0 -npx +networkx==3.2.1 nltk==3.8.1 -numcodecs==0.11.0 -numpy==1.24.2 -openai==0.27.8 -openapi3-parser==1.1.14 -packaging==23.0 -pathos==0.3.0 -Pillow==10.0.1 -pox==0.3.2 -ppft==1.7.6.6 -prompt-toolkit==3.0.38 +npx==0.1.1 +numcodecs==0.12.1 +numpy==1.26.2 +openai==1.6.1 +openapi-schema-validator==0.6.2 +openapi-spec-validator==0.6.0 +openapi3-parser==1.1.16 +packaging==23.2 +pathable==0.4.3 +pathos==0.3.1 +Pillow==10.1.0 +pluggy==1.3.0 +pox==0.3.3 +ppft==1.7.6.7 +prance==23.6.21.0 +prompt-toolkit==3.0.43 py==1.11.0 -pyasn1==0.4.8 -pycares==4.3.0 +pyasn1==0.5.1 +pycares==4.4.0 pycparser==2.21 -pycryptodomex==3.17 pycryptodome==3.19.0 -pydantic==1.10.5 -PyJWT==2.6.0 -pymongo==4.3.3 +pycryptodomex==3.19.0 +pydantic==2.5.3 +pydantic_core==2.14.6 +PyJWT==2.8.0 +pymongo==4.6.1 pyowm==3.3.0 PyPDF2==3.0.1 PySocks==1.7.1 -pytest +pytest==7.4.3 python-dateutil==2.8.2 python-dotenv==1.0.0 python-jose==3.3.0 -pytz==2022.7.1 -PyYAML==6.0 -redis==4.5.4 -regex==2022.10.31 +python-liquid==1.10.2 +pytz==2023.3.post1 +PyYAML==6.0.1 +redis==5.0.1 +regex==2023.10.3 requests==2.31.0 retry==0.9.2 +rfc3339-validator==0.1.4 +rpds-py==0.15.2 rsa==4.9 -scikit-learn==1.2.2 -scipy==1.10.1 -sentencepiece +ruamel.yaml==0.18.5 +ruamel.yaml.clib==0.2.8 +s3transfer==0.10.0 +safetensors==0.4.1 +scikit-learn==1.3.2 +scipy==1.11.4 +sentence-transformers +sentencepiece==0.1.99 six==1.16.0 -SQLAlchemy==1.4.46 -sympy==1.11.1 -tenacity==8.2.2 -threadpoolctl==3.1.0 +sniffio==1.3.0 +SQLAlchemy==2.0.23 +sympy==1.12 +tenacity==8.2.3 +threadpoolctl==3.2.0 tiktoken tokenizers==0.15.0 -tqdm==4.65.0 -transformers==4.36.0 -typer==0.7.0 -typing-inspect==0.8.0 -typing_extensions==4.5.0 -urllib3==1.26.18 -vine==5.0.0 -wcwidth==0.2.6 -yarl==1.8.2 -sentence-transformers==2.2.2 +torch==2.1.2 +torchvision==0.16.2 +tqdm==4.66.1 +transformers==4.36.2 +typer==0.9.0 +typing-inspect==0.9.0 +typing_extensions==4.9.0 +tzdata==2023.3 +vine==5.1.0 +wcwidth==0.2.12 +Werkzeug==3.0.1 +yarl==1.9.4 From f039d37c8aabf71f177381071e7208078e8c89fc Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 22 Dec 2023 14:03:43 +0000 Subject: [PATCH 2/3] fix: pydantic --- application/core/settings.py | 33 +++++++++++++++++---------------- application/requirements.txt | 1 + 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/application/core/settings.py b/application/core/settings.py index 25a3295d..da6de245 100644 --- a/application/core/settings.py +++ b/application/core/settings.py @@ -1,7 +1,8 @@ from pathlib import Path +from typing import Optional import os -from pydantic import BaseSettings +from pydantic_settings import BaseSettings current_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) @@ -18,25 +19,25 @@ class Settings(BaseSettings): API_URL: str = "http://localhost:7091" # backend url for celery worker - API_KEY: str = None # LLM api key - EMBEDDINGS_KEY: str = None # api key for embeddings (if using openai, just copy API_KEY) - OPENAI_API_BASE: str = None # azure openai api base url - OPENAI_API_VERSION: str = None # azure openai api version - AZURE_DEPLOYMENT_NAME: str = None # azure deployment name for answering - AZURE_EMBEDDINGS_DEPLOYMENT_NAME: str = None # azure deployment name for embeddings + API_KEY: Optional[str] = None # LLM api key + EMBEDDINGS_KEY: Optional[str] = None # api key for embeddings (if using openai, just copy API_KEY) + OPENAI_API_BASE: Optional[str] = None # azure openai api base url + OPENAI_API_VERSION: Optional[str] = None # azure openai api version + AZURE_DEPLOYMENT_NAME: Optional[str] = None # azure deployment name for answering + AZURE_EMBEDDINGS_DEPLOYMENT_NAME: Optional[str] = None # azure deployment name for embeddings # elasticsearch - ELASTIC_CLOUD_ID: str = None # cloud id for elasticsearch - ELASTIC_USERNAME: str = None # username for elasticsearch - ELASTIC_PASSWORD: str = None # password for elasticsearch - ELASTIC_URL: str = None # url for elasticsearch - ELASTIC_INDEX: str = "docsgpt" # index name for elasticsearch + ELASTIC_CLOUD_ID: Optional[str] = None # cloud id for elasticsearch + ELASTIC_USERNAME: Optional[str] = None # username for elasticsearch + ELASTIC_PASSWORD: Optional[str] = None # password for elasticsearch + ELASTIC_URL: Optional[str] = None # url for elasticsearch + ELASTIC_INDEX: Optional[str] = "docsgpt" # index name for elasticsearch # SageMaker config - SAGEMAKER_ENDPOINT: str = None # SageMaker endpoint name - SAGEMAKER_REGION: str = None # SageMaker region name - SAGEMAKER_ACCESS_KEY: str = None # SageMaker access key - SAGEMAKER_SECRET_KEY: str = None # SageMaker secret key + SAGEMAKER_ENDPOINT: Optional[str] = None # SageMaker endpoint name + SAGEMAKER_REGION: Optional[str] = None # SageMaker region name + SAGEMAKER_ACCESS_KEY: Optional[str] = None # SageMaker access key + SAGEMAKER_SECRET_KEY: Optional[str] = None # SageMaker secret key path = Path(__file__).parent.parent.absolute() diff --git a/application/requirements.txt b/application/requirements.txt index c0a16bfa..ede39bcf 100644 --- a/application/requirements.txt +++ b/application/requirements.txt @@ -96,6 +96,7 @@ pycryptodome==3.19.0 pycryptodomex==3.19.0 pydantic==2.5.3 pydantic_core==2.14.6 +pydantic_settings-2.1.0 PyJWT==2.8.0 pymongo==4.6.1 pyowm==3.3.0 From c81b0200eb5a5d8153871848877b26b01058ed10 Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 22 Dec 2023 14:08:03 +0000 Subject: [PATCH 3/3] fix: Update pydantic_settings version to 2.1.0 --- application/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/application/requirements.txt b/application/requirements.txt index ede39bcf..bb7f0356 100644 --- a/application/requirements.txt +++ b/application/requirements.txt @@ -96,7 +96,7 @@ pycryptodome==3.19.0 pycryptodomex==3.19.0 pydantic==2.5.3 pydantic_core==2.14.6 -pydantic_settings-2.1.0 +pydantic_settings==2.1.0 PyJWT==2.8.0 pymongo==4.6.1 pyowm==3.3.0