From 898c24949bfcaee24e08198566afb15125679c4b Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 8 Apr 2021 20:07:52 +0200 Subject: [PATCH 1/5] Add chown method to support docker --- freqtrade/commands/build_config_commands.py | 2 ++ .../configuration/directory_operations.py | 16 +++++++++++++ tests/test_directory_operations.py | 23 +++++++++++++++++-- 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/freqtrade/commands/build_config_commands.py b/freqtrade/commands/build_config_commands.py index 0dee480b3..03d095e12 100644 --- a/freqtrade/commands/build_config_commands.py +++ b/freqtrade/commands/build_config_commands.py @@ -5,6 +5,7 @@ from typing import Any, Dict, List from questionary import Separator, prompt +from freqtrade.configuration.directory_operations import chown_user_directory from freqtrade.constants import UNLIMITED_STAKE_AMOUNT from freqtrade.exceptions import OperationalException from freqtrade.exchange import MAP_EXCHANGE_CHILDCLASS, available_exchanges @@ -216,6 +217,7 @@ def start_new_config(args: Dict[str, Any]) -> None: """ config_path = Path(args['config'][0]) + chown_user_directory(config_path.parent) if config_path.exists(): overwrite = ask_user_overwrite(config_path) if overwrite: diff --git a/freqtrade/configuration/directory_operations.py b/freqtrade/configuration/directory_operations.py index 1ce8d1461..ca305c260 100644 --- a/freqtrade/configuration/directory_operations.py +++ b/freqtrade/configuration/directory_operations.py @@ -24,6 +24,21 @@ def create_datadir(config: Dict[str, Any], datadir: Optional[str] = None) -> Pat return folder +def chown_user_directory(directory: Path) -> None: + """ + Use Sudo to change permissions of the home-directory if necessary + Only applies when running in docker! + """ + import os + if os.environ.get('FT_APP_ENV') == 'docker': + try: + import subprocess + subprocess.check_output( + ['sudo', 'chown', '-R', 'ftuser:', str(directory.resolve())]) + except Exception: + logger.warning(f"Could not chown {directory}") + + def create_userdata_dir(directory: str, create_dir: bool = False) -> Path: """ Create userdata directory structure. @@ -37,6 +52,7 @@ def create_userdata_dir(directory: str, create_dir: bool = False) -> Path: sub_dirs = ["backtest_results", "data", "hyperopts", "hyperopt_results", "logs", "notebooks", "plot", "strategies", ] folder = Path(directory) + chown_user_directory(folder) if not folder.is_dir(): if create_dir: folder.mkdir(parents=True) diff --git a/tests/test_directory_operations.py b/tests/test_directory_operations.py index a8058c514..a11200526 100644 --- a/tests/test_directory_operations.py +++ b/tests/test_directory_operations.py @@ -1,11 +1,12 @@ # pragma pylint: disable=missing-docstring, protected-access, invalid-name +import os from pathlib import Path from unittest.mock import MagicMock import pytest -from freqtrade.configuration.directory_operations import (copy_sample_files, create_datadir, - create_userdata_dir) +from freqtrade.configuration.directory_operations import (chown_user_directory, copy_sample_files, + create_datadir, create_userdata_dir) from freqtrade.exceptions import OperationalException from tests.conftest import log_has, log_has_re @@ -31,6 +32,24 @@ def test_create_userdata_dir(mocker, default_conf, caplog) -> None: assert str(x) == str(Path("/tmp/bar")) +def test_create_userdata_dir_and_chown(mocker, tmpdir, caplog) -> None: + sp_mock = mocker.patch('subprocess.check_output') + path = Path(tmpdir / 'bar') + assert not path.is_dir() + + x = create_userdata_dir(str(path), create_dir=True) + assert sp_mock.call_count == 0 + assert log_has(f'Created user-data directory: {path}', caplog) + assert isinstance(x, Path) + assert path.is_dir() + assert (path / 'data').is_dir() + + os.environ['FT_APP_ENV'] = 'docker' + chown_user_directory(path / 'data') + assert sp_mock.call_count == 1 + del os.environ['FT_APP_ENV'] + + def test_create_userdata_dir_exists(mocker, default_conf, caplog) -> None: mocker.patch.object(Path, "is_dir", MagicMock(return_value=True)) md = mocker.patch.object(Path, 'mkdir', MagicMock()) From 4eb251ce416cf3a3bf40071b8843814f38726cfd Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 8 Apr 2021 20:17:53 +0200 Subject: [PATCH 2/5] Update dockerfiles to run as non-root --- Dockerfile | 26 ++++++++++++++++++-------- Dockerfile.armhf | 28 +++++++++++++++++----------- docker/Dockerfile.custom | 8 ++++---- docker/Dockerfile.develop | 4 ++-- docker/Dockerfile.jupyter | 2 +- docker/Dockerfile.plot | 2 +- 6 files changed, 43 insertions(+), 27 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4b399174b..ac48ea611 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,10 +5,19 @@ ENV LANG C.UTF-8 ENV LC_ALL C.UTF-8 ENV PYTHONDONTWRITEBYTECODE 1 ENV PYTHONFAULTHANDLER 1 -ENV PATH=/root/.local/bin:$PATH +ENV PATH=/home/ftuser/.local/bin:$PATH +ENV FT_APP_ENV="docker" # Prepare environment -RUN mkdir /freqtrade +RUN mkdir /freqtrade \ + && apt update \ + && apt install -y sudo \ + && apt-get clean \ + && useradd -u 1000 -G sudo -U -m ftuser \ + && chown ftuser:ftuser /freqtrade \ + # Allow sudoers + && echo "ftuser ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers + WORKDIR /freqtrade # Install dependencies @@ -24,7 +33,8 @@ RUN cd /tmp && /tmp/install_ta-lib.sh && rm -r /tmp/*ta-lib* ENV LD_LIBRARY_PATH /usr/local/lib # Install dependencies -COPY requirements.txt requirements-hyperopt.txt /freqtrade/ +COPY --chown=ftuser:ftuser requirements.txt requirements-hyperopt.txt /freqtrade/ +USER ftuser RUN pip install --user --no-cache-dir numpy \ && pip install --user --no-cache-dir -r requirements-hyperopt.txt @@ -33,13 +43,13 @@ FROM base as runtime-image COPY --from=python-deps /usr/local/lib /usr/local/lib ENV LD_LIBRARY_PATH /usr/local/lib -COPY --from=python-deps /root/.local /root/.local - - +COPY --from=python-deps /home/ftuser/.local /home/ftuser/.local +USER ftuser # Install and execute -COPY . /freqtrade/ -RUN pip install -e . --no-cache-dir \ +COPY --chown=ftuser:ftuser . /freqtrade/ + +RUN pip install -e . --user --no-cache-dir \ && mkdir /freqtrade/user_data/ \ && freqtrade install-ui diff --git a/Dockerfile.armhf b/Dockerfile.armhf index eecd9fdc0..62ef165c0 100644 --- a/Dockerfile.armhf +++ b/Dockerfile.armhf @@ -5,15 +5,20 @@ ENV LANG C.UTF-8 ENV LC_ALL C.UTF-8 ENV PYTHONDONTWRITEBYTECODE 1 ENV PYTHONFAULTHANDLER 1 -ENV PATH=/root/.local/bin:$PATH +ENV PATH=/home/ftuser/.local/bin:$PATH +ENV FT_APP_ENV="docker" # Prepare environment -RUN mkdir /freqtrade -WORKDIR /freqtrade +RUN mkdir /freqtrade \ + && apt-get update \ + && apt-get -y install libatlas3-base curl sqlite3 libhdf5-serial-dev sudo \ + && apt-get clean \ + && useradd -u 1000 -G sudo -U -m ftuser \ + && chown ftuser:ftuser /freqtrade \ + # Allow sudoers + && echo "ftuser ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers -RUN apt-get update \ - && apt-get -y install libatlas3-base curl sqlite3 \ - && apt-get clean +WORKDIR /freqtrade # Install dependencies FROM base as python-deps @@ -37,13 +42,14 @@ FROM base as runtime-image COPY --from=python-deps /usr/local/lib /usr/local/lib ENV LD_LIBRARY_PATH /usr/local/lib -COPY --from=python-deps /root/.local /root/.local +COPY --from=python-deps /home/ftuser/.local /home/ftuser/.local +USER ftuser # Install and execute -COPY . /freqtrade/ -RUN apt-get install -y libhdf5-serial-dev \ - && apt-get clean \ - && pip install -e . --no-cache-dir \ +COPY --chown=ftuser:ftuser . /freqtrade/ + +RUN pip install -e . --user --no-cache-dir \ + && mkdir /freqtrade/user_data/ \ && freqtrade install-ui ENTRYPOINT ["freqtrade"] diff --git a/docker/Dockerfile.custom b/docker/Dockerfile.custom index 10620e6b8..a7c599fa8 100644 --- a/docker/Dockerfile.custom +++ b/docker/Dockerfile.custom @@ -1,7 +1,7 @@ FROM freqtradeorg/freqtrade:develop -RUN apt-get update \ - && apt-get -y install git \ - && apt-get clean \ +RUN sudo apt-get update \ + && sudo apt-get -y install git \ + && sudo apt-get clean \ # The below dependency - pyti - serves as an example. Please use whatever you need! - && pip install pyti + && pip install --user pyti diff --git a/docker/Dockerfile.develop b/docker/Dockerfile.develop index cb49984e2..7c580f234 100644 --- a/docker/Dockerfile.develop +++ b/docker/Dockerfile.develop @@ -3,8 +3,8 @@ FROM freqtradeorg/freqtrade:develop # Install dependencies COPY requirements-dev.txt /freqtrade/ -RUN pip install numpy --no-cache-dir \ - && pip install -r requirements-dev.txt --no-cache-dir +RUN pip install numpy --user --no-cache-dir \ + && pip install -r requirements-dev.txt --user --no-cache-dir # Empty the ENTRYPOINT to allow all commands ENTRYPOINT [] diff --git a/docker/Dockerfile.jupyter b/docker/Dockerfile.jupyter index b7499eeef..7d603c667 100644 --- a/docker/Dockerfile.jupyter +++ b/docker/Dockerfile.jupyter @@ -1,7 +1,7 @@ FROM freqtradeorg/freqtrade:develop_plot -RUN pip install jupyterlab --no-cache-dir +RUN pip install jupyterlab --user --no-cache-dir # Empty the ENTRYPOINT to allow all commands ENTRYPOINT [] diff --git a/docker/Dockerfile.plot b/docker/Dockerfile.plot index 40bc72bc5..d2fc3618a 100644 --- a/docker/Dockerfile.plot +++ b/docker/Dockerfile.plot @@ -4,4 +4,4 @@ FROM freqtradeorg/freqtrade:${sourceimage} # Install dependencies COPY requirements-plot.txt /freqtrade/ -RUN pip install -r requirements-plot.txt --no-cache-dir +RUN pip install -r requirements-plot.txt --user --no-cache-dir From 644dcc1641624700033b0865f8360a5264e51585 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 8 Apr 2021 20:36:10 +0200 Subject: [PATCH 3/5] Only allow chown via sudo --- Dockerfile | 2 +- Dockerfile.armhf | 2 +- docker/Dockerfile.custom | 13 ++++++++----- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index ac48ea611..a6dc9a991 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,7 @@ RUN mkdir /freqtrade \ && useradd -u 1000 -G sudo -U -m ftuser \ && chown ftuser:ftuser /freqtrade \ # Allow sudoers - && echo "ftuser ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers + && echo "ftuser ALL=(ALL) NOPASSWD: /bin/chown" >> /etc/sudoers WORKDIR /freqtrade diff --git a/Dockerfile.armhf b/Dockerfile.armhf index 62ef165c0..909c44eaa 100644 --- a/Dockerfile.armhf +++ b/Dockerfile.armhf @@ -16,7 +16,7 @@ RUN mkdir /freqtrade \ && useradd -u 1000 -G sudo -U -m ftuser \ && chown ftuser:ftuser /freqtrade \ # Allow sudoers - && echo "ftuser ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers + && echo "ftuser ALL=(ALL) NOPASSWD: /bin/chown" >> /etc/sudoers WORKDIR /freqtrade diff --git a/docker/Dockerfile.custom b/docker/Dockerfile.custom index a7c599fa8..3b55fcb0e 100644 --- a/docker/Dockerfile.custom +++ b/docker/Dockerfile.custom @@ -1,7 +1,10 @@ FROM freqtradeorg/freqtrade:develop -RUN sudo apt-get update \ - && sudo apt-get -y install git \ - && sudo apt-get clean \ - # The below dependency - pyti - serves as an example. Please use whatever you need! - && pip install --user pyti +# Switch user to root if you must install something from apt +# Don't forget to switch the user back below! +# USER root + +# The below dependency - pyti - serves as an example. Please use whatever you need! +RUN pip install --user pyti + +# USER ftuser From 4b2cec22ec645bd8a2ef0d75db1a3f2f0b586b1c Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 9 Apr 2021 19:33:40 +0200 Subject: [PATCH 4/5] Chown .local dir --- .dockerignore | 12 ++++++++++-- Dockerfile | 2 +- Dockerfile.armhf | 2 +- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/.dockerignore b/.dockerignore index 09f4c9f0c..889a4dfc7 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,9 +1,8 @@ .git .gitignore Dockerfile +Dockerfile.armhf .dockerignore -config.json* -*.sqlite .coveragerc .eggs .github @@ -13,4 +12,13 @@ CONTRIBUTING.md MANIFEST.in README.md freqtrade.service +freqtrade.egg-info + +config.json* +*.sqlite user_data +*.log + +.vscode +.mypy_cache +.ipynb_checkpoints diff --git a/Dockerfile b/Dockerfile index a6dc9a991..5d8bcdd41 100644 --- a/Dockerfile +++ b/Dockerfile @@ -43,7 +43,7 @@ FROM base as runtime-image COPY --from=python-deps /usr/local/lib /usr/local/lib ENV LD_LIBRARY_PATH /usr/local/lib -COPY --from=python-deps /home/ftuser/.local /home/ftuser/.local +COPY --from=python-deps --chown=ftuser:ftuser /home/ftuser/.local /home/ftuser/.local USER ftuser # Install and execute diff --git a/Dockerfile.armhf b/Dockerfile.armhf index 909c44eaa..df1c4bc80 100644 --- a/Dockerfile.armhf +++ b/Dockerfile.armhf @@ -42,7 +42,7 @@ FROM base as runtime-image COPY --from=python-deps /usr/local/lib /usr/local/lib ENV LD_LIBRARY_PATH /usr/local/lib -COPY --from=python-deps /home/ftuser/.local /home/ftuser/.local +COPY --from=python-deps --chown=ftuser:ftuser /home/ftuser/.local /home/ftuser/.local USER ftuser # Install and execute From 126127c1e11a09a4d4653c0ca9551ffc8cf35ab3 Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 9 Apr 2021 21:28:38 +0200 Subject: [PATCH 5/5] Fix armHF image to use ftuser on install too --- Dockerfile.armhf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile.armhf b/Dockerfile.armhf index df1c4bc80..332b91b35 100644 --- a/Dockerfile.armhf +++ b/Dockerfile.armhf @@ -33,7 +33,8 @@ RUN cd /tmp && /tmp/install_ta-lib.sh && rm -r /tmp/*ta-lib* ENV LD_LIBRARY_PATH /usr/local/lib # Install dependencies -COPY requirements.txt /freqtrade/ +COPY --chown=ftuser:ftuser requirements.txt /freqtrade/ +USER ftuser RUN pip install --user --no-cache-dir numpy \ && pip install --user --no-cache-dir -r requirements.txt