From 916a847639e15017a6d0ecfdb6f14d1a91407567 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 24 Dec 2023 08:30:21 +0100 Subject: [PATCH 1/7] add pytest-xdist to speed up tests --- .github/workflows/ci.yml | 8 ++++---- requirements-dev.txt | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4f8ece4cc..03dc406c9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,7 +66,7 @@ jobs: - name: Tests run: | - pytest --random-order --cov=freqtrade --cov-config=.coveragerc + pytest --random-order -n auto --cov=freqtrade --cov-config=.coveragerc - name: Coveralls if: (runner.os == 'Linux' && matrix.python-version == '3.10' && matrix.os == 'ubuntu-22.04') @@ -188,7 +188,7 @@ jobs: - name: Tests run: | - pytest --random-order + pytest --random-order -n auto - name: Check for repository changes run: | @@ -262,7 +262,7 @@ jobs: - name: Tests run: | - pytest --random-order + pytest --random-order -n auto - name: Check for repository changes run: | @@ -402,7 +402,7 @@ jobs: env: CI_WEB_PROXY: http://152.67.78.211:13128 run: | - pytest --random-order --cov=freqtrade --cov-config=.coveragerc --longrun + pytest --random-order --cov=freqtrade --cov-config=.coveragerc --longrun -n auto # Notify only once - when CI completes (and after deploy) in case it's successfull diff --git a/requirements-dev.txt b/requirements-dev.txt index d6da8f08f..f1e29f401 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -15,6 +15,7 @@ pytest-asyncio==0.21.1 pytest-cov==4.1.0 pytest-mock==3.12.0 pytest-random-order==1.1.0 +pytest-xdist==3.5.0 isort==5.13.2 # For datetime mocking time-machine==2.13.0 From 12e75e849df14bbcd58335cf543ee157db056518 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 24 Dec 2023 12:09:46 +0100 Subject: [PATCH 2/7] Don't do coverge on --longrun tests --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 03dc406c9..7a2fcd30f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -402,7 +402,7 @@ jobs: env: CI_WEB_PROXY: http://152.67.78.211:13128 run: | - pytest --random-order --cov=freqtrade --cov-config=.coveragerc --longrun -n auto + pytest --random-order --longrun -n auto # Notify only once - when CI completes (and after deploy) in case it's successfull From ddbfce01b54ec0858f6da2698238c82c5f268b8f Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 24 Dec 2023 12:13:02 +0100 Subject: [PATCH 3/7] Show slowest tests --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7a2fcd30f..777501999 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -402,7 +402,7 @@ jobs: env: CI_WEB_PROXY: http://152.67.78.211:13128 run: | - pytest --random-order --longrun -n auto + pytest --random-order --longrun --durations 20 -n auto # Notify only once - when CI completes (and after deploy) in case it's successfull From 0a96aa69b9f47912adceb0d457f200f74dd85612 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 24 Dec 2023 12:15:36 +0100 Subject: [PATCH 4/7] Only run parallel on online tests --- .github/workflows/ci.yml | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 777501999..f01ee4fa6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -44,7 +44,6 @@ jobs: - name: pip cache (linux) uses: actions/cache@v3 - if: runner.os == 'Linux' with: path: ~/.cache/pip key: test-${{ matrix.os }}-${{ matrix.python-version }}-pip @@ -55,7 +54,6 @@ jobs: cd build_helpers && ./install_ta-lib.sh ${HOME}/dependencies/; cd .. - name: Installation - *nix - if: runner.os == 'Linux' run: | python -m pip install --upgrade pip wheel export LD_LIBRARY_PATH=${HOME}/dependencies/lib:$LD_LIBRARY_PATH @@ -66,7 +64,7 @@ jobs: - name: Tests run: | - pytest --random-order -n auto --cov=freqtrade --cov-config=.coveragerc + pytest --random-order --cov=freqtrade --cov-config=.coveragerc - name: Coveralls if: (runner.os == 'Linux' && matrix.python-version == '3.10' && matrix.os == 'ubuntu-22.04') @@ -188,7 +186,7 @@ jobs: - name: Tests run: | - pytest --random-order -n auto + pytest --random-order - name: Check for repository changes run: | @@ -262,7 +260,7 @@ jobs: - name: Tests run: | - pytest --random-order -n auto + pytest --random-order - name: Check for repository changes run: | @@ -367,7 +365,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: "3.9" + python-version: "3.11" - name: Cache_dependencies uses: actions/cache@v3 @@ -378,7 +376,6 @@ jobs: - name: pip cache (linux) uses: actions/cache@v3 - if: runner.os == 'Linux' with: path: ~/.cache/pip key: test-${{ matrix.os }}-${{ matrix.python-version }}-pip @@ -389,7 +386,6 @@ jobs: cd build_helpers && ./install_ta-lib.sh ${HOME}/dependencies/; cd .. - name: Installation - *nix - if: runner.os == 'Linux' run: | python -m pip install --upgrade pip wheel export LD_LIBRARY_PATH=${HOME}/dependencies/lib:$LD_LIBRARY_PATH From 03fb204408ce036eccfeddd4024db761687df32f Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 24 Dec 2023 13:57:42 +0100 Subject: [PATCH 5/7] use Loadscope to improve parallel test performance --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f01ee4fa6..2d1df9a05 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -398,7 +398,7 @@ jobs: env: CI_WEB_PROXY: http://152.67.78.211:13128 run: | - pytest --random-order --longrun --durations 20 -n auto + pytest --random-order --longrun --durations 20 -n auto --dist loadscope # Notify only once - when CI completes (and after deploy) in case it's successfull From fc13a99b331e6897ed51e69a35eed75ab71ba034 Mon Sep 17 00:00:00 2001 From: Matthias Date: Sun, 24 Dec 2023 16:27:18 +0100 Subject: [PATCH 6/7] Use intelligent scheduling to improve online test performance --- tests/conftest.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index 2d7a805b1..80d164ee4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,6 +11,7 @@ from unittest.mock import MagicMock, Mock, PropertyMock import numpy as np import pandas as pd import pytest +from xdist.scheduler.loadscope import LoadScopeScheduling from freqtrade import constants from freqtrade.commands import Arguments @@ -56,6 +57,27 @@ def pytest_configure(config): setattr(config.option, 'markexpr', 'not longrun') +class FixtureScheduler(LoadScopeScheduling): + # Based on the suggestion in + # https://github.com/pytest-dev/pytest-xdist/issues/18 + + def _split_scope(self, nodeid): + if 'exchange_online' in nodeid: + try: + # Extract exchange ID from nodeid + exchange_id = nodeid.split('[')[1].split('-')[0].rstrip(']') + return exchange_id + except Exception as e: + print(e) + pass + + return nodeid + + +def pytest_xdist_make_scheduler(config, log): + return FixtureScheduler(config, log) + + def log_has(line, logs): """Check if line is found on some caplog's message.""" return any(line == message for message in logs.messages) From 97780ee5bf313665fe326376ab5acde8e6eaf12b Mon Sep 17 00:00:00 2001 From: Matthias Date: Mon, 25 Dec 2023 15:07:06 +0100 Subject: [PATCH 7/7] Revert build-online to 3.9 for now --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2d1df9a05..953cd6c07 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -365,7 +365,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: "3.11" + python-version: "3.9" - name: Cache_dependencies uses: actions/cache@v3