From 7d38963c61d7082330c64c8ed24c584fbdd2a4a6 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 2 Sep 2025 19:32:15 +0200 Subject: [PATCH 01/18] chore(ci): extract environment parts to ci file --- .github/workflows/docker-build.yml | 34 +++++++++++++++++++++++++++ build_helpers/publish_docker_multi.sh | 20 ++-------------- 2 files changed, 36 insertions(+), 18 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index a781dbd17..2ef681e17 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -24,6 +24,7 @@ env: IMAGE_NAME: "freqtradeorg/freqtrade" CACHE_IMAGE: "freqtradeorg/freqtrade_cache" GHCR_IMAGE_NAME: "ghcr.io/freqtrade/freqtrade" + PI_PLATFORM: "linux/arm/v7" jobs: deploy-docker: @@ -50,9 +51,37 @@ jobs: else BRANCH_NAME="${GITHUB_REF##*/}" fi + + # Replace / with _ to create a valid tag + TAG=$(echo "${BRANCH_NAME}" | sed -e "s/\//_/g") + TAG_PLOT=${TAG}_plot + TAG_FREQAI=${TAG}_freqai + TAG_FREQAI_RL=${TAG_FREQAI}rl + TAG_PI="${TAG}_pi" + + CACHE_TAG_PI=${CACHE_IMAGE}:${TAG_PI}_cache + echo "GITHUB_REF='${GITHUB_REF}'" echo "BRANCH_NAME='${BRANCH_NAME}'" + echo "TAG='${TAG}'" + echo "TAG_PLOT='${TAG_PLOT}'" + echo "TAG_FREQAI='${TAG_FREQAI}'" + echo "TAG_FREQAI_RL='${TAG_FREQAI_RL}'" + echo "TAG_PI='${TAG_PI}'" + echo "BRANCH_NAME=${BRANCH_NAME}" >> "$GITHUB_ENV" + echo "TAG=${TAG}" >> "$GITHUB_ENV" + echo "TAG_PLOT=${TAG_PLOT}" >> "$GITHUB_ENV" + echo "TAG_FREQAI=${TAG_FREQAI}" >> "$GITHUB_ENV" + echo "TAG_FREQAI_RL=${TAG_FREQAI_RL}" >> "$GITHUB_ENV" + echo "TAG_PI=${TAG_PI}" >> "$GITHUB_ENV" + + echo "CACHE_TAG_PI=${CACHE_TAG_PI}" >> "$GITHUB_ENV" + + - name: Save commit SHA to file + # Add commit to docker container + run: | + echo "${GITHUB_SHA}" > freqtrade_commit - name: Login to Docker Hub uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0 @@ -104,6 +133,11 @@ jobs: echo "BRANCH_NAME='${BRANCH_NAME}'" echo "BRANCH_NAME=${BRANCH_NAME}" >> "$GITHUB_ENV" + - name: Save commit SHA to file + # Add commit to docker container + run: | + echo "${GITHUB_SHA}" > freqtrade_commit + - name: Login to Docker Hub uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0 with: diff --git a/build_helpers/publish_docker_multi.sh b/build_helpers/publish_docker_multi.sh index 52dbe7244..188f84689 100755 --- a/build_helpers/publish_docker_multi.sh +++ b/build_helpers/publish_docker_multi.sh @@ -1,28 +1,12 @@ #!/bin/sh -# The below assumes a correctly setup docker buildx environment - -# Replace / with _ to create a valid tag -TAG=$(echo "${BRANCH_NAME}" | sed -e "s/\//_/g") -TAG_PLOT=${TAG}_plot -TAG_FREQAI=${TAG}_freqai -TAG_FREQAI_RL=${TAG_FREQAI}rl -TAG_PI="${TAG}_pi" - -PI_PLATFORM="linux/arm/v7" -echo "Running for ${TAG}" -CACHE_TAG=${CACHE_IMAGE}:${TAG_PI}_cache - -# Add commit and commit_message to docker container -echo "${GITHUB_SHA}" > freqtrade_commit - if [ "${GITHUB_EVENT_NAME}" = "schedule" ]; then echo "event ${GITHUB_EVENT_NAME}: full rebuild - skipping cache" # Build regular image docker build -t freqtrade:${TAG} . # Build PI image docker buildx build \ - --cache-to=type=registry,ref=${CACHE_TAG} \ + --cache-to=type=registry,ref=${CACHE_TAG_PI} \ -f docker/Dockerfile.armhf \ --platform ${PI_PLATFORM} \ -t ${IMAGE_NAME}:${TAG_PI} \ @@ -39,7 +23,7 @@ else # docker pull --platform ${PI_PLATFORM} ${IMAGE_NAME}:${TAG} # disable provenance due to https://github.com/docker/buildx/issues/1509 docker buildx build \ - --cache-from=type=registry,ref=${CACHE_TAG} \ + --cache-from=type=registry,ref=${CACHE_TAG_PI} \ --cache-to=type=registry,ref=${CACHE_TAG} \ -f docker/Dockerfile.armhf \ --platform ${PI_PLATFORM} \ From bb7a4c473926e10fff0cab31d3a915c92cf3b603 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 2 Sep 2025 19:39:21 +0200 Subject: [PATCH 02/18] chore(ci): move CI build to action * Reduce the need for a helper script * increase visibility into individual build times --- .github/workflows/docker-build.yml | 65 +++++++++++++++++++++++- build_helpers/publish_docker_multi.sh | 71 --------------------------- 2 files changed, 63 insertions(+), 73 deletions(-) delete mode 100755 build_helpers/publish_docker_multi.sh diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 2ef681e17..7ac98e0fb 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -102,9 +102,70 @@ jobs: env: PLATFORMS: ${{ steps.buildx.outputs.platforms }} - - name: Build and test and push docker images + - name: Build image without cache + if: github.event_name == 'schedule' run: | - build_helpers/publish_docker_multi.sh + docker build -t freqtrade:${TAG} . + + - name: Build ARMHF image without cache + if: github.event_name == 'schedule' + run: | + docker buildx build \ + --cache-to=type=registry,ref=${CACHE_TAG_PI} \ + -f docker/Dockerfile.armhf \ + --platform ${PI_PLATFORM} \ + -t ${IMAGE_NAME}:${TAG_PI} \ + --push \ + --provenance=false \ + . + + - name: Build image without cache + if: github.event_name != 'schedule' + run: | + docker pull ${IMAGE_NAME}:${TAG} + docker build --cache-from ${IMAGE_NAME}:${TAG} -t freqtrade:${TAG} . + + - name: Build ARMHF image without cache + if: github.event_name != 'schedule' + # disable provenance due to https://github.com/docker/buildx/issues/1509 + run: | + docker buildx build \ + --cache-from=type=registry,ref=${CACHE_TAG_PI} \ + --cache-to=type=registry,ref=${CACHE_TAG} \ + -f docker/Dockerfile.armhf \ + --platform ${PI_PLATFORM} \ + -t ${IMAGE_NAME}:${TAG_PI} \ + --push \ + --provenance=false \ + . + + - name: Run build for AI images + run: | + docker build --build-arg sourceimage=freqtrade --build-arg sourcetag=${TAG} -t freqtrade:${TAG_PLOT} -f docker/Dockerfile.plot . + docker build --build-arg sourceimage=freqtrade --build-arg sourcetag=${TAG} -t freqtrade:${TAG_FREQAI} -f docker/Dockerfile.freqai . + docker build --build-arg sourceimage=freqtrade --build-arg sourcetag=${TAG_FREQAI} -t freqtrade:${TAG_FREQAI_RL} -f docker/Dockerfile.freqai_rl . + + - name: Tag images for caching + run: | + docker tag freqtrade:$TAG ${CACHE_IMAGE}:$TAG + docker tag freqtrade:$TAG_PLOT ${CACHE_IMAGE}:$TAG_PLOT + docker tag freqtrade:$TAG_FREQAI ${CACHE_IMAGE}:$TAG_FREQAI + docker tag freqtrade:$TAG_FREQAI_RL ${CACHE_IMAGE}:$TAG_FREQAI_RL + + - name: Run backtest in Docker + run: | + docker run --rm -v $(pwd)/tests/testdata/config.tests.json:/freqtrade/config.json:ro -v $(pwd)/tests:/tests freqtrade:${TAG} backtesting --datadir /tests/testdata --strategy-path /tests/strategy/strats/ --strategy StrategyTestV3 + + - name: push cache images + run: | + docker push ${CACHE_IMAGE}:$TAG + docker push ${CACHE_IMAGE}:$TAG_PLOT + docker push ${CACHE_IMAGE}:$TAG_FREQAI + docker push ${CACHE_IMAGE}:$TAG_FREQAI_RL + + - name: list Images + run: | + docker images deploy-arm: name: "Deploy Docker ARM64" diff --git a/build_helpers/publish_docker_multi.sh b/build_helpers/publish_docker_multi.sh deleted file mode 100755 index 188f84689..000000000 --- a/build_helpers/publish_docker_multi.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/bin/sh - -if [ "${GITHUB_EVENT_NAME}" = "schedule" ]; then - echo "event ${GITHUB_EVENT_NAME}: full rebuild - skipping cache" - # Build regular image - docker build -t freqtrade:${TAG} . - # Build PI image - docker buildx build \ - --cache-to=type=registry,ref=${CACHE_TAG_PI} \ - -f docker/Dockerfile.armhf \ - --platform ${PI_PLATFORM} \ - -t ${IMAGE_NAME}:${TAG_PI} \ - --push \ - --provenance=false \ - . -else - echo "event ${GITHUB_EVENT_NAME}: building with cache" - # Build regular image - docker pull ${IMAGE_NAME}:${TAG} - docker build --cache-from ${IMAGE_NAME}:${TAG} -t freqtrade:${TAG} . - - # Pull last build to avoid rebuilding the whole image - # docker pull --platform ${PI_PLATFORM} ${IMAGE_NAME}:${TAG} - # disable provenance due to https://github.com/docker/buildx/issues/1509 - docker buildx build \ - --cache-from=type=registry,ref=${CACHE_TAG_PI} \ - --cache-to=type=registry,ref=${CACHE_TAG} \ - -f docker/Dockerfile.armhf \ - --platform ${PI_PLATFORM} \ - -t ${IMAGE_NAME}:${TAG_PI} \ - --push \ - --provenance=false \ - . -fi - -if [ $? -ne 0 ]; then - echo "failed building multiarch images" - return 1 -fi -# Tag image for upload and next build step -docker tag freqtrade:$TAG ${CACHE_IMAGE}:$TAG - -docker build --build-arg sourceimage=freqtrade --build-arg sourcetag=${TAG} -t freqtrade:${TAG_PLOT} -f docker/Dockerfile.plot . -docker build --build-arg sourceimage=freqtrade --build-arg sourcetag=${TAG} -t freqtrade:${TAG_FREQAI} -f docker/Dockerfile.freqai . -docker build --build-arg sourceimage=freqtrade --build-arg sourcetag=${TAG_FREQAI} -t freqtrade:${TAG_FREQAI_RL} -f docker/Dockerfile.freqai_rl . - -docker tag freqtrade:$TAG_PLOT ${CACHE_IMAGE}:$TAG_PLOT -docker tag freqtrade:$TAG_FREQAI ${CACHE_IMAGE}:$TAG_FREQAI -docker tag freqtrade:$TAG_FREQAI_RL ${CACHE_IMAGE}:$TAG_FREQAI_RL - -# Run backtest -docker run --rm -v $(pwd)/tests/testdata/config.tests.json:/freqtrade/config.json:ro -v $(pwd)/tests:/tests freqtrade:${TAG} backtesting --datadir /tests/testdata --strategy-path /tests/strategy/strats/ --strategy StrategyTestV3 - -if [ $? -ne 0 ]; then - echo "failed running backtest" - return 1 -fi - -docker images - -docker push ${CACHE_IMAGE}:$TAG -docker push ${CACHE_IMAGE}:$TAG_PLOT -docker push ${CACHE_IMAGE}:$TAG_FREQAI -docker push ${CACHE_IMAGE}:$TAG_FREQAI_RL - -docker images - -if [ $? -ne 0 ]; then - echo "failed building image" - return 1 -fi From 53328bc59567111cdbd60d2ca3105b50826ffd59 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 2 Sep 2025 19:41:21 +0200 Subject: [PATCH 03/18] chore(ci): allow failure on pull --- .github/workflows/docker-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 7ac98e0fb..a3b17f4e9 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -122,7 +122,7 @@ jobs: - name: Build image without cache if: github.event_name != 'schedule' run: | - docker pull ${IMAGE_NAME}:${TAG} + docker pull ${IMAGE_NAME}:${TAG} || true docker build --cache-from ${IMAGE_NAME}:${TAG} -t freqtrade:${TAG} . - name: Build ARMHF image without cache From 35c3ca91eb2b87e72d16d73185fccc94e8d6c9cc Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 2 Sep 2025 19:44:03 +0200 Subject: [PATCH 04/18] chore(ci): fix small typo --- .github/workflows/docker-build.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index a3b17f4e9..8062a9a00 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -89,7 +89,6 @@ jobs: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - - name: Set up QEMU uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0 @@ -131,7 +130,7 @@ jobs: run: | docker buildx build \ --cache-from=type=registry,ref=${CACHE_TAG_PI} \ - --cache-to=type=registry,ref=${CACHE_TAG} \ + --cache-to=type=registry,ref=${CACHE_TAG_PI} \ -f docker/Dockerfile.armhf \ --platform ${PI_PLATFORM} \ -t ${IMAGE_NAME}:${TAG_PI} \ From 94aba0f9acb75213d123b21bb1a721c7786a71c9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 2 Sep 2025 19:48:11 +0200 Subject: [PATCH 05/18] chore(ci) fix step naming --- .github/workflows/docker-build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 8062a9a00..473f82aa8 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -118,13 +118,13 @@ jobs: --provenance=false \ . - - name: Build image without cache + - name: Build image with cache if: github.event_name != 'schedule' run: | docker pull ${IMAGE_NAME}:${TAG} || true docker build --cache-from ${IMAGE_NAME}:${TAG} -t freqtrade:${TAG} . - - name: Build ARMHF image without cache + - name: Build ARMHF image with cache if: github.event_name != 'schedule' # disable provenance due to https://github.com/docker/buildx/issues/1509 run: | From a94d18a10bc27a9bccb2c2df9b80e9bba5229a1f Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 2 Sep 2025 19:57:33 +0200 Subject: [PATCH 06/18] chore(ci): use composite action to extract docker variables --- .github/actions/docker-tags/action.yml | 45 ++++++++++++++++++++++++++ .github/workflows/docker-build.yml | 37 ++------------------- 2 files changed, 47 insertions(+), 35 deletions(-) create mode 100644 .github/actions/docker-tags/action.yml diff --git a/.github/actions/docker-tags/action.yml b/.github/actions/docker-tags/action.yml new file mode 100644 index 000000000..60bb80cef --- /dev/null +++ b/.github/actions/docker-tags/action.yml @@ -0,0 +1,45 @@ +name: 'docker-tags' +description: 'Set Docker default Tag environment variables' +# inputs: +# outputs: +runs: + using: "composite" + steps: + - name: Extract branch name + shell: bash + env: + BRANCH_NAME_INPUT: ${{ github.event.inputs.branch_name }} + run: | + if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + BRANCH_NAME="${BRANCH_NAME_INPUT}" + else + BRANCH_NAME="${GITHUB_REF##*/}" + fi + + # Replace / with _ to create a valid tag + TAG=$(echo "${BRANCH_NAME}" | sed -e "s/\//_/g") + TAG_PLOT=${TAG}_plot + TAG_FREQAI=${TAG}_freqai + TAG_FREQAI_RL=${TAG_FREQAI}rl + TAG_PI="${TAG}_pi" + + CACHE_TAG_PI=${CACHE_IMAGE}:${TAG_PI}_cache + + echo "GITHUB_REF='${GITHUB_REF}'" + echo "BRANCH_NAME='${BRANCH_NAME}'" + echo "TAG='${TAG}'" + echo "TAG_PLOT='${TAG_PLOT}'" + echo "TAG_FREQAI='${TAG_FREQAI}'" + echo "TAG_FREQAI_RL='${TAG_FREQAI_RL}'" + echo "TAG_PI='${TAG_PI}'" + + echo "BRANCH_NAME=${BRANCH_NAME}" >> "$GITHUB_ENV" + echo "TAG=${TAG}" >> "$GITHUB_ENV" + echo "TAG_PLOT=${TAG_PLOT}" >> "$GITHUB_ENV" + echo "TAG_FREQAI=${TAG_FREQAI}" >> "$GITHUB_ENV" + echo "TAG_FREQAI_RL=${TAG_FREQAI_RL}" >> "$GITHUB_ENV" + echo "TAG_PI=${TAG_PI}" >> "$GITHUB_ENV" + + echo "CACHE_TAG_PI=${CACHE_TAG_PI}" >> "$GITHUB_ENV" + + diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 473f82aa8..67a6f6c22 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -42,41 +42,8 @@ jobs: with: python-version: "3.12" - - name: Extract branch name - env: - BRANCH_NAME_INPUT: ${{ github.event.inputs.branch_name }} - run: | - if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then - BRANCH_NAME="${BRANCH_NAME_INPUT}" - else - BRANCH_NAME="${GITHUB_REF##*/}" - fi - - # Replace / with _ to create a valid tag - TAG=$(echo "${BRANCH_NAME}" | sed -e "s/\//_/g") - TAG_PLOT=${TAG}_plot - TAG_FREQAI=${TAG}_freqai - TAG_FREQAI_RL=${TAG_FREQAI}rl - TAG_PI="${TAG}_pi" - - CACHE_TAG_PI=${CACHE_IMAGE}:${TAG_PI}_cache - - echo "GITHUB_REF='${GITHUB_REF}'" - echo "BRANCH_NAME='${BRANCH_NAME}'" - echo "TAG='${TAG}'" - echo "TAG_PLOT='${TAG_PLOT}'" - echo "TAG_FREQAI='${TAG_FREQAI}'" - echo "TAG_FREQAI_RL='${TAG_FREQAI_RL}'" - echo "TAG_PI='${TAG_PI}'" - - echo "BRANCH_NAME=${BRANCH_NAME}" >> "$GITHUB_ENV" - echo "TAG=${TAG}" >> "$GITHUB_ENV" - echo "TAG_PLOT=${TAG_PLOT}" >> "$GITHUB_ENV" - echo "TAG_FREQAI=${TAG_FREQAI}" >> "$GITHUB_ENV" - echo "TAG_FREQAI_RL=${TAG_FREQAI_RL}" >> "$GITHUB_ENV" - echo "TAG_PI=${TAG_PI}" >> "$GITHUB_ENV" - - echo "CACHE_TAG_PI=${CACHE_TAG_PI}" >> "$GITHUB_ENV" + - name: Set docker tag names + uses: ./.github/actions/docker-tags - name: Save commit SHA to file # Add commit to docker container From 488a608c90d877baa33017ceae123ba392330c97 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 2 Sep 2025 20:05:49 +0200 Subject: [PATCH 07/18] chore(ci): simplify build steps --- .github/actions/docker-tags/action.yml | 23 +++++++++++++++++++- .github/workflows/docker-build.yml | 29 ++------------------------ build_helpers/publish_docker_arm64.sh | 17 --------------- 3 files changed, 24 insertions(+), 45 deletions(-) diff --git a/.github/actions/docker-tags/action.yml b/.github/actions/docker-tags/action.yml index 60bb80cef..bac281d6d 100644 --- a/.github/actions/docker-tags/action.yml +++ b/.github/actions/docker-tags/action.yml @@ -21,6 +21,13 @@ runs: TAG_PLOT=${TAG}_plot TAG_FREQAI=${TAG}_freqai TAG_FREQAI_RL=${TAG_FREQAI}rl + TAG_FREQAI_TORCH=${TAG_FREQAI}torch + + TAG_ARM=${TAG}_arm + TAG_PLOT_ARM=${TAG_PLOT}_arm + TAG_FREQAI_ARM=${TAG_FREQAI}_arm + TAG_FREQAI_RL_ARM=${TAG_FREQAI_RL}_arm + TAG_PI="${TAG}_pi" CACHE_TAG_PI=${CACHE_IMAGE}:${TAG_PI}_cache @@ -31,6 +38,11 @@ runs: echo "TAG_PLOT='${TAG_PLOT}'" echo "TAG_FREQAI='${TAG_FREQAI}'" echo "TAG_FREQAI_RL='${TAG_FREQAI_RL}'" + echo "TAG_FREQAI_TORCH='${TAG_FREQAI_TORCH}'" + echo "TAG_ARM='${TAG_ARM}'" + echo "TAG_PLOT_ARM='${TAG_PLOT_ARM}'" + echo "TAG_FREQAI_ARM='${TAG_FREQAI_ARM}'" + echo "TAG_FREQAI_RL_ARM='${TAG_FREQAI_RL_ARM}'" echo "TAG_PI='${TAG_PI}'" echo "BRANCH_NAME=${BRANCH_NAME}" >> "$GITHUB_ENV" @@ -38,8 +50,17 @@ runs: echo "TAG_PLOT=${TAG_PLOT}" >> "$GITHUB_ENV" echo "TAG_FREQAI=${TAG_FREQAI}" >> "$GITHUB_ENV" echo "TAG_FREQAI_RL=${TAG_FREQAI_RL}" >> "$GITHUB_ENV" + echo "TAG_FREQAI_TORCH=${TAG_FREQAI_TORCH}" >> "$GITHUB_ENV" + echo "TAG_ARM=${TAG_ARM}" >> "$GITHUB_ENV" + echo "TAG_PLOT_ARM=${TAG_PLOT_ARM}" >> "$GITHUB_ENV" + echo "TAG_FREQAI_ARM=${TAG_FREQAI_ARM}" >> "$GITHUB_ENV" + echo "TAG_FREQAI_RL_ARM=${TAG_FREQAI_RL_ARM}" >> "$GITHUB_ENV" echo "TAG_PI=${TAG_PI}" >> "$GITHUB_ENV" echo "CACHE_TAG_PI=${CACHE_TAG_PI}" >> "$GITHUB_ENV" - + - name: Save commit SHA to file + shell: bash + # Add commit to docker container + run: | + echo "${GITHUB_SHA}" > freqtrade_commit diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 67a6f6c22..8f3c23665 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -37,19 +37,9 @@ jobs: with: persist-credentials: false - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - name: Set docker tag names uses: ./.github/actions/docker-tags - - name: Save commit SHA to file - # Add commit to docker container - run: | - echo "${GITHUB_SHA}" > freqtrade_commit - - name: Login to Docker Hub uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0 with: @@ -147,23 +137,8 @@ jobs: with: persist-credentials: false - - name: Extract branch name - env: - BRANCH_NAME_INPUT: ${{ github.event.inputs.branch_name }} - run: | - if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then - BRANCH_NAME="${BRANCH_NAME_INPUT}" - else - BRANCH_NAME="${GITHUB_REF##*/}" - fi - echo "GITHUB_REF='${GITHUB_REF}'" - echo "BRANCH_NAME='${BRANCH_NAME}'" - echo "BRANCH_NAME=${BRANCH_NAME}" >> "$GITHUB_ENV" - - - name: Save commit SHA to file - # Add commit to docker container - run: | - echo "${GITHUB_SHA}" > freqtrade_commit + - name: Set docker tag names + uses: ./.github/actions/docker-tags - name: Login to Docker Hub uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0 diff --git a/build_helpers/publish_docker_arm64.sh b/build_helpers/publish_docker_arm64.sh index 990f23e81..b5bebbf83 100755 --- a/build_helpers/publish_docker_arm64.sh +++ b/build_helpers/publish_docker_arm64.sh @@ -1,25 +1,8 @@ #!/bin/sh -# Use BuildKit, otherwise building on ARM fails - -# Replace / with _ to create a valid tag -TAG=$(echo "${BRANCH_NAME}" | sed -e "s/\//_/g") -TAG_PLOT=${TAG}_plot -TAG_FREQAI=${TAG}_freqai -TAG_FREQAI_RL=${TAG_FREQAI}rl -TAG_FREQAI_TORCH=${TAG_FREQAI}torch -TAG_PI="${TAG}_pi" - -TAG_ARM=${TAG}_arm -TAG_PLOT_ARM=${TAG_PLOT}_arm -TAG_FREQAI_ARM=${TAG_FREQAI}_arm -TAG_FREQAI_RL_ARM=${TAG_FREQAI_RL}_arm echo "Running for ${TAG}" -# Add commit and commit_message to docker container -echo "${GITHUB_SHA}" > freqtrade_commit - if [ "${GITHUB_EVENT_NAME}" = "schedule" ]; then echo "event ${GITHUB_EVENT_NAME}: full rebuild - skipping cache" # Build regular image From 23d4d25e397f21a652656bb7bc8f7c4b7bc9123c Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 2 Sep 2025 20:36:42 +0200 Subject: [PATCH 08/18] chore(ci): move everything to docker build action --- .github/workflows/docker-build.yml | 91 ++++++++++++++++++++++++- build_helpers/publish_docker_arm64.sh | 97 --------------------------- 2 files changed, 88 insertions(+), 100 deletions(-) delete mode 100755 build_helpers/publish_docker_arm64.sh diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 8f3c23665..06d84cc5d 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -112,7 +112,7 @@ jobs: run: | docker run --rm -v $(pwd)/tests/testdata/config.tests.json:/freqtrade/config.json:ro -v $(pwd)/tests:/tests freqtrade:${TAG} backtesting --datadir /tests/testdata --strategy-path /tests/strategy/strats/ --strategy StrategyTestV3 - - name: push cache images + - name: Push cache images run: | docker push ${CACHE_IMAGE}:$TAG docker push ${CACHE_IMAGE}:$TAG_PLOT @@ -146,12 +146,97 @@ jobs: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - - name: Build and test and push docker images + - name: Build image without cache + if: github.event_name == 'schedule' + run: | + docker build -t freqtrade:${TAG_ARM} . + + - name: Build image with cache + if: github.event_name != 'schedule' + run: | + docker pull ${IMAGE_NAME}:${TAG_ARM} || true + docker build --cache-from ${IMAGE_NAME}:${TAG_ARM} -t freqtrade:${TAG_ARM} . + + - name: Run build for AI images + run: | + docker build --build-arg sourceimage=freqtrade --build-arg sourcetag=${TAG_ARM} -t freqtrade:${TAG_PLOT_ARM} -f docker/Dockerfile.plot . + docker build --build-arg sourceimage=freqtrade --build-arg sourcetag=${TAG_ARM} -t freqtrade:${TAG_FREQAI_ARM} -f docker/Dockerfile.freqai . + docker build --build-arg sourceimage=freqtrade --build-arg sourcetag=${TAG_FREQAI_ARM} -t freqtrade:${TAG_FREQAI_RL_ARM} -f docker/Dockerfile.freqai_rl . + + - name: Tag images for caching + run: | + docker tag freqtrade:$TAG_ARM ${CACHE_IMAGE}:$TAG_ARM + docker tag freqtrade:$TAG_PLOT_ARM ${CACHE_IMAGE}:$TAG_PLOT_ARM + docker tag freqtrade:$TAG_FREQAI_ARM ${CACHE_IMAGE}:$TAG_FREQAI_ARM + docker tag freqtrade:$TAG_FREQAI_RL_ARM ${CACHE_IMAGE}:$TAG_FREQAI_RL_ARM + + - name: Run backtest in Docker + run: | + docker run --rm -v $(pwd)/tests/testdata/config.tests.json:/freqtrade/config.json:ro -v $(pwd)/tests:/tests freqtrade:${TAG_ARM} backtesting --datadir /tests/testdata --strategy-path /tests/strategy/strats/ --strategy StrategyTestV3 + + - name: Docker images + run: | + docker images + + - name: Push cache images + run: | + docker push ${CACHE_IMAGE}:$TAG_PLOT_ARM + docker push ${CACHE_IMAGE}:$TAG_FREQAI_ARM + docker push ${CACHE_IMAGE}:$TAG_FREQAI_RL_ARM + docker push ${CACHE_IMAGE}:$TAG_ARM + + - name: Create manifests + run: | + docker manifest create ${IMAGE_NAME}:${TAG} ${CACHE_IMAGE}:${TAG} ${CACHE_IMAGE}:${TAG_ARM} ${IMAGE_NAME}:${TAG_PI} + docker manifest push -p ${IMAGE_NAME}:${TAG} + + docker manifest create ${IMAGE_NAME}:${TAG_PLOT} ${CACHE_IMAGE}:${TAG_PLOT} ${CACHE_IMAGE}:${TAG_PLOT_ARM} + docker manifest push -p ${IMAGE_NAME}:${TAG_PLOT} + + docker manifest create ${IMAGE_NAME}:${TAG_FREQAI} ${CACHE_IMAGE}:${TAG_FREQAI} ${CACHE_IMAGE}:${TAG_FREQAI_ARM} + docker manifest push -p ${IMAGE_NAME}:${TAG_FREQAI} + + docker manifest create ${IMAGE_NAME}:${TAG_FREQAI_RL} ${CACHE_IMAGE}:${TAG_FREQAI_RL} ${CACHE_IMAGE}:${TAG_FREQAI_RL_ARM} + docker manifest push -p ${IMAGE_NAME}:${TAG_FREQAI_RL} + + # Create special Torch tag - which is identical to the RL tag. + docker manifest create ${IMAGE_NAME}:${TAG_FREQAI_TORCH} ${CACHE_IMAGE}:${TAG_FREQAI_RL} ${CACHE_IMAGE}:${TAG_FREQAI_RL_ARM} + docker manifest push -p ${IMAGE_NAME}:${TAG_FREQAI_TORCH} + + - name: Move images to ghcr.io env: GHCR_USERNAME: ${{ github.actor }} GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - build_helpers/publish_docker_arm64.sh + # setup Crane + alias crane="docker run --rm -i -v $(pwd)/.crane:/home/nonroot/.docker/ gcr.io/go-containerregistry/crane" + mkdir .crane + chmod a+rwx .crane + + echo "${GHCR_TOKEN}" | crane auth login ghcr.io -u "${GHCR_USERNAME}" --password-stdin + + crane copy ${IMAGE_NAME}:${TAG_FREQAI_RL} ${GHCR_IMAGE_NAME}:${TAG_FREQAI_RL} + crane copy ${IMAGE_NAME}:${TAG_FREQAI_RL} ${GHCR_IMAGE_NAME}:${TAG_FREQAI_TORCH} + crane copy ${IMAGE_NAME}:${TAG_FREQAI} ${GHCR_IMAGE_NAME}:${TAG_FREQAI} + crane copy ${IMAGE_NAME}:${TAG_PLOT} ${GHCR_IMAGE_NAME}:${TAG_PLOT} + crane copy ${IMAGE_NAME}:${TAG} ${GHCR_IMAGE_NAME}:${TAG} + + - name: Tag latest + if: env.TAG == 'develop' + run: | + # Tag image as latest + docker manifest create ${IMAGE_NAME}:latest ${CACHE_IMAGE}:${TAG_ARM} ${IMAGE_NAME}:${TAG_PI} ${CACHE_IMAGE}:${TAG} + docker manifest push -p ${IMAGE_NAME}:latest + crane copy ${IMAGE_NAME}:latest ${GHCR_IMAGE_NAME}:latest + + - name: Crane cleanup + run: | + rm -rf .crane + + + - name: Image cleanup + run: | + docker image prune -a --force --filter "until=24h" - name: Discord notification uses: rjstone/discord-webhook-notify@c2597273488aeda841dd1e891321952b51f7996f #v2.2.1 diff --git a/build_helpers/publish_docker_arm64.sh b/build_helpers/publish_docker_arm64.sh deleted file mode 100755 index b5bebbf83..000000000 --- a/build_helpers/publish_docker_arm64.sh +++ /dev/null @@ -1,97 +0,0 @@ -#!/bin/sh - - -echo "Running for ${TAG}" - -if [ "${GITHUB_EVENT_NAME}" = "schedule" ]; then - echo "event ${GITHUB_EVENT_NAME}: full rebuild - skipping cache" - # Build regular image - docker build -t freqtrade:${TAG_ARM} . - -else - echo "event ${GITHUB_EVENT_NAME}: building with cache" - # Build regular image - docker pull ${IMAGE_NAME}:${TAG_ARM} - docker build --cache-from ${IMAGE_NAME}:${TAG_ARM} -t freqtrade:${TAG_ARM} . - -fi - -if [ $? -ne 0 ]; then - echo "failed building multiarch images" - return 1 -fi - -docker build --build-arg sourceimage=freqtrade --build-arg sourcetag=${TAG_ARM} -t freqtrade:${TAG_PLOT_ARM} -f docker/Dockerfile.plot . -docker build --build-arg sourceimage=freqtrade --build-arg sourcetag=${TAG_ARM} -t freqtrade:${TAG_FREQAI_ARM} -f docker/Dockerfile.freqai . -docker build --build-arg sourceimage=freqtrade --build-arg sourcetag=${TAG_FREQAI_ARM} -t freqtrade:${TAG_FREQAI_RL_ARM} -f docker/Dockerfile.freqai_rl . - -# Tag image for upload and next build step -docker tag freqtrade:$TAG_ARM ${CACHE_IMAGE}:$TAG_ARM -docker tag freqtrade:$TAG_PLOT_ARM ${CACHE_IMAGE}:$TAG_PLOT_ARM -docker tag freqtrade:$TAG_FREQAI_ARM ${CACHE_IMAGE}:$TAG_FREQAI_ARM -docker tag freqtrade:$TAG_FREQAI_RL_ARM ${CACHE_IMAGE}:$TAG_FREQAI_RL_ARM - -# Run backtest -docker run --rm -v $(pwd)/tests/testdata/config.tests.json:/freqtrade/config.json:ro -v $(pwd)/tests:/tests freqtrade:${TAG_ARM} backtesting --datadir /tests/testdata --strategy-path /tests/strategy/strats/ --strategy StrategyTestV3 - -if [ $? -ne 0 ]; then - echo "failed running backtest" - return 1 -fi - -docker images - -docker push ${CACHE_IMAGE}:$TAG_PLOT_ARM -docker push ${CACHE_IMAGE}:$TAG_FREQAI_ARM -docker push ${CACHE_IMAGE}:$TAG_FREQAI_RL_ARM -docker push ${CACHE_IMAGE}:$TAG_ARM - -# Create multi-arch image -# Make sure that all images contained here are pushed to github first. -# Otherwise installation might fail. -echo "create manifests" - -docker manifest create ${IMAGE_NAME}:${TAG} ${CACHE_IMAGE}:${TAG} ${CACHE_IMAGE}:${TAG_ARM} ${IMAGE_NAME}:${TAG_PI} -docker manifest push -p ${IMAGE_NAME}:${TAG} - -docker manifest create ${IMAGE_NAME}:${TAG_PLOT} ${CACHE_IMAGE}:${TAG_PLOT} ${CACHE_IMAGE}:${TAG_PLOT_ARM} -docker manifest push -p ${IMAGE_NAME}:${TAG_PLOT} - -docker manifest create ${IMAGE_NAME}:${TAG_FREQAI} ${CACHE_IMAGE}:${TAG_FREQAI} ${CACHE_IMAGE}:${TAG_FREQAI_ARM} -docker manifest push -p ${IMAGE_NAME}:${TAG_FREQAI} - -docker manifest create ${IMAGE_NAME}:${TAG_FREQAI_RL} ${CACHE_IMAGE}:${TAG_FREQAI_RL} ${CACHE_IMAGE}:${TAG_FREQAI_RL_ARM} -docker manifest push -p ${IMAGE_NAME}:${TAG_FREQAI_RL} - -# Create special Torch tag - which is identical to the RL tag. -docker manifest create ${IMAGE_NAME}:${TAG_FREQAI_TORCH} ${CACHE_IMAGE}:${TAG_FREQAI_RL} ${CACHE_IMAGE}:${TAG_FREQAI_RL_ARM} -docker manifest push -p ${IMAGE_NAME}:${TAG_FREQAI_TORCH} - -# copy images to ghcr.io - -alias crane="docker run --rm -i -v $(pwd)/.crane:/home/nonroot/.docker/ gcr.io/go-containerregistry/crane" -mkdir .crane -chmod a+rwx .crane - -echo "${GHCR_TOKEN}" | crane auth login ghcr.io -u "${GHCR_USERNAME}" --password-stdin - -crane copy ${IMAGE_NAME}:${TAG_FREQAI_RL} ${GHCR_IMAGE_NAME}:${TAG_FREQAI_RL} -crane copy ${IMAGE_NAME}:${TAG_FREQAI_RL} ${GHCR_IMAGE_NAME}:${TAG_FREQAI_TORCH} -crane copy ${IMAGE_NAME}:${TAG_FREQAI} ${GHCR_IMAGE_NAME}:${TAG_FREQAI} -crane copy ${IMAGE_NAME}:${TAG_PLOT} ${GHCR_IMAGE_NAME}:${TAG_PLOT} -crane copy ${IMAGE_NAME}:${TAG} ${GHCR_IMAGE_NAME}:${TAG} - -# Tag as latest for develop builds -if [ "${TAG}" = "develop" ]; then - echo 'Tagging image as latest' - docker manifest create ${IMAGE_NAME}:latest ${CACHE_IMAGE}:${TAG_ARM} ${IMAGE_NAME}:${TAG_PI} ${CACHE_IMAGE}:${TAG} - docker manifest push -p ${IMAGE_NAME}:latest - - crane copy ${IMAGE_NAME}:latest ${GHCR_IMAGE_NAME}:latest -fi - -docker images -rm -rf .crane - -# Cleanup old images from arm64 node. -docker image prune -a --force --filter "until=24h" From 89272e9ddb06bb97a0b75483d4314bf4f36e4679 Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 2 Sep 2025 20:48:50 +0200 Subject: [PATCH 09/18] chore(ci): don't cache qemu image --- .github/workflows/docker-build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 06d84cc5d..766328630 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -48,6 +48,8 @@ jobs: - name: Set up QEMU uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0 + with: + cache-image: false - name: Set up Docker Buildx id: buildx From 662747c06c33f52dd036538f036607eea718597d Mon Sep 17 00:00:00 2001 From: Matthias Date: Tue, 2 Sep 2025 21:14:46 +0200 Subject: [PATCH 10/18] chore(ci): enable aliases --- .github/workflows/docker-build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 766328630..830b52642 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -210,6 +210,7 @@ jobs: GHCR_USERNAME: ${{ github.actor }} GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | + shopt -s expand_aliases # setup Crane alias crane="docker run --rm -i -v $(pwd)/.crane:/home/nonroot/.docker/ gcr.io/go-containerregistry/crane" mkdir .crane From 387431e4dfd8dc77daf15d20de2f048538c7893a Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 3 Sep 2025 07:25:09 +0200 Subject: [PATCH 11/18] chore(ci): use docker manifest to copy to ghcr --- .github/workflows/docker-build.yml | 41 ++++++++++++------------------ 1 file changed, 16 insertions(+), 25 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 830b52642..77025b7e8 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -205,37 +205,28 @@ jobs: docker manifest create ${IMAGE_NAME}:${TAG_FREQAI_TORCH} ${CACHE_IMAGE}:${TAG_FREQAI_RL} ${CACHE_IMAGE}:${TAG_FREQAI_RL_ARM} docker manifest push -p ${IMAGE_NAME}:${TAG_FREQAI_TORCH} - - name: Move images to ghcr.io - env: - GHCR_USERNAME: ${{ github.actor }} - GHCR_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Login to github + uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0 + with: + username: ${{ secrets.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + registry: ghcr.io + + - name: Push to ghcr.io run: | - shopt -s expand_aliases - # setup Crane - alias crane="docker run --rm -i -v $(pwd)/.crane:/home/nonroot/.docker/ gcr.io/go-containerregistry/crane" - mkdir .crane - chmod a+rwx .crane - - echo "${GHCR_TOKEN}" | crane auth login ghcr.io -u "${GHCR_USERNAME}" --password-stdin - - crane copy ${IMAGE_NAME}:${TAG_FREQAI_RL} ${GHCR_IMAGE_NAME}:${TAG_FREQAI_RL} - crane copy ${IMAGE_NAME}:${TAG_FREQAI_RL} ${GHCR_IMAGE_NAME}:${TAG_FREQAI_TORCH} - crane copy ${IMAGE_NAME}:${TAG_FREQAI} ${GHCR_IMAGE_NAME}:${TAG_FREQAI} - crane copy ${IMAGE_NAME}:${TAG_PLOT} ${GHCR_IMAGE_NAME}:${TAG_PLOT} - crane copy ${IMAGE_NAME}:${TAG} ${GHCR_IMAGE_NAME}:${TAG} + docker manifest create -t ${GHCR_IMAGE_NAME}:${TAG} ${IMAGE_NAME}:${TAG} + docker manifest create -t ${GHCR_IMAGE_NAME}:${TAG_PLOT} ${IMAGE_NAME}:${TAG_PLOT} + docker manifest create -t ${GHCR_IMAGE_NAME}:${TAG_FREQAI} ${IMAGE_NAME}:${TAG_FREQAI} + docker manifest create -t ${GHCR_IMAGE_NAME}:${TAG_FREQAI_RL} ${IMAGE_NAME}:${TAG_FREQAI_RL} + docker manifest create -t ${GHCR_IMAGE_NAME}:${TAG_FREQAI_TORCH} ${IMAGE_NAME}:${TAG_FREQAI_TORCH} - name: Tag latest if: env.TAG == 'develop' run: | # Tag image as latest - docker manifest create ${IMAGE_NAME}:latest ${CACHE_IMAGE}:${TAG_ARM} ${IMAGE_NAME}:${TAG_PI} ${CACHE_IMAGE}:${TAG} - docker manifest push -p ${IMAGE_NAME}:latest - crane copy ${IMAGE_NAME}:latest ${GHCR_IMAGE_NAME}:latest - - - name: Crane cleanup - run: | - rm -rf .crane - + docker manifest create -t ${IMAGE_NAME}:latest ${IMAGE_NAME}:${TAG} + docker manifest create -t ${GHCR_IMAGE_NAME}:latest ${IMAGE_NAME}:${TAG} - name: Image cleanup run: | From 84ec09565aa0b3d73854c416cab191d031b2d0a9 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 3 Sep 2025 08:01:19 +0200 Subject: [PATCH 12/18] chore(ci): Fix docker github login --- .github/workflows/docker-build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 77025b7e8..329956678 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -209,9 +209,9 @@ jobs: - name: Login to github uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0 with: - username: ${{ secrets.actor }} - password: ${{ secrets.GITHUB_TOKEN }} registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - name: Push to ghcr.io run: | From d40efb883c5ae69eed195c59554fdc72044e8d55 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 3 Sep 2025 08:50:02 +0200 Subject: [PATCH 13/18] chore(ci): use proper manifest command --- .github/workflows/docker-build.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 329956678..815331b9a 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -215,18 +215,18 @@ jobs: - name: Push to ghcr.io run: | - docker manifest create -t ${GHCR_IMAGE_NAME}:${TAG} ${IMAGE_NAME}:${TAG} - docker manifest create -t ${GHCR_IMAGE_NAME}:${TAG_PLOT} ${IMAGE_NAME}:${TAG_PLOT} - docker manifest create -t ${GHCR_IMAGE_NAME}:${TAG_FREQAI} ${IMAGE_NAME}:${TAG_FREQAI} - docker manifest create -t ${GHCR_IMAGE_NAME}:${TAG_FREQAI_RL} ${IMAGE_NAME}:${TAG_FREQAI_RL} - docker manifest create -t ${GHCR_IMAGE_NAME}:${TAG_FREQAI_TORCH} ${IMAGE_NAME}:${TAG_FREQAI_TORCH} + docker manifest create ${GHCR_IMAGE_NAME}:${TAG} ${IMAGE_NAME}:${TAG} + docker manifest create ${GHCR_IMAGE_NAME}:${TAG_PLOT} ${IMAGE_NAME}:${TAG_PLOT} + docker manifest create ${GHCR_IMAGE_NAME}:${TAG_FREQAI} ${IMAGE_NAME}:${TAG_FREQAI} + docker manifest create ${GHCR_IMAGE_NAME}:${TAG_FREQAI_RL} ${IMAGE_NAME}:${TAG_FREQAI_RL} + docker manifest create ${GHCR_IMAGE_NAME}:${TAG_FREQAI_TORCH} ${IMAGE_NAME}:${TAG_FREQAI_TORCH} - name: Tag latest if: env.TAG == 'develop' run: | # Tag image as latest - docker manifest create -t ${IMAGE_NAME}:latest ${IMAGE_NAME}:${TAG} - docker manifest create -t ${GHCR_IMAGE_NAME}:latest ${IMAGE_NAME}:${TAG} + docker manifest create ${IMAGE_NAME}:latest ${IMAGE_NAME}:${TAG} + docker manifest create ${GHCR_IMAGE_NAME}:latest ${IMAGE_NAME}:${TAG} - name: Image cleanup run: | From 3934aa118f3e0ef0539f5abbc3382594c92d3a94 Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 3 Sep 2025 19:36:23 +0200 Subject: [PATCH 14/18] chore(ci): use docker buildx imagetool create --- .github/workflows/docker-build.yml | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 815331b9a..157cc3b05 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -215,18 +215,23 @@ jobs: - name: Push to ghcr.io run: | - docker manifest create ${GHCR_IMAGE_NAME}:${TAG} ${IMAGE_NAME}:${TAG} - docker manifest create ${GHCR_IMAGE_NAME}:${TAG_PLOT} ${IMAGE_NAME}:${TAG_PLOT} - docker manifest create ${GHCR_IMAGE_NAME}:${TAG_FREQAI} ${IMAGE_NAME}:${TAG_FREQAI} - docker manifest create ${GHCR_IMAGE_NAME}:${TAG_FREQAI_RL} ${IMAGE_NAME}:${TAG_FREQAI_RL} - docker manifest create ${GHCR_IMAGE_NAME}:${TAG_FREQAI_TORCH} ${IMAGE_NAME}:${TAG_FREQAI_TORCH} + docker buildx imagetools create --tag ${GHCR_IMAGE_NAME}:${TAG} ${IMAGE_NAME}:${TAG} + docker buildx imagetools create --tag ${GHCR_IMAGE_NAME}:${TAG_PLOT} ${IMAGE_NAME}:${TAG_PLOT} + docker buildx imagetools create --tag ${GHCR_IMAGE_NAME}:${TAG_FREQAI} ${IMAGE_NAME}:${TAG_FREQAI} + docker buildx imagetools create --tag ${GHCR_IMAGE_NAME}:${TAG_FREQAI_RL} --tag ${GHCR_IMAGE_NAME}:${TAG_FREQAI_TORCH} ${IMAGE_NAME}:${TAG_FREQAI_RL} - name: Tag latest if: env.TAG == 'develop' run: | # Tag image as latest - docker manifest create ${IMAGE_NAME}:latest ${IMAGE_NAME}:${TAG} - docker manifest create ${GHCR_IMAGE_NAME}:latest ${IMAGE_NAME}:${TAG} + docker buildx imagetools create \ + --tag ${GHCR_IMAGE_NAME}:${TAG} \ + --tag ${GHCR_IMAGE_NAME}:latest \ + ${IMAGE_NAME}:${TAG} + + - name: Docker images + run: | + docker images - name: Image cleanup run: | From 48ff0c5744c3007e3dc2011ac082ae77b91e52bd Mon Sep 17 00:00:00 2001 From: Matthias Date: Wed, 3 Sep 2025 20:51:50 +0200 Subject: [PATCH 15/18] chore(ci): Use buildx imagetools only --- .github/workflows/docker-build.yml | 61 ++++++++++++++++-------------- 1 file changed, 33 insertions(+), 28 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 157cc3b05..6710889a9 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -148,6 +148,13 @@ jobs: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} + - name: Login to github + uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build image without cache if: github.event_name == 'schedule' run: | @@ -189,36 +196,34 @@ jobs: - name: Create manifests run: | - docker manifest create ${IMAGE_NAME}:${TAG} ${CACHE_IMAGE}:${TAG} ${CACHE_IMAGE}:${TAG_ARM} ${IMAGE_NAME}:${TAG_PI} - docker manifest push -p ${IMAGE_NAME}:${TAG} + docker buildx imagetools create \ + --tag ${IMAGE_NAME}:${TAG} \ + --tag ${GHCR_IMAGE_NAME}:${TAG} \ + ${CACHE_IMAGE}:${TAG} ${CACHE_IMAGE}:${TAG_ARM} ${IMAGE_NAME}:${TAG_PI} - docker manifest create ${IMAGE_NAME}:${TAG_PLOT} ${CACHE_IMAGE}:${TAG_PLOT} ${CACHE_IMAGE}:${TAG_PLOT_ARM} - docker manifest push -p ${IMAGE_NAME}:${TAG_PLOT} - - docker manifest create ${IMAGE_NAME}:${TAG_FREQAI} ${CACHE_IMAGE}:${TAG_FREQAI} ${CACHE_IMAGE}:${TAG_FREQAI_ARM} - docker manifest push -p ${IMAGE_NAME}:${TAG_FREQAI} - - docker manifest create ${IMAGE_NAME}:${TAG_FREQAI_RL} ${CACHE_IMAGE}:${TAG_FREQAI_RL} ${CACHE_IMAGE}:${TAG_FREQAI_RL_ARM} - docker manifest push -p ${IMAGE_NAME}:${TAG_FREQAI_RL} - - # Create special Torch tag - which is identical to the RL tag. - docker manifest create ${IMAGE_NAME}:${TAG_FREQAI_TORCH} ${CACHE_IMAGE}:${TAG_FREQAI_RL} ${CACHE_IMAGE}:${TAG_FREQAI_RL_ARM} - docker manifest push -p ${IMAGE_NAME}:${TAG_FREQAI_TORCH} - - - - name: Login to github - uses: docker/login-action@184bdaa0721073962dff0199f1fb9940f07167d1 # v3.5.0 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Push to ghcr.io + - name: Create multiarch image - Plot run: | - docker buildx imagetools create --tag ${GHCR_IMAGE_NAME}:${TAG} ${IMAGE_NAME}:${TAG} - docker buildx imagetools create --tag ${GHCR_IMAGE_NAME}:${TAG_PLOT} ${IMAGE_NAME}:${TAG_PLOT} - docker buildx imagetools create --tag ${GHCR_IMAGE_NAME}:${TAG_FREQAI} ${IMAGE_NAME}:${TAG_FREQAI} - docker buildx imagetools create --tag ${GHCR_IMAGE_NAME}:${TAG_FREQAI_RL} --tag ${GHCR_IMAGE_NAME}:${TAG_FREQAI_TORCH} ${IMAGE_NAME}:${TAG_FREQAI_RL} + docker buildx imagetools create \ + --tag ${IMAGE_NAME}:${TAG_PLOT} \ + --tag ${GHCR_IMAGE_NAME}:${TAG_PLOT} \ + ${CACHE_IMAGE}:${TAG_PLOT} ${CACHE_IMAGE}:${TAG_PLOT_ARM} + + - name: Create multiarch image - FreqAI + run: | + docker buildx imagetools create \ + --tag ${IMAGE_NAME}:${TAG_FREQAI} \ + --tag ${GHCR_IMAGE_NAME}:${TAG_FREQAI} \ + ${CACHE_IMAGE}:${TAG_FREQAI} ${CACHE_IMAGE}:${TAG_FREQAI_ARM} + + - name: Create multiarch image - FreqAI RL + run: | + # Create special Torch tag - which is identical to the RL tag. + docker buildx imagetools create \ + --tag ${IMAGE_NAME}:${TAG_FREQAI_RL} \ + --tag ${GHCR_IMAGE_NAME}:${TAG_FREQAI_RL} \ + --tag ${IMAGE_NAME}:${TAG_FREQAI_TORCH} \ + --tag ${GHCR_IMAGE_NAME}:${TAG_FREQAI_TORCH} \ + ${CACHE_IMAGE}:${TAG_FREQAI_RL} ${CACHE_IMAGE}:${TAG_FREQAI_RL_ARM} - name: Tag latest if: env.TAG == 'develop' From 44b65daf1ce851719acf321a55278efc451a6e6f Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 4 Sep 2025 19:49:38 +0200 Subject: [PATCH 16/18] chore(ci): Switch to use outputs instead of global environments --- .github/actions/docker-tags/action.yml | 37 ++++------- .github/workflows/docker-build.yml | 89 +++++++++++++++++++------- 2 files changed, 78 insertions(+), 48 deletions(-) diff --git a/.github/actions/docker-tags/action.yml b/.github/actions/docker-tags/action.yml index bac281d6d..3ddd2fadf 100644 --- a/.github/actions/docker-tags/action.yml +++ b/.github/actions/docker-tags/action.yml @@ -32,32 +32,21 @@ runs: CACHE_TAG_PI=${CACHE_IMAGE}:${TAG_PI}_cache - echo "GITHUB_REF='${GITHUB_REF}'" - echo "BRANCH_NAME='${BRANCH_NAME}'" - echo "TAG='${TAG}'" - echo "TAG_PLOT='${TAG_PLOT}'" - echo "TAG_FREQAI='${TAG_FREQAI}'" - echo "TAG_FREQAI_RL='${TAG_FREQAI_RL}'" - echo "TAG_FREQAI_TORCH='${TAG_FREQAI_TORCH}'" - echo "TAG_ARM='${TAG_ARM}'" - echo "TAG_PLOT_ARM='${TAG_PLOT_ARM}'" - echo "TAG_FREQAI_ARM='${TAG_FREQAI_ARM}'" - echo "TAG_FREQAI_RL_ARM='${TAG_FREQAI_RL_ARM}'" - echo "TAG_PI='${TAG_PI}'" + echo "BRANCH_NAME=${BRANCH_NAME}" >> "$GITHUB_OUTPUT" + echo "TAG=${TAG}" >> "$GITHUB_OUTPUT" + echo "TAG_PLOT=${TAG_PLOT}" >> "$GITHUB_OUTPUT" + echo "TAG_FREQAI=${TAG_FREQAI}" >> "$GITHUB_OUTPUT" + echo "TAG_FREQAI_RL=${TAG_FREQAI_RL}" >> "$GITHUB_OUTPUT" + echo "TAG_FREQAI_TORCH=${TAG_FREQAI_TORCH}" >> "$GITHUB_OUTPUT" + echo "TAG_ARM=${TAG_ARM}" >> "$GITHUB_OUTPUT" + echo "TAG_PLOT_ARM=${TAG_PLOT_ARM}" >> "$GITHUB_OUTPUT" + echo "TAG_FREQAI_ARM=${TAG_FREQAI_ARM}" >> "$GITHUB_OUTPUT" + echo "TAG_FREQAI_RL_ARM=${TAG_FREQAI_RL_ARM}" >> "$GITHUB_OUTPUT" + echo "TAG_PI=${TAG_PI}" >> "$GITHUB_OUTPUT" - echo "BRANCH_NAME=${BRANCH_NAME}" >> "$GITHUB_ENV" - echo "TAG=${TAG}" >> "$GITHUB_ENV" - echo "TAG_PLOT=${TAG_PLOT}" >> "$GITHUB_ENV" - echo "TAG_FREQAI=${TAG_FREQAI}" >> "$GITHUB_ENV" - echo "TAG_FREQAI_RL=${TAG_FREQAI_RL}" >> "$GITHUB_ENV" - echo "TAG_FREQAI_TORCH=${TAG_FREQAI_TORCH}" >> "$GITHUB_ENV" - echo "TAG_ARM=${TAG_ARM}" >> "$GITHUB_ENV" - echo "TAG_PLOT_ARM=${TAG_PLOT_ARM}" >> "$GITHUB_ENV" - echo "TAG_FREQAI_ARM=${TAG_FREQAI_ARM}" >> "$GITHUB_ENV" - echo "TAG_FREQAI_RL_ARM=${TAG_FREQAI_RL_ARM}" >> "$GITHUB_ENV" - echo "TAG_PI=${TAG_PI}" >> "$GITHUB_ENV" + echo "CACHE_TAG_PI=${CACHE_TAG_PI}" >> "$GITHUB_OUTPUT" - echo "CACHE_TAG_PI=${CACHE_TAG_PI}" >> "$GITHUB_ENV" + cat "$GITHUB_OUTPUT" - name: Save commit SHA to file shell: bash diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 6710889a9..4be322b2e 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -38,6 +38,7 @@ jobs: persist-credentials: false - name: Set docker tag names + id: tags uses: ./.github/actions/docker-tags - name: Login to Docker Hub @@ -62,11 +63,16 @@ jobs: - name: Build image without cache if: github.event_name == 'schedule' + env: + TAG: ${{ steps.tags.outputs.TAG }} run: | - docker build -t freqtrade:${TAG} . + docker build -t ${IMAGE_NAME}:${TAG} . - name: Build ARMHF image without cache if: github.event_name == 'schedule' + env: + TAG_PI: ${{ steps.tags.outputs.TAG_PI }} + CACHE_TAG_PI: ${{ steps.tags.outputs.CACHE_TAG_PI }} run: | docker buildx build \ --cache-to=type=registry,ref=${CACHE_TAG_PI} \ @@ -79,9 +85,11 @@ jobs: - name: Build image with cache if: github.event_name != 'schedule' + env: + TAG: ${{ steps.tags.outputs.TAG }} run: | docker pull ${IMAGE_NAME}:${TAG} || true - docker build --cache-from ${IMAGE_NAME}:${TAG} -t freqtrade:${TAG} . + docker build --cache-from ${IMAGE_NAME}:${TAG} -t ${CACHE_IMAGE}:${TAG} . - name: Build ARMHF image with cache if: github.event_name != 'schedule' @@ -98,23 +106,29 @@ jobs: . - name: Run build for AI images + env: + TAG: ${{ steps.tags.outputs.TAG }} + TAG_PLOT: ${{ steps.tags.outputs.TAG_PLOT }} + TAG_FREQAI: ${{ steps.tags.outputs.TAG_FREQAI }} + TAG_FREQAI_RL: ${{ steps.tags.outputs.TAG_FREQAI_RL }} run: | - docker build --build-arg sourceimage=freqtrade --build-arg sourcetag=${TAG} -t freqtrade:${TAG_PLOT} -f docker/Dockerfile.plot . - docker build --build-arg sourceimage=freqtrade --build-arg sourcetag=${TAG} -t freqtrade:${TAG_FREQAI} -f docker/Dockerfile.freqai . - docker build --build-arg sourceimage=freqtrade --build-arg sourcetag=${TAG_FREQAI} -t freqtrade:${TAG_FREQAI_RL} -f docker/Dockerfile.freqai_rl . + docker build --build-arg sourceimage=${CACHE_IMAGE} --build-arg sourcetag=${TAG} -t ${CACHE_IMAGE}:${TAG_PLOT} -f docker/Dockerfile.plot . + docker build --build-arg sourceimage=${CACHE_IMAGE} --build-arg sourcetag=${TAG} -t ${CACHE_IMAGE}:${TAG_FREQAI} -f docker/Dockerfile.freqai . + docker build --build-arg sourceimage=${CACHE_IMAGE} --build-arg sourcetag=${TAG_FREQAI} -t ${CACHE_IMAGE}:${TAG_FREQAI_RL} -f docker/Dockerfile.freqai_rl . - - name: Tag images for caching - run: | - docker tag freqtrade:$TAG ${CACHE_IMAGE}:$TAG - docker tag freqtrade:$TAG_PLOT ${CACHE_IMAGE}:$TAG_PLOT - docker tag freqtrade:$TAG_FREQAI ${CACHE_IMAGE}:$TAG_FREQAI - docker tag freqtrade:$TAG_FREQAI_RL ${CACHE_IMAGE}:$TAG_FREQAI_RL - name: Run backtest in Docker + env: + TAG: ${{ steps.tags.outputs.TAG }} run: | - docker run --rm -v $(pwd)/tests/testdata/config.tests.json:/freqtrade/config.json:ro -v $(pwd)/tests:/tests freqtrade:${TAG} backtesting --datadir /tests/testdata --strategy-path /tests/strategy/strats/ --strategy StrategyTestV3 + docker run --rm -v $(pwd)/tests/testdata/config.tests.json:/freqtrade/config.json:ro -v $(pwd)/tests:/tests ${CACHE_IMAGE}:${TAG} backtesting --datadir /tests/testdata --strategy-path /tests/strategy/strats/ --strategy StrategyTestV3 - name: Push cache images + env: + TAG: ${{ steps.tags.outputs.TAG }} + TAG_PLOT: ${{ steps.tags.outputs.TAG_PLOT }} + TAG_FREQAI: ${{ steps.tags.outputs.TAG_FREQAI }} + TAG_FREQAI_RL: ${{ steps.tags.outputs.TAG_FREQAI_RL }} run: | docker push ${CACHE_IMAGE}:$TAG docker push ${CACHE_IMAGE}:$TAG_PLOT @@ -140,6 +154,7 @@ jobs: persist-credentials: false - name: Set docker tag names + id: tags uses: ./.github/actions/docker-tags - name: Login to Docker Hub @@ -157,37 +172,46 @@ jobs: - name: Build image without cache if: github.event_name == 'schedule' + env: + TAG_ARM: ${{ steps.tags.outputs.TAG_ARM }} run: | - docker build -t freqtrade:${TAG_ARM} . + docker build -t ${IMAGE_NAME}:${TAG_ARM} . - name: Build image with cache if: github.event_name != 'schedule' + env: + TAG_ARM: ${{ steps.tags.outputs.TAG_ARM }} run: | docker pull ${IMAGE_NAME}:${TAG_ARM} || true - docker build --cache-from ${IMAGE_NAME}:${TAG_ARM} -t freqtrade:${TAG_ARM} . + docker build --cache-from ${IMAGE_NAME}:${TAG_ARM} -t ${CACHE_IMAGE}:${TAG_ARM} . - name: Run build for AI images + env: + TAG_PLOT_ARM: ${{ steps.tags.outputs.TAG_PLOT_ARM }} + TAG_FREQAI_ARM: ${{ steps.tags.outputs.TAG_FREQAI_ARM }} + TAG_FREQAI_RL_ARM: ${{ steps.tags.outputs.TAG_FREQAI_RL_ARM }} run: | - docker build --build-arg sourceimage=freqtrade --build-arg sourcetag=${TAG_ARM} -t freqtrade:${TAG_PLOT_ARM} -f docker/Dockerfile.plot . - docker build --build-arg sourceimage=freqtrade --build-arg sourcetag=${TAG_ARM} -t freqtrade:${TAG_FREQAI_ARM} -f docker/Dockerfile.freqai . - docker build --build-arg sourceimage=freqtrade --build-arg sourcetag=${TAG_FREQAI_ARM} -t freqtrade:${TAG_FREQAI_RL_ARM} -f docker/Dockerfile.freqai_rl . + docker build --build-arg sourceimage=${CACHE_IMAGE} --build-arg sourcetag=${TAG_ARM} -t ${CACHE_IMAGE}:${TAG_PLOT_ARM} -f docker/Dockerfile.plot . + docker build --build-arg sourceimage=${CACHE_IMAGE} --build-arg sourcetag=${TAG_ARM} -t ${CACHE_IMAGE}:${TAG_FREQAI_ARM} -f docker/Dockerfile.freqai . + docker build --build-arg sourceimage=${CACHE_IMAGE} --build-arg sourcetag=${TAG_FREQAI_ARM} -t ${CACHE_IMAGE}:${TAG_FREQAI_RL_ARM} -f docker/Dockerfile.freqai_rl . - - name: Tag images for caching - run: | - docker tag freqtrade:$TAG_ARM ${CACHE_IMAGE}:$TAG_ARM - docker tag freqtrade:$TAG_PLOT_ARM ${CACHE_IMAGE}:$TAG_PLOT_ARM - docker tag freqtrade:$TAG_FREQAI_ARM ${CACHE_IMAGE}:$TAG_FREQAI_ARM - docker tag freqtrade:$TAG_FREQAI_RL_ARM ${CACHE_IMAGE}:$TAG_FREQAI_RL_ARM - name: Run backtest in Docker + env: + TAG_ARM: ${{ steps.tags.outputs.TAG_ARM }} run: | - docker run --rm -v $(pwd)/tests/testdata/config.tests.json:/freqtrade/config.json:ro -v $(pwd)/tests:/tests freqtrade:${TAG_ARM} backtesting --datadir /tests/testdata --strategy-path /tests/strategy/strats/ --strategy StrategyTestV3 + docker run --rm -v $(pwd)/tests/testdata/config.tests.json:/freqtrade/config.json:ro -v $(pwd)/tests:/tests ${CACHE_IMAGE}:${TAG_ARM} backtesting --datadir /tests/testdata --strategy-path /tests/strategy/strats/ --strategy StrategyTestV3 - name: Docker images run: | docker images - name: Push cache images + env: + TAG_PLOT_ARM: ${{ steps.tags.outputs.TAG_PLOT_ARM }} + TAG_FREQAI_ARM: ${{ steps.tags.outputs.TAG_FREQAI_ARM }} + TAG_FREQAI_RL_ARM: ${{ steps.tags.outputs.TAG_FREQAI_RL_ARM }} + TAG_ARM: ${{ steps.tags.outputs.TAG_ARM }} run: | docker push ${CACHE_IMAGE}:$TAG_PLOT_ARM docker push ${CACHE_IMAGE}:$TAG_FREQAI_ARM @@ -195,6 +219,11 @@ jobs: docker push ${CACHE_IMAGE}:$TAG_ARM - name: Create manifests + env: + TAG_PLOT_ARM: ${{ steps.tags.outputs.TAG_PLOT_ARM }} + TAG_FREQAI_ARM: ${{ steps.tags.outputs.TAG_FREQAI_ARM }} + TAG_FREQAI_RL_ARM: ${{ steps.tags.outputs.TAG_FREQAI_RL_ARM }} + TAG_ARM: ${{ steps.tags.outputs.TAG_ARM }} run: | docker buildx imagetools create \ --tag ${IMAGE_NAME}:${TAG} \ @@ -202,6 +231,9 @@ jobs: ${CACHE_IMAGE}:${TAG} ${CACHE_IMAGE}:${TAG_ARM} ${IMAGE_NAME}:${TAG_PI} - name: Create multiarch image - Plot + env: + TAG_PLOT: ${{ steps.tags.outputs.TAG_PLOT }} + TAG_PLOT_ARM: ${{ steps.tags.outputs.TAG_PLOT_ARM }} run: | docker buildx imagetools create \ --tag ${IMAGE_NAME}:${TAG_PLOT} \ @@ -209,6 +241,9 @@ jobs: ${CACHE_IMAGE}:${TAG_PLOT} ${CACHE_IMAGE}:${TAG_PLOT_ARM} - name: Create multiarch image - FreqAI + env: + TAG_FREQAI: ${{ steps.tags.outputs.TAG_FREQAI }} + TAG_FREQAI_ARM: ${{ steps.tags.outputs.TAG_FREQAI_ARM }} run: | docker buildx imagetools create \ --tag ${IMAGE_NAME}:${TAG_FREQAI} \ @@ -216,6 +251,10 @@ jobs: ${CACHE_IMAGE}:${TAG_FREQAI} ${CACHE_IMAGE}:${TAG_FREQAI_ARM} - name: Create multiarch image - FreqAI RL + env: + TAG_FREQAI_RL: ${{ steps.tags.outputs.TAG_FREQAI_RL }} + TAG_FREQAI_RL_ARM: ${{ steps.tags.outputs.TAG_FREQAI_RL_ARM }} + TAG_FREQAI_TORCH: ${{ steps.tags.outputs.TAG_FREQAI_TORCH }} run: | # Create special Torch tag - which is identical to the RL tag. docker buildx imagetools create \ @@ -227,6 +266,8 @@ jobs: - name: Tag latest if: env.TAG == 'develop' + env: + TAG: ${{ steps.tags.outputs.TAG }} run: | # Tag image as latest docker buildx imagetools create \ From 67774c71e5d2ac9ed663b165ad8a67063b00c518 Mon Sep 17 00:00:00 2001 From: Matthias Date: Thu, 4 Sep 2025 19:54:02 +0200 Subject: [PATCH 17/18] chore(ci): set composite action outputs --- .github/actions/docker-tags/action.yml | 39 +++++++++++++++++++++++++- .github/workflows/docker-build.yml | 5 +++- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/.github/actions/docker-tags/action.yml b/.github/actions/docker-tags/action.yml index 3ddd2fadf..1a563aade 100644 --- a/.github/actions/docker-tags/action.yml +++ b/.github/actions/docker-tags/action.yml @@ -1,12 +1,49 @@ name: 'docker-tags' description: 'Set Docker default Tag environment variables' # inputs: -# outputs: +outputs: + BRANCH_NAME: + description: 'The branch name' + value: ${{ steps.tags.outputs.BRANCH_NAME }} + TAG: + description: 'The Docker tag' + value: ${{ steps.tags.outputs.TAG }} + TAG_PLOT: + description: 'The Docker tag for the plot' + value: ${{ steps.tags.outputs.TAG_PLOT }} + TAG_FREQAI: + description: 'The Docker tag for the freqai' + value: ${{ steps.tags.outputs.TAG_FREQAI }} + TAG_FREQAI_RL: + description: 'The Docker tag for the freqai_rl' + value: ${{ steps.tags.outputs.TAG_FREQAI_RL }} + TAG_FREQAI_TORCH: + description: 'The Docker tag for the freqai_torch' + value: ${{ steps.tags.outputs.TAG_FREQAI_TORCH }} + TAG_ARM: + description: 'The Docker tag for the arm' + value: ${{ steps.tags.outputs.TAG_ARM }} + TAG_PLOT_ARM: + description: 'The Docker tag for the plot arm' + value: ${{ steps.tags.outputs.TAG_PLOT_ARM }} + TAG_FREQAI_ARM: + description: 'The Docker tag for the freqai arm' + value: ${{ steps.tags.outputs.TAG_FREQAI_ARM }} + TAG_FREQAI_RL_ARM: + description: 'The Docker tag for the freqai_rl arm' + value: ${{ steps.tags.outputs.TAG_FREQAI_RL_ARM }} + TAG_PI: + description: 'The Docker tag for the pi' + value: ${{ steps.tags.outputs.TAG_PI }} + CACHE_TAG_PI: + description: 'The Docker cache tag for the pi' + value: ${{ steps.tags.outputs.CACHE_TAG_PI }} runs: using: "composite" steps: - name: Extract branch name shell: bash + id: tags env: BRANCH_NAME_INPUT: ${{ github.event.inputs.branch_name }} run: | diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 4be322b2e..59d5bb968 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -66,7 +66,7 @@ jobs: env: TAG: ${{ steps.tags.outputs.TAG }} run: | - docker build -t ${IMAGE_NAME}:${TAG} . + docker build -t ${CACHE_IMAGE}:${TAG} . - name: Build ARMHF image without cache if: github.event_name == 'schedule' @@ -94,6 +94,9 @@ jobs: - name: Build ARMHF image with cache if: github.event_name != 'schedule' # disable provenance due to https://github.com/docker/buildx/issues/1509 + env: + TAG_PI: ${{ steps.tags.outputs.TAG_PI }} + CACHE_TAG_PI: ${{ steps.tags.outputs.CACHE_TAG_PI }} run: | docker buildx build \ --cache-from=type=registry,ref=${CACHE_TAG_PI} \ From 31204a37a5ab549687242f76acb4bc66750713bb Mon Sep 17 00:00:00 2001 From: Matthias Date: Fri, 5 Sep 2025 06:38:42 +0200 Subject: [PATCH 18/18] chore(ci): set missed env --- .github/workflows/docker-build.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 59d5bb968..1fc50315b 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -190,6 +190,7 @@ jobs: - name: Run build for AI images env: + TAG_ARM: ${{ steps.tags.outputs.TAG_ARM }} TAG_PLOT_ARM: ${{ steps.tags.outputs.TAG_PLOT_ARM }} TAG_FREQAI_ARM: ${{ steps.tags.outputs.TAG_FREQAI_ARM }} TAG_FREQAI_RL_ARM: ${{ steps.tags.outputs.TAG_FREQAI_RL_ARM }} @@ -211,10 +212,10 @@ jobs: - name: Push cache images env: + TAG_ARM: ${{ steps.tags.outputs.TAG_ARM }} TAG_PLOT_ARM: ${{ steps.tags.outputs.TAG_PLOT_ARM }} TAG_FREQAI_ARM: ${{ steps.tags.outputs.TAG_FREQAI_ARM }} TAG_FREQAI_RL_ARM: ${{ steps.tags.outputs.TAG_FREQAI_RL_ARM }} - TAG_ARM: ${{ steps.tags.outputs.TAG_ARM }} run: | docker push ${CACHE_IMAGE}:$TAG_PLOT_ARM docker push ${CACHE_IMAGE}:$TAG_FREQAI_ARM @@ -223,10 +224,9 @@ jobs: - name: Create manifests env: - TAG_PLOT_ARM: ${{ steps.tags.outputs.TAG_PLOT_ARM }} - TAG_FREQAI_ARM: ${{ steps.tags.outputs.TAG_FREQAI_ARM }} - TAG_FREQAI_RL_ARM: ${{ steps.tags.outputs.TAG_FREQAI_RL_ARM }} TAG_ARM: ${{ steps.tags.outputs.TAG_ARM }} + TAG: ${{ steps.tags.outputs.TAG }} + TAG_PI: ${{ steps.tags.outputs.TAG_PI }} run: | docker buildx imagetools create \ --tag ${IMAGE_NAME}:${TAG} \