From b3af4ee50b68c93e2807cf67ff39202dbbe7a7be Mon Sep 17 00:00:00 2001 From: Ankit Matth Date: Sun, 24 Aug 2025 08:59:19 +0530 Subject: [PATCH 1/4] speed up scripts by using docker hub --- deployment/docker-compose-hub.yaml | 71 ++++++++++++++++++++++++++++++ setup.ps1 | 24 +++++----- setup.sh | 10 ++--- 3 files changed, 89 insertions(+), 16 deletions(-) create mode 100644 deployment/docker-compose-hub.yaml diff --git a/deployment/docker-compose-hub.yaml b/deployment/docker-compose-hub.yaml new file mode 100644 index 00000000..517fb68e --- /dev/null +++ b/deployment/docker-compose-hub.yaml @@ -0,0 +1,71 @@ +name: docsgpt-oss +services: + frontend: + image: arc53/docsgpt:develop # some changes required here + environment: + - VITE_API_HOST=http://localhost:7091 + - VITE_API_STREAMING=$VITE_API_STREAMING + ports: + - "5173:5173" + depends_on: + - backend + + backend: + user: root + image: arc53/docsgpt:develop # some changes required here + environment: + - API_KEY=$API_KEY + - EMBEDDINGS_KEY=$API_KEY + - LLM_PROVIDER=$LLM_PROVIDER + - LLM_NAME=$LLM_NAME + - CELERY_BROKER_URL=redis://redis:6379/0 + - CELERY_RESULT_BACKEND=redis://redis:6379/1 + - MONGO_URI=mongodb://mongo:27017/docsgpt + - CACHE_REDIS_URL=redis://redis:6379/2 + - OPENAI_BASE_URL=$OPENAI_BASE_URL + ports: + - "7091:7091" + volumes: + - ../application/indexes:/app/indexes + - ../application/inputs:/app/inputs + - ../application/vectors:/app/vectors + depends_on: + - redis + - mongo + + worker: + user: root + image: arc53/docsgpt:develop # some changes required here + command: celery -A application.app.celery worker -l INFO -B + environment: + - API_KEY=$API_KEY + - EMBEDDINGS_KEY=$API_KEY + - LLM_PROVIDER=$LLM_PROVIDER + - LLM_NAME=$LLM_NAME + - CELERY_BROKER_URL=redis://redis:6379/0 + - CELERY_RESULT_BACKEND=redis://redis:6379/1 + - MONGO_URI=mongodb://mongo:27017/docsgpt + - API_URL=http://backend:7091 + - CACHE_REDIS_URL=redis://redis:6379/2 + volumes: + - ../application/indexes:/app/indexes + - ../application/inputs:/app/inputs + - ../application/vectors:/app/vectors + depends_on: + - redis + - mongo + + redis: + image: redis:6-alpine + ports: + - 6379:6379 + + mongo: + image: mongo:6 + ports: + - 27017:27017 + volumes: + - mongodb_data_container:/data/db + +volumes: + mongodb_data_container: diff --git a/setup.ps1 b/setup.ps1 index 8572484f..01a969ab 100644 --- a/setup.ps1 +++ b/setup.ps1 @@ -9,7 +9,7 @@ $ErrorActionPreference = "Stop" # Get current script directory $SCRIPT_DIR = Split-Path -Parent $MyInvocation.MyCommand.Definition -$COMPOSE_FILE = Join-Path -Path $SCRIPT_DIR -ChildPath "deployment\docker-compose.yaml" +$COMPOSE_FILE = Join-Path -Path $SCRIPT_DIR -ChildPath "deployment\docker-compose-hub.yaml" $ENV_FILE = Join-Path -Path $SCRIPT_DIR -ChildPath ".env" # Function to write colored text @@ -304,9 +304,9 @@ function Use-DocsPublicAPIEndpoint { # Run Docker compose commands try { - & docker compose --env-file "$ENV_FILE" -f "$COMPOSE_FILE" build + & docker compose --env-file "$ENV_FILE" -f "$COMPOSE_FILE" pull if ($LASTEXITCODE -ne 0) { - throw "Docker compose build failed with exit code $LASTEXITCODE" + throw "Docker compose pull failed with exit code $LASTEXITCODE" } & docker compose --env-file "$ENV_FILE" -f "$COMPOSE_FILE" up -d @@ -415,10 +415,10 @@ function Serve-LocalOllama { Write-Host "" Write-ColorText "Starting Docker Compose with Ollama ($docker_compose_file_suffix)..." -ForegroundColor "White" - # Build the containers - & docker compose --env-file "$ENV_FILE" -f "$COMPOSE_FILE" -f "$optional_compose" build + # Pull the containers + & docker compose --env-file "$ENV_FILE" -f "$COMPOSE_FILE" -f "$optional_compose" pull if ($LASTEXITCODE -ne 0) { - throw "Docker compose build failed with exit code $LASTEXITCODE" + throw "Docker compose pull failed with exit code $LASTEXITCODE" } # Start the containers @@ -575,10 +575,10 @@ function Connect-LocalInferenceEngine { Write-Host "" Write-ColorText "Starting Docker Compose..." -ForegroundColor "White" - # Build the containers - & docker compose --env-file "$ENV_FILE" -f "$COMPOSE_FILE" build + # Pull the containers + & docker compose --env-file "$ENV_FILE" -f "$COMPOSE_FILE" pull if ($LASTEXITCODE -ne 0) { - throw "Docker compose build failed with exit code $LASTEXITCODE" + throw "Docker compose pull failed with exit code $LASTEXITCODE" } # Start the containers @@ -706,10 +706,12 @@ function Connect-CloudAPIProvider { Write-ColorText "Starting Docker Compose..." -ForegroundColor "White" # Run Docker compose commands - & docker compose --env-file "$ENV_FILE" -f "$COMPOSE_FILE" up -d --build + & docker compose --env-file "$ENV_FILE" -f "$COMPOSE_FILE" pull if ($LASTEXITCODE -ne 0) { - throw "Docker compose build or up failed with exit code $LASTEXITCODE" + throw "Docker compose pull failed with exit code $LASTEXITCODE" } + + & docker compose --env-file "$ENV_FILE" -f "$COMPOSE_FILE" up -d Write-Host "" Write-ColorText "DocsGPT is now configured to use $provider_name on http://localhost:5173" -ForegroundColor "Green" diff --git a/setup.sh b/setup.sh index b072d546..f322884e 100755 --- a/setup.sh +++ b/setup.sh @@ -9,7 +9,7 @@ NC='\033[0m' BOLD='\033[1m' # Base Compose file (relative to script location) -COMPOSE_FILE="$(dirname "$(readlink -f "$0")")/deployment/docker-compose.yaml" +COMPOSE_FILE="$(dirname "$(readlink -f "$0")")/deployment/docker-compose-hub.yaml" ENV_FILE="$(dirname "$(readlink -f "$0")")/.env" # Animation function @@ -176,7 +176,7 @@ use_docs_public_api_endpoint() { check_and_start_docker echo -e "\n${NC}Starting Docker Compose...${NC}" - docker compose --env-file "${ENV_FILE}" -f "${COMPOSE_FILE}" build && docker compose --env-file "${ENV_FILE}" -f "${COMPOSE_FILE}" up -d + docker compose --env-file "${ENV_FILE}" -f "${COMPOSE_FILE}" pull && docker compose --env-file "${ENV_FILE}" -f "${COMPOSE_FILE}" up -d docker_compose_status=$? # Capture exit status of docker compose echo "Docker Compose Exit Status: $docker_compose_status" @@ -252,7 +252,7 @@ serve_local_ollama() { ) echo -e "\n${NC}Starting Docker Compose with Ollama (${docker_compose_file_suffix})...${NC}" - docker compose --env-file "${ENV_FILE}" "${compose_files[@]}" build + docker compose --env-file "${ENV_FILE}" "${compose_files[@]}" pull docker compose --env-file "${ENV_FILE}" "${compose_files[@]}" up -d docker_compose_status=$? @@ -360,7 +360,7 @@ connect_local_inference_engine() { check_and_start_docker echo -e "\n${NC}Starting Docker Compose...${NC}" - docker compose --env-file "${ENV_FILE}" -f "${COMPOSE_FILE}" build && docker compose -f "${COMPOSE_FILE}" up -d + docker compose --env-file "${ENV_FILE}" -f "${COMPOSE_FILE}" pull && docker compose -f "${COMPOSE_FILE}" up -d docker_compose_status=$? echo "Docker Compose Exit Status: $docker_compose_status" # Debug output @@ -449,7 +449,7 @@ connect_cloud_api_provider() { check_and_start_docker echo -e "\n${NC}Starting Docker Compose...${NC}" - docker compose --env-file "${ENV_FILE}" -f "${COMPOSE_FILE}" up -d --build + docker compose --env-file "${ENV_FILE}" -f "${COMPOSE_FILE}" pull && docker compose --env-file "${ENV_FILE}" -f "${COMPOSE_FILE}" up -d docker_compose_status=$? echo "Docker Compose Exit Status: $docker_compose_status" # Debug output From 9da4215d1f25b433d59d0f90aa2661b76058b6dd Mon Sep 17 00:00:00 2001 From: Siddhant Rai Date: Thu, 28 Aug 2025 12:01:04 +0530 Subject: [PATCH 2/4] feat: implement Docker Hub integration for building and pushing images in CI/CD workflow --- .github/workflows/docker-publish.yml | 37 ++++++++++++++++++++++++++++ deployment/docker-compose-hub.yaml | 6 ++--- setup.ps1 | 27 +++++++++++++------- setup.sh | 19 +++++++++++--- 4 files changed, 73 insertions(+), 16 deletions(-) create mode 100644 .github/workflows/docker-publish.yml diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml new file mode 100644 index 00000000..60ddba1c --- /dev/null +++ b/.github/workflows/docker-publish.yml @@ -0,0 +1,37 @@ +# This GitHub Actions workflow builds and pushes Docker images to Docker Hub +# only when a GitHub release is published and the tag name contains 'latest'. +# The image is tagged as both the release tag and 'latest'. + + +name: Build and Push Docker Images (Release: latest only) + +on: + release: + types: [published] + +jobs: + build-and-push: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Build and push Docker image (only for releases with 'latest' in tag) + if: github.event_name == 'release' && contains(github.event.release.tag_name, 'latest') + uses: docker/build-push-action@v5 + with: + context: . + file: ./application/Dockerfile + push: true + tags: | + arc53/docsgpt:${{ github.event.release.tag_name }} + arc53/docsgpt:latest diff --git a/deployment/docker-compose-hub.yaml b/deployment/docker-compose-hub.yaml index 517fb68e..d1efc22e 100644 --- a/deployment/docker-compose-hub.yaml +++ b/deployment/docker-compose-hub.yaml @@ -1,7 +1,7 @@ name: docsgpt-oss services: frontend: - image: arc53/docsgpt:develop # some changes required here + image: arc53/docsgpt:latest environment: - VITE_API_HOST=http://localhost:7091 - VITE_API_STREAMING=$VITE_API_STREAMING @@ -12,7 +12,7 @@ services: backend: user: root - image: arc53/docsgpt:develop # some changes required here + image: arc53/docsgpt:latest environment: - API_KEY=$API_KEY - EMBEDDINGS_KEY=$API_KEY @@ -35,7 +35,7 @@ services: worker: user: root - image: arc53/docsgpt:develop # some changes required here + image: arc53/docsgpt:latest command: celery -A application.app.celery worker -l INFO -B environment: - API_KEY=$API_KEY diff --git a/setup.ps1 b/setup.ps1 index 01a969ab..d675536e 100644 --- a/setup.ps1 +++ b/setup.ps1 @@ -9,7 +9,9 @@ $ErrorActionPreference = "Stop" # Get current script directory $SCRIPT_DIR = Split-Path -Parent $MyInvocation.MyCommand.Definition -$COMPOSE_FILE = Join-Path -Path $SCRIPT_DIR -ChildPath "deployment\docker-compose-hub.yaml" +$COMPOSE_FILE_HUB = Join-Path -Path $SCRIPT_DIR -ChildPath "deployment\docker-compose-hub.yaml" +$COMPOSE_FILE_LOCAL = Join-Path -Path $SCRIPT_DIR -ChildPath "deployment\docker-compose.yaml" +$COMPOSE_FILE = $COMPOSE_FILE_HUB $ENV_FILE = Join-Path -Path $SCRIPT_DIR -ChildPath ".env" # Function to write colored text @@ -223,12 +225,15 @@ function Prompt-MainMenu { Write-Host "" Write-ColorText "Welcome to DocsGPT Setup!" -ForegroundColor "White" -Bold Write-ColorText "How would you like to proceed?" -ForegroundColor "White" - Write-ColorText "1) Use DocsGPT Public API Endpoint (simple and free)" -ForegroundColor "Yellow" + Write-ColorText "1) Use DocsGPT Public API Endpoint (simple and free, uses pre-built Docker images from Docker Hub for fastest setup)" -ForegroundColor "Yellow" Write-ColorText "2) Serve Local (with Ollama)" -ForegroundColor "Yellow" Write-ColorText "3) Connect Local Inference Engine" -ForegroundColor "Yellow" Write-ColorText "4) Connect Cloud API Provider" -ForegroundColor "Yellow" + Write-ColorText "5) Advanced: Build images locally (for developers)" -ForegroundColor "Yellow" Write-Host "" - $script:main_choice = Read-Host "Choose option (1-4)" + Write-ColorText "By default, DocsGPT uses pre-built images from Docker Hub for a fast, reliable, and consistent experience. This avoids local build errors and speeds up onboarding. Advanced users can choose to build images locally if needed." -ForegroundColor "White" + Write-Host "" + $script:main_choice = Read-Host "Choose option (1-5)" } # Function to prompt for Local Inference Engine options @@ -737,13 +742,13 @@ while ($true) { switch ($main_choice) { "1" { + $COMPOSE_FILE = $COMPOSE_FILE_HUB Use-DocsPublicAPIEndpoint $exitLoop = $true # Set flag to true on completion break } "2" { Serve-LocalOllama - # Only exit the loop if user didn't press "b" to go back if ($ollama_choice -ne "b" -and $ollama_choice -ne "B") { $exitLoop = $true } @@ -751,7 +756,6 @@ while ($true) { } "3" { Connect-LocalInferenceEngine - # Only exit the loop if user didn't press "b" to go back if ($engine_choice -ne "b" -and $engine_choice -ne "B") { $exitLoop = $true } @@ -759,20 +763,25 @@ while ($true) { } "4" { Connect-CloudAPIProvider - # Only exit the loop if user didn't press "b" to go back if ($provider_choice -ne "b" -and $provider_choice -ne "B") { $exitLoop = $true } break } + "5" { + Write-Host "" + Write-ColorText "You have selected to build images locally. This is recommended for developers or if you want to test local changes." -ForegroundColor "Yellow" + $COMPOSE_FILE = $COMPOSE_FILE_LOCAL + Use-DocsPublicAPIEndpoint + $exitLoop = $true + break + } default { Write-Host "" - Write-ColorText "Invalid choice. Please choose 1-4." -ForegroundColor "Red" + Write-ColorText "Invalid choice. Please choose 1-5." -ForegroundColor "Red" Start-Sleep -Seconds 1 } } - - # Only break out of the loop if a function completed successfully if ($exitLoop) { break } diff --git a/setup.sh b/setup.sh index f322884e..23aeb717 100755 --- a/setup.sh +++ b/setup.sh @@ -10,6 +10,7 @@ BOLD='\033[1m' # Base Compose file (relative to script location) COMPOSE_FILE="$(dirname "$(readlink -f "$0")")/deployment/docker-compose-hub.yaml" +COMPOSE_FILE_LOCAL="$(dirname "$(readlink -f "$0")")/deployment/docker-compose.yaml" ENV_FILE="$(dirname "$(readlink -f "$0")")/.env" # Animation function @@ -111,12 +112,15 @@ check_and_start_docker() { prompt_main_menu() { echo -e "\n${DEFAULT_FG}${BOLD}Welcome to DocsGPT Setup!${NC}" echo -e "${DEFAULT_FG}How would you like to proceed?${NC}" - echo -e "${YELLOW}1) Use DocsGPT Public API Endpoint (simple and free)${NC}" + echo -e "${YELLOW}1) Use DocsGPT Public API Endpoint (simple and free, uses pre-built Docker images from Docker Hub for fastest setup)${NC}" echo -e "${YELLOW}2) Serve Local (with Ollama)${NC}" echo -e "${YELLOW}3) Connect Local Inference Engine${NC}" echo -e "${YELLOW}4) Connect Cloud API Provider${NC}" + echo -e "${YELLOW}5) Advanced: Build images locally (for developers)${NC}" echo - read -p "$(echo -e "${DEFAULT_FG}Choose option (1-4): ${NC}")" main_choice + echo -e "${DEFAULT_FG}By default, DocsGPT uses pre-built images from Docker Hub for a fast, reliable, and consistent experience. This avoids local build errors and speeds up onboarding. Advanced users can choose to build images locally if needed.${NC}" + echo + read -p "$(echo -e "${DEFAULT_FG}Choose option (1-5): ${NC}")" main_choice } # Function to prompt for Local Inference Engine options @@ -468,12 +472,14 @@ connect_cloud_api_provider() { # Main script execution animate_dino + while true; do # Main menu loop clear # Clear screen before showing main menu again prompt_main_menu case $main_choice in - 1) # Use DocsGPT Public API Endpoint + 1) # Use DocsGPT Public API Endpoint (Docker Hub images) + COMPOSE_FILE="$(dirname "$(readlink -f "$0")")/deployment/docker-compose-hub.yaml" use_docs_public_api_endpoint break ;; 2) # Serve Local (with Ollama) @@ -485,8 +491,13 @@ while true; do # Main menu loop 4) # Connect Cloud API Provider connect_cloud_api_provider break ;; + 5) # Advanced: Build images locally + echo -e "\n${YELLOW}You have selected to build images locally. This is recommended for developers or if you want to test local changes.${NC}" + COMPOSE_FILE="$COMPOSE_FILE_LOCAL" + use_docs_public_api_endpoint + break ;; *) - echo -e "\n${RED}Invalid choice. Please choose 1-4.${NC}" ; sleep 1 ;; + echo -e "\n${RED}Invalid choice. Please choose 1-5.${NC}" ; sleep 1 ;; esac done From 5b7c7a4471964621e8fe195acec8fb96eb288a1d Mon Sep 17 00:00:00 2001 From: Siddhant Rai Date: Thu, 28 Aug 2025 12:11:06 +0530 Subject: [PATCH 3/4] fix: update Docker images in docker-compose to use 'develop' tag --- .github/workflows/docker-publish.yml | 37 ---------------------------- deployment/docker-compose-hub.yaml | 6 ++--- 2 files changed, 3 insertions(+), 40 deletions(-) delete mode 100644 .github/workflows/docker-publish.yml diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml deleted file mode 100644 index 60ddba1c..00000000 --- a/.github/workflows/docker-publish.yml +++ /dev/null @@ -1,37 +0,0 @@ -# This GitHub Actions workflow builds and pushes Docker images to Docker Hub -# only when a GitHub release is published and the tag name contains 'latest'. -# The image is tagged as both the release tag and 'latest'. - - -name: Build and Push Docker Images (Release: latest only) - -on: - release: - types: [published] - -jobs: - build-and-push: - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Log in to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Build and push Docker image (only for releases with 'latest' in tag) - if: github.event_name == 'release' && contains(github.event.release.tag_name, 'latest') - uses: docker/build-push-action@v5 - with: - context: . - file: ./application/Dockerfile - push: true - tags: | - arc53/docsgpt:${{ github.event.release.tag_name }} - arc53/docsgpt:latest diff --git a/deployment/docker-compose-hub.yaml b/deployment/docker-compose-hub.yaml index d1efc22e..f798571a 100644 --- a/deployment/docker-compose-hub.yaml +++ b/deployment/docker-compose-hub.yaml @@ -1,7 +1,7 @@ name: docsgpt-oss services: frontend: - image: arc53/docsgpt:latest + image: arc53/docsgpt:develop environment: - VITE_API_HOST=http://localhost:7091 - VITE_API_STREAMING=$VITE_API_STREAMING @@ -12,7 +12,7 @@ services: backend: user: root - image: arc53/docsgpt:latest + image: arc53/docsgpt:develop environment: - API_KEY=$API_KEY - EMBEDDINGS_KEY=$API_KEY @@ -35,7 +35,7 @@ services: worker: user: root - image: arc53/docsgpt:latest + image: arc53/docsgpt:develop command: celery -A application.app.celery worker -l INFO -B environment: - API_KEY=$API_KEY From 56e5aba55904b477a00cf57681a537fa8621ac5d Mon Sep 17 00:00:00 2001 From: Siddhant Rai Date: Thu, 28 Aug 2025 18:21:54 +0530 Subject: [PATCH 4/4] fix: correct frontend image name in docker-compose configuration --- deployment/docker-compose-hub.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/deployment/docker-compose-hub.yaml b/deployment/docker-compose-hub.yaml index f798571a..ddfaf26c 100644 --- a/deployment/docker-compose-hub.yaml +++ b/deployment/docker-compose-hub.yaml @@ -1,7 +1,8 @@ name: docsgpt-oss services: + frontend: - image: arc53/docsgpt:develop + image: arc53/docsgpt-fe:develop environment: - VITE_API_HOST=http://localhost:7091 - VITE_API_STREAMING=$VITE_API_STREAMING @@ -10,6 +11,7 @@ services: depends_on: - backend + backend: user: root image: arc53/docsgpt:develop @@ -33,6 +35,7 @@ services: - redis - mongo + worker: user: root image: arc53/docsgpt:develop