Add MinIO cluster (#15)

Co-authored-by: Fedor Batonogov <f.batonogov@yandex.ru>

ansible/install_minio.yml

@@ -0,0 +1,24 @@
+---
+- name: Подготовка узлов
+  become: true
+  hosts:
+    - minio_hosts
+  roles:
+    - docker_install
+
+- name: Развертывание MinIO Multi-Node Multi-Drive
+  become: true
+  hosts:
+    - minio_hosts
+  vars:
+    filesystem: xfs
+    device_name:
+      - /dev/vdb
+    device:
+      - { src: /dev/vdb, path: /mnt/disk1 }
+    minio_username: minio
+  roles:
+    - create_filesystem
+    - mount
+    - minio_start
+    - docker_cleaner

ansible/install_nginx.yml

@@ -0,0 +1,27 @@
+- name: Подготавливаю узлы
+  become: true
+  hosts:
+    - nginx_hosts
+  roles:
+    - docker_install
+    - nginx_install
+
+- name: Настраиваю keepalived
+  become: true
+  hosts:
+    - nginx-01
+  roles:
+    - role: keepalived
+      unit_file: "keepalived.master.conf.j2"
+      virtual_ip: "10.0.75.90/24"
+      virtual_router_id: 10
+
+- name: Настраиваю keepalived
+  become: true
+  hosts:
+    - nginx-02
+  roles:
+    - role: keepalived
+      unit_file: "keepalived.backup.conf.j2"
+      virtual_ip: "10.0.75.90/24"
+      virtual_router_id: 10

ansible/inventory.yml

@@ -47,3 +47,27 @@ patroni_postgresql_cluster:
   vars:
     ansible_user: infra
     ansible_port: 22
+
+minio_hosts:
+  hosts:
+    minio1:
+      ansible_host: 10.0.75.55
+    minio2:
+      ansible_host: 10.0.75.56
+    minio3:
+      ansible_host: 10.0.75.57
+    minio4:
+      ansible_host: 10.0.75.58
+  vars:
+    ansible_user: infra
+    ansible_port: 22
+
+nginx_hosts:
+  hosts:
+    nginx-01:
+      ansible_host: 10.0.75.91
+    nginx-02:
+      ansible_host: 10.0.75.92
+  vars:
+    ansible_user: infra
+    ansible_port: 22

ansible/roles/create_filesystem/tasks/main.yml

@@ -0,0 +1,7 @@
+---
+# tasks file for create_filesystem
+- name: Create a filesystem
+  community.general.filesystem:
+    fstype: "{{ filesystem }}"
+    dev: '{{ item["src"] }}'
+  loop: "{{ device }}"

ansible/roles/docker_cleaner/tasks/docker.yml

@@ -0,0 +1,10 @@
+---
+- name: Навожу порядок
+  community.docker.docker_prune:
+    containers: false
+    images: true
+    images_filters:
+      dangling: false
+    networks: false
+    volumes: false
+    builder_cache: false

ansible/roles/docker_cleaner/tasks/main.yml

@@ -0,0 +1,4 @@
+---
+# tasks file for  docker_cleaner
+- name: Чищу докер
+  ansible.builtin.import_tasks: docker.yml

ansible/roles/docker_install/files/daemon.json

@@ -1,7 +1,11 @@
 {
-    "registry-mirrors": [
-        "https://dockerhub.timeweb.cloud",
-        "https://mirror.gcr.io",
-        "https://public.ecr.aws"
-    ]
+  "registry-mirrors": [
+    "https://dockerhub.timeweb.cloud",
+    "https://mirror.gcr.io",
+    "https://public.ecr.aws"
+  ],
+  "log-driver": "json-file",
+  "log-opts": {
+    "max-size": "1g"
+  }
 }

ansible/roles/minio_start/handlers/main.yml

@@ -0,0 +1,13 @@
+---
+- name: Загружаю образ
+  community.docker.docker_image:
+    name: "quay.io/minio/minio"
+    tag: "{{ minio_version }}"
+    source: pull
+
+- name: Перезапускаю minio.service
+  ansible.builtin.systemd:
+    name: minio.service
+    state: restarted
+    enabled: true
+    daemon_reload: true

ansible/roles/minio_start/tasks/add_dirs.yml

@@ -0,0 +1,8 @@
+---
+- name: Create a directory if it does not exist
+  ansible.builtin.file:
+    path: /var/lib/minio
+    state: directory
+    owner: "{{ minio_username }}"
+    group: "{{ minio_username }}"
+    mode: "755"

ansible/roles/minio_start/tasks/add_user.yml

@@ -0,0 +1,8 @@
+---
+- name: Создаю пользователя
+  ansible.builtin.user:
+    name: "{{ minio_username }}"
+    shell: /sbin/nologin
+    create_home: true
+    groups: docker
+    uid: "{{ minio_uid }}"

ansible/roles/minio_start/tasks/main.yml

@@ -0,0 +1,10 @@
+---
+# tasks file for minio_start
+- name: Создаю пользователя
+  ansible.builtin.import_tasks: add_user.yml
+
+- name: Создаю директории
+  ansible.builtin.import_tasks: add_dirs.yml
+
+- name: Запускаю minio
+  ansible.builtin.import_tasks: start_minio.yml

ansible/roles/minio_start/tasks/start_minio.yml

@@ -0,0 +1,34 @@
+---
+- name: Наливаю переменные
+  ansible.builtin.template:
+    src: minio.config.j2
+    dest: /etc/default/minio
+    owner: "{{ minio_username }}"
+    group: "{{ minio_username }}"
+    mode: "644"
+  notify:
+    - Перезапускаю minio.service
+
+- name: Наливаю юнит файл
+  ansible.builtin.template:
+    src: minio.service.j2
+    dest: /etc/systemd/system/minio.service
+    mode: "644"
+  notify:
+    - Загружаю образ
+    - Перезапускаю minio.service
+
+- name: Create a directory if it does not exist
+  ansible.builtin.file:
+    path: '{{ item["path"] }}/minio'
+    state: directory
+    owner: "{{ minio_username }}"
+    group: "{{ minio_username }}"
+    mode: "755"
+  loop: "{{ device }}"
+
+- name: Настраиваю minio.service
+  ansible.builtin.systemd:
+    name: minio.service
+    state: started
+    enabled: true

ansible/roles/minio_start/templates/minio.config.j2

@@ -0,0 +1,21 @@
+# MINIO_ROOT_USER and MINIO_ROOT_PASSWORD sets the root account for the MinIO server.
+# This user has unrestricted permissions to perform S3 and administrative API operations on any resource in the deployment.
+# Omit to use the default values 'minioadmin:minioadmin'.
+# MinIO recommends setting non-default values as a best practice, regardless of environment.
+
+MINIO_ROOT_USER=admin
+MINIO_ROOT_PASSWORD={{ lookup('password', 'secrets/minio/admin_secret length=64') }}
+
+# MINIO_VOLUMES sets the storage volumes or paths to use for the MinIO server.
+# The specified path uses MinIO expansion notation to denote a sequential series of drives between 1 and 4, inclusive.
+# All drives or paths included in the expanded drive list must exist *and* be empty or freshly formatted for MinIO to start successfully.
+
+MINIO_VOLUMES="http://minio-node{1...4}:9000/mnt/disk1/minio"
+
+# MINIO_SERVER_URL sets the hostname of the local machine for use with the MinIO Server.
+# MinIO assumes your network control plane can correctly resolve this hostname to the local machine.
+
+# Uncomment the following line and replace the value with the correct hostname for the local machine.
+
+MINIO_SERVER_URL="http://10.0.75.90"
+MINIO_BROWSER_REDIRECT_URL="https://s3.example.local/minio/ui"

ansible/roles/minio_start/templates/minio.service.j2

@@ -0,0 +1,29 @@
+[Unit]
+Description=minio
+Requires=docker.service
+After=docker.service
+
+[Service]
+User={{ minio_username }}
+Group={{ minio_username }}
+Restart=always
+ExecStartPre=-/usr/bin/docker rm -f minio
+ExecStart=/usr/bin/docker run \
+  --rm \
+  --network host \
+  --user {{ minio_uid }}:{{ minio_uid }} \
+  --name minio \
+  --env	"MINIO_CONFIG_ENV_FILE=/etc/config.env" \
+  --add-host "minio-node1:10.0.75.55" \
+  --add-host "minio-node2:10.0.75.56" \
+  --add-host "minio-node3:10.0.75.57" \
+  --add-host "minio-node4:10.0.75.58" \
+  --volume /etc/default/minio:/etc/config.env:ro \
+  --volume /var/lib/minio:/var/lib/minio \
+  --volume /mnt/disk1/minio:/mnt/disk1/minio \
+  quay.io/minio/minio:{{ minio_version }} \
+  server /var/lib/minio --console-address ":9001"
+ExecStop=/usr/bin/docker stop -t 10 minio
+
+[Install]
+WantedBy=multi-user.target

ansible/roles/minio_start/vars/main.yml

@@ -0,0 +1,5 @@
+---
+# vars file for minio_start
+# Мы качаем docker image отсюда: https://quay.io/minio/minio/
+minio_version: RELEASE.2024-07-16T23-46-41Z
+minio_uid: 1111

ansible/roles/mount/tasks/main.yml

@@ -0,0 +1,8 @@
+---
+- name: Mount up device
+  ansible.posix.mount:
+    path: '{{ item["path"] }}'
+    src: '{{ item["src"] }}'
+    fstype: "{{ filesystem }}"
+    state: mounted
+  loop: "{{ device }}"

ansible/roles/nginx_install/files/conf.d/minio.conf

@@ -0,0 +1,73 @@
+upstream minio_s3 {
+    least_conn;
+    server 10.0.75.55:9000;
+    server 10.0.75.56:9000;
+    server 10.0.75.57:9000;
+    server 10.0.75.58:9000;
+}
+
+upstream minio_console {
+    least_conn;
+    server 10.0.75.55:9001;
+    server 10.0.75.56:9001;
+    server 10.0.75.57:9001;
+    server 10.0.75.58:9001;
+}
+
+server {
+    listen        80;
+    listen   [::]:80;
+    listen *:443 ssl;
+    server_name  s3.example.local www.s3.example.local;
+    ssl_certificate /etc/ssl/private/minio.crt;
+    ssl_certificate_key /etc/ssl/private/private.key;
+
+    server_tokens off;
+
+    # Allow special characters in headers
+    ignore_invalid_headers off;
+    # Allow any size file to be uploaded.
+    # Set to a value such as 1000m; to restrict file size to a specific value
+    client_max_body_size 0;
+    # Disable buffering
+    proxy_buffering off;
+    proxy_request_buffering off;
+
+    location / {
+        proxy_set_header Host $http_host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+
+        proxy_connect_timeout 300;
+        # Default is HTTP/1, keepalive is only enabled in HTTP/1.1
+        proxy_http_version 1.1;
+        proxy_set_header Connection "";
+        chunked_transfer_encoding off;
+
+        proxy_pass http://minio_s3; # This uses the upstream directive definition to load balance
+    }
+
+    location /minio/ui/ {
+        rewrite ^/minio/ui/(.*) /$1 break;
+        proxy_set_header Host $http_host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_set_header X-NginX-Proxy true;
+
+        # This is necessary to pass the correct IP to be hashed
+        real_ip_header X-Real-IP;
+
+        proxy_connect_timeout 300;
+
+        # To support websockets in MinIO versions released after January 2023
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection "upgrade";
+
+        chunked_transfer_encoding off;
+
+        proxy_pass http://minio_console; # This uses the upstream directive definition to load balance
+    }
+}

ansible/roles/nginx_install/handlers/main.yml

@@ -0,0 +1,8 @@
+---
+- name: Перезапускаю сервер nginx
+  community.docker.docker_container_exec:
+    container: nginx
+    command: "{{ item }}"
+  loop:
+    - nginx -t
+    - nginx -s reload

ansible/roles/nginx_install/tasks/cert.yml

@@ -0,0 +1,34 @@
+---
+- name: Создаю директорию для ключей
+  ansible.builtin.file:
+    path: /etc/ssl/private
+    state: directory
+    owner: "{{ nginx_user }}"
+    group: "{{ nginx_user }}"
+    mode: "755"
+
+- name: Генерирую приватный ключ
+  community.crypto.openssl_privatekey:
+    path: "/etc/ssl/private/private.key"
+    mode: "0600"
+    owner: "{{ nginx_user }}"
+    group: "{{ nginx_user }}"
+
+- name: Создаю запроса на подписание сертификата (CSR) для самоподписанного сертификата
+  community.crypto.openssl_csr_pipe:
+    privatekey_path: "/etc/ssl/private/private.key"
+    common_name: "minio"
+    organization_name: Example, Inc.
+    subject_alt_name:
+      - "DNS:s3.example.local"
+  register: csr
+
+- name: Создаю самоподписанный сертификат из CSR
+  community.crypto.x509_certificate:
+    path: "/etc/ssl/private/minio.crt"
+    csr_content: "{{ csr.csr }}"
+    privatekey_path: "/etc/ssl/private/private.key"
+    provider: selfsigned
+    mode: "0640"
+    owner: "{{ nginx_user }}"
+    group: "{{ nginx_user }}"

ansible/roles/nginx_install/tasks/config.yml

@@ -0,0 +1,19 @@
+---
+- name: Создаю директорию для nginx conf
+  ansible.builtin.file:
+    path: /etc/nginx/conf.d
+    state: directory
+    owner: "{{ nginx_user }}"
+    group: "{{ nginx_user }}"
+    mode: "755"
+
+- name: Синхронизирую конфигурационные файлы
+  ansible.posix.synchronize:
+    src: "conf.d/"
+    dest: "/etc/nginx/conf.d/"
+    delete: true
+    rsync_opts:
+      - "--chown={{ nginx_user }}:{{ nginx_user }}"
+      - "--chmod=F640"
+  notify:
+    - Перезапускаю сервер nginx

ansible/roles/nginx_install/tasks/main.yml

@@ -0,0 +1,16 @@
+---
+- name: Создаю пользователя nginx
+  ansible.builtin.import_tasks: user.yml
+  when: ansible_distribution == 'Ubuntu'
+
+- name: Копирую конфигурацию
+  ansible.builtin.import_tasks: config.yml
+  when: ansible_distribution == 'Ubuntu'
+
+- name: Генерирую самоподписанный сертификат
+  ansible.builtin.import_tasks: cert.yml
+  when: ansible_distribution == 'Ubuntu'
+
+- name: Устанавливаю nginx
+  ansible.builtin.import_tasks: nginx.yml
+  when: ansible_distribution == 'Ubuntu'

ansible/roles/nginx_install/tasks/nginx.yml

@@ -0,0 +1,23 @@
+---
+- name: Наливаю docker-compose файл nginx
+  ansible.builtin.template:
+    src: docker-compose.yml.j2
+    dest: "/home/{{ nginx_user }}/docker-compose.yml"
+    owner: "{{ nginx_user }}"
+    group: "{{ nginx_user }}"
+    mode: "640"
+
+- name: Налаживаю права для директории /etc/letsencrypt/
+  ansible.builtin.file:
+    path: "{{ item }}"
+    owner: "{{ nginx_user }}"
+    group: "{{ nginx_user }}"
+    recurse: true
+  with_items:
+    - "/etc/letsencrypt/"
+
+- name: Поднимаю nginx
+  community.docker.docker_compose_v2:
+    project_src: "/home/{{ nginx_user }}/"
+    remove_orphans: true
+    wait: true

ansible/roles/nginx_install/tasks/user.yml

@@ -0,0 +1,7 @@
+---
+- name: Создаю пользователя {{ nginx_user }}
+  ansible.builtin.user:
+    name: "{{ nginx_user }}"
+    state: present
+    groups: docker
+    shell: /bin/bash

ansible/roles/nginx_install/templates/docker-compose.yml.j2

@@ -0,0 +1,28 @@
+services:
+  nginx:
+    image: nginx:{{ nginx_version }}
+    container_name: nginx
+    restart: always
+    ports:
+      - "80:80"
+      - "443:443"
+    environment:
+      - TZ=Europe/Moscow
+    volumes:
+      - /etc/nginx/conf.d/:/etc/nginx/conf.d/:ro
+      - /etc/nginx/include.d/:/etc/nginx/include.d/:ro
+      - /etc/ssl/private/:/etc/ssl/private/:ro
+      - /var/lib/letsencrypt/:/var/lib/letsencrypt/:ro
+      - /etc/letsencrypt:/etc/letsencrypt:ro
+    depends_on:
+      - certbot
+
+  certbot:
+    image: certbot/certbot:{{ certbot_version }}
+    container_name: certbot
+    restart: always
+    environment:
+      - TZ=Europe/Moscow
+    volumes:
+      - /etc/letsencrypt:/etc/letsencrypt
+    entrypoint: "/bin/sh -c 'trap exit TERM; while :; do certbot renew --webroot -w /etc/letsencrypt/ -n; sleep 12h & wait $${!}; done;'"

ansible/roles/nginx_install/vars/main.yml

@@ -0,0 +1,5 @@
+---
+# vars file for nginx-install
+nginx_version: 1.27.0-alpine
+nginx_user: nginx
+certbot_version: v2.11.0