Add MinIO cluster (#15)

Co-authored-by: Fedor Batonogov <f.batonogov@yandex.ru>

ansible/install_minio.yml

@@ -0,0 +1,24 @@
+---
+- name: Подготовка узлов
+  become: true
+  hosts:
+    - minio_hosts
+  roles:
+    - docker_install
+
+- name: Развертывание MinIO Multi-Node Multi-Drive
+  become: true
+  hosts:
+    - minio_hosts
+  vars:
+    filesystem: xfs
+    device_name:
+      - /dev/vdb
+    device:
+      - { src: /dev/vdb, path: /mnt/disk1 }
+    minio_username: minio
+  roles:
+    - create_filesystem
+    - mount
+    - minio_start
+    - docker_cleaner

ansible/install_nginx.yml

@@ -0,0 +1,27 @@
+- name: Подготавливаю узлы
+  become: true
+  hosts:
+    - nginx_hosts
+  roles:
+    - docker_install
+    - nginx_install
+
+- name: Настраиваю keepalived
+  become: true
+  hosts:
+    - nginx-01
+  roles:
+    - role: keepalived
+      unit_file: "keepalived.master.conf.j2"
+      virtual_ip: "10.0.75.90/24"
+      virtual_router_id: 10
+
+- name: Настраиваю keepalived
+  become: true
+  hosts:
+    - nginx-02
+  roles:
+    - role: keepalived
+      unit_file: "keepalived.backup.conf.j2"
+      virtual_ip: "10.0.75.90/24"
+      virtual_router_id: 10

ansible/inventory.yml

@@ -47,3 +47,27 @@ patroni_postgresql_cluster:
   vars:
     ansible_user: infra
     ansible_port: 22
+
+minio_hosts:
+  hosts:
+    minio1:
+      ansible_host: 10.0.75.55
+    minio2:
+      ansible_host: 10.0.75.56
+    minio3:
+      ansible_host: 10.0.75.57
+    minio4:
+      ansible_host: 10.0.75.58
+  vars:
+    ansible_user: infra
+    ansible_port: 22
+
+nginx_hosts:
+  hosts:
+    nginx-01:
+      ansible_host: 10.0.75.91
+    nginx-02:
+      ansible_host: 10.0.75.92
+  vars:
+    ansible_user: infra
+    ansible_port: 22

ansible/roles/create_filesystem/tasks/main.yml

@@ -0,0 +1,7 @@
+---
+# tasks file for create_filesystem
+- name: Create a filesystem
+  community.general.filesystem:
+    fstype: "{{ filesystem }}"
+    dev: '{{ item["src"] }}'
+  loop: "{{ device }}"

ansible/roles/docker_cleaner/tasks/docker.yml

@@ -0,0 +1,10 @@
+---
+- name: Навожу порядок
+  community.docker.docker_prune:
+    containers: false
+    images: true
+    images_filters:
+      dangling: false
+    networks: false
+    volumes: false
+    builder_cache: false

ansible/roles/docker_cleaner/tasks/main.yml

@@ -0,0 +1,4 @@
+---
+# tasks file for  docker_cleaner
+- name: Чищу докер
+  ansible.builtin.import_tasks: docker.yml

ansible/roles/docker_install/files/daemon.json

@@ -1,7 +1,11 @@
 {
-    "registry-mirrors": [
-        "https://dockerhub.timeweb.cloud",
-        "https://mirror.gcr.io",
-        "https://public.ecr.aws"
-    ]
+  "registry-mirrors": [
+    "https://dockerhub.timeweb.cloud",
+    "https://mirror.gcr.io",
+    "https://public.ecr.aws"
+  ],
+  "log-driver": "json-file",
+  "log-opts": {
+    "max-size": "1g"
+  }
 }

ansible/roles/minio_start/handlers/main.yml

@@ -0,0 +1,13 @@
+---
+- name: Загружаю образ
+  community.docker.docker_image:
+    name: "quay.io/minio/minio"
+    tag: "{{ minio_version }}"
+    source: pull
+
+- name: Перезапускаю minio.service
+  ansible.builtin.systemd:
+    name: minio.service
+    state: restarted
+    enabled: true
+    daemon_reload: true

ansible/roles/minio_start/tasks/add_dirs.yml

@@ -0,0 +1,8 @@
+---
+- name: Create a directory if it does not exist
+  ansible.builtin.file:
+    path: /var/lib/minio
+    state: directory
+    owner: "{{ minio_username }}"
+    group: "{{ minio_username }}"
+    mode: "755"

ansible/roles/minio_start/tasks/add_user.yml

@@ -0,0 +1,8 @@
+---
+- name: Создаю пользователя
+  ansible.builtin.user:
+    name: "{{ minio_username }}"
+    shell: /sbin/nologin
+    create_home: true
+    groups: docker
+    uid: "{{ minio_uid }}"

ansible/roles/minio_start/tasks/main.yml

@@ -0,0 +1,10 @@
+---
+# tasks file for minio_start
+- name: Создаю пользователя
+  ansible.builtin.import_tasks: add_user.yml
+
+- name: Создаю директории
+  ansible.builtin.import_tasks: add_dirs.yml
+
+- name: Запускаю minio
+  ansible.builtin.import_tasks: start_minio.yml

ansible/roles/minio_start/tasks/start_minio.yml

@@ -0,0 +1,34 @@
+---
+- name: Наливаю переменные
+  ansible.builtin.template:
+    src: minio.config.j2
+    dest: /etc/default/minio
+    owner: "{{ minio_username }}"
+    group: "{{ minio_username }}"
+    mode: "644"
+  notify:
+    - Перезапускаю minio.service
+
+- name: Наливаю юнит файл
+  ansible.builtin.template:
+    src: minio.service.j2
+    dest: /etc/systemd/system/minio.service
+    mode: "644"
+  notify:
+    - Загружаю образ
+    - Перезапускаю minio.service
+
+- name: Create a directory if it does not exist
+  ansible.builtin.file:
+    path: '{{ item["path"] }}/minio'
+    state: directory
+    owner: "{{ minio_username }}"
+    group: "{{ minio_username }}"
+    mode: "755"
+  loop: "{{ device }}"
+
+- name: Настраиваю minio.service
+  ansible.builtin.systemd:
+    name: minio.service
+    state: started
+    enabled: true

ansible/roles/minio_start/templates/minio.config.j2

@@ -0,0 +1,21 @@
+# MINIO_ROOT_USER and MINIO_ROOT_PASSWORD sets the root account for the MinIO server.
+# This user has unrestricted permissions to perform S3 and administrative API operations on any resource in the deployment.
+# Omit to use the default values 'minioadmin:minioadmin'.
+# MinIO recommends setting non-default values as a best practice, regardless of environment.
+
+MINIO_ROOT_USER=admin
+MINIO_ROOT_PASSWORD={{ lookup('password', 'secrets/minio/admin_secret length=64') }}
+
+# MINIO_VOLUMES sets the storage volumes or paths to use for the MinIO server.
+# The specified path uses MinIO expansion notation to denote a sequential series of drives between 1 and 4, inclusive.
+# All drives or paths included in the expanded drive list must exist *and* be empty or freshly formatted for MinIO to start successfully.
+
+MINIO_VOLUMES="http://minio-node{1...4}:9000/mnt/disk1/minio"
+
+# MINIO_SERVER_URL sets the hostname of the local machine for use with the MinIO Server.
+# MinIO assumes your network control plane can correctly resolve this hostname to the local machine.
+
+# Uncomment the following line and replace the value with the correct hostname for the local machine.
+
+MINIO_SERVER_URL="http://10.0.75.90"
+MINIO_BROWSER_REDIRECT_URL="https://s3.example.local/minio/ui"

ansible/roles/minio_start/templates/minio.service.j2

@@ -0,0 +1,29 @@
+[Unit]
+Description=minio
+Requires=docker.service
+After=docker.service
+
+[Service]
+User={{ minio_username }}
+Group={{ minio_username }}
+Restart=always
+ExecStartPre=-/usr/bin/docker rm -f minio
+ExecStart=/usr/bin/docker run \
+  --rm \
+  --network host \
+  --user {{ minio_uid }}:{{ minio_uid }} \
+  --name minio \
+  --env	"MINIO_CONFIG_ENV_FILE=/etc/config.env" \
+  --add-host "minio-node1:10.0.75.55" \
+  --add-host "minio-node2:10.0.75.56" \
+  --add-host "minio-node3:10.0.75.57" \
+  --add-host "minio-node4:10.0.75.58" \
+  --volume /etc/default/minio:/etc/config.env:ro \
+  --volume /var/lib/minio:/var/lib/minio \
+  --volume /mnt/disk1/minio:/mnt/disk1/minio \
+  quay.io/minio/minio:{{ minio_version }} \
+  server /var/lib/minio --console-address ":9001"
+ExecStop=/usr/bin/docker stop -t 10 minio
+
+[Install]
+WantedBy=multi-user.target

ansible/roles/minio_start/vars/main.yml

@@ -0,0 +1,5 @@
+---
+# vars file for minio_start
+# Мы качаем docker image отсюда: https://quay.io/minio/minio/
+minio_version: RELEASE.2024-07-16T23-46-41Z
+minio_uid: 1111

ansible/roles/mount/tasks/main.yml

@@ -0,0 +1,8 @@
+---
+- name: Mount up device
+  ansible.posix.mount:
+    path: '{{ item["path"] }}'
+    src: '{{ item["src"] }}'
+    fstype: "{{ filesystem }}"
+    state: mounted
+  loop: "{{ device }}"

ansible/roles/nginx_install/files/conf.d/minio.conf

@@ -0,0 +1,73 @@
+upstream minio_s3 {
+    least_conn;
+    server 10.0.75.55:9000;
+    server 10.0.75.56:9000;
+    server 10.0.75.57:9000;
+    server 10.0.75.58:9000;
+}
+
+upstream minio_console {
+    least_conn;
+    server 10.0.75.55:9001;
+    server 10.0.75.56:9001;
+    server 10.0.75.57:9001;
+    server 10.0.75.58:9001;
+}
+
+server {
+    listen        80;
+    listen   [::]:80;
+    listen *:443 ssl;
+    server_name  s3.example.local www.s3.example.local;
+    ssl_certificate /etc/ssl/private/minio.crt;
+    ssl_certificate_key /etc/ssl/private/private.key;
+
+    server_tokens off;
+
+    # Allow special characters in headers
+    ignore_invalid_headers off;
+    # Allow any size file to be uploaded.
+    # Set to a value such as 1000m; to restrict file size to a specific value
+    client_max_body_size 0;
+    # Disable buffering
+    proxy_buffering off;
+    proxy_request_buffering off;
+
+    location / {
+        proxy_set_header Host $http_host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+
+        proxy_connect_timeout 300;
+        # Default is HTTP/1, keepalive is only enabled in HTTP/1.1
+        proxy_http_version 1.1;
+        proxy_set_header Connection "";
+        chunked_transfer_encoding off;
+
+        proxy_pass http://minio_s3; # This uses the upstream directive definition to load balance
+    }
+
+    location /minio/ui/ {
+        rewrite ^/minio/ui/(.*) /$1 break;
+        proxy_set_header Host $http_host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header X-Forwarded-Proto $scheme;
+        proxy_set_header X-NginX-Proxy true;
+
+        # This is necessary to pass the correct IP to be hashed
+        real_ip_header X-Real-IP;
+
+        proxy_connect_timeout 300;
+
+        # To support websockets in MinIO versions released after January 2023
+        proxy_http_version 1.1;
+        proxy_set_header Upgrade $http_upgrade;
+        proxy_set_header Connection "upgrade";
+
+        chunked_transfer_encoding off;
+
+        proxy_pass http://minio_console; # This uses the upstream directive definition to load balance
+    }
+}

ansible/roles/nginx_install/handlers/main.yml

@@ -0,0 +1,8 @@
+---
+- name: Перезапускаю сервер nginx
+  community.docker.docker_container_exec:
+    container: nginx
+    command: "{{ item }}"
+  loop:
+    - nginx -t
+    - nginx -s reload

ansible/roles/nginx_install/tasks/cert.yml

@@ -0,0 +1,34 @@
+---
+- name: Создаю директорию для ключей
+  ansible.builtin.file:
+    path: /etc/ssl/private
+    state: directory
+    owner: "{{ nginx_user }}"
+    group: "{{ nginx_user }}"
+    mode: "755"
+
+- name: Генерирую приватный ключ
+  community.crypto.openssl_privatekey:
+    path: "/etc/ssl/private/private.key"
+    mode: "0600"
+    owner: "{{ nginx_user }}"
+    group: "{{ nginx_user }}"
+
+- name: Создаю запроса на подписание сертификата (CSR) для самоподписанного сертификата
+  community.crypto.openssl_csr_pipe:
+    privatekey_path: "/etc/ssl/private/private.key"
+    common_name: "minio"
+    organization_name: Example, Inc.
+    subject_alt_name:
+      - "DNS:s3.example.local"
+  register: csr
+
+- name: Создаю самоподписанный сертификат из CSR
+  community.crypto.x509_certificate:
+    path: "/etc/ssl/private/minio.crt"
+    csr_content: "{{ csr.csr }}"
+    privatekey_path: "/etc/ssl/private/private.key"
+    provider: selfsigned
+    mode: "0640"
+    owner: "{{ nginx_user }}"
+    group: "{{ nginx_user }}"

ansible/roles/nginx_install/tasks/config.yml

@@ -0,0 +1,19 @@
+---
+- name: Создаю директорию для nginx conf
+  ansible.builtin.file:
+    path: /etc/nginx/conf.d
+    state: directory
+    owner: "{{ nginx_user }}"
+    group: "{{ nginx_user }}"
+    mode: "755"
+
+- name: Синхронизирую конфигурационные файлы
+  ansible.posix.synchronize:
+    src: "conf.d/"
+    dest: "/etc/nginx/conf.d/"
+    delete: true
+    rsync_opts:
+      - "--chown={{ nginx_user }}:{{ nginx_user }}"
+      - "--chmod=F640"
+  notify:
+    - Перезапускаю сервер nginx

ansible/roles/nginx_install/tasks/main.yml

@@ -0,0 +1,16 @@
+---
+- name: Создаю пользователя nginx
+  ansible.builtin.import_tasks: user.yml
+  when: ansible_distribution == 'Ubuntu'
+
+- name: Копирую конфигурацию
+  ansible.builtin.import_tasks: config.yml
+  when: ansible_distribution == 'Ubuntu'
+
+- name: Генерирую самоподписанный сертификат
+  ansible.builtin.import_tasks: cert.yml
+  when: ansible_distribution == 'Ubuntu'
+
+- name: Устанавливаю nginx
+  ansible.builtin.import_tasks: nginx.yml
+  when: ansible_distribution == 'Ubuntu'

ansible/roles/nginx_install/tasks/nginx.yml

@@ -0,0 +1,23 @@
+---
+- name: Наливаю docker-compose файл nginx
+  ansible.builtin.template:
+    src: docker-compose.yml.j2
+    dest: "/home/{{ nginx_user }}/docker-compose.yml"
+    owner: "{{ nginx_user }}"
+    group: "{{ nginx_user }}"
+    mode: "640"
+
+- name: Налаживаю права для директории /etc/letsencrypt/
+  ansible.builtin.file:
+    path: "{{ item }}"
+    owner: "{{ nginx_user }}"
+    group: "{{ nginx_user }}"
+    recurse: true
+  with_items:
+    - "/etc/letsencrypt/"
+
+- name: Поднимаю nginx
+  community.docker.docker_compose_v2:
+    project_src: "/home/{{ nginx_user }}/"
+    remove_orphans: true
+    wait: true

ansible/roles/nginx_install/tasks/user.yml

@@ -0,0 +1,7 @@
+---
+- name: Создаю пользователя {{ nginx_user }}
+  ansible.builtin.user:
+    name: "{{ nginx_user }}"
+    state: present
+    groups: docker
+    shell: /bin/bash

ansible/roles/nginx_install/templates/docker-compose.yml.j2

@@ -0,0 +1,28 @@
+services:
+  nginx:
+    image: nginx:{{ nginx_version }}
+    container_name: nginx
+    restart: always
+    ports:
+      - "80:80"
+      - "443:443"
+    environment:
+      - TZ=Europe/Moscow
+    volumes:
+      - /etc/nginx/conf.d/:/etc/nginx/conf.d/:ro
+      - /etc/nginx/include.d/:/etc/nginx/include.d/:ro
+      - /etc/ssl/private/:/etc/ssl/private/:ro
+      - /var/lib/letsencrypt/:/var/lib/letsencrypt/:ro
+      - /etc/letsencrypt:/etc/letsencrypt:ro
+    depends_on:
+      - certbot
+
+  certbot:
+    image: certbot/certbot:{{ certbot_version }}
+    container_name: certbot
+    restart: always
+    environment:
+      - TZ=Europe/Moscow
+    volumes:
+      - /etc/letsencrypt:/etc/letsencrypt
+    entrypoint: "/bin/sh -c 'trap exit TERM; while :; do certbot renew --webroot -w /etc/letsencrypt/ -n; sleep 12h & wait $${!}; done;'"

ansible/roles/nginx_install/vars/main.yml

@@ -0,0 +1,5 @@
+---
+# vars file for nginx-install
+nginx_version: 1.27.0-alpine
+nginx_user: nginx
+certbot_version: v2.11.0

opentofu/minio/.terraform.lock.hcl

@@ -0,0 +1,25 @@
+# This file is maintained automatically by "tofu init".
+# Manual edits may be lost in future updates.
+
+provider "registry.opentofu.org/bpg/proxmox" {
+  version     = "0.61.1"
+  constraints = ">= 0.57.0"
+  hashes = [
+    "h1:SQSHTHj2ThcF08cON2gHYcnkS/XLmoF8E4cRIgpagtE=",
+    "zh:27d8b589a2dc1e0a5b0f8ab299b9f3704a2f0b69799d1d4d8845c68056986d1f",
+    "zh:46dfa6b33ddd7007a2144f38090457604eb56a59a303b37bb0ad1be5c84ddaca",
+    "zh:47a1b14a759393c5ecc76f2feb950677c418c910b8c677fde0dd3e4675c41579",
+    "zh:582e49d109d1c2b1f3b1268a7cbc43548f3c6d96a87c92a5428767097a5e383e",
+    "zh:5e98ad6afae5969a4c3ffb14c0484936550c66c8313d7686551c29b633ff32f2",
+    "zh:7b9e24b76f947ab8f1e571cf61beefc983b7d2aa1b85df35c4f015728fe37a38",
+    "zh:8255ca210f279a0f7b8ca2762df26d2ea1a01704298c5e3d5cf601bd39a743f0",
+    "zh:85d7655fdc95dedced9cf8105a0beeb0d7bc8f668c55f62019a7215a76d60300",
+    "zh:8aeea5a1d001b06baaf923b754e1a14d06c75eb8c8b87a7f65a3c8205fc8b079",
+    "zh:a9cfab6c06f613658c5fdd83742cd22c0eb7563778924b1407965ef8c36c1ce0",
+    "zh:ceaab67801d49a92eb5858b1ddae6df2569462e5ffbe31f9dbd79dcb684ea142",
+    "zh:dc25b506d5c55d1d78a335d3ebd03213c99b4b2a5859812349a955c2f746ff7e",
+    "zh:e04b477fd77a0d37a0bdb76a7cf69184dad9e7fbba9b4f3a378a8901b82b75e5",
+    "zh:f1e6838d9141557f73340df9b21fce5a82b41cc16ae36f063a920ccc36bc0758",
+    "zh:f26e0763dbe6a6b2195c94b44696f2110f7f55433dc142839be16b9697fa5597",
+  ]
+}

opentofu/minio/minio.tf

@@ -0,0 +1,122 @@
+variable "minio_vms" {
+  type = list(object({
+    name      = string
+    address   = string
+    node_name = string
+  }))
+  default = [
+    {
+      name      = "minio1"
+      address   = "10.0.75.55/24"
+      node_name = "pve-01"
+    },
+    {
+      name      = "minio2"
+      address   = "10.0.75.56/24"
+      node_name = "pve-02"
+    },
+    {
+      name      = "minio3"
+      address   = "10.0.75.57/24"
+      node_name = "pve-01"
+    },
+    {
+      name      = "minio4"
+      address   = "10.0.75.58/24"
+      node_name = "pve-02"
+    }
+  ]
+}
+
+# Создание виртуальных машин
+resource "proxmox_virtual_environment_vm" "minio" {
+  for_each = { for vm in var.minio_vms : vm.name => vm }
+
+  name    = each.value.name
+  migrate = true
+  # protection  = true
+  description = "Managed by OpenTofu"
+  tags        = ["minio", "opentofu"]
+  on_boot     = true
+  node_name   = each.value.node_name
+
+  clone {
+    vm_id     = "2404"
+    node_name = "pve-01"
+    retries   = 3
+  }
+
+  agent {
+    enabled = true
+  }
+
+  operating_system {
+    type = "l26"
+  }
+
+  cpu {
+    cores = 2
+    type  = "host"
+    numa  = true
+  }
+
+  memory {
+    dedicated = 2048
+  }
+
+  vga {
+    memory = 4
+    type   = "serial0"
+  }
+
+  disk {
+    size         = "20"
+    interface    = "virtio0"
+    datastore_id = "proxmox-data-02"
+    file_format  = "raw"
+  }
+
+  disk {
+    size         = "450"
+    interface    = "virtio1"
+    datastore_id = "minio-data-01"
+    file_format  = "raw"
+  }
+
+  network_device {
+    bridge = "vmbr0"
+    model  = "virtio"
+  }
+
+  initialization {
+    datastore_id = "proxmox-data-02"
+    ip_config {
+      ipv4 {
+        address = each.value.address
+        gateway = "10.0.75.1"
+      }
+    }
+    dns {
+      servers = [
+        "10.0.75.65",
+        "10.0.75.66"
+      ]
+    }
+    user_account {
+      username = "infra"
+      keys = [
+        var.ssh_public_key
+      ]
+    }
+  }
+}
+
+# Создание ресурсов высокой доступности
+# resource "proxmox_virtual_environment_haresource" "patroni" {
+#   for_each = { for vm in var.minio_vms : vm.name => vm }
+
+#   resource_id = "vm:${proxmox_virtual_environment_vm.patroni[each.key].vm_id}"
+#   state       = "started"
+#   group       = "prod"
+#   comment     = "Managed by OpenTofu"
+# }

opentofu/minio/provider.tf

@@ -0,0 +1,17 @@
+terraform {
+  required_providers {
+    proxmox = {
+      source  = "bpg/proxmox"
+      version = ">= 0.57.0"
+    }
+  }
+}
+
+provider "proxmox" {
+  endpoint  = var.virtual_environment_endpoint
+  api_token = var.virtual_environment_api_token
+  insecure  = true
+  ssh {
+    agent = false
+  }
+}

opentofu/minio/terraform.tfvars.example

@@ -0,0 +1,3 @@
+virtual_environment_api_token = "fedor@pve!opentofu=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
+virtual_environment_endpoint = "https://x.x.x.x:8006/"
+ssh_public_key = "ssh-rsa ..."

opentofu/minio/variables.tf

@@ -0,0 +1,14 @@
+variable "virtual_environment_endpoint" {
+  type        = string
+  description = "The endpoint for the Proxmox Virtual Environment API (example: https://host:port)"
+}
+
+variable "virtual_environment_api_token" {
+  type        = string
+  description = "The api roken the Proxmox Virtual Environment API (example: root@pam!for-terraform-provider=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx)"
+}
+
+variable "ssh_public_key" {
+  type        = string
+  description = "SSH Puclic key for VMs (example: ssh-rsa ...)"
+}

opentofu/nginx/.terraform.lock.hcl

@@ -0,0 +1,25 @@
+# This file is maintained automatically by "tofu init".
+# Manual edits may be lost in future updates.
+
+provider "registry.opentofu.org/bpg/proxmox" {
+  version     = "0.61.1"
+  constraints = ">= 0.57.0"
+  hashes = [
+    "h1:SQSHTHj2ThcF08cON2gHYcnkS/XLmoF8E4cRIgpagtE=",
+    "zh:27d8b589a2dc1e0a5b0f8ab299b9f3704a2f0b69799d1d4d8845c68056986d1f",
+    "zh:46dfa6b33ddd7007a2144f38090457604eb56a59a303b37bb0ad1be5c84ddaca",
+    "zh:47a1b14a759393c5ecc76f2feb950677c418c910b8c677fde0dd3e4675c41579",
+    "zh:582e49d109d1c2b1f3b1268a7cbc43548f3c6d96a87c92a5428767097a5e383e",
+    "zh:5e98ad6afae5969a4c3ffb14c0484936550c66c8313d7686551c29b633ff32f2",
+    "zh:7b9e24b76f947ab8f1e571cf61beefc983b7d2aa1b85df35c4f015728fe37a38",
+    "zh:8255ca210f279a0f7b8ca2762df26d2ea1a01704298c5e3d5cf601bd39a743f0",
+    "zh:85d7655fdc95dedced9cf8105a0beeb0d7bc8f668c55f62019a7215a76d60300",
+    "zh:8aeea5a1d001b06baaf923b754e1a14d06c75eb8c8b87a7f65a3c8205fc8b079",
+    "zh:a9cfab6c06f613658c5fdd83742cd22c0eb7563778924b1407965ef8c36c1ce0",
+    "zh:ceaab67801d49a92eb5858b1ddae6df2569462e5ffbe31f9dbd79dcb684ea142",
+    "zh:dc25b506d5c55d1d78a335d3ebd03213c99b4b2a5859812349a955c2f746ff7e",
+    "zh:e04b477fd77a0d37a0bdb76a7cf69184dad9e7fbba9b4f3a378a8901b82b75e5",
+    "zh:f1e6838d9141557f73340df9b21fce5a82b41cc16ae36f063a920ccc36bc0758",
+    "zh:f26e0763dbe6a6b2195c94b44696f2110f7f55433dc142839be16b9697fa5597",
+  ]
+}

opentofu/nginx/nginx.tf

@@ -0,0 +1,105 @@
+variable "minio_vms" {
+  type = list(object({
+    name      = string
+    address   = string
+    node_name = string
+  }))
+  default = [
+    {
+      name      = "nginx-01"
+      address   = "10.0.75.91/24"
+      node_name = "pve-01"
+    },
+    {
+      name      = "nginx-02"
+      address   = "10.0.75.92/24"
+      node_name = "pve-02"
+    }
+  ]
+}
+
+# Создание виртуальных машин
+resource "proxmox_virtual_environment_vm" "minio" {
+  for_each = { for vm in var.minio_vms : vm.name => vm }
+
+  name    = each.value.name
+  migrate = true
+  # protection  = true
+  description = "Managed by OpenTofu"
+  tags        = ["nignx", "opentofu"]
+  on_boot     = true
+  node_name   = each.value.node_name
+
+  clone {
+    vm_id     = "2404"
+    node_name = "pve-01"
+    retries   = 3
+  }
+
+  agent {
+    enabled = true
+  }
+
+  operating_system {
+    type = "l26"
+  }
+
+  cpu {
+    cores = 1
+    type  = "host"
+    numa  = true
+  }
+
+  memory {
+    dedicated = 1024
+  }
+
+  vga {
+    memory = 4
+    type   = "serial0"
+  }
+
+  disk {
+    size         = "20"
+    interface    = "virtio0"
+    datastore_id = "proxmox-data-02"
+    file_format  = "raw"
+  }
+
+  network_device {
+    bridge = "vmbr0"
+    model  = "virtio"
+  }
+
+  initialization {
+    datastore_id = "proxmox-data-02"
+    ip_config {
+      ipv4 {
+        address = each.value.address
+        gateway = "10.0.75.1"
+      }
+    }
+    dns {
+      servers = [
+        "10.0.75.65",
+        "10.0.75.66"
+      ]
+    }
+    user_account {
+      username = "infra"
+      keys = [
+        var.ssh_public_key
+      ]
+    }
+  }
+}
+
+# Создание ресурсов высокой доступности
+# resource "proxmox_virtual_environment_haresource" "patroni" {
+#   for_each = { for vm in var.minio_vms : vm.name => vm }
+
+#   resource_id = "vm:${proxmox_virtual_environment_vm.patroni[each.key].vm_id}"
+#   state       = "started"
+#   group       = "prod"
+#   comment     = "Managed by OpenTofu"
+# }

opentofu/nginx/provider.tf

@@ -0,0 +1,17 @@
+terraform {
+  required_providers {
+    proxmox = {
+      source  = "bpg/proxmox"
+      version = ">= 0.57.0"
+    }
+  }
+}
+
+provider "proxmox" {
+  endpoint  = var.virtual_environment_endpoint
+  api_token = var.virtual_environment_api_token
+  insecure  = true
+  ssh {
+    agent = false
+  }
+}

opentofu/nginx/terraform.tfvars.example

@@ -0,0 +1,3 @@
+virtual_environment_api_token = "fedor@pve!opentofu=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
+virtual_environment_endpoint = "https://x.x.x.x:8006/"
+ssh_public_key = "ssh-rsa ..."

opentofu/nginx/variables.tf

@@ -0,0 +1,14 @@
+variable "virtual_environment_endpoint" {
+  type        = string
+  description = "The endpoint for the Proxmox Virtual Environment API (example: https://host:port)"
+}
+
+variable "virtual_environment_api_token" {
+  type        = string
+  description = "The api roken the Proxmox Virtual Environment API (example: root@pam!for-terraform-provider=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx)"
+}
+
+variable "ssh_public_key" {
+  type        = string
+  description = "SSH Puclic key for VMs (example: ssh-rsa ...)"
+}