Add L2 Load Balancer (#16)

Co-authored-by: Fedor Batonogov <f.batonogov@yandex.ru>
2025-11-29 16:53:01 +00:00 · 2024-07-27 11:35:15 +03:00
parent 55deb994c6
commit 52a3d0eb3b
31 changed files with 782 additions and 739 deletions
--- a/ansible/inventory.yml
+++ b/ansible/inventory.yml
@@ -3,27 +3,19 @@ kubeadm:
    kubeadm_control_plane:
      hosts:
        kubeadm-cp-01:
-          ansible_host: 10.0.70.70
+          ansible_host: 10.0.75.81
        kubeadm-cp-02:
-          ansible_host: 10.0.70.78
+          ansible_host: 10.0.75.82
        kubeadm-cp-03:
-          ansible_host: 10.0.70.79
+          ansible_host: 10.0.75.83
    kubeadm_nodes:
      hosts:
        kubeadm-node-01:
-          ansible_host: 10.0.70.71
+          ansible_host: 10.0.75.84
        kubeadm-node-02:
-          ansible_host: 10.0.70.77
+          ansible_host: 10.0.75.85
        kubeadm-node-03:
-          ansible_host: 10.0.70.74
-  vars:
-    ansible_user: infra
-    ansible_port: 22
-
-test_hosts:
-  hosts:
-    kubeadm-cp-01:
-      ansible_host: 10.0.75.203
+          ansible_host: 10.0.75.86
  vars:
    ansible_user: infra
    ansible_port: 22
--- a/ansible/kubeadm.yml
+++ b/ansible/kubeadm.yml
@@ -1,44 +1,29 @@
-# Запустите сервисы как статические подсистемы
-# https://github.com/kubernetes/kubeadm/blob/main/docs/ha-considerations.md#option-2-run-the-services-as-static-pods
- name: Настраиваю keepalived + haproxy
-  become: true
-  hosts:
-    - kubeadm_control_plane
-  roles:
-    - haproxy_static_pods
-
- name: Разворачиваю kubernetes кластер
+# Подготовка к запуску Kubernetes кластера
+- name: Подготоваливаю узлы для kubernetes кластера
  become: true
  hosts:
    - kubeadm
-  handlers:
-    - name: Перезагружаю виртуальные машины
-      ansible.builtin.reboot:
  tasks:
    - name: Добавляю модули br_netfilter и overlay
      community.general.modprobe:
-        name: '{{ item }}'
+        name: "{{ item }}"
        state: present
      with_items:
        - br_netfilter
        - overlay
-      # notify:
-      #   - Перезагружаю виртуальные машины

    - name: Добавляю модули br_netfilter и overlay в /etc/modules
      ansible.builtin.lineinfile:
        path: /etc/modules
-        line: '{{ item }}'
+        line: "{{ item }}"
        create: true
      with_items:
        - br_netfilter
        - overlay
-      # notify:
-      #   - Перезагружаю виртуальные машины

    - name: Включаю маршрутизацию IP и iptables для моста
      ansible.posix.sysctl:
-        name: '{{ item }}'
+        name: "{{ item }}"
        value: 1
        state: present
      with_items:
@@ -54,6 +39,9 @@
          - gpg
          - software-properties-common
        update_cache: true
+      register: apt_res
+      retries: 5
+      until: apt_res is success

    - name: Добавляю gpg ключ для репозиториев Kubernetes и cri-o
      ansible.builtin.apt_key:
@@ -61,15 +49,21 @@
        state: present
        keyring: '{{ item["keyring"] }}'
      with_items:
-        - {url: "https://pkgs.k8s.io/core:/stable:/v1.30/deb/Release.key", keyring: "/etc/apt/keyrings/kubernetes-apt-keyring.gpg"}
-        - {url: "https://pkgs.k8s.io/addons:/cri-o:/prerelease:/main/deb/Release.key", keyring: "/etc/apt/keyrings/cri-o-apt-keyring.gpg"}
+        - {
+            url: "https://pkgs.k8s.io/core:/stable:/v1.30/deb/Release.key",
+            keyring: "/etc/apt/keyrings/kubernetes-apt-keyring.gpg",
+          }
+        - {
+            url: "https://pkgs.k8s.io/addons:/cri-o:/prerelease:/main/deb/Release.key",
+            keyring: "/etc/apt/keyrings/cri-o-apt-keyring.gpg",
+          }

    - name: Добавляю репозитории Kubernetes и cri-o
      ansible.builtin.apt_repository:
-        repo: '{{ item }}'
+        repo: "{{ item }}"
        state: present
      with_items:
-        - deb [signed-by=/etc/apt/keyrings/cri-o-apt-keyring.gpg] https://pkgs.k8s.io/addons:/cri-o:/prerelease:/main/deb/ /
+        - deb [signed-by=/etc/apt/keyrings/cri-o-apt-keyring.gpg] https://pkgs.k8s.io/addons:/cri-o:/stable:/v1.30/deb/ /
        - deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.30/deb/ /

    - name: Устанавливаю пакеты kubelet, kubeadm, kubectl и cri-o
@@ -84,7 +78,7 @@

    - name: Предотвращаю обновление kubelet, kubeadm и kubectl
      ansible.builtin.dpkg_selections:
-        name: '{{ item }}'
+        name: "{{ item }}"
        selection: hold
      with_items:
        - kubelet
@@ -93,33 +87,88 @@

    - name: Включаю и запускаю службы kubelet и cri-o
      ansible.builtin.systemd:
-        name: '{{ item }}'
+        name: "{{ item }}"
        enabled: true
        state: started
      with_items:
        - kubelet
        - crio

- name: Устанавливаю пакеты для Longhorn
+# Запуск сервисов keepalived и haproxy как статических подсистем
+# https://github.com/kubernetes/kubeadm/blob/main/docs/ha-considerations.md#option-2-run-the-services-as-static-pods
+- name: Настраиваю keepalived + haproxy и инициализирую кластер
  become: true
  hosts:
-    - kubeadm_nodes
+    - kubeadm_control_plane
+  roles:
+    - haproxy_static_pods
  tasks:
-    - name: Устанавливаю нужные пакеты
-      ansible.builtin.apt:
-        name:
-          - open-iscsi
-          - nfs-common
-          - bash
-          - curl
-          - grep
-        state: present
-        update_cache: true
+    - name: Инициализирую высокодоступный кластер
+      run_once: true
+      ansible.builtin.command: |
+        kubeadm init \
+          --pod-network-cidr=10.244.0.0/16 \
+          --control-plane-endpoint=10.0.75.80:8888 \
+          --upload-certs \
+          --skip-phases=addon/kube-proxy
+      args:
+        creates: /etc/kubernetes/kubelet.conf
+      notify:
+        - Создаю token для control-plane
+        - Создаю token для node
+        - Добавляю control-plane узлы в кластер
+        - Добавляю node узлы в кластер
+  handlers:
+    - name: Создаю token для control-plane
+      ansible.builtin.shell:
+        cmd: |
+          set -o pipefail
+          echo $(kubeadm token create --print-join-command) \
+            --control-plane \
+            --certificate-key \
+            $(kubeadm init phase upload-certs --upload-certs | grep -vw -e certificate -e Namespace)
+        executable: /bin/bash
+      register: join_control_plane_raw
+    - name: Создаю token для node
+      ansible.builtin.command: kubeadm token create --print-join-command
+      register: join_node_raw
+    - name: Добавляю control-plane узлы в кластер
+      ansible.builtin.command: "{{ join_control_plane_raw.stdout }}"
+      args:
+        creates: /etc/kubernetes/kubelet.conf
+      delegate_to: "{{ item }}"
+      loop: "{{ groups['kubeadm_control_plane'] }}"
+    - name: Добавляю node узлы в кластер
+      ansible.builtin.command: "{{ join_node_raw.stdout }}"
+      args:
+        creates: /etc/kubernetes/kubelet.conf
+      delegate_to: "{{ item }}"
+      loop: "{{ groups['kubeadm_nodes'] }}"

-    - name: Включаю и запускаю службы
-      ansible.builtin.systemd:
-        name: '{{ item }}'
-        enabled: true
-        state: started
-      with_items:
-        - iscsid
+# Подготовка control-plane узлов
+- name: Подготовка control-plane узлов для работы с kubectl
+  become: true
+  gather_facts: false
+  hosts:
+    - kubeadm_control_plane
+  tasks:
+    - name: Создаю директорию .kube
+      become_user: infra
+      ansible.builtin.file:
+        path: $HOME/.kube
+        state: directory
+        mode: "755"
+    - name: Копирую admin.conf в директорию .kube
+      ansible.builtin.copy:
+        src: /etc/kubernetes/admin.conf
+        dest: /home/infra/.kube/config
+        remote_src: true
+        owner: infra
+        group: infra
+        mode: "600"
+    - name: Копирую kube/config
+      run_once: true
+      ansible.posix.synchronize:
+        src: "~/.kube/config" # remote host
+        dest: "~/.kube/config" # localhost
+        mode: pull
--- a/ansible/roles/haproxy_static_pods/files/check_apiserver.sh
+++ b/ansible/roles/haproxy_static_pods/files/check_apiserver.sh
@@ -5,7 +5,7 @@ errorExit() {
    exit 1
 }

-curl --silent --max-time 2 --insecure https://localhost:7443/ -o /dev/null || errorExit "Error GET https://localhost:7443/"
-if ip addr | grep -q 10.0.70.85; then
-    curl --silent --max-time 2 --insecure https://10.0.70.85:7443/ -o /dev/null || errorExit "Error GET https://10.0.70.85:7443/"
+curl --silent --max-time 2 --insecure https://localhost:8888/ -o /dev/null || errorExit "Error GET https://localhost:8888/"
+if ip addr | grep -q 10.0.75.80; then
+    curl --silent --max-time 2 --insecure https://10.0.75.80:8888/ -o /dev/null || errorExit "Error GET https://10.0.75.80:8888/"
 fi
--- a/ansible/roles/haproxy_static_pods/files/haproxy.cfg
+++ b/ansible/roles/haproxy_static_pods/files/haproxy.cfg
@@ -32,7 +32,7 @@ defaults
 # apiserver frontend which proxys to the control plane nodes
 #---------------------------------------------------------------------
 frontend apiserver
-    bind *:7443
+    bind *:8888
    mode tcp
    option tcplog
    default_backend apiserverbackend
@@ -46,7 +46,7 @@ backend apiserverbackend
    mode tcp
    option ssl-hello-chk
    balance     roundrobin
-        server 10.0.70.70 10.0.70.70:6443 check
-        server 10.0.70.78 10.0.70.78:6443 check
-        server 10.0.70.79 10.0.70.79:6443 check
+        server 10.0.75.81 10.0.75.81:6443 check
+        server 10.0.75.82 10.0.75.82:6443 check
+        server 10.0.75.83 10.0.75.83:6443 check
        # [...]
--- a/ansible/roles/haproxy_static_pods/files/haproxy.yaml
+++ b/ansible/roles/haproxy_static_pods/files/haproxy.yaml
@@ -5,23 +5,23 @@ metadata:
  namespace: kube-system
 spec:
  containers:
-  - image: haproxy:2.9.7
-    name: haproxy
-    livenessProbe:
-      failureThreshold: 8
-      httpGet:
-        host: localhost
-        path: /healthz
-        port: 7443
-        scheme: HTTPS
-    volumeMounts:
-    - mountPath: /usr/local/etc/haproxy/haproxy.cfg
-      name: haproxyconf
-      readOnly: true
+    - image: haproxy:3.0.2
+      name: haproxy
+      livenessProbe:
+        failureThreshold: 8
+        httpGet:
+          host: localhost
+          path: /healthz
+          port: 8888
+          scheme: HTTPS
+      volumeMounts:
+        - mountPath: /usr/local/etc/haproxy/haproxy.cfg
+          name: haproxyconf
+          readOnly: true
  hostNetwork: true
  volumes:
-  - hostPath:
-      path: /etc/haproxy/haproxy.cfg
-      type: FileOrCreate
-    name: haproxyconf
+    - hostPath:
+        path: /etc/haproxy/haproxy.cfg
+        type: FileOrCreate
+      name: haproxyconf
 status: {}
--- a/ansible/roles/haproxy_static_pods/files/keepalived.yaml
+++ b/ansible/roles/haproxy_static_pods/files/keepalived.yaml
@@ -6,26 +6,26 @@ metadata:
  namespace: kube-system
 spec:
  containers:
-  - image: osixia/keepalived:2.0.20
-    name: keepalived
-    resources: {}
-    securityContext:
-      capabilities:
-        add:
-        - NET_ADMIN
-        - NET_BROADCAST
-        - NET_RAW
-    volumeMounts:
-    - mountPath: /usr/local/etc/keepalived/keepalived.conf
-      name: config
-    - mountPath: /etc/keepalived/check_apiserver.sh
-      name: check
+    - image: ghcr.io/batonogov/docker-keepalived:2.0.20
+      name: keepalived
+      resources: {}
+      securityContext:
+        capabilities:
+          add:
+            - NET_ADMIN
+            - NET_BROADCAST
+            - NET_RAW
+      volumeMounts:
+        - mountPath: /usr/local/etc/keepalived/keepalived.conf
+          name: config
+        - mountPath: /etc/keepalived/check_apiserver.sh
+          name: check
  hostNetwork: true
  volumes:
-  - hostPath:
-      path: /etc/keepalived/keepalived.conf
-    name: config
-  - hostPath:
-      path: /etc/keepalived/check_apiserver.sh
-    name: check
+    - hostPath:
+        path: /etc/keepalived/keepalived.conf
+      name: config
+    - hostPath:
+        path: /etc/keepalived/check_apiserver.sh
+      name: check
 status: {}
--- a/ansible/roles/haproxy_static_pods/tasks/main.yml
+++ b/ansible/roles/haproxy_static_pods/tasks/main.yml
@@ -1,9 +1,9 @@
 # tasks file for haproxy_static_pods
 - name: Создать директории /etc/kubernetes/manifests и /etc/keepalived
  ansible.builtin.file:
-    path: '{{ item }}'
+    path: "{{ item }}"
    state: directory
-    mode: '755'
+    mode: "755"
  with_items:
    - /etc/kubernetes/manifests
    - /etc/keepalived
@@ -19,22 +19,22 @@
  ansible.builtin.copy:
    src: check_apiserver.sh
    dest: /etc/keepalived/check_apiserver.sh
-    mode: '644'
+    mode: "644"

 - name: Наливаю haproxy.cfg
  ansible.builtin.copy:
    src: haproxy.cfg
    dest: /etc/haproxy/haproxy.cfg
-    mode: '644'
+    mode: "644"

 - name: Наливаю keepalived static pods manifest
  ansible.builtin.copy:
    src: keepalived.yaml
    dest: /etc/kubernetes/manifests/keepalived.yaml
-    mode: '644'
+    mode: "644"

 - name: Наливаю haproxy static pods manifest
  ansible.builtin.copy:
    src: haproxy.yaml
    dest: /etc/kubernetes/manifests/haproxy.yaml
-    mode: '644'
+    mode: "644"
--- a/ansible/roles/haproxy_static_pods/templates/keepalived.conf.j2
+++ b/ansible/roles/haproxy_static_pods/templates/keepalived.conf.j2
@@ -21,7 +21,7 @@ vrrp_instance VI_1 {
        auth_pass {{ lookup('password', 'secrets/kubeadm/keepalived/auth_pass length=64') }}
    }
    virtual_ipaddress {
-        10.0.70.85/24
+        10.0.75.80/24
    }
    track_script {
        check_apiserver