Add kubeadm example (#4)

* Add kubeadm example

* Fix dns

---------

Co-authored-by: Fedor Batonogov <f.batonogov@yandex.ru>

ansible/ansible.cfg

@@ -0,0 +1,8 @@
+[defaults]
+timeout=60
+roles_path=./roles
+inventory=./inventory.yml
+forks=100
+serial=10
+host_key_checking=False
+callbacks_enabled=ansible.posix.profile_tasks

ansible/inventory.yml

@@ -0,0 +1,21 @@
+kubeadm:
+  children:
+    kubeadm_control_plane:
+      hosts:
+        kubeadm-cp-01:
+          ansible_host: 10.0.70.70
+        kubeadm-cp-02:
+          ansible_host: 10.0.70.78
+        kubeadm-cp-03:
+          ansible_host: 10.0.70.79
+    kubeadm_nodes:
+      hosts:
+        kubeadm-node-01:
+          ansible_host: 10.0.70.71
+        kubeadm-node-02:
+          ansible_host: 10.0.70.77
+        kubeadm-node-03:
+          ansible_host: 10.0.70.74
+  vars:
+    ansible_user: infra
+    ansible_port: 22

ansible/kubeadm.yml

@@ -0,0 +1,101 @@
+# Запустите сервисы как статические подсистемы
+# https://github.com/kubernetes/kubeadm/blob/main/docs/ha-considerations.md#option-2-run-the-services-as-static-pods
+- name: Настраиваю keepalived + haproxy
+  become: true
+  hosts:
+    - kubeadm_control_plane
+  roles:
+    - haproxy_static_pods
+
+- name: Разворачиваю kubernetes кластер
+  become: true
+  hosts:
+    - kubeadm
+  handlers:
+    - name: Перезагружаю виртуальные машины
+      ansible.builtin.reboot:
+  tasks:
+    - name: Добавляю модули br_netfilter и overlay
+      community.general.modprobe:
+        name: '{{ item }}'
+        state: present
+      with_items:
+        - br_netfilter
+        - overlay
+      # notify:
+      #   - Перезагружаю виртуальные машины
+
+    - name: Добавляю модули br_netfilter и overlay в /etc/modules
+      ansible.builtin.lineinfile:
+        path: /etc/modules
+        line: '{{ item }}'
+        create: true
+      with_items:
+        - br_netfilter
+        - overlay
+      # notify:
+      #   - Перезагружаю виртуальные машины
+
+    - name: Включаю маршрутизацию IP и iptables для моста
+      ansible.posix.sysctl:
+        name: '{{ item }}'
+        value: 1
+        state: present
+      with_items:
+        - net.ipv4.ip_forward
+        - net.bridge.bridge-nf-call-iptables
+
+    - name: Устанавливаю пакеты
+      ansible.builtin.apt:
+        name:
+          - apt-transport-https
+          - ca-certificates
+          - curl
+          - gpg
+          - software-properties-common
+        update_cache: true
+
+    - name: Добавляю gpg ключ для репозиториев Kubernetes и cri-o
+      ansible.builtin.apt_key:
+        url: '{{ item["url"] }}'
+        state: present
+        keyring: '{{ item["keyring"] }}'
+      with_items:
+        - {url: "https://pkgs.k8s.io/core:/stable:/v1.30/deb/Release.key", keyring: "/etc/apt/keyrings/kubernetes-apt-keyring.gpg"}
+        - {url: "https://pkgs.k8s.io/addons:/cri-o:/prerelease:/main/deb/Release.key", keyring: "/etc/apt/keyrings/cri-o-apt-keyring.gpg"}
+
+    - name: Добавляю репозитории Kubernetes и cri-o
+      ansible.builtin.apt_repository:
+        repo: '{{ item }}'
+        state: present
+      with_items:
+        - deb [signed-by=/etc/apt/keyrings/cri-o-apt-keyring.gpg] https://pkgs.k8s.io/addons:/cri-o:/prerelease:/main/deb/ /
+        - deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.30/deb/ /
+
+    - name: Устанавливаю пакеты kubelet, kubeadm, kubectl и cri-o
+      ansible.builtin.apt:
+        name:
+          - kubelet
+          - kubeadm
+          - kubectl
+          - cri-o
+        state: present
+        update_cache: true
+
+    - name: Предотвращаю обновление kubelet, kubeadm и kubectl
+      ansible.builtin.dpkg_selections:
+        name: '{{ item }}'
+        selection: hold
+      with_items:
+        - kubelet
+        - kubeadm
+        - kubectl
+
+    - name: Включаю и запускаю службы kubelet и cri-o
+      ansible.builtin.systemd:
+        name: '{{ item }}'
+        enabled: true
+        state: started
+      with_items:
+        - kubelet
+        - crio

ansible/roles/haproxy_static_pods/files/check_apiserver.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+errorExit() {
+    echo "*** $*" 1>&2
+    exit 1
+}
+
+curl --silent --max-time 2 --insecure https://localhost:7443/ -o /dev/null || errorExit "Error GET https://localhost:7443/"
+if ip addr | grep -q 10.0.70.85; then
+    curl --silent --max-time 2 --insecure https://10.0.70.85:7443/ -o /dev/null || errorExit "Error GET https://10.0.70.85:7443/"
+fi

ansible/roles/haproxy_static_pods/files/haproxy.cfg

@@ -0,0 +1,52 @@
+# /etc/haproxy/haproxy.cfg
+#---------------------------------------------------------------------
+# Global settings
+#---------------------------------------------------------------------
+global
+    log /dev/log local0
+    log /dev/log local1 notice
+    daemon
+
+#---------------------------------------------------------------------
+# common defaults that all the 'listen' and 'backend' sections will
+# use if not designated in their block
+#---------------------------------------------------------------------
+defaults
+    mode                    http
+    log                     global
+    option                  httplog
+    option                  dontlognull
+    option http-server-close
+    option forwardfor       except 127.0.0.0/8
+    option                  redispatch
+    retries                 1
+    timeout http-request    10s
+    timeout queue           20s
+    timeout connect         5s
+    timeout client          20s
+    timeout server          20s
+    timeout http-keep-alive 10s
+    timeout check           10s
+
+#---------------------------------------------------------------------
+# apiserver frontend which proxys to the control plane nodes
+#---------------------------------------------------------------------
+frontend apiserver
+    bind *:7443
+    mode tcp
+    option tcplog
+    default_backend apiserverbackend
+
+#---------------------------------------------------------------------
+# round robin balancing for apiserver
+#---------------------------------------------------------------------
+backend apiserverbackend
+    option httpchk GET /healthz
+    http-check expect status 200
+    mode tcp
+    option ssl-hello-chk
+    balance     roundrobin
+        server 10.0.70.70 10.0.70.70:6443 check
+        server 10.0.70.78 10.0.70.78:6443 check
+        server 10.0.70.79 10.0.70.79:6443 check
+        # [...]

ansible/roles/haproxy_static_pods/files/haproxy.yaml

@@ -0,0 +1,27 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: haproxy
+  namespace: kube-system
+spec:
+  containers:
+  - image: haproxy:2.9.7
+    name: haproxy
+    livenessProbe:
+      failureThreshold: 8
+      httpGet:
+        host: localhost
+        path: /healthz
+        port: 7443
+        scheme: HTTPS
+    volumeMounts:
+    - mountPath: /usr/local/etc/haproxy/haproxy.cfg
+      name: haproxyconf
+      readOnly: true
+  hostNetwork: true
+  volumes:
+  - hostPath:
+      path: /etc/haproxy/haproxy.cfg
+      type: FileOrCreate
+    name: haproxyconf
+status: {}

ansible/roles/haproxy_static_pods/files/keepalived.yaml

@@ -0,0 +1,31 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  creationTimestamp: null
+  name: keepalived
+  namespace: kube-system
+spec:
+  containers:
+  - image: osixia/keepalived:2.0.20
+    name: keepalived
+    resources: {}
+    securityContext:
+      capabilities:
+        add:
+        - NET_ADMIN
+        - NET_BROADCAST
+        - NET_RAW
+    volumeMounts:
+    - mountPath: /usr/local/etc/keepalived/keepalived.conf
+      name: config
+    - mountPath: /etc/keepalived/check_apiserver.sh
+      name: check
+  hostNetwork: true
+  volumes:
+  - hostPath:
+      path: /etc/keepalived/keepalived.conf
+    name: config
+  - hostPath:
+      path: /etc/keepalived/check_apiserver.sh
+    name: check
+status: {}

ansible/roles/haproxy_static_pods/tasks/main.yml

@@ -0,0 +1,40 @@
+# tasks file for haproxy_static_pods
+- name: Создать директории /etc/kubernetes/manifests и /etc/keepalived
+  ansible.builtin.file:
+    path: '{{ item }}'
+    state: directory
+    mode: '755'
+  with_items:
+    - /etc/kubernetes/manifests
+    - /etc/keepalived
+    - /etc/haproxy
+
+- name: Наливаю конфигурацию keepalived
+  ansible.builtin.template:
+    src: keepalived.conf.j2
+    dest: /etc/keepalived/keepalived.conf
+    mode: "644"
+
+- name: Наливаю check_apiserver.sh
+  ansible.builtin.copy:
+    src: check_apiserver.sh
+    dest: /etc/keepalived/check_apiserver.sh
+    mode: '644'
+
+- name: Наливаю haproxy.cfg
+  ansible.builtin.copy:
+    src: haproxy.cfg
+    dest: /etc/haproxy/haproxy.cfg
+    mode: '644'
+
+- name: Наливаю keepalived static pods manifest
+  ansible.builtin.copy:
+    src: keepalived.yaml
+    dest: /etc/kubernetes/manifests/keepalived.yaml
+    mode: '644'
+
+- name: Наливаю haproxy static pods manifest
+  ansible.builtin.copy:
+    src: haproxy.yaml
+    dest: /etc/kubernetes/manifests/haproxy.yaml
+    mode: '644'

ansible/roles/haproxy_static_pods/templates/keepalived.conf.j2

@@ -0,0 +1,29 @@
+! /etc/keepalived/keepalived.conf
+! Configuration File for keepalived
+global_defs {
+    router_id LVS_DEVEL
+}
+vrrp_script check_apiserver {
+  script "/etc/keepalived/check_apiserver.sh"
+  interval 3
+  weight -2
+  fall 10
+  rise 2
+}
+
+vrrp_instance VI_1 {
+    state MASTER
+    interface eth0
+    virtual_router_id 51
+    priority 101
+    authentication {
+        auth_type PASS
+        auth_pass {{ lookup('password', 'secrets/kubeadm/keepalived/auth_pass length=64') }}
+    }
+    virtual_ipaddress {
+        10.0.70.85/24
+    }
+    track_script {
+        check_apiserver
+    }
+}