作者： Ju4t
兼容 k8s v1.24.0 以后的版本
本文中 ansibe-playbook 是通过ssh登录到服务器上去部署集群的，所以需要一台客户端运行playbook。
OS：CentOS 7.9 CentOS-7-x86_64-Minimal-2009.iso
硬件：Kubernetes 官方要求不能低于2C4G
数量：
master ≥ 1台，数量建议为基数
worker ≥ 1台，总数不要好过1万台
也可以参考 inventory.ini
您可能会用到的一条修改IP到命令，仅供参考（批量复制出来的虚拟机场景下使用）。
# 将网卡eth0的ip 192.168.8.10 修改为 192.168.8.50
sed -i 's/^IPADDR="192.168.8.10"/IPADDR="192.168.8.50"/g' /etc/sysconfig/network-scripts/ifcfg-eth0 && service netowrk restart
yum install -y python3 unzip expect
pip3 install -U pip setuptools -i https://mirrors.aliyun.com/pypi/simple/
pip3 install ansible -i https://pypi.tuna.tsinghua.edu.cn/simple
inventory.ini
[all:vars]
ansible_ssh_user=root
ansible_ssh_port=22
KUBERNETES_VIP=192.168.8.60
# 安装完后 kubeadm 后，可执行 kubeadm config images list 获取最新版本号
KUBERNETES_VER=v1.23.5
[kube_master:children]
kube_master_init
kube_master_backup
[kube_master_init]
192.168.8.61 hostname=k8s-master01
[kube_master_backup]
# 192.168.8.62 hostname=k8s-master02
# 192.168.8.63 hostname=k8s-master03
[kube_worker]
192.168.8.71 hostname=k8s-worker01
192.168.8.72 hostname=k8s-worker02
[kube_etcd]
192.168.8.61
192.168.8.62
192.168.8.63
# 生成 RSA
ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
# 执行权限
chmod -x ssh-copy.sh
# 执行（修改文件中对于的IP和解析名，和inventory.ini中保持一致）
./ssh-copy.sh
ssh-copy.sh
#!/bin/bash
user=root
password=PassWord
hosts="
192.168.8.61
192.168.8.71
192.168.8.72
"
for host in $hosts
do
echo ============= $host =============
expect <<-EOF
set timeout 3
spawn ssh-copy-id -f "$user@$host"
expect {
"yes/no" { send "yes\n"; exp_continue }
"password:" { send "$password\n"; }
}
expect off;
EOF
done
# k8s 先决条件
ansible-playbook -i inventory 1[tab].yml
# 安装 keepalived，单 master 可忽略
ansible-playbook -i inventory 2[tab].yml
# 引导 master 节点
ansible-playbook -i inventory 3[tab].yml
# 4、5 修改密钥，查看 master01 节点的 kubeadm-init.log
# 引导 master backup，单 master 可忽略
ansible-playbook -i inventory 4[tab].yml
# 引导 work 节点
ansible-playbook -i inventory 5[tab].yml
# 部署 flannel
ansible-playbook -i inventory 6[tab].yml
# 重置所有服务器
ansible-playbook -i inventory 9[tab].yml
# 验证集群版本
kubectl version
# 验证节点就绪 (Ready) 状态
kubectl get node
# 验证集群pod状态，默认已安装网络插件等
kubectl get pod -A
# 验证集群服务状态
kubectl get svc -A
# 2022/11/10 更新 v1.25.3
git clone https://github.com/Ju4t/ansible-kubernetes.git
.
├── 1.k8s-base.yml
├── 2.k8s-keepalived.yml
├── 3.k8s-init-master.yml
├── 4.k8s-join-master.yml
├── 5.k8s-join-works.yml
├── 6.k8s-flannel.yml
├── 9.k8s-reset.yml
├── inventory.ini
├── k8s
│ └── kube-flannel.yaml
└── ssh-copy.sh
---
- hosts:
- kube_master
- kube_worker
# - kube_new
gather_facts: no
# vars:
# - APISERVER_VIP: 192.168.8.80
# - docker_version: 20.10.12
tasks:
- name: 修改主机名
shell: |
hostnamectl set-hostname {{ hostname }}
- name: 关闭交换区
shell: swapoff -a && sed -ri 's/.*swap.*/#&/' /etc/fstab
- name: 临时关闭 SELINUX
shell: /sbin/setenforce 0
ignore_errors: True
- name: 永久关闭 SELINUX
lineinfile:
path: /etc/selinux/config
regexp: '^SELINUX='
line: SELINUX=permissive
- name: 关闭防火墙
service:
name: firewalld
state: stopped
enabled: no
- name: 删除冲突包
shell: |
yum remove -y podman docker docker-client docker-client-latest docker-common docker-latest docker-latest-logrotate docker-logrotate docker-selinux docker-engine-selinux docker-engine containerd.io
ignore_errors: True
- name: 关闭系统不需要的服务
shell: systemctl stop postfix && systemctl disable postfix
- name: 生成 hosts
shell:
cmd: |
cat <<EOF | sudo tee /etc/hosts
{{ KUBERNETES_VIP }} k8s-master # Keepalived VIP
192.168.8.61 k8s-master01
192.168.8.62 k8s-master02
192.168.8.63 k8s-master03
192.168.8.71 k8s-worker01
192.168.8.72 k8s-worker02
192.168.8.73 k8s-worker03
192.168.8.80 cr.labdoc.cc
EOF
- name: IPVS 安装和配置的先决条件
shell:
cmd: |
cat <<EOF | sudo tee /etc/sysconfig/modules/ipvs.modules
#!/bin/bash
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack_ipv4
EOF
- name: IPVS 设置生效
shell: chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules
ignore_errors: True
- name: Containerd 安装和配置的先决条件
shell:
cmd: |
cat <<EOF | sudo tee /etc/modules-load.d/containerd.conf
overlay
br_netfilter
EOF
- name: Containerd 设置生效
shell: sudo modprobe overlay && sudo modprobe br_netfilter
- name: kubernetes 安装和配置的先决条件
shell:
cmd: |
cat <<EOF > /etc/sysctl.d/kubernetes.conf
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-ip6tables = 1
EOF
- name: kubernetes 设置生效
shell: sysctl -p /etc/sysctl.d/kubernetes.conf
ignore_errors: True
- name: 安装 yum-utils ipset ipvsadm device-mapper-persistent-data lvm2 net-tools openssh-clients ntpdate
yum:
name: "{{ packages }}"
vars:
packages:
- yum-utils
- device-mapper-persistent-data
- lvm2
- ipset
- ipvsadm
- net-tools
- ntpdate
- openssh-clients
- name: 同步时间
shell: ntpdate time.windows.com
ignore_errors: True
- name: 添加 CentOS 7 阿里云 yum 源
shell: yum-config-manager --add-repo https://mirrors.aliyun.com/repo/Centos-7.repo
- name: 添加 Docker-CE 阿里云 yum 源
shell: yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
- name: 添加 kubernetes 阿里云 yum 源
shell:
cmd: |
cat <<EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF
# v1.24 弃用 dockershim
# - name: 安装 Docker
# yum:
# name: docker
# - name: 配置 docker
# shell: if [ ! -d /etc/docker ] ; then mkdir /etc/docker ; fi
# - name: 配置 docker
# shell:
# cmd: |
# cat <<EOF | sudo tee /etc/docker/daemon.json
# {
# "exec-opts": ["native.cgroupdriver=systemd"],
# "log-driver": "json-file",
# "log-opts": {
# "max-size": "100m"
# },
# "registry-mirrors" : ["https://5g2xk4rj.mirror.aliyuncs.com", "https://docker.mirrors.ustc.edu.cn", "http://hub-mirror.c.163.com"]
# }
# EOF
# - name: 配置 docker.service.d
# shell: mkdir -p /etc/systemd/system/docker.service.d
# - name: 启动 docker
# shell: systemctl daemon-reload && systemctl restart docker && systemctl enable docker
- name: 安装 containerd kubeadm kubectl kubelet
yum:
name:
- containerd
- kubeadm
- kubectl
- kubelet
- name: 配置 containerd
shell:
cmd: |
mkdir -p /etc/containerd \
&& containerd config default > /etc/containerd/config.toml \
&& sed -i "s#k8s.gcr.io#registry.aliyuncs.com/google_containers#g" /etc/containerd/config.toml \
&& sed -i "s#https://registry-1.docker.io#https://registry.aliyuncs.com#g" /etc/containerd/config.toml \
&& sed -i "s#registry.k8s.io#registry.aliyuncs.com/google_containers#g" /etc/containerd/config.toml
- name: 启动 Containerd
shell: systemctl daemon-reload && systemctl restart containerd && systemctl enable containerd
- name: 启动 kubelet
shell: systemctl enable kubelet && systemctl start kubelet
- name: 配置 crictl runtime
shell: crictl config runtime-endpoint unix:///run/containerd/containerd.sock
---
- hosts:
- kube_master
gather_facts: no
vars:
- KUBERNETES_PORT: 6443
- PRIORITY: 100
- STATE: MASTER
- INTERFACE: eth0
- AUTH_PASS: 1111
tasks:
- name: 安装 keepalived
yum:
name: keepalived
- name: 配置 check_apiserver
shell:
cmd: |
cat <<EOF | sudo tee /etc/keepalived/check_apiserver.sh
#!/bin/sh
errorExit() {
echo "*** $*" 1>&2
exit 1
}
curl --silent --max-time 2 --insecure https://localhost:{{ KUBERNETES_PORT }}/ -o /dev/null || errorExit "Error GET https://localhost:{{ KUBERNETES_PORT }}/"
if ip addr | grep -q {{ KUBERNETES_VIP }}; then
curl --silent --max-time 2 --insecure https://{{ KUBERNETES_VIP }}:{{ KUBERNETES_PORT }}/ -o /dev/null || errorExit "Error GET https://{{ KUBERNETES_VIP }}:{{ KUBERNETES_PORT }}/"
fi
EOF
- name: 配置 keepalived(MASTER)
shell:
cmd: |
cat <<EOF | sudo tee /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
router_id LVS_DEVEL
}
vrrp_script check_apiserver {
script "/etc/keepalived/check_apiserver.sh"
interval 3
weight -2
fall 10
rise 2
}
vrrp_instance VIP {
interface {{ INTERFACE }} # 网卡名字
virtual_router_id 50 # 对于所有集群主机应该是相同的，keepalived而在同一子网中的所有集群中是唯一的。许多发行版将其值预配置为51
state {{ STATE }} # 备用
priority {{ PRIORITY }} # 权重
advert_int 1
authentication {
auth_type PASS
auth_pass {{ AUTH_PASS }}
}
virtual_ipaddress {
{{ KUBERNETES_VIP }} #VIP
}
track_script {
check_apiserver
}
}
EOF
- name: 启动 keepalived
shell: service keepalived start && systemctl enable keepalived
- hosts: kube_backup
vars:
# - KUBERNETES_VIP: 192.168.8.60
- KUBERNETES_PORT: 6443
- PRIORITY: 80
- STATE: BACKUP
- INTERFACE: eth0
- AUTH_PASS: 1111
tasks:
- name: 安装 keepalived
yum:
name: keepalived
- name: 配置 check_apiserver
shell:
cmd: |
cat <<EOF | sudo tee /etc/keepalived/check_apiserver.sh
#!/bin/sh
errorExit() {
echo "*** $*" 1>&2
exit 1
}
curl --silent --max-time 2 --insecure https://localhost:{{ KUBERNETES_PORT }}/ -o /dev/null || errorExit "Error GET https://localhost:{{ KUBERNETES_PORT }}/"
if ip addr | grep -q {{ KUBERNETES_VIP }}; then
curl --silent --max-time 2 --insecure https://{{ KUBERNETES_VIP }}:{{ KUBERNETES_PORT }}/ -o /dev/null || errorExit "Error GET https://{{ KUBERNETES_VIP }}:{{ KUBERNETES_PORT }}/"
fi
EOF
- name: 配置 keepalived(BACKUP)
shell:
cmd: |
cat <<EOF | sudo tee /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
router_id LVS_DEVEL
}
vrrp_script check_apiserver {
script "/etc/keepalived/check_apiserver.sh"
interval 3
weight -2
fall 10
rise 2
}
vrrp_instance VIP {
interface {{ INTERFACE }} # 网卡名字
virtual_router_id 50
state {{ STATE }} # 备用
priority {{ PRIORITY }} # 权重
advert_int 1
authentication {
auth_type PASS
auth_pass {{ AUTH_PASS }}
}
virtual_ipaddress {
{{ KUBERNETES_VIP }} #VIP
}
track_script {
check_apiserver
}
}
EOF
- name: 启动 keepalived
shell: service keepalived start && systemctl enable keepalived
---
- hosts:
- kube_master_init
gather_facts: yes
tasks:
- name: k8s check
shell: |
if [ ! -d ~/k8s ] ; then mkdir ~/k8s ; fi
- name: kubeadm init
shell: |
kubeadm init \
--control-plane-endpoint k8s-master:6443 \
--image-repository registry.aliyuncs.com/google_containers \
--cri-socket=/run/containerd/containerd.sock \
--kubernetes-version={{ KUBERNETES_VER }} \
--service-dns-domain="cluster.local" \
--service-cidr=10.96.0.0/12 \
--pod-network-cidr=10.244.0.0/16 \
--upload-certs | tee ~/k8s/kubeadm-init.log
- name: kube config
shell: |
mkdir -p $HOME/.kube
sudo \cp /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
# when:
# - hostname.find("master01") > -1
# kubeadm init 参考：https://kubernetes.io/zh/docs/reference/setup-tools/kubeadm/kubeadm-init/
---
- hosts:
- kube_master_backup
gather_facts: no
tasks:
- name: kubeadm reset
shell: kubeadm reset -f
- name: kubeadm jion
shell: |
kubeadm join k8s-master:6443 --token 5q40sr.s2zkk4davs90x02j \
--discovery-token-ca-cert-hash sha256:ecd22e90b568fabfdd20d86ebad313dda59dd6b3dbbecb8e6674907231ac744e \
--control-plane --certificate-key ac8ce94658454ca0d0b0f705979f4a43ae309e94784b94656ac6fea2a413bc3d
- name: kube config
shell: |
mkdir -p $HOME/.kube
sudo \cp /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
---
- hosts:
- kube_worker
# - kube_new
gather_facts: no
tasks:
- name: kubeadm reset
shell: |
kubeadm reset -f
- name: kubeadm jion
shell: |
kubeadm join k8s-master:6443 --token 5q40sr.s2zkk4davs90x02j \
--discovery-token-ca-cert-hash sha256:ecd22e90b568fabfdd20d86ebad313dda59dd6b3dbbecb8e6674907231ac744e
# when:
# - hostname.find("master01") == -1
---
- hosts:
- kube_master_init
gather_facts: no
tasks:
- name: Copy kube-flannel
copy:
src: ./k8s/kube-flannel.yaml
dest: ~/k8s/kube-flannel.yaml
- name: 应用 flannel
shell: kubectl apply -f ~/k8s/kube-flannel.yaml
---
- hosts:
- kube_master
- kube_backup
- kube_worker
gather_facts: no
tasks:
- name: kubeadm reset
shell: kubeadm reset -f
---
- hosts:
- kube_master
gather_facts: no
vars:
- KUBERNETES_PORT: 6443
- PRIORITY: 100
- STATE: MASTER
- INTERFACE: eth0
- AUTH_PASS: 1111
tasks:
- name: 安装 keepalived
yum:
name: keepalived
- name: 配置 check_apiserver
shell:
cmd: |
cat <<EOF | sudo tee /etc/keepalived/check_apiserver.sh
#!/bin/sh
errorExit() {
echo "*** $*" 1>&2
exit 1
}
curl --silent --max-time 2 --insecure https://localhost:{{ KUBERNETES_PORT }}/ -o /dev/null || errorExit "Error GET https://localhost:{{ KUBERNETES_PORT }}/"
if ip addr | grep -q {{ KUBERNETES_VIP }}; then
curl --silent --max-time 2 --insecure https://{{ KUBERNETES_VIP }}:{{ KUBERNETES_PORT }}/ -o /dev/null || errorExit "Error GET https://{{ KUBERNETES_VIP }}:{{ KUBERNETES_PORT }}/"
fi
EOF
- name: 配置 keepalived(MASTER)
shell:
cmd: |
cat <<EOF | sudo tee /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
router_id LVS_DEVEL
}
vrrp_script check_apiserver {
script "/etc/keepalived/check_apiserver.sh"
interval 3
weight -2
fall 10
rise 2
}
vrrp_instance VIP {
interface {{ INTERFACE }} # 网卡名字
virtual_router_id 50 # 对于所有集群主机应该是相同的，keepalived而在同一子网中的所有集群中是唯一的。许多发行版将其值预配置为51
state {{ STATE }} # 备用
priority {{ PRIORITY }} # 权重
advert_int 1
authentication {
auth_type PASS
auth_pass {{ AUTH_PASS }}
}
virtual_ipaddress {
{{ KUBERNETES_VIP }} #VIP
}
track_script {
check_apiserver
}
}
EOF
- name: 启动 keepalived
shell: service keepalived start && systemctl enable keepalived
- hosts: kube_backup
vars:
# - KUBERNETES_VIP: 192.168.8.60
- KUBERNETES_PORT: 6443
- PRIORITY: 80
- STATE: BACKUP
- INTERFACE: eth0
- AUTH_PASS: 1111
tasks:
- name: 安装 keepalived
yum:
name: keepalived
- name: 配置 check_apiserver
shell:
cmd: |
cat <<EOF | sudo tee /etc/keepalived/check_apiserver.sh
#!/bin/sh
errorExit() {
echo "*** $*" 1>&2
exit 1
}
curl --silent --max-time 2 --insecure https://localhost:{{ KUBERNETES_PORT }}/ -o /dev/null || errorExit "Error GET https://localhost:{{ KUBERNETES_PORT }}/"
if ip addr | grep -q {{ KUBERNETES_VIP }}; then
curl --silent --max-time 2 --insecure https://{{ KUBERNETES_VIP }}:{{ KUBERNETES_PORT }}/ -o /dev/null || errorExit "Error GET https://{{ KUBERNETES_VIP }}:{{ KUBERNETES_PORT }}/"
fi
EOF
- name: 配置 keepalived(BACKUP)
shell:
cmd: |
cat <<EOF | sudo tee /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
router_id LVS_DEVEL
}
vrrp_script check_apiserver {
script "/etc/keepalived/check_apiserver.sh"
interval 3
weight -2
fall 10
rise 2
}
vrrp_instance VIP {
interface {{ INTERFACE }} # 网卡名字
virtual_router_id 50
state {{ STATE }} # 备用
priority {{ PRIORITY }} # 权重
advert_int 1
authentication {
auth_type PASS
auth_pass {{ AUTH_PASS }}
}
virtual_ipaddress {
{{ KUBERNETES_VIP }} #VIP
}
track_script {
check_apiserver
}
}
EOF
- name: 启动 keepalived
shell: service keepalived start && systemctl enable keepalived
https://kubernetes.io/zh-cn/docs/setup/production-environment/tools/kops/