Kubernetes v1.19.0 高可用安装部署

共计 17592 个字符，预计需要花费 44 分钟才能阅读完成。

1 系统准备

1.1 环境准备

用户执行命令

服务器均采用腾讯云CVM，需提前创建好VPC、子网等

1.2 硬件信息

1.3 软件信息

CentOS Linux release 7.3.1611 (Core)

Kubernetesv 1.19.0

Docker 19.03.12

1.4 安装流程说明

• 服务器环境配置

• docker

• Kubernetes

• 和keepalived

• 下载镜像

• master01初始化

• master02和master01加入集群

• node01加入集群

• 高可用测试

1.5 修改主机名

所有节点执行

修改所有主机名，对应1.2中硬件信息的hostname

hostnamectl set-hostname k8s-master01

1.6 关闭防火墙

所有节点执行

systemctl disable --now firewalld

1.7 禁用swap

所有节点执行

swapoff -a # 临时禁用
sed -i 's/.*swap.*/#&/' /etc/fstab # 永久禁用

1.8 关闭 SELinux

所有节点执行

setenforce 0
sed -i 's/^SELINUX=.*/SELINUX=disabled/' /etc/selinux/config

1.9 设置系统时区、同步时间

所有节点执行

timedatectl set-timezone Asia/Shanghai
systemctl enable --now chronyd
​
# 将当前的 UTC 时间写入硬件时钟
timedatectl set-local-rtc 0
# 重启依赖于系统时间的服务
systemctl restart rsyslog && systemctl restart crond

1.10 配置内核参数

所有节点执行

cat >> /etc/sysctl.d/k8s.conf << EOF
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
vm.swappiness=0
EOF
​
sysctl -p

1.11 内核升级

所有节点执行

yum update kernel systemd -y

1.12 配置免密登录

k8s-master01执行

ssh-keygen -t rsa
ssh-copy-id -i ~/.ssh/id_rsa.pub root@k8s-master02
ssh-copy-id -i ~/.ssh/id_rsa.pub root@k8s-master03
ssh-copy-id -i ~/.ssh/id_rsa.pub root@k8s-node01

2 部署docker

2.1 添加docker yum源

所有节点执行

# 安装必要依赖
yum install -y yum-utils device-mapper-persistent-data lvm2
# 添加aliyun docker-ce yum源
yum -y install yum-utils
yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
# 重建yum缓存
yum makecache fast

2.2 安装指定版本docker

所有节点执行

yum install -y docker-ce-19.03.12-3.el7

2.3 配置docker

mkdir -p /etc/docker
tee /etc/docker/daemon.json <<-'EOF'
{
  "registry-mirrors": ["https://su9ppkb0.mirror.aliyuncs.com"]
}
EOF
#添加开机自启，立即启动
systemctl enable --now docker
systemctl daemon-reload
systemctl restart docker

3 安装ipvs

3.1 安装软件

所有节点执行

yum install ipvsadm ipset sysstat conntrack libseccomp -y

3.2 加载模块

所有节点执行

cat > /etc/sysconfig/modules/ipvs.modules <<EOF
#!/bin/bash
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack
modprobe -- ip_tables
modprobe -- ip_set
modprobe -- xt_set
modprobe -- ipt_set
modprobe -- ipt_rpfilter
modprobe -- ipt_REJECT
modprobe -- ipip
EOF

3.3 配置重启自动加载

所有节点执行

chmod 755 /etc/sysconfig/modules/ipvs.modules && bash /etc/sysconfig/modules/ipvs.modules && lsmod | grep -e ip_vs -e nf_conntrack

4 部署kubernetes

4.1 添加kubernetes源

所有节点执行

cat > /etc/yum.repos.d/kubernetes.repo <<EOF
[kubernetes]
name=Kubernetes
baseurl=http://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64/
enabled=1
gpgcheck=1
repo_gpgcheck=1
gpgkey=http://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg http://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF
#重建yum缓存，输入y添加证书认证
yum makecache fast

4.2 查看安装版本

yum list |grep kubelet
yum list |grep kubeadm
yum list |grep kubectl

4.3 安装kubeadm、kubelet、kubectl

所有master节点执行

yum install -y kubelet-1.19.0-0 --disableexcludes=kubernetes
yum install -y kubeadm-1.19.0-0 --disableexcludes=kubernetes
yum install -y kubectl-1.19.0-0 --disableexcludes=kubernetes
# 将 kubelet 设置为开机自动启动
systemctl enable --now kubelet

4.4 配置自动补全命令

所有master节点执行

#安装bash自动补全插件
yum install bash-completion -y

4.5 设置kubectl与kubeadm命令补全，下次login生效

K8s-master0[1:3]执行

kubectl completion bash >/etc/bash_completion.d/kubectl
kubeadm completion bash > /etc/bash_completion.d/kubeadm

5 集群初始化

5.1 配置集群高可用

节点部署

安装软件

yum install keepalived haproxy -y

配置 Haproxy

#---------------------------------------------------------------------
# Global settings
#---------------------------------------------------------------------
global
    # to have these messages end up in /var/log/haproxy.log you will
    # need to:
    #
    # 1) configure syslog to accept network log events.  This is done
    #    by adding the '-r' option to the SYSLOGD_OPTIONS in
    #    /etc/sysconfig/syslog
    #
    # 2) configure local2 events to go to the /var/log/haproxy.log
    #   file. A line like the following can be added to
    #   /etc/sysconfig/syslog
    #
    #    local2.*                       /var/log/haproxy.log
    #
    log         127.0.0.1 local2
​
    chroot      /var/lib/haproxy
    pidfile     /var/run/haproxy.pid
    maxconn     4000
    user        haproxy
    group       haproxy
    daemon
​
    # turn on stats unix socket
    stats socket /var/lib/haproxy/stats
​
#---------------------------------------------------------------------
# common defaults that all the 'listen' and 'backend' sections will
# use if not designated in their block
#---------------------------------------------------------------------
defaults
    mode                    http
    log                     global
    option                  httplog
    option                  dontlognull
    option http-server-close
    option                  redispatch
    retries                 3
    timeout http-request    10s
    timeout queue           1m
    timeout connect         10s
    timeout client          1m
    timeout server          1m
    timeout http-keep-alive 10s
    timeout check           10s
    maxconn                 3000
​
#---------------------------------------------------------------------
# kubernetes apiserver frontend which proxys to the backends
#---------------------------------------------------------------------
frontend kubernetes
    mode                 tcp
    bind                 *:16443
    option               tcplog
    default_backend      kubernetes-apiserver
​
#---------------------------------------------------------------------
# round robin balancing between the various backends
#---------------------------------------------------------------------
backend kubernetes-apiserver
    mode        tcp
    balance     roundrobin
# 修改集群IP地址
    server  k8s-master01 172.20.5.11:6443 check
    server  k8s-master02 172.20.5.12:6443 check
    server  k8s-master03 172.20.5.13:6443 check
​
#---------------------------------------------------------------------
# collection haproxy statistics message
#---------------------------------------------------------------------
listen stats
    bind                 *:9999
    stats auth           admin:P@ssW0rd
    stats refresh        5s
    stats realm          HAProxy\ Statistics
    stats uri            /admin?stats

和k8s-master03

scp /etc/haproxy/haproxy.cfg root@k8s-master02:/etc/haproxy
scp /etc/haproxy/haproxy.cfg root@k8s-master03:/etc/haproxy

安装 keepalived

k8s-master01配置

vim /etc/keepalived/keepalived.conf
! Configuration File for keepalived
​
global_defs {
   notification_email {
     acassen@firewall.loc
     failover@firewall.loc
     sysadmin@firewall.loc
   }
   notification_email_from Alexandre.Cassen@firewall.loc
   smtp_server 192.168.200.1
   smtp_connect_timeout 30
   router_id LVS_DEVEL
   vrrp_skip_check_adv_addr
   vrrp_garp_interval 0
   vrrp_gna_interval 0
}
# 定义脚本
vrrp_script check_apiserver {
    script "/etc/keepalived/check_apiserver.sh"
    interval 2
    weight -5
    fall 3
    rise 2
}
​
vrrp_instance VI_1 {
    state MASTER
    interface eth0
    virtual_router_id 51
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
   172.20.5.10
    }
​
    # 调用脚本
    #track_script {
    #    check_apiserver
    #}
}

k8s-master02配置

! Configuration File for keepalived
​
global_defs {
   notification_email {
     acassen@firewall.loc
     failover@firewall.loc
     sysadmin@firewall.loc
   }
   notification_email_from Alexandre.Cassen@firewall.loc
   smtp_server 192.168.200.1
   smtp_connect_timeout 30
   router_id LVS_DEVEL
   vrrp_skip_check_adv_addr
   vrrp_garp_interval 0
   vrrp_gna_interval 0
}
# 定义脚本
vrrp_script check_apiserver {
    script "/etc/keepalived/check_apiserver.sh"
    interval 2
    weight -5
    fall 3
    rise 2
}
​
vrrp_instance VI_1 {
    state MASTER
    interface eth0
    virtual_router_id 51
    priority 99
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
   172.20.5.10
    }
​
    # 调用脚本
    #track_script {
    #    check_apiserver
    #}
}

k8s-master03配置

! Configuration File for keepalived
​
global_defs {
   notification_email {
     acassen@firewall.loc
     failover@firewall.loc
     sysadmin@firewall.loc
   }
   notification_email_from Alexandre.Cassen@firewall.loc
   smtp_server 192.168.200.1
   smtp_connect_timeout 30
   router_id LVS_DEVEL
   vrrp_skip_check_adv_addr
   vrrp_garp_interval 0
   vrrp_gna_interval 0
}
# 定义脚本
vrrp_script check_apiserver {
    script "/etc/keepalived/check_apiserver.sh"
    interval 2
    weight -5
    fall 3
    rise 2
}
​
vrrp_instance VI_1 {
    state MASTER
    interface eth0
    virtual_router_id 51
    priority 98
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
   172.20.5.10
    }
​
    # 调用脚本
    #track_script {
    #    check_apiserver
    #}
}

编写健康检测脚本

所有master节点执行

vim /etc/keepalived/check-apiserver.sh
#!/bin/bash
​
function check_apiserver(){
 for ((i=0;i<5;i++))
 do
  apiserver_job_id=${pgrep kube-apiserver}
  if [[ ! -z ${apiserver_job_id} ]];then
   return
  else
   sleep 2
  fi
  apiserver_job_id=0
 done
}
​
# 1->running    0->stopped
check_apiserver
if [[ $apiserver_job_id -eq 0 ]];then
 /usr/bin/systemctl stop keepalived
 exit 1
else
 exit 0
fi

和keepalived

所有master节点执行

systemctl enable --now keepalived
systemctl enable --now haproxy