[系统运维]centos7使用kubeadm部署高可用k8s集群

一、环境准备

??版本信息

??版本要求
??使用containerd，官方建议4.x以上内核，centos7默认使用3.1版本内核，建议升级内核。升级内核参考之前的文章。
??centos7升级内核

1.k8s集群高可用拓扑选项

??堆叠拓扑

??外部 etcd 拓扑

使用堆叠拓扑，控制平面和 etcd 成员在同一节点上，设置和管理相对简单，缺点在于如果出现单点故障，etcd 成员和控制平面实例都将丢失。
使用外部 etcd 拓扑，etcd 分布式数据存储集群在独立于控制平面节点的其他节点上运行，可用性更高，但是需要两倍于堆叠 拓扑的主机数量。

2.节点环境初始化（所有节点都执行）

??选择纯净的机器，静态配置IP地址。运行以下脚本：

#!/bin/bash

# 更新
yum update -y

# 卸载 firewalld
systemctl stop firewalld
yum remove firewalld -y

# 卸载 networkmanager
systemctl stop NetworkManager
yum remove NetworkManager -y

# 同步服务器时间
yum install chrony -y
systemctl enable --now chronyd
chronyc sources

# 安装iptables(搭建完集群后再安装，比较省事)
#yum install -y iptables iptables-services && systemctl enable --now iptables.service

# 关闭 selinux
setenforce 0
sed -i '/^SELINUX=/cSELINUX=disabled' /etc/selinux/config
getenforce

# 关闭swap分区
swapoff -a # 临时
sed -i '/ swap / s/^/# /g' /etc/fstab #永久

# 安装其他必要组件和常用工具包
yum install -y yum-utils zlib zlib-devel openssl openssl-devel \
net-tools vim wget lsof unzip zip bind-utils lrzsz telnet

# 如果是从安装过docker的服务器升级k8s，建议将/etc/sysctl.conf配置清掉
# 这条命令会清除所有没被注释的行
# sed -i '/^#/!d' /etc/sysctl.conf

# 安装ipvs
cat > /etc/sysconfig/modules/ipvs.modules <<EOF
#!/bin/bash
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack_ipv4
EOF
chmod 755 /etc/sysconfig/modules/ipvs.modules 
bash /etc/sysconfig/modules/ipvs.modules 
lsmod | grep -e ip_vs -e nf_conntrack_ipv4
yum install ipset ipvsadm -y

# 允许 iptables 检查桥接流量
cat <<EOF | tee /etc/modules-load.d/k8s.conf
br_netfilter
EOF
sysctl --system

cat <<EOF | tee /etc/sysctl.d/k8s.conf
vm.swappiness = 0
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
EOF
modprobe br_netfilter
lsmod | grep netfilter
sysctl -p /etc/sysctl.d/k8s.conf

# 安装 containerd
yum install -y yum-utils device-mapper-persistent-data lvm2
yum-config-manager --add-repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
yum list containerd.io --showduplicates
yum install -y containerd.io
mkdir -p /etc/containerd
containerd config default | tee /etc/containerd/config.toml

cat <<EOF | tee /etc/crictl.yaml
runtime-endpoint: "unix:///run/containerd/containerd.sock"
image-endpoint: "unix:///run/containerd/containerd.sock"
timeout: 10
debug: false
pull-image-on-create: false
disable-pull-on-run: false
EOF

# 使用 systemd cgroup驱动程序
sed -i "s#k8s.gcr.io#registry.aliyuncs.com/google_containers#g"  /etc/containerd/config.toml
sed -i '/containerd.runtimes.runc.options/a\ \ \ \ \ \ \ \ \ \ \ \ SystemdCgroup = true' /etc/containerd/config.toml
sed -i "s#https://registry-1.docker.io#https://registry.aliyuncs.com#g"  /etc/containerd/config.toml
systemctl daemon-reload
systemctl enable containerd
systemctl restart containerd

# 添加kubernetes yum软件源
cat > /etc/yum.repos.d/kubernetes.repo << EOF
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF

# 安装kubeadm,kubelet和kubectl
yum list kubeadm --showduplicates
yum install -y kubelet-1.22.7 kubeadm-1.22.7 kubectl-1.22.7 --disableexcludes=kubernetes

# 设置开机自启
systemctl daemon-reload
systemctl enable --now kubelet
# kubelet每隔几秒就会重启，陷入等待 kubeadm 指令的死循环

# 命令自动补全
yum install -y bash-completion
source <(crictl completion bash)
crictl completion bash >/etc/bash_completion.d/crictl
source <(kubectl completion bash)
kubectl completion bash >/etc/bash_completion.d/kubectl
source /usr/share/bash-completion/bash_completion

3.重新命名主机，修改hosts文件

#分别修改主机名称（根据实际情况选择不同服务器重命名）
hostnamectl set-hostname k8s-master1
hostnamectl set-hostname k8s-node1
	
#修改hosts文件（按实际情况修改后，所有节点执行）
cat <<EOF | tee /etc/hosts
127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
192.168.1.15 k8s-master1
192.168.1.123 k8s-master2
192.168.1.124 k8s-master3
192.168.1.65 k8s-node1
192.168.1.26 k8s-node2
192.168.1.128 k8s-node3
192.168.1.129 k8s-node4
EOF

??以上步骤k8s集群所有主机都需要执行

4.安装keepalived、haproxy

配置文件
??k8s-master1
vim /etc/keepalived/keepalived.conf

! Configuration File for keepalived

global_defs {
   router_id k8s-master1

# 添加如下内容
   script_user root
   enable_script_security
}

vrrp_script check_haproxy {
    script "/etc/keepalived/check_haproxy.sh"         # 检测脚本路径
    interval 3
    weight -2 
    fall 10
    rise 2
}

vrrp_instance VI_1 {
    state MASTER #主
    interface eth0 #网卡名
    virtual_router_id 51
    priority 100 #权重
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.1.223 #vip
    }
    track_script {
        check_haproxy       # 模块
    }
}

??k8s-master2

! Configuration File for keepalived

global_defs {
   router_id k8s-master2
   
   # 添加如下内容
   script_user root
   enable_script_security
}

vrrp_script check_haproxy {
    script "/etc/keepalived/check_haproxy.sh"         # 检测脚本路径
    interval 3
    weight -2 
    fall 10
    rise 2
}

vrrp_instance VI_1 {
    state BACKUP #备
    interface eth0 #网卡名
    virtual_router_id 51
    priority 99 #权重
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.1.223 #vip
    }
    track_script {
        check_haproxy       # 模块
    }
}

??k8s-master3

! Configuration File for keepalived

global_defs {
   router_id k8s-master3
   
   # 添加如下内容
   script_user root
   enable_script_security
}

vrrp_script check_haproxy {
    script "/etc/keepalived/check_haproxy.sh"         # 检测脚本路径
    interval 3
    weight -2 
    fall 10
    rise 2
}

vrrp_instance VI_1 {
    state BACKUP #备
    interface eth0 #网卡名
    virtual_router_id 51
    priority 98 #权重
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.1.223 #vip
    }
    track_script {
        check_haproxy       # 模块
    }
}

??haproxy配置（3台master都一样）
vim /etc/haproxy/haproxy.cfg

#---------------------------------------------------------------------
# Example configuration for a possible web application.  See the
# full configuration options online.
#
#   http://haproxy.1wt.eu/download/1.4/doc/configuration.txt
#
#---------------------------------------------------------------------

#---------------------------------------------------------------------
# Global settings
#---------------------------------------------------------------------
global
    # to have these messages end up in /var/log/haproxy.log you will
    # need to:
    #
    # 1) configure syslog to accept network log events.  This is done
    #    by adding the '-r' option to the SYSLOGD_OPTIONS in
    #    /etc/sysconfig/syslog
    #
    # 2) configure local2 events to go to the /var/log/haproxy.log
    #   file. A line like the following can be added to
    #   /etc/sysconfig/syslog
    #
    #    local2.*                       /var/log/haproxy.log
    #
    log         127.0.0.1 local2

    chroot      /var/lib/haproxy
    pidfile     /var/run/haproxy.pid
    maxconn     4000
    user        haproxy
    group       haproxy
    daemon

    # turn on stats unix socket
    stats socket /var/lib/haproxy/stats

#---------------------------------------------------------------------
# common defaults that all the 'listen' and 'backend' sections will
# use if not designated in their block
#---------------------------------------------------------------------
defaults
    mode                    http
    log                     global
    option                  httplog
    option                  dontlognull
    option http-server-close
    option forwardfor       except 127.0.0.0/8
    option                  redispatch
    retries                 3
    timeout http-request    10s
    timeout queue           1m
    timeout connect         10s
    timeout client          1m
    timeout server          1m
    timeout http-keep-alive 10s
    timeout check           10s
    maxconn                 3000

#---------------------------------------------------------------------
# main frontend which proxys to the backends
#---------------------------------------------------------------------
frontend  kubernetes-apiserver
    mode                        tcp
    bind                        *:16443
    option                      tcplog
    default_backend             kubernetes-apiserver

#---------------------------------------------------------------------
# static backend for serving up images, stylesheets and such
#---------------------------------------------------------------------
listen stats
    bind            *:1080
    stats auth      admin:awesomePassword
    stats refresh   5s
    stats realm     HAProxy\ Statistics
    stats uri       /admin?stats

#---------------------------------------------------------------------
# round robin balancing between the various backends
#---------------------------------------------------------------------
backend kubernetes-apiserver
    mode        tcp
    balance     roundrobin
    server  k8s-master1 192.168.1.15:6443 check
    server  k8s-master2 192.168.1.123:6443 check
    server  k8s-master3 192.168.1.124:6443 check

??健康检查脚本

vim /etc/keepalived/check_haproxy.sh

#!/bin/sh
# HAPROXY down
A=`ps -C haproxy --no-header | wc -l`
if [ $A -eq 0 ]
then
systmectl start haproxy
if [ ps -C haproxy --no-header | wc -l -eq 0 ]
then
killall -9 haproxy
echo "HAPROXY down"#需要告警的话根据实际情况填写
sleep 3600
fi 

fi

启动服务，并设置开机自启

systemctl enable --now keepalived
systemctl enable --now haproxy

二、部署Kubernetes Master

1.kubeadm常用命令

2.推荐使用配置文件部署

??默认配置文件

# 查看默认配置文件
kubeadm config print init-defaults
# 查看所需镜像
kubeadm config images list --image-repository registry.aliyuncs.com
# 导出默认配置文件到当前目录
kubeadm config print init-defaults > kubeadm.yaml

??配置文件参考

kubeadm-config.yaml

apiVersion: kubeadm.k8s.io/v1beta3
bootstrapTokens:
- groups:
  - system:bootstrappers:kubeadm:default-node-token
  token: qjbajd.zp1ta327pwur2k8g
  ttl: 24h0m0s
  usages:
  - signing
  - authentication
kind: InitConfiguration
localAPIEndpoint:
  advertiseAddress: 192.168.1.15
  bindPort: 6443
nodeRegistration:
  criSocket: /run/containerd/containerd.sock 
  name: k8s-master1
  taints:
  - effect: NoSchedule
    key: node-role.kubernetes.io/master
---
apiServer:
  extraArgs:
    authorization-mode: "Node,RBAC"
  certSANs:   
  - 192.168.1.223
  - 192.168.1.15
  - 192.168.1.123
  - 192.168.1.124
  timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta3
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controllerManager: {}
dns: {}
etcd:
  local:
    dataDir: /var/lib/etcd
imageRepository: registry.aliyuncs.com/google_containers
kind: ClusterConfiguration
kubernetesVersion: v1.22.7
controlPlaneEndpoint: "192.168.1.223:16443"    # 虚拟IP和haproxy端口
networking:
  dnsDomain: cluster.local
  podSubnet: 10.244.0.0/16
  serviceSubnet: 10.96.0.0/12
scheduler: {}
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
mode: ipvs
---
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
cgroupDriver: systemd

添加certSANs、controlPlaneEndpoint等配置项，选择IPVS 代理模式 ，cgroup选择systemd（kubelet和containerd使用相同的cgroup驱动，官方建议使用systemd），镜像库使用阿里云镜像站，其他的根据实际情况填写

??提前拉取镜像

kubeadm config images pull --config kubeadm-config.yaml

3.集群初始化

kubeadm init --config=kubeadm-config.yaml --upload-certs

??部分信息

Your Kubernetes control-plane has initialized successfully!

To start using your cluster, you need to run the following as a regular user:

  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config

Alternatively, if you are the root user, you can run:

  export KUBECONFIG=/etc/kubernetes/admin.conf

You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  https://kubernetes.io/docs/concepts/cluster-administration/addons/

You can now join any number of the control-plane node running the following command on each as root:

  kubeadm join 192.168.1.223:16443 --token qjbajd.zp1ta327pwur2k8g \
	--discovery-token-ca-cert-hash sha256:d21d770c1bfc059280c005096e5fd0e4133ef0e69ac724980005c9e821b45fe9 \
	--control-plane --certificate-key ba37092cbcaa1784e46cb2827bc3603d200c6149db28577e9fe433403126734b

Please note that the certificate-key gives access to cluster sensitive data, keep it secret!
As a safeguard, uploaded-certs will be deleted in two hours; If necessary, you can use
"kubeadm init phase upload-certs --upload-certs" to reload certs afterward.

Then you can join any number of worker nodes by running the following on each as root:

kubeadm join 192.168.1.223:16443 --token qjbajd.zp1ta327pwur2k8g \
	--discovery-token-ca-cert-hash sha256:d21d770c1bfc059280c005096e5fd0e4133ef0e69ac724980005c9e821b45fe9

??根据提示操作

mkdir -p $HOME/.kube && \
cp -i /etc/kubernetes/admin.conf $HOME/.kube/config && \
chown $(id -u):$(id -g) $HOME/.kube/config

??master节点执行命令加入集群
示例：

[root@k8s-master2 ~]# kubeadm join 192.168.1.223:16443 --token qjbajd.zp1ta327pwur2k8g \
> --discovery-token-ca-cert-hash sha256:d21d770c1bfc059280c005096e5fd0e4133ef0e69ac724980005c9e821b45fe9 \
> --control-plane --certificate-key ba37092cbcaa1784e46cb2827bc3603d200c6149db28577e9fe433403126734b
[preflight] Running pre-flight checks
[preflight] Reading configuration from the cluster...
[preflight] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -o yaml'
[preflight] Running pre-flight checks before initializing the new control plane instance
[preflight] Pulling images required for setting up a Kubernetes cluster
[preflight] This might take a minute or two, depending on the speed of your internet connection
[preflight] You can also perform this action in beforehand using 'kubeadm config images pull'
[download-certs] Downloading the certificates in Secret "kubeadm-certs" in the "kube-system" Namespace
[certs] Using certificateDir folder "/etc/kubernetes/pki"
[certs] Generating "apiserver-kubelet-client" certificate and key
[certs] Generating "apiserver" certificate and key
[certs] apiserver serving cert is signed for DNS names [k8s-master2 kubernetes kubernetes.default kubernetes.default.svc kubernetes.default.svc.cluster.local] and IPs [10.96.0.1 192.168.1.123 192.168.1.223 192.168.1.15 192.168.1.124]
[certs] Generating "front-proxy-client" certificate and key
[certs] Generating "apiserver-etcd-client" certificate and key
[certs] Generating "etcd/server" certificate and key
[certs] etcd/server serving cert is signed for DNS names [k8s-master2 localhost] and IPs [192.168.1.123 127.0.0.1 ::1]
[certs] Generating "etcd/healthcheck-client" certificate and key
[certs] Generating "etcd/peer" certificate and key
[certs] etcd/peer serving cert is signed for DNS names [k8s-master2 localhost] and IPs [192.168.1.123 127.0.0.1 ::1]
[certs] Valid certificates and keys now exist in "/etc/kubernetes/pki"
[certs] Using the existing "sa" key
[kubeconfig] Generating kubeconfig files
[kubeconfig] Using kubeconfig folder "/etc/kubernetes"
[endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address
[kubeconfig] Writing "admin.conf" kubeconfig file
[endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address
[kubeconfig] Writing "controller-manager.conf" kubeconfig file
[endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address
[kubeconfig] Writing "scheduler.conf" kubeconfig file
[control-plane] Using manifest folder "/etc/kubernetes/manifests"
[control-plane] Creating static Pod manifest for "kube-apiserver"
[control-plane] Creating static Pod manifest for "kube-controller-manager"
[control-plane] Creating static Pod manifest for "kube-scheduler"
[check-etcd] Checking that the etcd cluster is healthy
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[kubelet-start] Starting the kubelet
[kubelet-start] Waiting for the kubelet to perform the TLS Bootstrap...
[etcd] Announced new etcd member joining to the existing etcd cluster
[etcd] Creating static Pod manifest for "etcd"
[etcd] Waiting for the new etcd member to join the cluster. This can take up to 40s
The 'update-status' phase is deprecated and will be removed in a future release. Currently it performs no operation
[mark-control-plane] Marking the node k8s-master2 as control-plane by adding the labels: [node-role.kubernetes.io/master(deprecated) node-role.kubernetes.io/control-plane node.kubernetes.io/exclude-from-external-load-balancers]
[mark-control-plane] Marking the node k8s-master2 as control-plane by adding the taints [node-role.kubernetes.io/master:NoSchedule]

This node has joined the cluster and a new control plane instance was created:

* Certificate signing request was sent to apiserver and approval was received.
* The Kubelet was informed of the new secure connection details.
* Control plane (master) label and taint were applied to the new node.
* The Kubernetes control plane instances scaled up.
* A new etcd member was added to the local/stacked etcd cluster.

To start administering your cluster from this node, you need to run the following as a regular user:

	mkdir -p $HOME/.kube
	sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
	sudo chown $(id -u):$(id -g) $HOME/.kube/config

Run 'kubectl get nodes' to see this node join the cluster.

??master节点加入后也要执行命令生成配置文件，才能使用kubectl管理集群

mkdir -p $HOME/.kube && \
cp -i /etc/kubernetes/admin.conf $HOME/.kube/config && \
chown $(id -u):$(id -g) $HOME/.kube/config

??node节点执行命令加入集群
示例：

[root@k8s-node1 ~]# kubeadm join 192.168.1.223:16443 --token qjbajd.zp1ta327pwur2k8g \
> --discovery-token-ca-cert-hash sha256:d21d770c1bfc059280c005096e5fd0e4133ef0e69ac724980005c9e821b45fe9
[preflight] Running pre-flight checks
[preflight] Reading configuration from the cluster...
[preflight] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -o yaml'
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[kubelet-start] Starting the kubelet
[kubelet-start] Waiting for the kubelet to perform the TLS Bootstrap...

This node has joined the cluster:
* Certificate signing request was sent to apiserver and a response was received.
* The Kubelet was informed of the new secure connection details.

Run 'kubectl get nodes' on the control-plane to see this node join the cluster.

4.部署calico插件

??部署网络插件，使用 Kubernetes API 数据存储进行安装——50 个节点或更少

??使用3.21版本，下载

curl -O https://docs.projectcalico.org/archive/v3.21/manifests/calico.yaml

使用calico插件，网卡名称最好是常用的，比如说eth、eno等。如果是enp0s6这种，可能导致部分pod报错。使用服务器部署容易遇到这个问题，虚拟机、云主机网卡名称一般是统一的。

??使用apply部署

kubectl apply -f calico.yaml

??查看pod

kubectl get pods --all-namespaces -o wide

??等待一段时间，查看集群运行情况，查看node节点

kubectl get nodes -o wide

??效果示例

[root@k8s-master1 yaml]# kubectl get pods --all-namespaces -o wide
NAMESPACE     NAME                                     READY   STATUS    RESTARTS        AGE     IP               NODE          NOMINATED NODE   READINESS GATES
kube-system   calico-kube-controllers-76c5bc74-476tl   1/1     Running   0               4m32s   10.244.159.131   k8s-master1   <none>           <none>
kube-system   calico-node-48zjc                        1/1     Running   0               2m51s   192.168.1.124    k8s-master3   <none>           <none>
kube-system   calico-node-4c4v9                        1/1     Running   0               2m45s   192.168.1.65     k8s-node1     <none>           <none>
kube-system   calico-node-5pz6d                        1/1     Running   0               4m33s   192.168.1.15     k8s-master1   <none>           <none>
kube-system   calico-node-7r4lq                        1/1     Running   0               2m38s   192.168.1.129    k8s-node4     <none>           <none>
kube-system   calico-node-gb95k                        1/1     Running   0               3m42s   192.168.1.123    k8s-master2   <none>           <none>
kube-system   calico-node-hqtgk                        1/1     Running   0               2m49s   192.168.1.26     k8s-node2     <none>           <none>
kube-system   calico-node-s5z6k                        1/1     Running   0               2m46s   192.168.1.128    k8s-node3     <none>           <none>
kube-system   coredns-7f6cbbb7b8-rqrqn                 1/1     Running   0               5m54s   10.244.159.130   k8s-master1   <none>           <none>
kube-system   coredns-7f6cbbb7b8-zlw7s                 1/1     Running   0               5m54s   10.244.159.129   k8s-master1   <none>           <none>
kube-system   etcd-k8s-master1                         1/1     Running   5               6m8s    192.168.1.15     k8s-master1   <none>           <none>
kube-system   etcd-k8s-master2                         1/1     Running   0               3m41s   192.168.1.123    k8s-master2   <none>           <none>
kube-system   etcd-k8s-master3                         1/1     Running   0               3m21s   192.168.1.124    k8s-master3   <none>           <none>
kube-system   kube-apiserver-k8s-master1               1/1     Running   5               6m8s    192.168.1.15     k8s-master1   <none>           <none>
kube-system   kube-apiserver-k8s-master2               1/1     Running   1               3m41s   192.168.1.123    k8s-master2   <none>           <none>
kube-system   kube-apiserver-k8s-master3               1/1     Running   1               2m59s   192.168.1.124    k8s-master3   <none>           <none>
kube-system   kube-controller-manager-k8s-master1      1/1     Running   7 (3m30s ago)   6m1s    192.168.1.15     k8s-master1   <none>           <none>
kube-system   kube-controller-manager-k8s-master2      1/1     Running   1               3m41s   192.168.1.123    k8s-master2   <none>           <none>
kube-system   kube-controller-manager-k8s-master3      1/1     Running   1               3m21s   192.168.1.124    k8s-master3   <none>           <none>
kube-system   kube-proxy-2f55t                         1/1     Running   0               2m51s   192.168.1.124    k8s-master3   <none>           <none>
kube-system   kube-proxy-dcbzb                         1/1     Running   0               3m42s   192.168.1.123    k8s-master2   <none>           <none>
kube-system   kube-proxy-lqkw8                         1/1     Running   0               2m47s   192.168.1.128    k8s-node3     <none>           <none>
kube-system   kube-proxy-lvpnz                         1/1     Running   0               2m38s   192.168.1.129    k8s-node4     <none>           <none>
kube-system   kube-proxy-pwz5m                         1/1     Running   0               2m49s   192.168.1.26     k8s-node2     <none>           <none>
kube-system   kube-proxy-rdfw2                         1/1     Running   0               2m46s   192.168.1.65     k8s-node1     <none>           <none>
kube-system   kube-proxy-vqksb                         1/1     Running   0               5m54s   192.168.1.15     k8s-master1   <none>           <none>
kube-system   kube-scheduler-k8s-master1               1/1     Running   7 (3m30s ago)   6m1s    192.168.1.15     k8s-master1   <none>           <none>
kube-system   kube-scheduler-k8s-master2               1/1     Running   1               3m41s   192.168.1.123    k8s-master2   <none>           <none>
kube-system   kube-scheduler-k8s-master3               1/1     Running   1               3m9s    192.168.1.124    k8s-master3   <none>           <none>
[root@k8s-master1 yaml]# kubectl get nodes -o wide
NAME          STATUS   ROLES                  AGE     VERSION   INTERNAL-IP     EXTERNAL-IP   OS-IMAGE                KERNEL-VERSION                CONTAINER-RUNTIME
k8s-master1   Ready    control-plane,master   6m43s   v1.22.7   192.168.1.15    <none>        CentOS Linux 7 (Core)   5.4.180-1.el7.elrepo.x86_64   containerd://1.4.12
k8s-master2   Ready    control-plane,master   4m16s   v1.22.7   192.168.1.123   <none>        CentOS Linux 7 (Core)   5.4.180-1.el7.elrepo.x86_64   containerd://1.4.13
k8s-master3   Ready    control-plane,master   4m6s    v1.22.7   192.168.1.124   <none>        CentOS Linux 7 (Core)   5.4.180-1.el7.elrepo.x86_64   containerd://1.4.13
k8s-node1     Ready    <none>                 3m21s   v1.22.7   192.168.1.65    <none>        CentOS Linux 7 (Core)   5.4.180-1.el7.elrepo.x86_64   containerd://1.4.12
k8s-node2     Ready    <none>                 3m23s   v1.22.7   192.168.1.26    <none>        CentOS Linux 7 (Core)   5.4.180-1.el7.elrepo.x86_64   containerd://1.4.12
k8s-node3     Ready    <none>                 3m22s   v1.22.7   192.168.1.128   <none>        CentOS Linux 7 (Core)   5.4.180-1.el7.elrepo.x86_64   containerd://1.4.13
k8s-node4     Ready    <none>                 3m14s   v1.22.7   192.168.1.129   <none>        CentOS Linux 7 (Core)   5.4.180-1.el7.elrepo.x86_64   containerd://1.4.13

??node和pod都显示ready，k8s集群部署成功

三、常用扩展工具部署

1.部署metrics-server

??下载地址

https://github.com/kubernetes-sigs/metrics-server/releases/download/v0.5.2/components.yaml

若被墙，下载链接

https://download.csdn.net/download/weixin_44254035/83205018

??修改yaml文件

spec:
  selector:
    matchLabels:
      k8s-app: metrics-server
  strategy:
    rollingUpdate:
      maxUnavailable: 0
  template:
    metadata:
      labels:
        k8s-app: metrics-server
    spec:
      containers:
      - args:
        - --cert-dir=/tmp
        - --secure-port=4443
        - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
        - --kubelet-use-node-status-port
        - --metric-resolution=15s
        - --kubelet-insecure-tls    #忽略证书要求，使用阿里云镜像站
        image: registry.cn-hangzhou.aliyuncs.com/google_containers/metrics-server:v0.5.2
        imagePullPolicy: IfNotPresent
        livenessProbe:
          failureThreshold: 3
          httpGet:
            path: /livez
            port: https
            scheme: HTTPS
          periodSeconds: 10
        name: metrics-server

??使用kubectl apply -f部署

kubectl apply -f metrics-0.5.2.yaml
# 查看pod，等待ready
kubectl get pods -n kube-system

若报错，可以查看日志定位问题
kubectl logs pod名称 -n kube-system

??目前默认的命名空间没有pod在运行，运行nginx、srs测试一下
vim nginx-test.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: nginx-deployment
spec:
  selector:
    matchLabels:
      app: nginx
  replicas: 8
  template:
    metadata:
      labels:
        app: nginx
    spec:
      containers:
      - name: nginx
        image: nginx:1.20.2
        ports:
        - containerPort: 80

vim srs-deployment.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: srs-deploy
  labels:
    app: srs
spec:
  replicas: 1
  selector:
    matchLabels:
      app: srs
  template:
    metadata:
      labels:
        app: srs
    spec:
      containers:
      - name: srs
        image: ossrs/srs
        imagePullPolicy: IfNotPresent
        ports:
        - containerPort: 1935
        - containerPort: 1985
        - containerPort: 8080

vim srs-service.yaml

apiVersion: v1
kind: Service
metadata:
  name: srs-origin-service
spec:
  type: NodePort
  selector:
    app: srs
  ports:
  - name: srs-origin-service-1935-1935
    port: 1935
    protocol: TCP
    targetPort: 1935
    nodePort: 31935 # 新增
  - name: srs-origin-service-1985-1985
    port: 1985
    protocol: TCP
    targetPort: 1985
    nodePort: 31985 # 新增
  - name: srs-origin-service-8080-8080
    port: 8080
    protocol: TCP
    targetPort: 8080
    nodePort: 30080 # 新增

部署

kubectl apply -f nginx-test.yaml -f srs-deployment.yaml -f srs-service.yaml

??使用kubectl top命令

kubectl top nodes
kubectl top pods

效果示例

[root@k8s-master1 test]# kubectl top nodes
NAME          CPU(cores)   CPU%   MEMORY(bytes)   MEMORY%   
k8s-master1   433m         10%    1367Mi          35%       
k8s-master2   356m         8%     1018Mi          26%       
k8s-master3   325m         8%     1088Mi          28%       
k8s-node1     206m         0%     856Mi           2%        
k8s-node2     189m         0%     950Mi           0%        
k8s-node3     133m         3%     738Mi           19%       
k8s-node4     125m         3%     717Mi           18%       
[root@k8s-master1 test]# kubectl top pods
NAME                               CPU(cores)   MEMORY(bytes)   
nginx-deployment-cc4b758d6-bz6n7   0m           23Mi            
nginx-deployment-cc4b758d6-ct7jg   0m           5Mi             
nginx-deployment-cc4b758d6-dpf5d   0m           5Mi             
nginx-deployment-cc4b758d6-jw9xf   0m           44Mi            
nginx-deployment-cc4b758d6-mfzjh   0m           44Mi            
nginx-deployment-cc4b758d6-mg2np   0m           23Mi            
nginx-deployment-cc4b758d6-rb7jr   0m           5Mi             
nginx-deployment-cc4b758d6-t6b9m   0m           5Mi             
srs-deploy-68f79458dc-7jt6l        60m          12Mi

2.部署Dashboard

mkdir -p /home/yaml/dashboard && cd /home/yaml/dashboard
# 下载
wget https://raw.githubusercontent.com/kubernetes/dashboard/v2.5.0/aio/deploy/recommended.yaml
# 改名
mv recommended.yaml dashboard-recommended.yaml

若被墙，下载链接

https://download.csdn.net/download/weixin_44254035/83056768

??修改dashboard-recommended.yaml，修改service的类型为NodePort

kind: Service
apiVersion: v1
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard
  namespace: kubernetes-dashboard
spec:
  type: NodePort    # 新增
  ports:
    - port: 443
      targetPort: 8443
      nodePort: 30005    # 新增
  selector:
    k8s-app: kubernetes-dashboard

kubectl apply -f dashboard-recommended.yaml

??创建Service Account 及 ClusterRoleBinding
vim auth.yaml

apiVersion: v1
kind: ServiceAccount
metadata:
  name: admin-user
  namespace: kubernetes-dashboard
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: admin-user
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-admin
subjects:
- kind: ServiceAccount
  name: admin-user
  namespace: kubernetes-dashboard

kubectl apply -f auth.yaml

??查看运行情况

kubectl get po,svc -n kubernetes-dashboard -o wide

NAME                                            READY   STATUS    RESTARTS   AGE   IP               NODE        NOMINATED NODE   READINESS GATES
pod/dashboard-metrics-scraper-c45b7869d-vx249   1/1     Running   0          69s   10.244.169.133   k8s-node2   <none>           <none>
pod/kubernetes-dashboard-764b4dd7-q7cwl         1/1     Running   0          71s   10.244.169.132   k8s-node2   <none>           <none>

NAME                                TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)         AGE   SELECTOR
service/dashboard-metrics-scraper   ClusterIP   10.104.248.183   <none>        8000/TCP        71s   k8s-app=dashboard-metrics-scraper
service/kubernetes-dashboard        NodePort    10.103.213.23    <none>        443:30005/TCP   75s   k8s-app=kubernetes-dashboard

??获取访问 Kubernetes Dashboard所需的 Token

kubectl -n kubernetes-dashboard describe secret $(kubectl -n kubernetes-dashboard get secret | grep admin-user | awk '{print $1}')

Name:         admin-user-token-6gzhs
Namespace:    kubernetes-dashboard
Labels:       <none>
Annotations:  kubernetes.io/service-account.name: admin-user
              kubernetes.io/service-account.uid: 6977202e-a4b9-47b6-8a5b-428fc29f44b5

Type:  kubernetes.io/service-account-token

Data
====
ca.crt:     1099 bytes
namespace:  20 bytes
token:      eyJhbGciOiJSUzI1NiIsImtpZCI6Im1Kc3J3Vi03VEE3Q1Z4RWZ4U1lybHhkdFEzMlBIYUlQUzd5WkZ6V2I0SFkifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlcm5ldGVzLWRhc2hib2FyZCIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJhZG1pbi11c2VyLXRva2VuLTZnemhzIiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZXJ2aWNlLWFjY291bnQubmFtZSI6ImFkbWluLXVzZXIiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC51aWQiOiI2OTc3MjAyZS1hNGI5LTQ3YjYtOGE1Yi00MjhmYzI5ZjQ0YjUiLCJzdWIiOiJzeXN0ZW06c2VydmljZWFjY291bnQ6a3ViZXJuZXRlcy1kYXNoYm9hcmQ6YWRtaW4tdXNlciJ9.g0AqarDbfOukXv-B9w8U8e-I1RhQTq6C2Z96Ae_6c4OpwJgyTL8yz3MwoEYv3HoGsXQgxGeEWn7lyBe4xIdOBSwQ__u9TRwFr4CLLVlJQCEhSwJUnys3aAg1nbQztk8IJxprl8blcYFwDrumvwVF8gDgJzBU6CD4d_zAk9xXW7tGXFKoZuNww_v4K2YNQUXAIJ4bSunDAJJ5sTtdvgaZty_0lXwgcUzdSxpPDOtkCAAlUt0cPXJZiv-lrRAllrlloTv0Cip50s9MagaHrgkuzmPPkZuXJ-y9XUJuP0D_QoAAJVCynTfgCgTNTksuOilA23NnHT5f60xdQYNrT3ophA

??浏览器访问

https://192.168.1.223:30005

复制token粘贴后进入

??下面通过kubeconfig登陆

获取token

kubectl get secret -n kubernetes-dashboard

NAME                               TYPE                                  DATA   AGE
admin-user-token-6gzhs             kubernetes.io/service-account-token   3      6m47s
default-token-df4j6                kubernetes.io/service-account-token   3      7m5s
kubernetes-dashboard-certs         Opaque                                0      7m4s
kubernetes-dashboard-csrf          Opaque                                1      7m4s
kubernetes-dashboard-key-holder    Opaque                                2      7m3s
kubernetes-dashboard-token-xn9kp   kubernetes.io/service-account-token   3      7m6s

使用admin-user-token-**

DASH_TOCKEN=$(kubectl -n kubernetes-dashboard get secret admin-user-token-6gzhs -o jsonpath={.data.token} |base64 -d)

获取配置文件

cd /etc/kubernetes/pki

??设置集群条目

kubectl config set-cluster kubernetes --certificate-authority=./ca.crt --server="https://192.168.1.223:16443" --embed-certs=true  --kubeconfig=/root/dashboard-admin.conf

??设置用户条目

kubectl config set-credentials dashboard-admin --token=$DASH_TOCKEN --kubeconfig=/root/dashboard-admin.conf

??设置上下文条目

kubectl config set-context dashboard-admin@kubernetes --cluster=kubernetes --user=dashboard-admin --kubeconfig=/root/dashboard-admin.conf

??设置当前上下文

kubectl config use-context dashboard-admin@kubernetes --kubeconfig=/root/dashboard-admin.conf

??传出文件

sz /root/dashboard-admin.conf