Ubuntu 20.04版本安装k8s控制节点与控制节点升级

一、环境配置

服务器配置：2核4G
IP：192.168.10.23
主机名：master4
将改主机加入此 集群

# 1.修改主机名
hostnamectl set-hostname master4 && bash

# 2.添加hosts
127.0.1.1 master4
192.168.10.20 master
192.168.10.21 master2
192.168.10.22 master3
192.168.10.23 master4
192.168.10.24 node1
192.168.10.25 node2

# 3.ssh信任
ssh-keygen -t rsa
ssh-copy-id master

# 4.关闭交换分区
swapoff -a  # 临时关闭
永久关闭为注销/etc/fstab中swap一行

# 5.修改机器内核参数
modprobe br_netfilter
echo "modprobe br_netfilter" >> /etc/profile

cat > /etc/sysctl.d/k8s.conf <<EOF
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
EOF

sysctl -p /etc/sysctl.d/k8s.conf

# 6. 关闭防火墙
systemctl stop firewalld ; systemctl disable firewalld

# 7.关闭selinux,修改 x selinux  配置文件之后，重启：ubuntu如果没有安装不用管
sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config

# 8.配置阿里云源
# step 1: 安装必要的一些系统工具
sudo apt-get update
sudo apt-get -y install apt-transport-https ca-certificates curl software-properties-common
# step 2: 安装GPG证书
curl -fsSL https://mirrors.aliyun.com/docker-ce/linux/ubuntu/gpg | sudo apt-key add -
# Step 3: 写入软件源信息
sudo add-apt-repository "deb [arch=amd64] https://mirrors.aliyun.com/docker-ce/linux/ubuntu $(lsb_release -cs) stable"
# Step 4: 更新并安装Docker-CE
sudo apt-get -y update

# 9.配置kubernets源
# (1).更新 apt 包索引并安装使用 Kubernetes apt 仓库所需要的包
sudo apt-get update
# apt-transport-https 可能是一个虚拟包（dummy package）；如果是的话，你可以跳过安装这个包
sudo apt-get install -y apt-transport-https ca-certificates curl gpg
# (2).下载用于 Kubernetes 软件包仓库的公共签名密钥。所有仓库都使用相同的签名密钥，因此你可以忽略URL中的版本：
curl -fsSL https://pkgs.k8s.io/core:/stable:/v1.28/deb/Release.key | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
# (3).添加 Kubernetes apt 仓库。 请注意，此仓库仅包含适用于 Kubernetes 1.28 的软件包； 对于其他 Kubernetes 次要版本，则需要更改 URL 中的 Kubernetes 次要版本以匹配你所需的次要版本，如果之前有会覆盖
echo 'deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/v1.28/deb/ /' | sudo tee /etc/apt/sources.list.d/kubernetes.list

# 10.时间同步并定时同步
cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
ntpdate time1.aliyun.com

二、基础软件包安装

sudo apt -y install ipvsadm ipset sysstat conntrack g++ openssl libssl-dev zlib zlib1g-dev libpcre3 libpcre3-dev build-essential

2.1 安装containerd

# 1.安装containerd服务
yum -y install containerd

# 2.生成containerd配置文件
mkdir -p /etc/containerd
containerd config default > /etc/containerd/config.toml

# 3.修改配置文件
vim /etc/containerd/config.toml
SystemdCgroup = true   # false改为true
sandbox_image = "registry.aliyuncs.com/google_containers/pause:3.9"   # 如果版本不清楚后面kubeadm config images list --config=kubeadm.yml时可以看了再修改

# 4.配置为开机启动
systemctl enable containerd --now

# 5.修改/etc/crictl.yaml 文件
cat > /etc/crictl.yaml <<EOF
runtime-endpoint: unix:///run/containerd/containerd.sock
image-endpoint: unix:///run/containerd/containerd.sock
timeout: 10
debug: false
EOF

systemctl restart containerd

# 6.配置镜像加速器
# 编辑 vim /etc/containerd/config.toml 文件，修改
config_path = "/etc/containerd/certs.d"

mkdir /etc/containerd/certs.d/docker.io/ -p
vim /etc/containerd/certs.d/docker.io/hosts.toml
[host."https://pft7f97f.mirror.aliyuncs.com",host."https://registry.docker-cn.com",host."https://docker.mirrors.ustc.edu.cn"]
  capabilities = ["pull"]

systemctl restart containerd

# 设置容器运行时

# 1.设置容器运行时，master,node
crictl config runtime-endpoint unix:///run/containerd/containerd.sock

2.2 安装k8s所需软件

# 安装 kubelet、kubeadm 和 kubectl，并锁定其版本：
sudo apt-get update -y
# 1.查看kubeadm、kubectl、kubelet有哪些版本
root@master4:/etc/containerd# apt-cache madison kubeadm kubelet kubectl
   kubeadm | 1.28.5-1.1 | https://pkgs.k8s.io/core:/stable:/v1.28/deb  Packages
   kubeadm | 1.28.4-1.1 | https://pkgs.k8s.io/core:/stable:/v1.28/deb  Packages
   kubeadm | 1.28.3-1.1 | https://pkgs.k8s.io/core:/stable:/v1.28/deb  Packages
   kubeadm | 1.28.2-1.1 | https://pkgs.k8s.io/core:/stable:/v1.28/deb  Packages
   kubeadm | 1.28.1-1.1 | https://pkgs.k8s.io/core:/stable:/v1.28/deb  Packages
   kubeadm | 1.28.0-1.1 | https://pkgs.k8s.io/core:/stable:/v1.28/deb  Packages
   kubelet | 1.28.5-1.1 | https://pkgs.k8s.io/core:/stable:/v1.28/deb  Packages
   kubelet | 1.28.4-1.1 | https://pkgs.k8s.io/core:/stable:/v1.28/deb  Packages
   kubelet | 1.28.3-1.1 | https://pkgs.k8s.io/core:/stable:/v1.28/deb  Packages
   kubelet | 1.28.2-1.1 | https://pkgs.k8s.io/core:/stable:/v1.28/deb  Packages
   kubelet | 1.28.1-1.1 | https://pkgs.k8s.io/core:/stable:/v1.28/deb  Packages
   kubelet | 1.28.0-1.1 | https://pkgs.k8s.io/core:/stable:/v1.28/deb  Packages
   kubectl | 1.28.5-1.1 | https://pkgs.k8s.io/core:/stable:/v1.28/deb  Packages
   kubectl | 1.28.4-1.1 | https://pkgs.k8s.io/core:/stable:/v1.28/deb  Packages
   kubectl | 1.28.3-1.1 | https://pkgs.k8s.io/core:/stable:/v1.28/deb  Packages
   kubectl | 1.28.2-1.1 | https://pkgs.k8s.io/core:/stable:/v1.28/deb  Packages
   kubectl | 1.28.1-1.1 | https://pkgs.k8s.io/core:/stable:/v1.28/deb  Packages
   kubectl | 1.28.0-1.1 | https://pkgs.k8s.io/core:/stable:/v1.28/deb  Packages

# 2.安装指定版本
root@master4:/etc/containerd# apt-get -y install kubelet=1.28.2-1.1 kubeadm=1.28.2-1.1 kubectl=1.28.2-1.1

# 阻止软件自动更新
apt-mark hold kubelet kubeadm kubectl

systemctl enable kubelet

2.3 安装配置nginx+keepalived

# 安装nginx、keepalived
apt-get install nginx keepalived

# 配置文件
[root@master service]# scp /etc/keepalived/keepalived.conf master4:/etc/keepalived/
keepalived.conf                                                                                                                  100%  806    85.4KB/s   00:00
[root@master service]# scp /etc/keepalived/check_nginx.sh master4:/etc/keepalived/
check_nginx.sh
[root@master service]# scp /etc/nginx/nginx.conf master4:/etc/nginx/
nginx.conf

# 在master4的nginx.conf中添加一条启用stream模块
include /etc/nginx/modules-enabled/*.conf;

systemctl enable nginx && systemctl start nginx
systemctl enable keepalived && systemctl start keepalived

2.4 加入到集群中作为控制节点

# 2.将master节点证书拷贝到master4节点，master4节点建立目录
mkdir -p /etc/kubernetes/pki/etcd/

scp /etc/kubernetes/pki/ca.* master4:/etc/kubernetes/pki/
scp /etc/kubernetes/pki/sa.* master4:/etc/kubernetes/pki/
scp /etc/kubernetes/pki/front-proxy-ca.* master4:/etc/kubernetes/pki/
scp /etc/kubernetes/pki/etcd/ca.* master4:/etc/kubernetes/pki/etcd/

# 将kubeadm.yaml文件传输到master4上下载kubernetes所需镜像
root@master4:~# kubeadm config images list --config=kubeadm.yaml
registry.cn-hangzhou.aliyuncs.com/google_containers/kube-apiserver:v1.28.2
registry.cn-hangzhou.aliyuncs.com/google_containers/kube-controller-manager:v1.28.2
registry.cn-hangzhou.aliyuncs.com/google_containers/kube-scheduler:v1.28.2
registry.cn-hangzhou.aliyuncs.com/google_containers/kube-proxy:v1.28.2
registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.9
registry.cn-hangzhou.aliyuncs.com/google_containers/etcd:3.5.9-0
registry.cn-hangzhou.aliyuncs.com/google_containers/coredns:v1.10.1

root@master4:~# kubeadm config images pull--config=kubeadm.yaml

# 在master节点生成token
[root@master k8s]# kubeadm token create --print-join-command
kubeadm join 192.168.10.19:16443 --token 03b1fu.tyfrsq69b28jcduj --discovery-token-ca-cert-hash sha256:3d2052ebcdc58cce07aeb55f9e5987d8d406e3b0d0370299283cdb4fdc216eeb

# 将master4加入集群中
kubeadm join 192.168.10.19:16443 --token 03b1fu.tyfrsq69b28jcduj --discovery-token-ca-cert-hash sha256:3d2052ebcdc58cce07aeb55f9e5987d8d406e3b0d0370299283cdb4fdc216eeb --control-plane

# master4上执行
    mkdir -p $HOME/.kube
    sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
    sudo chown $(id -u):$(id -g) $HOME/.kube/config

# 查看nodes
root@master4:/etc/nginx# kubectl get nodes
NAME      STATUS   ROLES           AGE     VERSION
master    Ready    control-plane   2d11h   v1.28.5
master2   Ready    control-plane   2d11h   v1.28.5
master4   Ready    control-plane   4m3s    v1.28.2
node1     Ready    worker          2d10h   v1.28.5
node2     Ready    worker          30h     v1.28.5

三、版本升级：从1.28.2升级到1.28.5

# 1.将master4设置为维护模式
kubectl cordon master4

# 2.驱逐节点上数据
kubectl  drain master4 --ignore-daemonsets --force --delete-empty-data

# 3.查看可升级的版本
apt update
apt-cache madison kubeadm

# 4.升级kubeadm,用最新的补丁版本号替换 1.28.x-* 中的 x
apt-mark unhold kubeadm && \
apt-get update && apt-get install -y kubeadm='1.28.x-*' && \
apt-mark hold kubeadm

# 查看版本号
kubectl version

# 5.验证版本升级计划
root@master4:/etc/apt/keyrings# kubeadm upgrade plan
[upgrade/config] Making sure the configuration is correct:
[upgrade/config] Reading configuration from the cluster...
[upgrade/config] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -o yaml'
[preflight] Running pre-flight checks.
[upgrade] Running cluster health checks
[upgrade] Fetching available versions to upgrade to
[upgrade/versions] Cluster version: v1.28.5
[upgrade/versions] kubeadm version: v1.28.5
I0113 11:32:45.140007  382936 version.go:256] remote version is much newer: v1.29.0; falling back to: stable-1.28
[upgrade/versions] Target version: v1.28.5
[upgrade/versions] Latest version in the v1.28 series: v1.28.5

# 6.选择要升级到的目标版本，运行合适的命令
root@master4:/etc/apt/keyrings# kubeadm upgrade apply v1.28.5
成功命令：
[upgrade/successful] SUCCESS! Your cluster was upgraded to "v1.28.5". Enjoy!

root@master4:/etc/apt/keyrings# kubeadm upgrade apply v1.28.5
[upgrade/config] Making sure the configuration is correct:
[upgrade/config] Reading configuration from the cluster...
[upgrade/config] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -o yaml'
[preflight] Running pre-flight checks.
[upgrade] Running cluster health checks
[upgrade/version] You have chosen to change the cluster version to "v1.28.5"
[upgrade/versions] Cluster version: v1.28.5
[upgrade/versions] kubeadm version: v1.28.5
[upgrade] Are you sure you want to proceed? [y/N]: y
[upgrade/prepull] Pulling images required for setting up a Kubernetes cluster
[upgrade/prepull] This might take a minute or two, depending on the speed of your internet connection
[upgrade/prepull] You can also perform this action in beforehand using 'kubeadm config images pull'
W0113 11:41:29.087629  387705 checks.go:835] detected that the sandbox image "registry.aliyuncs.com/google_containers/pause:3.7" of the container runtime is inconsistent with that used by kubeadm. It is recommended that using "registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.9" as the CRI sandbox image.
[upgrade/apply] Upgrading your Static Pod-hosted control plane to version "v1.28.5" (timeout: 5m0s)...
[upgrade/etcd] Upgrading to TLS for etcd
[upgrade/staticpods] Preparing for "etcd" upgrade
[upgrade/staticpods] Renewing etcd-server certificate
[upgrade/staticpods] Renewing etcd-peer certificate
[upgrade/staticpods] Renewing etcd-healthcheck-client certificate
[upgrade/staticpods] Moved new manifest to "/etc/kubernetes/manifests/etcd.yaml" and backed up old manifest to "/etc/kubernetes/tmp/kubeadm-backup-manifests-2024-01-13-11-41-52/etcd.yaml"
[upgrade/staticpods] Waiting for the kubelet to restart the component
[upgrade/staticpods] This might take a minute or longer depending on the component/version gap (timeout 5m0s)
[apiclient] Found 3 Pods for label selector component=etcd
[upgrade/staticpods] Component "etcd" upgraded successfully!
[upgrade/etcd] Waiting for etcd to become available
[upgrade/staticpods] Writing new Static Pod manifests to "/etc/kubernetes/tmp/kubeadm-upgraded-manifests3806239562"
[upgrade/staticpods] Preparing for "kube-apiserver" upgrade
[upgrade/staticpods] Current and new manifests of kube-apiserver are equal, skipping upgrade
[upgrade/staticpods] Preparing for "kube-controller-manager" upgrade
[upgrade/staticpods] Current and new manifests of kube-controller-manager are equal, skipping upgrade
[upgrade/staticpods] Preparing for "kube-scheduler" upgrade
[upgrade/staticpods] Current and new manifests of kube-scheduler are equal, skipping upgrade
[upload-config] Storing the configuration used in ConfigMap "kubeadm-config" in the "kube-system" Namespace
[kubelet] Creating a ConfigMap "kubelet-config" in namespace kube-system with the configuration for the kubelets in the cluster
[upgrade] Backing up kubelet config file to /etc/kubernetes/tmp/kubeadm-kubelet-config188327240/config.yaml
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[bootstrap-token] Configured RBAC rules to allow Node Bootstrap tokens to get nodes
[bootstrap-token] Configured RBAC rules to allow Node Bootstrap tokens to post CSRs in order for nodes to get long term certificate credentials
[bootstrap-token] Configured RBAC rules to allow the csrapprover controller automatically approve CSRs from a Node Bootstrap Token
[bootstrap-token] Configured RBAC rules to allow certificate rotation for all node client certificates in the cluster
[addons] Applied essential addon: CoreDNS
W0113 11:44:01.589812  387705 endpoint.go:57] [endpoint] WARNING: port specified in controlPlaneEndpoint overrides bindPort in the controlplane address
[addons] Applied essential addon: kube-proxy

[upgrade/successful] SUCCESS! Your cluster was upgraded to "v1.28.5". Enjoy!

[upgrade/kubelet] Now that your control plane is upgraded, please proceed with upgrading your kubelets if you haven't already done so.

kubeadm upgrade apply v1.28.5

# 7.升级 kubelet 和 kubectl
apt-mark unhold kubeadm kubectl
apt-get update && apt-get install -y kubelet='1.28.5-*' kubectl='1.28.5-*'

验证版本：
kubectl version
kubelet --version

apt-mark hold kubelet kubectl

# 7.重启服务
sudo systemctl daemon-reload
sudo systemctl restart kubelet

# 8.解除保护，将节点加入集群
root@master4:/etc/apt/keyrings# kubectl uncordon master4
node/master4 uncordoned
root@master4:/etc/apt/keyrings# kubectl get nodes
NAME      STATUS   ROLES           AGE     VERSION
master    Ready    control-plane   2d22h   v1.28.5
master2   Ready    control-plane   2d22h   v1.28.5
master4   Ready    control-plane   10h     v1.28.5
node1     Ready    worker          2d20h   v1.28.5
node2     Ready    worker          41h     v1.28.5