2.生产环境k8s-1.28.2集群小版本升级到1.28.5

环境：https://www.cnblogs.com/yangmeichong/p/17956335

# 流程：先升级master，再升级node

# 1.备份组件
参考：https://kubernetes.io/zh-cn/docs/tasks/administer-cluster/configure-upgrade-etcd/

[root@master ~]# ETCDCTL_API=3 etcdctl --endpoints=https://192.168.10.20:2379 --cacert=/etc/kubernetes/pki/etcd/ca.crt --cert=/etc/kubernetes/pki/etcd/server.crt --key=/etc/kubernetes/pki/etcd/server.key snapshot save master.db
{"level":"info","ts":"2024-01-11T09:09:45.954024+0800","caller":"snapshot/v3_snapshot.go:65","msg":"created temporary db file","path":"master.db.part"}
{"level":"info","ts":"2024-01-11T09:09:45.994705+0800","logger":"client","caller":"v3@v3.5.9/maintenance.go:212","msg":"opened snapshot stream; downloading"}
{"level":"info","ts":"2024-01-11T09:09:45.994821+0800","caller":"snapshot/v3_snapshot.go:73","msg":"fetching snapshot","endpoint":"https://192.168.10.20:2379"}
{"level":"info","ts":"2024-01-11T09:09:46.174339+0800","logger":"client","caller":"v3@v3.5.9/maintenance.go:220","msg":"completed snapshot read; closing"}
{"level":"info","ts":"2024-01-11T09:09:46.181942+0800","caller":"snapshot/v3_snapshot.go:88","msg":"fetched snapshot","endpoint":"https://192.168.10.20:2379","size":"3.3 MB","took":"now"}
{"level":"info","ts":"2024-01-11T09:09:46.18219+0800","caller":"snapshot/v3_snapshot.go:97","msg":"saved","path":"master.db"}
Snapshot saved at master.db

# 先升级master3
# 2.腾空节点，驱逐master
# 2.1 节点设置为维护状态
[root@master ~]# kubectl cordon master3
node/master3 cordoned

[root@master ~]# kubectl get nodes
NAME      STATUS                     ROLES           AGE     VERSION
master    Ready                      control-plane   5h2m    v1.28.2
master2   Ready                      control-plane   4h50m   v1.28.2
master3   Ready,SchedulingDisabled   control-plane   4h49m   v1.28.2
node1     Ready                      worker          3h14m   v1.28.2

# 2.2 驱逐节点上的pod
[root@master ~]# kubectl drain master3 --delete-emptydir-data --ignore-daemonsets --force
node/master3 already cordoned
Warning: ignoring DaemonSet-managed Pods: kube-system/calico-node-c56kn, kube-system/kube-proxy-phdlz
evicting pod kube-system/coredns-6554b8b87f-cjtsk
evicting pod kube-system/calico-kube-controllers-7ddc4f45bc-76zdb
evicting pod kube-system/coredns-6554b8b87f-ccvtm
pod/calico-kube-controllers-7ddc4f45bc-76zdb evicted
pod/coredns-6554b8b87f-cjtsk evicted
pod/coredns-6554b8b87f-ccvtm evicted
node/master3 drained

# 2.3 查看可升级的版本
参考：https://v1-28.docs.kubernetes.io/zh-cn/docs/tasks/administer-cluster/kubeadm/kubeadm-upgrade/
[root@master3 yum.repos.d]# yum list --showduplicates kubeadm --disableexcludes=kubernetes

# 2.4 升级kubeadm
yum install -y kubeadm-'1.28.5-*' --disableexcludes=kubernetes

# 2.5 验证升级计划
[root@master3 yum.repos.d]# kubeadm version
kubeadm version: &version.Info{Major:"1", Minor:"28", GitVersion:"v1.28.5", GitCommit:"506050d61cf291218dfbd41ac93913945c9aa0da", GitTreeState:"clean", BuildDate:"2023-12-19T13:40:52Z", GoVersion:"go1.20.12", Compiler:"gc", Platform:"linux/amd64"}
# 此命令检查你的集群是否可被升级，并取回你要升级的目标版本。 命令也会显示一个包含组件配置版本状态的表格
[root@master3 yum.repos.d]# kubeadm upgrade plan

# 2.6 选择升级版本v1.28.5 ,忽略etcd升级
[root@master3 ~]# kubeadm upgrade apply v1.28.5 --etcd-upgrade=false

成功显示：
[upgrade/successful] SUCCESS! Your cluster was upgraded to "v1.28.5". Enjoy!

# 2.7 升级其他组件kubelet,kubectl
[root@master3 ~]# yum install -y kubelet-1.28.5 kubectl-1.28.5 --disableexcludes=kubernetes

[root@master3 ~]# kubelet --version
Kubernetes v1.28.5
[root@master3 ~]# kubectl version
Client Version: v1.28.5
Kustomize Version: v5.0.4-0.20230601165947-6ce0bf390ce3
Server Version: v1.28.5

#2.8 重启服务
systemctl daemon-reload
systemctl restart kubelet

# 2.9 将节点设置为可调度状态
[root@master3 ~]# kubectl uncordon master3
node/master3 uncordoned

[root@master3 ~]# kubectl get nodes
NAME      STATUS   ROLES           AGE     VERSION
master    Ready    control-plane   5h25m   v1.28.2
master2   Ready    control-plane   5h13m   v1.28.2
master3   Ready    control-plane   5h12m   v1.28.5
node1     Ready    worker          3h37m   v1.28.2

[root@master3 yum.repos.d]# kubeadm upgrade plan
[upgrade/config] Making sure the configuration is correct:
[upgrade/config] Reading configuration from the cluster...
[upgrade/config] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -o yaml'
[preflight] Running pre-flight checks.
[upgrade] Running cluster health checks
[upgrade] Fetching available versions to upgrade to
[upgrade/versions] Cluster version: v1.28.2
[upgrade/versions] kubeadm version: v1.28.5
I0110 18:31:04.904312  103974 version.go:256] remote version is much newer: v1.29.0; falling back to: stable-1.28
[upgrade/versions] Target version: v1.28.5
[upgrade/versions] Latest version in the v1.28 series: v1.28.5

Components that must be upgraded manually after you have upgraded the control plane with 'kubeadm upgrade apply':
COMPONENT   CURRENT       TARGET
kubelet     4 x v1.28.2   v1.28.5

Upgrade to the latest version in the v1.28 series:

COMPONENT                 CURRENT   TARGET
kube-apiserver            v1.28.2   v1.28.5
kube-controller-manager   v1.28.2   v1.28.5
kube-scheduler            v1.28.2   v1.28.5
kube-proxy                v1.28.2   v1.28.5
CoreDNS                   v1.10.1   v1.10.1
etcd                      3.5.9-0   3.5.9-0

You can now apply the upgrade by executing the following command:

    kubeadm upgrade apply v1.28.5

_____________________________________________________________________

The table below shows the current state of component configs as understood by this version of kubeadm.
Configs that have a "yes" mark in the "MANUAL UPGRADE REQUIRED" column require manual config upgrade or
resetting to kubeadm defaults before a successful upgrade can be performed. The version to manually
upgrade to is denoted in the "PREFERRED VERSION" column.

API GROUP                 CURRENT VERSION   PREFERRED VERSION   MANUAL UPGRADE REQUIRED
kubeproxy.config.k8s.io   v1alpha1          v1alpha1            no
kubelet.config.k8s.io     v1beta1           v1beta1             no
_____________________________________________________________________

kubeadm upgrade plan

[root@master3 ~]# kubeadm upgrade apply v1.28.5 --etcd-upgrade=false
[upgrade/config] Making sure the configuration is correct:
[upgrade/config] Reading configuration from the cluster...
[upgrade/config] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -o yaml'
[preflight] Running pre-flight checks.
[upgrade] Running cluster health checks
[upgrade/version] You have chosen to change the cluster version to "v1.28.5"
[upgrade/versions] Cluster version: v1.28.2
[upgrade/versions] kubeadm version: v1.28.5
[upgrade] Are you sure you want to proceed? [y/N]: y
[upgrade/prepull] Pulling images required for setting up a Kubernetes cluster
[upgrade/prepull] This might take a minute or two, depending on the speed of your internet connection
[upgrade/prepull] You can also perform this action in beforehand using 'kubeadm config images pull'
W0110 18:36:50.707322  106437 checks.go:835] detected that the sandbox image "registry.aliyuncs.com/google_containers/pause:3.7" of the container runtime is inconsistent with that used by kubeadm. It is recommended that using "registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.9" as the CRI sandbox image.
[upgrade/apply] Upgrading your Static Pod-hosted control plane to version "v1.28.5" (timeout: 5m0s)...
[upgrade/staticpods] Writing new Static Pod manifests to "/etc/kubernetes/tmp/kubeadm-upgraded-manifests1651974058"
[upgrade/staticpods] Preparing for "kube-apiserver" upgrade
[upgrade/staticpods] Renewing apiserver certificate
[upgrade/staticpods] Renewing apiserver-kubelet-client certificate
[upgrade/staticpods] Renewing front-proxy-client certificate
[upgrade/staticpods] Renewing apiserver-etcd-client certificate
[upgrade/staticpods] Moved new manifest to "/etc/kubernetes/manifests/kube-apiserver.yaml" and backed up old manifest to "/etc/kubernetes/tmp/kubeadm-backup-manifests-2024-01-10-18-36-50/kube-apiserver.yaml"
[upgrade/staticpods] Waiting for the kubelet to restart the component
[upgrade/staticpods] This might take a minute or longer depending on the component/version gap (timeout 5m0s)
[apiclient] Found 3 Pods for label selector component=kube-apiserver
[upgrade/staticpods] Component "kube-apiserver" upgraded successfully!
[upgrade/staticpods] Preparing for "kube-controller-manager" upgrade
[upgrade/staticpods] Renewing controller-manager.conf certificate
[upgrade/staticpods] Moved new manifest to "/etc/kubernetes/manifests/kube-controller-manager.yaml" and backed up old manifest to "/etc/kubernetes/tmp/kubeadm-backup-manifests-2024-01-10-18-36-50/kube-controller-manager.yaml"
[upgrade/staticpods] Waiting for the kubelet to restart the component
[upgrade/staticpods] This might take a minute or longer depending on the component/version gap (timeout 5m0s)
[apiclient] Found 3 Pods for label selector component=kube-controller-manager
[upgrade/staticpods] Component "kube-controller-manager" upgraded successfully!
[upgrade/staticpods] Preparing for "kube-scheduler" upgrade
[upgrade/staticpods] Renewing scheduler.conf certificate
[upgrade/staticpods] Moved new manifest to "/etc/kubernetes/manifests/kube-scheduler.yaml" and backed up old manifest to "/etc/kubernetes/tmp/kubeadm-backup-manifests-2024-01-10-18-36-50/kube-scheduler.yaml"
[upgrade/staticpods] Waiting for the kubelet to restart the component
[upgrade/staticpods] This might take a minute or longer depending on the component/version gap (timeout 5m0s)
[apiclient] Found 3 Pods for label selector component=kube-scheduler
[upgrade/staticpods] Component "kube-scheduler" upgraded successfully!
[upload-config] Storing the configuration used in ConfigMap "kubeadm-config" in the "kube-system" Namespace
[kubelet] Creating a ConfigMap "kubelet-config" in namespace kube-system with the configuration for the kubelets in the cluster
[upgrade] Backing up kubelet config file to /etc/kubernetes/tmp/kubeadm-kubelet-config690952238/config.yaml
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[bootstrap-token] Configured RBAC rules to allow Node Bootstrap tokens to get nodes
[bootstrap-token] Configured RBAC rules to allow Node Bootstrap tokens to post CSRs in order for nodes to get long term certificate credentials
[bootstrap-token] Configured RBAC rules to allow the csrapprover controller automatically approve CSRs from a Node Bootstrap Token
[bootstrap-token] Configured RBAC rules to allow certificate rotation for all node client certificates in the cluster
[upgrade/addons] skip upgrade addons because control plane instances [master master2] have not been upgraded

[upgrade/successful] SUCCESS! Your cluster was upgraded to "v1.28.5". Enjoy!

[upgrade/kubelet] Now that your control plane is upgraded, please proceed with upgrading your kubelets if you haven't already done so.

kubeadm upgrade apply v1.28.5 --etcd-upgrade=false

二、node工作节点升级

参考：https://v1-28.docs.kubernetes.io/zh-cn/docs/tasks/administer-cluster/kubeadm/upgrading-linux-nodes/

# 1.将节点设置成维护状态,
[root@master ~]# kubectl cordon node1
node/node1 cordoned

# 2.将节点标记为不可调度并驱逐所有负载，准备节点的维护：
[root@master ~]# kubectl drain --ignore-daemonsets node1 --delete-emptydir-data --force
node/node1 already cordoned
Warning: ignoring DaemonSet-managed Pods: kube-system/calico-node-nw8hw, kube-system/kube-proxy-qhlbv
evicting pod kube-system/coredns-6554b8b87f-v55xj
evicting pod kube-system/calico-kube-controllers-7ddc4f45bc-5whqq
pod/calico-kube-controllers-7ddc4f45bc-5whqq evicted
pod/coredns-6554b8b87f-v55xj evicted
node/node1 drained

# 3.升级kubeadm
yum list --showduplicates kubeadm --disableexcludes=kubernetes
yum install -y kubeadm-'1.28.5-*' --disableexcludes=kubernetes

kubeadm version

# 4.升级本地的 kubelet 配置
[root@node1 ~]# kubeadm upgrade node
[upgrade] Reading configuration from the cluster...
[upgrade] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -o yaml'
[preflight] Running pre-flight checks
[preflight] Skipping prepull. Not a control plane node.
[upgrade] Skipping phase. Not a control plane node.
[upgrade] Backing up kubelet config file to /etc/kubernetes/tmp/kubeadm-kubelet-config2371242324/config.yaml
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[upgrade] The configuration for this node was successfully updated!
[upgrade] Now you should go ahead and upgrade the kubelet package using your package manager.

# 5.升级 kubelet 和 kubectl
yum install -y kubelet-'1.28.5-*' kubectl-'1.28.5-*' --disableexcludes=kubernetes

# 查看升级后的版本
kubectl version
kubelet --version

# 6.重启服务
[root@node1 ~]# systemctl daemon-reload
[root@node1 ~]# systemctl restart kubelet

# 7.将节点标记为可调度
[root@master ~]# kubectl uncordon node1
node/node1 uncordoned
[root@master ~]# kubectl get nodes
NAME      STATUS   ROLES           AGE   VERSION
master    Ready    control-plane   20h   v1.28.5
master2   Ready    control-plane   20h   v1.28.5
master3   Ready    control-plane   20h   v1.28.5
node1     Ready    worker          19h   v1.28.5

升级完成