解决一次calico异常情况，pod之间访问pod ip不通

k8s 集群采用二进制安装，cni网络插件用calico通讯
问题描述：
发现有些pod不是很正常例如: ht13.node

正常系统采样

[root@ht6 ~]# cat /etc/redhat-release
CentOS Linux release 7.9.2009 (Core)

[root@ht6 ~]# uname -r
3.10.0-1160.42.2.el7.x86_64

[root@ht6 ~]# docker ps | grep calico
3c5a2fe1f2d3 9f355e076ea7 "/install-cni.sh" 3 months ago Up 3 months k8s_install-cni_calico-node-rfns5_kube-system_7e3f2319-1c3d-11ec-9a8a-065c62000e9f_1
735699701a90 k8s.gcr.io/pause-amd64:3.0 "/pause" 3 months ago Up 3 months k8s_POD_calico-node-rfns5_kube-system_7e3f2319-1c3d-11ec-9a8a-065c62000e9f_1
24db12a21b59 quay.io/calico/node:v3.1.0 "start_runit" 3 months ago Up 3 months calico-node

//ipip模式采样

[root@ht6 ~]# ifconfig tunl0
tunl0: flags=193<UP,RUNNING,NOARP> mtu 1440
inet 172.17.244.192 netmask 255.255.255.255
tunnel txqueuelen 1000 (IPIP Tunnel)
RX packets 37971826 bytes 12993721945 (12.1 GiB)
RX errors 0 dropped 0 overruns 0 frame 0
TX packets 41952855 bytes 5357352541 (4.9 GiB)
TX errors 0 dropped 0 overruns 0 carrier 0 collisions 0

在master上查看,貌似正常 ht13.node

[root@master-web-38 ~]# kubectl get pod -n kube-system -o wide | grep calico

calico-node-rfns5 1/1 Running 1 148d 10.121.55.131 ht6.node

calico-node-wfvd6 1/1 Running 0 81d 10.121.52.193 ht13.node

node节点异常系统采样

[root@ht13 calico]# cat /etc/redhat-release
CentOS Linux release 7.9.2009 (Core)

[root@ht13 calico]# uname -r
3.10.0-327.el7.x86_64

[root@ht13 ~]# docker images | grep calico
quay.io/calico/cni v3.1.3 9f355e076ea7 3 years ago 68.8MB
quay.io/calico/node v3.1.0 ad8d84fa5f8c 3 years ago 277MB
//关注这里，因为，我们看到calico-node 没有创建起来

重新创建calico

[root@ht13 calico]# ll
total 392880
drwxr-xr-x 2 root root 4096 Nov 28 18:51 calico
-rw-r--r-- 1 root root 11829 Nov 28 18:51 calico.example.yaml
-rw-r--r-- 1 root root 2483 Nov 29 19:37 calico-node.service.sh
-rw-r--r-- 1 root root 277872640 Nov 28 18:51 caliconode.v3.1.0.tar
-rw-r--r-- 1 root root 1081 Nov 28 18:51 calico.sh
-rw-r--r-- 1 root root 19389 Nov 28 18:51 calico.yaml
-rw-r--r-- 1 root root 69124608 Nov 28 18:51 cni.v3.1.3.tar
-rw-r--r-- 1 root root 55256576 Nov 28 18:51 kube-controllers.v3.1.3.tar
-rw-r--r-- 1 root root 1247 Nov 28 18:51 rbac.yaml

//执行重新创建calico-node
[root@ht13 calico]# sh calico-node.service.sh
//查看calico
[root@ht13 calico]# docker ps | grep calico

 

[root@ht13 calico]# vi calico-node.service.sh

K8S_MASTER_IP="10.121.52.193"
HOSTNAME='ht13.node'
#HOSTNAME=`cat /etc/hostname`
ETCD_ENDPOINTS="https://10.121.52.111"  //这个是master

cat > /lib/systemd/system/calico-node.service <<EOF
[Unit]
Description=calico node
After=docker.service
Requires=docker.service

[Service]
User=root
Environment=ETCD_ENDPOINTS=${ETCD_ENDPOINTS}
PermissionsStartOnly=true
ExecStart=/usr/bin/docker run --net=host --privileged --name=calico-node \\
-e ETCD_ENDPOINTS=\${ETCD_ENDPOINTS} \\
-e ETCD_CA_CERT_FILE=/etc/etcd/ssl/etcd-root-ca.pem \\
-e ETCD_CERT_FILE=/etc/etcd/ssl/etcd.pem \\
-e ETCD_KEY_FILE=/etc/etcd/ssl/etcd-key.pem \\
-e NODENAME=${HOSTNAME} \\
-e IP= \\
-e IP_AUTODETECTION_METHOD=can-reach=${K8S_MASTER_IP} \\
-e AS=64512 \\
-e CLUSTER_TYPE=k8s,bgp \\
-e CALICO_IPV4POOL_CIDR=172.17.0.0/16 \\
-e CALICO_IPV4POOL_IPIP=always \\

-e CALICO_LIBNETWORK_ENABLED=true \\
-e CALICO_NETWORKING_BACKEND=bird \\
-e CALICO_DISABLE_FILE_LOGGING=true \\
-e FELIX_IPV6SUPPORT=false \\
-e FELIX_DEFAULTENDPOINTTOHOSTACTION=ACCEPT \\
-e FELIX_LOGSEVERITYSCREEN=info \\
-e FELIX_IPINIPMTU=1440 \\
-e FELIX_HEALTHENABLED=true \\
-e CALICO_K8S_NODE_REF=${HOSTNAME} \\
-v /etc/calico/etcd-root-ca.pem:/etc/etcd/ssl/etcd-root-ca.pem \\
-v /etc/calico/etcd.pem:/etc/etcd/ssl/etcd.pem \\
-v /etc/calico/etcd-key.pem:/etc/etcd/ssl/etcd-key.pem \\
-v /lib/modules:/lib/modules \\
-v /var/lib/calico:/var/lib/calico \\
-v /var/run/calico:/var/run/calico \\
quay.io/calico/node:v3.1.0
ExecStop=/usr/bin/docker rm -f calico-node
Restart=always
RestartSec=10

[Install]
WantedBy=multi-user.target
EOF

//现在尝试把一个pod在该node节点上重建
1、把该节点设置为可调度（通过rancher，rancher这方面调度非常方便）
2、把一个pod重新创建在该node上