k8s集群的搭建之三：flannel

一介绍

flannel是CoreOS提供用于解决Dokcer集群跨主机通讯的覆盖网络工具。它的主要思路是：预先留出一个网段，每个主机使用其中一部分，然后每个容器被分配不同的ip；
让所有的容器认为大家在同一个直连的网络，底层通过UDP/VxLAN等进行报文的封装和转发。
flannel是Overlay网络的一种，也是将源数据包封装在另一种网络包里面进行路由转发和通信，目前已经支持UDP,VXLAN,host-gw,AWS,VPC,GCE路由等级传输,推荐 vxlan或者host-gw

二搭建（这里以两台为例）

2.1 分配网段，封装类型为vxlan,高效，稳定（master上执行）

etcdctl --ca-file=/etc/etcd/ssl/etcd-root-ca.pem --cert-file=/etc/etcd/ssl/etcd.pem --key-file=/etc/etcd/ssl/etcd-key.pem   \
 --endpoints=https://192.168.137.66:2379,https://192.168.137.16:2379  set  /coreos.com/network/config '{ "Network": "172.17.0.0/16", "Backend": {"Type": "vxlan"}}

查看分配的结果：

[root@k8s-master ~]# etcdctl --ca-file=/etc/etcd/ssl/etcd-root-ca.pem --cert-file=/etc/etcd/ssl/etcd.pem --key-file=/etc/etcd/ssl/etcd-key.pem  \
  --endpoints=https://192.168.137.66:2379,https://192.168.137.16:2379  get  /coreos.com/network/config '{ "Network": "172.17.0.0/16", "Backend": {"Type": "vxlan"}}'
{ "Network": "172.17.0.0/16", "Backend": {"Type": "vxlan"}}

2.2 下载并安装(master上执行)

mkdir  /home/softs
cd /home/softs
wget https://github.com/coreos/flannel/releases/download/v0.11.0/flannel-v0.11.0-linux-amd64.tar.gz
tar xf flannel-v0.11.0-linux-amd64.tar.gz
mv flanneld mk-docker-opts.sh /opt/kubernetes/bin/

[root@k8s-master softs]# cat flanneld.sh    #这里我们继续使用上一篇中安装etcd时生成的pem证书
#!/bin/bash

ETCD_ENDPOINTS=${1:-"http://127.0.0.1:2379"}
cat <<EOF >/opt/kubernetes/cfg/flanneld
FLANNEL_OPTIONS="--etcd-endpoints=${ETCD_ENDPOINTS} \
-etcd-cafile=/etc/etcd/ssl/etcd-root-ca.pem \
-etcd-certfile=/etc/etcd/ssl/etcd.pem \
-etcd-keyfile=/etc/etcd/ssl/etcd-key.pem"
EOF

cat <<EOF >/usr/lib/systemd/system/flanneld.service
[Unit]
Description=Flanneld overlay address etcd agent
After=network-online.target network.target
Before=docker.service
[Service]
Type=notify
EnvironmentFile=/opt/kubernetes/cfg/flanneld
ExecStart=/opt/kubernetes/bin/flanneld --ip-masq \$FLANNEL_OPTIONS
ExecStartPost=/opt/kubernetes/bin/mk-docker-opts.sh -k DOCKER_NETWORK_OPTIONS -d /run/flannel/subnet.env
Restart=on-failure
[Install]
WantedBy=multi-user.target
EOF

cat <<EOF >/usr/lib/systemd/system/docker.service
[Unit]
Description=Docker Application Container Engine
Documentation=https://docs.docker.com
After=network-online.target firewalld.service
Wants=network-online.target
[Service]
Type=notify
EnvironmentFile=/run/flannel/subnet.env
ExecStart=/usr/bin/dockerd  \$DOCKER_NETWORK_OPTIONS
ExecReload=/bin/kill -s HUP \$MAINPID
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity
TimeoutStartSec=0
Delegate=yes
KillMode=process
Restart=on-failure
StartLimitBurst=3
StartLimitInterval=60s
[Install]
WantedBy=multi-user.target
EOF

systemctl daemon-reload
systemctl enable flanneld
systemctl restart flanneld
systemctl restart docker

sh -x flanneld.sh https://192.168.137.66:2379,https://192.168.137.16:2379

分发到节点上执行（master 上执行）

cd /home/softs
scp flanneld.sh root@k8s-node1:/home/softs/
cd /opt/kubernetes/bin/ 
scp flanneld mk-docker-opts.sh root@master-node1:/opt/kubernetes/bin/

节点上执行（各节点执行）

sh -x flanneld.sh https://192.168.137.66:2379,https://192.168.137.16:2379

2.3 查看flannel状态（都可查看）

[root@k8s-master softs]# etcdctl --ca-file=/etc/etcd/ssl/etcd-root-ca.pem --cert-file=/etc/etcd/ssl/etcd.pem --key-file=/etc/etcd/ssl/etcd-key.pem \
 --endpoints="https://192.168.137.66:2379,https://192.168.137.16:2379" ls /coreos.com/network/subnets
/coreos.com/network/subnets/172.17.90.0-24
/coreos.com/network/subnets/172.17.84.0-24

[root@k8s-master softs]# etcdctl --ca-file=/etc/etcd/ssl/etcd-root-ca.pem --cert-file=/etc/etcd/ssl/etcd.pem --key-file=/etc/etcd/ssl/etcd-key.pem \
 --endpoints="https://192.168.137.66:2379,https://192.168.137.16:2379" get /coreos.com/network/subnets/172.17.84.0-24 docker
{"PublicIP":"192.168.137.16","BackendType":"vxlan","BackendData":{"VtepMAC":"76:ec:3f:7c:1b:81"}}

[root@k8s-master softs]# etcdctl --ca-file=/etc/etcd/ssl/etcd-root-ca.pem --cert-file=/etc/etcd/ssl/etcd.pem --key-file=/etc/etcd/ssl/etcd-key.pem \ 
 --endpoints="https://192.168.137.66:2379,https://192.168.137.16:2379" get /coreos.com/network/subnets/172.17.90.0-24 docker
{"PublicIP":"192.168.137.66","BackendType":"vxlan","BackendData":{"VtepMAC":"9e:51:aa:45:24:7f"}}

查看路由情况

#master
[root@k8s-master bin]# route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         192.168.137.1   0.0.0.0         UG    100    0        0 ens33
172.17.84.0     172.17.84.0     255.255.255.0   UG    0      0        0 flannel.1
172.17.90.0     0.0.0.0         255.255.255.0   U     0      0        0 docker0
192.168.137.0   0.0.0.0         255.255.255.0   U     100    0        0 ens33

#node
[root@k8s-node1 softs]# route -n
Kernel IP routing table
Destination     Gateway         Genmask         Flags Metric Ref    Use Iface
0.0.0.0         192.168.137.1   0.0.0.0         UG    100    0        0 ens33
172.17.84.0     0.0.0.0         255.255.255.0   U     0      0        0 docker0
172.17.90.0     172.17.90.0     255.255.255.0   UG    0      0        0 flannel.1
192.168.137.0   0.0.0.0         255.255.255.0   U     100    0        0 ens33