062、如何使用flannel host-gw backend（2019-04-02 周二）

参考https://www.cnblogs.com/CloudMan6/p/7457653.html

 

flannel 支持多种backend，前面学习的是 vxlan backend ，host-gw 是 flannel的另一个backend。

 

与vxlan不同，host-gw 不会封装数据包，而是在主机的路由表中创建到其他主机的subnet 路由条目，从而实现容器跨主机通信。要使用host-gw 首先要修改flannel的配置  flannel-config.json ： 将type中的vxlan改为host-gw

 

#    1、配置etcd数据库，更改type

 

[root@docker-machine ~]# cat flannel-config.json

{

  "Network": "10.2.0.0/16",

  "SubnetLen": 24,

  "Backend": {

    "Type": "host-gw"

  }

}

[root@docker-machine ~]# etcdctl --endpoints=10.12.31.213:2379 set /docker-test/network/config < flannel-config.json

{

  "Network": "10.2.0.0/16",

  "SubnetLen": 24,

  "Backend": {

    "Type": "host-gw"

  }

}

 

 

#    2、host1上重启flannel，修改mtu，重启docker

 

root@host1:~# ps -ef | grep flannel

root      7315  7226  0 17:36 pts/0    00:00:00 /usr/local/bin/flanneld-amd64 -etcd-endpoints=http://10.12.31.213:2379 -iface=ens160 -etcd-prefix=/docker-test/network

root      7437  7226  0 17:38 pts/0    00:00:00 grep --color=auto flannel

root@host1:~# kill -9 7315

root@host1:~# /usr/local/bin/flanneld-amd64 -etcd-endpoints=http://10.12.31.213:2379 -iface=ens160 -etcd-prefix=/docker-test/network &

[1] 7440

root@host1:~# I0402 17:38:43.723057    7440 main.go:529] Using interface with name ens160 and address 10.12.31.211

I0402 17:38:43.723121    7440 main.go:546] Defaulting external address to interface address (10.12.31.211)

I0402 17:38:43.723289    7440 main.go:244] Created subnet manager: Etcd Local Manager with Previous Subnet: 10.2.46.0/24

I0402 17:38:43.723307    7440 main.go:247] Installing signal handlers

I0402 17:38:43.725268    7440 main.go:388] Found network config - Backend type: host-gw

I0402 17:38:43.739204    7440 local_manager.go:147] Found lease (10.2.46.0/24) for current IP (10.12.31.211), reusing

I0402 17:38:43.751344    7440 main.go:311] Changing default FORWARD chain policy to ACCEPT

I0402 17:38:43.751523    7440 main.go:319] Wrote subnet file to /run/flannel/subnet.env

I0402 17:38:43.751546    7440 main.go:323] Running backend.

I0402 17:38:43.751616    7440 route_network.go:53] Watching for new subnet leases

I0402 17:38:43.756410    7440 main.go:431] Waiting for 22h59m59.974453402s to renew lease

I0402 17:38:43.756617    7440 route_network.go:85] Subnet added: 10.2.44.0/24 via 10.12.31.212

W0402 17:38:43.756637    7440 route_network.go:88] Ignoring non-host-gw subnet: type=vxlan

root@host1:~# cat /run/flannel/subnet.env

FLANNEL_NETWORK=10.2.0.0/16

FLANNEL_SUBNET=10.2.46.1/24

FLANNEL_MTU=1500

FLANNEL_IPMASQ=false

root@host1:~# ip r

default via 10.12.28.6 dev ens160 onlink

10.2.44.0/24 via 10.2.44.0 dev flannel.1 onlink

10.2.46.0/24 dev docker0  proto kernel  scope link  src 10.2.46.1

10.12.28.0/22 dev ens160  proto kernel  scope link  src 10.12.31.211

172.22.0.0/16 via 10.12.28.1 dev ens160

root@host1:~# cat /etc/systemd/system/docker.service.d/10-machine.conf

[Service]

ExecStart=

ExecStart=/usr/bin/dockerd -H tcp://0.0.0.0:2376 -H unix:///var/run/docker.sock --storage-driver overlay2 --tlsverify --tlscacert /etc/docker/ca.pem --tlscert /etc/docker/server.pem --tlskey /etc/docker/server-key.pem --label provider=generic --bip=10.2.46.1/24 --mtu=1500

#--cluster-store=consul://10.12.31.213:8500 --cluster-advertise=ens160:2376

Environment=

root@host1:~# systemctl daemon-reload

root@host1:~# systemctl restart docker.service

 

#    3、host1上重启flannel，修改mtu，重启docker

 

root@host2:~# ps -ef | grep flannel

root      1572     1  0 Apr01 ?        00:00:33 /usr/local/bin/flanneld-amd64 -etcd-endpoints=http://10.12.31.213:2379 -iface=ens160 -etcd-prefix=/docker-test/network

root     18111 17898  0 17:41 pts/0    00:00:00 grep --color=auto flannel

root@host2:~# kill -9 1572

root@host2:~# /usr/local/bin/flanneld-amd64 -etcd-endpoints=http://10.12.31.213:2379 -iface=ens160 -etcd-prefix=/docker-test/network &

[1] 18120

root@host2:~# I0402 17:41:52.208836   18120 main.go:529] Using interface with name ens160 and address 10.12.31.212

I0402 17:41:52.208929   18120 main.go:546] Defaulting external address to interface address (10.12.31.212)

I0402 17:41:52.209142   18120 main.go:244] Created subnet manager: Etcd Local Manager with Previous Subnet: 10.2.44.0/24

I0402 17:41:52.209168   18120 main.go:247] Installing signal handlers

I0402 17:41:52.211324   18120 main.go:388] Found network config - Backend type: host-gw

I0402 17:41:52.237102   18120 local_manager.go:147] Found lease (10.2.44.0/24) for current IP (10.12.31.212), reusing

I0402 17:41:52.253167   18120 main.go:311] Changing default FORWARD chain policy to ACCEPT

I0402 17:41:52.253345   18120 main.go:319] Wrote subnet file to /run/flannel/subnet.env

I0402 17:41:52.253369   18120 main.go:323] Running backend.

I0402 17:41:52.253604   18120 route_network.go:53] Watching for new subnet leases

I0402 17:41:52.269068   18120 route_network.go:85] Subnet added: 10.2.46.0/24 via 10.12.31.211

W0402 17:41:52.271450   18120 route_network.go:102] Replacing existing route to 10.2.46.0/24 via 10.2.46.0 dev index 6 with 10.2.46.0/24 via 10.12.31.211 dev index 2.

I0402 17:41:52.272686   18120 main.go:431] Waiting for 22h59m59.965316418s to renew lease

root@host2:~# cat /run/flannel/subnet.env

FLANNEL_NETWORK=10.2.0.0/16

FLANNEL_SUBNET=10.2.44.1/24

FLANNEL_MTU=1500

FLANNEL_IPMASQ=false

root@host2:~# ip r

default via 10.12.28.6 dev ens160 onlink

10.2.44.0/24 dev docker0  proto kernel  scope link  src 10.2.44.1

10.2.46.0/24 via 10.12.31.211 dev ens160

10.12.28.0/22 dev ens160  proto kernel  scope link  src 10.12.31.212

172.22.0.0/16 via 10.12.28.1 dev ens160

 

root@host2:~# cat /etc/systemd/system/docker.service.d/10-machine.conf

[Service]

ExecStart=

ExecStart=/usr/bin/dockerd -H tcp://0.0.0.0:2376 -H unix:///var/run/docker.sock --storage-driver overlay2 --tlsverify --tlscacert /etc/docker/ca.pem --tlscert /etc/docker/server.pem --tlskey /etc/docker/server-key.pem --label provider=generic --bip 10.2.44.1/24 --mtu=1500

# --cluster-store=consul://10.12.31.213:8500 --cluster-advertise=ens160:2376

Environment=

root@host2:~# systemctl daemon-reload

root@host2:~# systemctl restart docker.service

 

重新进行连通性测试

 

root@host1:~# docker exec bbox1 ip r

default via 10.2.46.1 dev eth0

10.2.46.0/24 dev eth0 scope link  src 10.2.46.2

root@host1:~# docker exec bbox1 ping -c 2 10.2.44.2

PING 10.2.44.2 (10.2.44.2): 56 data bytes

64 bytes from 10.2.44.2: seq=0 ttl=62 time=0.641 ms

64 bytes from 10.2.44.2: seq=1 ttl=62 time=0.462 ms

--- 10.2.44.2 ping statistics ---

2 packets transmitted, 2 packets received, 0% packet loss

round-trip min/avg/max = 0.462/0.551/0.641 ms

 

root@host2:~# docker exec bbox2 ip r

default via 10.2.44.1 dev eth0

10.2.44.0/24 dev eth0 scope link  src 10.2.44.2

 

 

host-gw 的MTU 为1500，所以需要修改docker启动参数--mtu值

 

下面对比 host-gw 和 vxlan 两种backend：

 

1、host-gw 把每个主机都配置成网关，主机知道其他主机的subnet和转发地址。vxlan则在主机间建立隧道，不同主机的容器都在一个大的网段内

 

2、虽然vxlan与host-gw使用不同的机制建立主机之间连接，但对于容器则无需任何改变，bbox1仍然可以与bbox2通信

 

3、由于vxlan需要对数据进行额外打包和拆包，性能稍逊于host-gw