052、overlay如何实现跨主机通信？（2019-03-19 周二）

参考https://www.cnblogs.com/CloudMan6/p/7305989.html

 

今天开始学习 overlay 网络跨主机通信的原理

 

root@host01:~# ufw allow 4789/udp

root@host01:~# ufw status numbered

Status: active

     To                         Action      From

     --                         ------      ----

[ 1] 22                         ALLOW IN    Anywhere                  

[ 2] 2376                       ALLOW IN    Anywhere                  

[ 3] 4789/udp                   ALLOW IN    Anywhere  

 

root@host01:~# docker run -itd --name bbox1 --network ov_net1 busybox

root@host01:~# docker exec bbox1 ip a

1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue qlen 1

    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00

    inet 127.0.0.1/8 scope host lo

       valid_lft forever preferred_lft forever

8: eth0@if9: <BROADCAST,MULTICAST,UP,LOWER_UP,M-DOWN> mtu 1450 qdisc noqueue

    link/ether 02:42:0a:00:00:02 brd ff:ff:ff:ff:ff:ff

    inet 10.0.0.2/24 brd 10.0.0.255 scope global eth0

       valid_lft forever preferred_lft forever

11: eth1@if12: <BROADCAST,MULTICAST,UP,LOWER_UP,M-DOWN> mtu 1500 qdisc noqueue

    link/ether 02:42:ac:12:00:02 brd ff:ff:ff:ff:ff:ff

    inet 172.18.0.2/16 brd 172.18.255.255 scope global eth1

       valid_lft forever preferred_lft forever

 

root@host01:~# docker exec bbox1 ping -c 2  bbox2

PING bbox2 (10.0.0.3): 56 data bytes

64 bytes from 10.0.0.3: seq=0 ttl=64 time=0.348 ms

64 bytes from 10.0.0.3: seq=1 ttl=64 time=0.440 ms

--- bbox2 ping statistics ---

2 packets transmitted, 2 packets received, 0% packet loss

round-trip min/avg/max = 0.348/0.394/0.440 ms

 

root@host01:~# ln -s /var/run/docker/netns /var/run/netns

root@host01:~# ip netns

a8d468c12df8 (id: 1)

1-609020e03f (id: 0)

root@host01:~# ip netns exec 1-609020e03f brctl show

bridge name    bridge id        STP enabled    interfaces

br0        8000.0af427b2de24    no        veth0

                            vxlan0

root@host01:~# ip netns exec 1-609020e03f ip -d l show vxlan0

7: vxlan0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue master br0 state UNKNOWN mode DEFAULT group default

    link/ether 1a:3d:67:93:45:5d brd ff:ff:ff:ff:ff:ff link-netnsid 0 promiscuity 1

    vxlan id 256 srcport 0 0 dstport 4789 proxy l2miss l3miss ageing 300

    bridge_slave state forwarding priority 32 cost 100 hairpin off guard off root_block off fastleave off learning on flood on addrgenmode eui64

 

--------------------------------------------------------------------------------------------------------

 

root@host02:~# ufw allow 4789/udp

root@host02:~# ufw status numbered

Status: active

     To                         Action      From

     --                         ------      ----

[ 1] 22                         ALLOW IN    Anywhere                  

[ 2] 2376                       ALLOW IN    Anywhere                  

[ 3] 4789/udp                   ALLOW IN    Anywhere                  

 

root@host02:~# docker run -itd --name bbox2 --network ov_net1 busybox

root@host02:~# docker exec bbox2 ip a

1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue qlen 1

    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00

    inet 127.0.0.1/8 scope host lo

       valid_lft forever preferred_lft forever

8: eth0@if9: <BROADCAST,MULTICAST,UP,LOWER_UP,M-DOWN> mtu 1450 qdisc noqueue

    link/ether 02:42:0a:00:00:03 brd ff:ff:ff:ff:ff:ff

    inet 10.0.0.3/24 brd 10.0.0.255 scope global eth0

       valid_lft forever preferred_lft forever

11: eth1@if12: <BROADCAST,MULTICAST,UP,LOWER_UP,M-DOWN> mtu 1500 qdisc noqueue

    link/ether 02:42:ac:12:00:02 brd ff:ff:ff:ff:ff:ff

    inet 172.18.0.2/16 brd 172.18.255.255 scope global eth1

       valid_lft forever preferred_lft forever

 

root@host02:~# docker exec bbox2 ping -c 2 bbox1

PING bbox1 (10.0.0.2): 56 data bytes

64 bytes from 10.0.0.2: seq=0 ttl=64 time=0.382 ms

64 bytes from 10.0.0.2: seq=1 ttl=64 time=0.353 ms

--- bbox1 ping statistics ---

2 packets transmitted, 2 packets received, 0% packet loss

round-trip min/avg/max = 0.353/0.367/0.382 ms

 

root@host02:~# ln -s /var/run/docker/netns /var/run/netns

root@host02:~# ip netns

9e3d32ba4934 (id: 1)

1-609020e03f (id: 0)

root@host02:~# ip netns exec 1-609020e03f brctl show

bridge name    bridge id        STP enabled    interfaces

br0        8000.1ad722007738    no        veth0

                            vxlan0

root@host02:~# ip netns exec 1-609020e03f ip -d l show vxlan0

7: vxlan0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1450 qdisc noqueue master br0 state UNKNOWN mode DEFAULT group default

    link/ether 1a:d7:22:00:77:38 brd ff:ff:ff:ff:ff:ff link-netnsid 0 promiscuity 1

    vxlan id 256 srcport 0 0 dstport 4789 proxy l2miss l3miss ageing 300

    bridge_slave state forwarding priority 32 cost 100 hairpin off guard off root_block off fastleave off learning on flood on addrgenmode eui64

 

 

以上实验，可见overlay网络中的容器可以直接通信，即使两个容器不在同一台host上，且支持 docker dns

 

docker 会为每个 overlay 网络创建一个独立的 network namespace ，其中会有一个 linux bridge br0 ，endpoint 还是由 veth pair 实现，一端连接到容器中（即eth0），另一端连接到namespace的br0上。

 

br0 除了连接所有的endpoint，还会连接一个vxlan 设备，用于与其他host建立vxlan tunnel。容器之间的数据就是通过这个tunnel通信的。

 

要查看overlay 网络的namespace 可以在 host01 和 host02 上执行 ipnetns （需要先执行ln -s /var/run/docker/netns /var/run/netns），可以看到两个 host上有一个相同名称的namespace，这就是 ov_net1 的namespace，可以在该namespace中查看br0设备，还可以在该namespace中查看到具有相同 id 的vxlan