[ovs][dpdk] ovs-dpdk, dpdk port 大量丢包

gdb了ovs的代码，发现是 dpdk的imiss计数在不断的丢包。

看了ovs-openvswitchd的日志，重启时发现如下行：

 --21T11::.427Z||timeval|WARN|Unreasonably long 22418ms poll interval (474ms user, 21612ms system)
 --21T11::.427Z||timeval|WARN|faults:  minor,  major
 --21T11::.427Z||timeval|WARN|disk:  reads,  writes
 --21T11::.427Z||timeval|WARN|context switches:  voluntary,  involuntary

开启debug

[root@vrouter1 ~]# ovs-appctl vlog/set file:dbg

重装新版dpdk

[root@vrouter1 ovs-dpdk]# ls
dpdk-17.11..tar.xz  dpdk-stable-17.11.  openvswitch-2.9.  openvswitch-2.9..tar.gz

1. 编译dpdk

[root@vrouter1 dpdk-stable-17.11.]# make config T=$RTE_TARGET O=$RTE_TARGET
Configuration done using x86_64-native-linuxapp-gcc
[root@vrouter1 dpdk-stable-17.11.]# cd x86_64-native-linuxapp-gcc/
[root@vrouter1 x86_64-native-linuxapp-gcc]# make

2. 编译ovs

[root@vrouter1 openvswitch-2.9.]# ./boot.sh
[root@vrouter1 openvswitch-2.9.]# ./configure --with-dpdk=$RTE_SDK/$RTE_TARGET
[root@vrouter1 openvswitch-2.9.]# make
[root@vrouter1 openvswitch-2.9.]# make install

3. 运行

[root@vrouter1 ovs-dpdk]# cat ovs.sh
export PATH=$PATH:/usr/local/share/openvswitch/scripts
export DB_SOCK=/usr/local/var/run/openvswitch/db.sock

ovs-ctl --no-ovs-vswitchd start
#ovs-vsctl --no-wait set Open_vSwitch . other_config:dpdk-init=true
#ovs-ctl --no-ovsdb-server --db-sock="$DB_SOCK" start
ovs-ctl --no-ovsdb-server start
[root@vrouter1 ovs-dpdk]# 

4. 配置

[root@vrouter1 Datapath]# dpdk-devbind -b vfio-pci ::00.0
[root@vrouter1 ovs-dpdk]# ovs-vsctl add-br br-phy -- set bridge br-phy datapath_type=netdev
[root@vrouter1 ovs-dpdk]# ovs-vsctl add-port br-phy dpdk-p0 -- set Interface dpdk-p0 type=dpdk options:dpdk-devargs=::00.0

[root@vrouter1 ovs-dpdk]# ovs-vsctl add-br br0 -- set bridge br0 datapath_type=netdev
[root@vrouter1 ovs-dpdk]# ovs-vsctl add-port br0 vxlan0 -- set Interface vxlan0 type=vxlan options:remote_ip=10.0.0.163 options:local_ip=10.0.0.161 options:in_key=flow options:out_key=flow

 #>ovs-appctl ovs/route/add 10.0.0.163/ br-phy
[root@vrouter1 ~]# ip a add 10.0.0.161/ dev br-phy

5. 绑定dpdk core

[root@vrouter1 ~]# ovs-vsctl set Interface dpdk-p0 options:n_rxq=
[root@vrouter1 ~]# ovs-vsctl set Open_vSwitch . other_config:pmd-cpu-mask=0x154

6. 不丢包了。

7. vhost user client

7.1 启动vhost iommu

[root@vrouter1 ~]# ovs-vsctl set Open_vSwitch . other_config:vhost-iommu-support=true

8 添加vhostuserclient网卡

[root@vrouter1 ~]# ovs-vsctl add-port br0 vhost0 -- set Interface vhost0 type=dpdkvhostuserclient options:vhost-server-path=/tmp/nlb_vm0.sock
[root@vrouter1 ~]# ovs-vsctl add-port br0 vhost1 -- set Interface vhost1 type=dpdkvhostuserclient options:vhost-server-path=/tmp/nlb_vm1.sock

9 加流表

[root@vrouter1 ~]# ovs-appctl dpif/show
netdev@ovs-netdev: hit: missed:
    br-phy:
        br-phy /: (tap)
        dpdk-p0 /: (dpdk: configured_rx_queues=, configured_rxq_descriptors=, configured_tx_queues=, configured_txq_descriptors=, lsc_interrupt_mode=false, mtu=, requested_rx_queues=, requested_rxq_descriptors=, requested_tx_queues=, requested_txq_descriptors=, rx_csum_offload=true)
    br0:
        br0 /: (tap)
        vhost0 /: (dpdkvhostuserclient: configured_rx_queues=, configured_tx_queues=, mtu=, requested_rx_queues=, requested_tx_queues=)
        vhost1 /: (dpdkvhostuserclient: configured_rx_queues=, configured_tx_queues=, mtu=, requested_rx_queues=, requested_tx_queues=)
        vxlan0 /: (vxlan: key=flow, local_ip=10.0.0.161, remote_ip=10.0.0.163)

[root@vrouter1 ~]# ovs-ofctl add-flow br0 "cookie=0x1111,table=0, priority=100, tun_id=200,dl_dst=00:00:00:11:22:41,nw_dst=192.168.77.161,actions=move:NXM_NX_TUN_ID[0..23]->NXM_NX_REG0[0..23],resubmit(,1)"
[root@vrouter1 ~]# ovs-ofctl add-flow br0 "cookie=0x1111,table=0, priority=100, tun_id=200,dl_dst=00:00:00:11:22:41,nw_dst=192.168.77.161,actions=move:NXM_NX_TUN_ID[0..23]->NXM_NX_REG0[0..23],resubmit(,1)"

10， 查看队列与core的mapping关系

[root@vrouter1 ~]# ovs-appctl dpif-netdev/pmd-rxq-show
pmd thread numa_id  core_id :
    isolated : false
    port: dpdk-p0             queue-id:      pmd usage:  %
    port: vhost1              queue-id:      pmd usage:   %
    port: vhost1              queue-id:      pmd usage:   %
pmd thread numa_id  core_id :
    isolated : false
    port: dpdk-p0             queue-id:      pmd usage:  %
    port: vhost1              queue-id:      pmd usage:   %
pmd thread numa_id  core_id :
    isolated : false
    port: dpdk-p0             queue-id:      pmd usage:  %
    port: vhost0              queue-id:      pmd usage:   %
pmd thread numa_id  core_id :
    isolated : false
    port: vhost0              queue-id:      pmd usage:   %
    port: vhost0              queue-id:      pmd usage:   %
pmd thread numa_id  core_id :
    isolated : false
    port: dpdk-p0             queue-id:      pmd usage:  %
    port: vhost1              queue-id:      pmd usage:   %
    port: vhost1              queue-id:      pmd usage:   %
pmd thread numa_id  core_id :
    isolated : false
    port: vhost0              queue-id:      pmd usage:   %
    port: vhost0              queue-id:      pmd usage:   %

总结：

丢包只要是丢在了内核，因为top的时候看绑定core的cpu占用，可以看见大约80%的占用是sys，20%是user

正常的情况是包都在dpdk用户态走，所有应该100%是user。

理解了路由，流表，vxlan的原理之后，可以逐个梳理，保证包不会被流转进内核，便可以消除丢包。

总之，原因就是由于流表路由的设置问题使数据包被转发入了内核。